From 14ad46ed54596b2adbb4a97d6c686d4a834f0934 Mon Sep 17 00:00:00 2001 From: Tobias Grigo Date: Wed, 29 Jun 2022 00:00:59 +0200 Subject: [PATCH] Add optimize mode for sync tasks closes #564 Co-Authored-By: Quirin Pamp This will change the default behaviour of the sync to use optimize unless it is explicitly set to False. --- CHANGES/564.feature | 1 + pulp_deb/app/serializers/__init__.py | 6 +- .../app/serializers/repository_serializers.py | 24 +- pulp_deb/app/tasks/synchronizing.py | 115 ++++++++- pulp_deb/app/viewsets/repository.py | 13 +- .../functional/api/test_download_content.py | 4 +- .../functional/api/test_download_policies.py | 12 +- pulp_deb/tests/functional/api/test_publish.py | 4 +- .../api/test_publish_complex_dists.py | 4 +- .../api/test_publish_empty_repository.py | 4 +- .../api/test_publish_flat_repo_format.py | 4 +- .../tests/functional/api/test_pulpexport.py | 4 +- .../tests/functional/api/test_pulpimport.py | 4 +- pulp_deb/tests/functional/api/test_sync.py | 8 +- .../api/test_sync_missing_architecture.py | 4 +- .../functional/api/test_sync_optimize.py | 231 ++++++++++++++++++ pulp_deb/tests/functional/constants.py | 10 + requirements.txt | 2 +- 18 files changed, 411 insertions(+), 43 deletions(-) create mode 100644 CHANGES/564.feature create mode 100644 pulp_deb/tests/functional/api/test_sync_optimize.py diff --git a/CHANGES/564.feature b/CHANGES/564.feature new file mode 100644 index 000000000..f0ea9a2de --- /dev/null +++ b/CHANGES/564.feature @@ -0,0 +1 @@ +Added the option to synchronize repositories using an optimized mode (enabled by default). diff --git a/pulp_deb/app/serializers/__init__.py b/pulp_deb/app/serializers/__init__.py index b792264f3..83cf769d9 100644 --- a/pulp_deb/app/serializers/__init__.py +++ b/pulp_deb/app/serializers/__init__.py @@ -23,4 +23,8 @@ from .remote_serializers import AptRemoteSerializer -from .repository_serializers import AptRepositorySerializer, CopySerializer +from .repository_serializers import ( + AptRepositorySerializer, + AptRepositorySyncURLSerializer, + CopySerializer, +) diff --git a/pulp_deb/app/serializers/repository_serializers.py b/pulp_deb/app/serializers/repository_serializers.py index 1b08daa18..ab9211fd0 100644 --- a/pulp_deb/app/serializers/repository_serializers.py +++ b/pulp_deb/app/serializers/repository_serializers.py @@ -1,5 +1,9 @@ from gettext import gettext as _ -from pulpcore.plugin.serializers import RepositorySerializer, validate_unknown_fields +from pulpcore.plugin.serializers import ( + RepositorySerializer, + RepositorySyncURLSerializer, + validate_unknown_fields, +) from pulp_deb.app.models import AptRepository @@ -18,6 +22,24 @@ class Meta: model = AptRepository +class AptRepositorySyncURLSerializer(RepositorySyncURLSerializer): + """ + A Serializer for AptRepository Sync. + """ + + optimize = serializers.BooleanField( + help_text=_( + "Using optimize sync, will skip the processing of metadata if the checksum has not " + "changed since the last sync. This greately improves re-sync performance in such " + "situations. If you feel the sync is missing something that has changed about the " + "remote repository you are syncing, try using optimize=False for a full re-sync. " + "Consider opening an issue on why we should not optimize in your use case." + ), + required=False, + default=True, + ) + + class CopySerializer(serializers.Serializer): """ A serializer for Content Copy API. diff --git a/pulp_deb/app/tasks/synchronizing.py b/pulp_deb/app/tasks/synchronizing.py index 8a7a74f39..06ff4258f 100644 --- a/pulp_deb/app/tasks/synchronizing.py +++ b/pulp_deb/app/tasks/synchronizing.py @@ -22,7 +22,6 @@ Artifact, ProgressReport, Remote, - Repository, ) from pulpcore.plugin.stages import ( @@ -51,6 +50,7 @@ PackageReleaseComponent, InstallerPackage, AptRemote, + AptRepository, ) from pulp_deb.app.serializers import ( @@ -137,7 +137,7 @@ def __init__(self, release_file_path, unknown_value, *args, **kwargs): pass -def synchronize(remote_pk, repository_pk, mirror): +def synchronize(remote_pk, repository_pk, mirror, optimize): """ Sync content from the remote repository. @@ -147,18 +147,20 @@ def synchronize(remote_pk, repository_pk, mirror): remote_pk (str): The remote PK. repository_pk (str): The repository PK. mirror (bool): True for mirror mode, False for additive. + optimize (bool): Optimize mode. Raises: ValueError: If the remote does not specify a URL to sync """ remote = AptRemote.objects.get(pk=remote_pk) - repository = Repository.objects.get(pk=repository_pk) + repository = AptRepository.objects.get(pk=repository_pk) + previous_repo_version = repository.latest_version() if not remote.url: raise ValueError(_("A remote must have a url specified to synchronize.")) - first_stage = DebFirstStage(remote) + first_stage = DebFirstStage(remote, optimize, mirror, previous_repo_version) DebDeclarativeVersion(first_stage, repository, mirror=mirror).create() @@ -209,6 +211,7 @@ def pipeline_stages(self, new_version): list: List of :class:`~pulpcore.plugin.stages.Stage` instances """ + self.first_stage.new_version = new_version pipeline = [ self.first_stage, QueryExistingArtifacts(), @@ -498,17 +501,32 @@ class DebFirstStage(Stage): The first stage of a pulp_deb sync pipeline. """ - def __init__(self, remote, *args, **kwargs): + def __init__(self, remote, optimize, mirror, previous_repo_version, *args, **kwargs): """ The first stage of a pulp_deb sync pipeline. Args: - remote (FileRemote): The remote data to be used when syncing - + remote (AptRemote): The remote data to be used when syncing + optimize (Boolean): If optimize mode is enabled or not + previous_repo_version repository (RepositoryVersion): The previous RepositoryVersion. """ super().__init__(*args, **kwargs) self.remote = remote + self.optimize = optimize + self.previous_repo_version = previous_repo_version + self.previous_sync_info = defaultdict(dict, previous_repo_version.info) + self.sync_info = defaultdict() + self.sync_info["remote_options"] = self._gen_remote_options() + self.sync_info["sync_options"] = { + "optimize": optimize, + "mirror": mirror, + } self.parsed_url = urlparse(remote.url) + self.sync_options_unchanged = ( + self.previous_sync_info["remote_options"] == self.sync_info["remote_options"] + and self.previous_sync_info["sync_options"]["mirror"] + == self.sync_info["sync_options"]["mirror"] + ) async def run(self): """ @@ -521,6 +539,8 @@ async def run(self): *[self._handle_distribution(dist) for dist in self.remote.distributions.split()] ) + self.new_version.info = self.sync_info + async def _create_unit(self, d_content): await self.put(d_content) return await d_content.resolution() @@ -536,6 +556,19 @@ def _to_d_artifact(self, relative_path, data=None): deferred_download=False, ) + def _gen_remote_options(self): + return { + "distributions": self.remote.distributions, + "components": self.remote.components, + "architectures": self.remote.architectures, + "policy": self.remote.policy, + "sync_sources": self.remote.sync_sources, + "sync_udebs": self.remote.sync_udebs, + "sync_installer": self.remote.sync_installer, + "gpgkey": self.remote.gpgkey, + "ignore_missing_package_indices": self.remote.ignore_missing_package_indices, + } + async def _handle_distribution(self, distribution): log.info(_('Downloading Release file for distribution: "{}"').format(distribution)) # Create release_file @@ -553,7 +586,23 @@ async def _handle_distribution(self, distribution): release_file = await self._create_unit(release_file_dc) if release_file is None: return - # Create release object + if self.optimize and self.sync_options_unchanged: + previous_release_file = await _get_previous_release_file( + self.previous_repo_version, distribution + ) + if previous_release_file.artifact_set_sha256 == release_file.artifact_set_sha256: + await _readd_previous_package_indices( + self.previous_repo_version, self.new_version, distribution + ) + message = 'ReleaseFile has not changed for distribution="{}". Skipping.' + log.info(_(message).format(distribution)) + async with ProgressReport( + message="Skipping ReleaseFile sync (no change from previous sync)", + code="sync.release_file.was_skipped", + ) as pb: + await pb.aincrement() + return + release_unit = Release( codename=release_file.codename, suite=release_file.suite, distribution=distribution ) @@ -799,6 +848,20 @@ async def _handle_package_index( else: raise NoPackageIndexFile(relative_dir=package_index_dir) + if self.optimize and self.sync_options_unchanged: + previous_package_index = await _get_previous_package_index( + self.previous_repo_version, relative_path + ) + if previous_package_index.artifact_set_sha256 == package_index.artifact_set_sha256: + message = 'PackageIndex has not changed for relative_path="{}". Skipped.' + log.info(_(message).format(relative_path)) + async with ProgressReport( + message="Skipping PackageIndex processing (no change from previous sync)", + code="sync.package_index.was_skipped", + ) as pb: + await pb.aincrement() + return + # Interpret policy to download Artifacts or not deferred_download = self.remote.policy != Remote.IMMEDIATE # parse package_index @@ -1017,6 +1080,42 @@ async def _handle_translation_files(self, release_file, release_component, file_ ) +@sync_to_async +def _readd_previous_package_indices(previous_version, new_version, distribution): + new_version.add_content( + previous_version.get_content( + PackageIndex.objects.filter(relative_path__contains=distribution) + ) + ) + new_version.add_content( + previous_version.get_content( + InstallerFileIndex.objects.filter(relative_path__contains=distribution) + ) + ) + + +@sync_to_async +def _get_previous_release_file(previous_version, distribution): + previous_release_file_qs = previous_version.get_content( + ReleaseFile.objects.filter(distribution=distribution) + ) + if previous_release_file_qs.count() > 1: + message = "Previous ReleaseFile count: {}. There should only be one." + raise Exception(message.format(previous_release_file_qs.count())) + return previous_release_file_qs.first() + + +@sync_to_async +def _get_previous_package_index(previous_version, relative_path): + previous_package_index_qs = previous_version.get_content( + PackageIndex.objects.filter(relative_path=relative_path) + ) + if previous_package_index_qs.count() > 1: + message = "Previous PackageIndex count: {}. There should only be one." + raise Exception(message.format(previous_package_index_qs.count())) + return previous_package_index_qs.first() + + @sync_to_async def _get_main_artifact_blocking(content): return content.main_artifact diff --git a/pulp_deb/app/viewsets/repository.py b/pulp_deb/app/viewsets/repository.py index c18b86a26..2d62c809f 100644 --- a/pulp_deb/app/viewsets/repository.py +++ b/pulp_deb/app/viewsets/repository.py @@ -5,11 +5,10 @@ from rest_framework import viewsets from rest_framework.serializers import ValidationError as DRFValidationError +from pulp_deb.app.serializers import AptRepositorySyncURLSerializer + from pulpcore.plugin.actions import ModifyRepositoryActionMixin -from pulpcore.plugin.serializers import ( - AsyncOperationResponseSerializer, - RepositorySyncURLSerializer, -) +from pulpcore.plugin.serializers import AsyncOperationResponseSerializer from pulpcore.plugin.models import RepositoryVersion from pulpcore.plugin.tasking import dispatch from pulpcore.plugin.viewsets import ( @@ -42,13 +41,13 @@ class AptRepositoryViewSet(RepositoryViewSet, ModifyRepositoryActionMixin): summary="Sync from remote", responses={202: AsyncOperationResponseSerializer}, ) - @action(detail=True, methods=["post"], serializer_class=RepositorySyncURLSerializer) + @action(detail=True, methods=["post"], serializer_class=AptRepositorySyncURLSerializer) def sync(self, request, pk): """ Dispatches a sync task. """ repository = self.get_object() - serializer = RepositorySyncURLSerializer( + serializer = AptRepositorySyncURLSerializer( data=request.data, context={"request": request, "repository_pk": pk} ) @@ -56,6 +55,7 @@ def sync(self, request, pk): serializer.is_valid(raise_exception=True) remote = serializer.validated_data.get("remote", repository.remote) mirror = serializer.validated_data.get("mirror", True) + optimize = serializer.validated_data.get("optimize") result = dispatch( func=tasks.synchronize, @@ -65,6 +65,7 @@ def sync(self, request, pk): "remote_pk": remote.pk, "repository_pk": repository.pk, "mirror": mirror, + "optimize": optimize, }, ) return OperationPostponedResponse(result, request) diff --git a/pulp_deb/tests/functional/api/test_download_content.py b/pulp_deb/tests/functional/api/test_download_content.py index 0212e73f3..4540debc1 100644 --- a/pulp_deb/tests/functional/api/test_download_content.py +++ b/pulp_deb/tests/functional/api/test_download_content.py @@ -15,7 +15,7 @@ get_deb_verbatim_content_unit_paths, ) -from pulpcore.client.pulp_deb import RepositorySyncURL, DebAptPublication, DebVerbatimPublication +from pulpcore.client.pulp_deb import AptRepositorySyncURL, DebAptPublication, DebVerbatimPublication @pytest.mark.parametrize( @@ -125,7 +125,7 @@ def gen_repository_with_synced_remote( def _gen_repository_with_synced_remote(): repo = gen_object_with_cleanup(apt_repository_api, gen_repo()) remote = gen_object_with_cleanup(apt_remote_api, gen_deb_remote()) - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = apt_repository_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) return apt_repository_api.read(repo.pulp_href) diff --git a/pulp_deb/tests/functional/api/test_download_policies.py b/pulp_deb/tests/functional/api/test_download_policies.py index 980bd18db..ccc4276a1 100644 --- a/pulp_deb/tests/functional/api/test_download_policies.py +++ b/pulp_deb/tests/functional/api/test_download_policies.py @@ -27,7 +27,7 @@ from pulpcore.client.pulp_deb import ( DebAptPublication, - RepositorySyncURL, + AptRepositorySyncURL, ) @@ -98,7 +98,7 @@ def do_sync(self, download_policy): # Sync the repository. self.assertEqual(repo.latest_version_href, f"{repo.pulp_href}versions/0/") - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) repo = repo_api.read(repo.pulp_href) @@ -129,7 +129,7 @@ def do_publish(self, download_policy): remote = remote_api.create(body) self.addCleanup(remote_api.delete, remote.pulp_href) - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) repo = repo_api.read(repo.pulp_href) @@ -188,7 +188,7 @@ def do_test(self, policy): # Sync the repository. self.assertEqual(repo.latest_version_href, f"{repo.pulp_href}versions/0/") - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) repo = repo_api.read(repo.pulp_href) @@ -245,7 +245,7 @@ def do_test(self, policy): self.addCleanup(remote_api.delete, remote.pulp_href) # Sync the repository using a lazy download policy - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) artifacts = artifact_api.list() @@ -258,7 +258,7 @@ def do_test(self, policy): self.assertEqual(remote.policy, "immediate") # Sync using immediate download policy - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) diff --git a/pulp_deb/tests/functional/api/test_publish.py b/pulp_deb/tests/functional/api/test_publish.py index 77faca430..f3bb1bdf0 100644 --- a/pulp_deb/tests/functional/api/test_publish.py +++ b/pulp_deb/tests/functional/api/test_publish.py @@ -22,7 +22,7 @@ ) from pulpcore.client.pulp_deb import ( - RepositorySyncURL, + AptRepositorySyncURL, DebAptPublication, DebVerbatimPublication, ) @@ -70,7 +70,7 @@ def test_all(self): repo = repo_api.create(gen_repo()) self.addCleanup(repo_api.delete, repo.pulp_href) - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) diff --git a/pulp_deb/tests/functional/api/test_publish_complex_dists.py b/pulp_deb/tests/functional/api/test_publish_complex_dists.py index b21c4edcd..d2a65b4d1 100644 --- a/pulp_deb/tests/functional/api/test_publish_complex_dists.py +++ b/pulp_deb/tests/functional/api/test_publish_complex_dists.py @@ -33,7 +33,7 @@ ) from pulpcore.client.pulp_deb import ( - RepositorySyncURL, + AptRepositorySyncURL, DebAptPublication, ) @@ -104,7 +104,7 @@ def do_publish(self, expected_values): self.addCleanup(deb_remote_api.delete, remote.pulp_href) # Sync the repository: - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = deb_repository_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) repo = deb_repository_api.read(repo.pulp_href) diff --git a/pulp_deb/tests/functional/api/test_publish_empty_repository.py b/pulp_deb/tests/functional/api/test_publish_empty_repository.py index f8d53d445..1de00694a 100644 --- a/pulp_deb/tests/functional/api/test_publish_empty_repository.py +++ b/pulp_deb/tests/functional/api/test_publish_empty_repository.py @@ -27,7 +27,7 @@ ) from pulpcore.client.pulp_deb import ( - RepositorySyncURL, + AptRepositorySyncURL, DebAptPublication, ) @@ -62,7 +62,7 @@ def test_publish(self): self.addCleanup(deb_remote_api.delete, remote.pulp_href) # Sync the repository: - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = deb_repository_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) repo = deb_repository_api.read(repo.pulp_href) diff --git a/pulp_deb/tests/functional/api/test_publish_flat_repo_format.py b/pulp_deb/tests/functional/api/test_publish_flat_repo_format.py index 8f8718494..bea5dd7e5 100644 --- a/pulp_deb/tests/functional/api/test_publish_flat_repo_format.py +++ b/pulp_deb/tests/functional/api/test_publish_flat_repo_format.py @@ -31,7 +31,7 @@ ) from pulpcore.client.pulp_deb import ( - RepositorySyncURL, + AptRepositorySyncURL, DebAptPublication, DebVerbatimPublication, ) @@ -160,7 +160,7 @@ def do_publish(self, expected_values, modus): self.addCleanup(deb_remote_api.delete, remote.pulp_href) # Sync the repository: - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = deb_repository_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) repo = deb_repository_api.read(repo.pulp_href) diff --git a/pulp_deb/tests/functional/api/test_pulpexport.py b/pulp_deb/tests/functional/api/test_pulpexport.py index 4e713a208..c86804bf6 100644 --- a/pulp_deb/tests/functional/api/test_pulpexport.py +++ b/pulp_deb/tests/functional/api/test_pulpexport.py @@ -26,7 +26,7 @@ from pulpcore.client.pulp_deb import ( RepositoriesAptApi, - RepositorySyncURL, + AptRepositorySyncURL, RemotesAptApi, ) @@ -49,7 +49,7 @@ def _setup_repositories(cls): # give it a remote and sync it body = gen_deb_remote() remote = cls.remote_api.create(body) - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = cls.repo_api.sync(a_repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) # remember it diff --git a/pulp_deb/tests/functional/api/test_pulpimport.py b/pulp_deb/tests/functional/api/test_pulpimport.py index 8a1bb7c47..c211fda11 100644 --- a/pulp_deb/tests/functional/api/test_pulpimport.py +++ b/pulp_deb/tests/functional/api/test_pulpimport.py @@ -27,7 +27,7 @@ ) from pulpcore.client.pulp_deb import ( RepositoriesAptApi, - RepositorySyncURL, + AptRepositorySyncURL, RemotesAptApi, ) @@ -56,7 +56,7 @@ def _setup_repositories(cls, url=None): body = gen_deb_remote() remote = cls.remote_api.create(body) - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = cls.repo_api.sync(export_repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) # remember it diff --git a/pulp_deb/tests/functional/api/test_sync.py b/pulp_deb/tests/functional/api/test_sync.py index 85c92efa5..ef4f1d662 100644 --- a/pulp_deb/tests/functional/api/test_sync.py +++ b/pulp_deb/tests/functional/api/test_sync.py @@ -25,7 +25,7 @@ deb_repository_api, ) -from pulpcore.client.pulp_deb import RepositorySyncURL +from pulpcore.client.pulp_deb import AptRepositorySyncURL class BasicSyncTestCase(unittest.TestCase): @@ -80,7 +80,7 @@ def do_sync(self, sync_udebs, fixture_summary): # Sync the repository. self.assertEqual(repo.latest_version_href, f"{repo.pulp_href}versions/0/") - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) repo = repo_api.read(repo.pulp_href) @@ -91,7 +91,7 @@ def do_sync(self, sync_udebs, fixture_summary): # Sync the repository again. latest_version_href = repo.latest_version_href - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) repo = repo_api.read(repo.pulp_href) @@ -177,6 +177,6 @@ def do_test(self, url=DEB_FIXTURE_URL, distribution=DEB_FIXTURE_DISTRIBUTIONS, * remote = remote_api.create(body) self.addCleanup(remote_api.delete, remote.pulp_href) - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = repo_api.sync(repo.pulp_href, repository_sync_data) return monitor_task(sync_response.task) diff --git a/pulp_deb/tests/functional/api/test_sync_missing_architecture.py b/pulp_deb/tests/functional/api/test_sync_missing_architecture.py index f85bc04ad..cf6953d0a 100644 --- a/pulp_deb/tests/functional/api/test_sync_missing_architecture.py +++ b/pulp_deb/tests/functional/api/test_sync_missing_architecture.py @@ -30,7 +30,7 @@ ) from pulpcore.client.pulp_deb import ( - RepositorySyncURL, + AptRepositorySyncURL, DebAptPublication, ) @@ -87,7 +87,7 @@ def do_publish(self, expected_values): self.addCleanup(deb_remote_api.delete, remote.pulp_href) # Sync the repository: - repository_sync_data = RepositorySyncURL(remote=remote.pulp_href) + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) sync_response = deb_repository_api.sync(repo.pulp_href, repository_sync_data) monitor_task(sync_response.task) repo = deb_repository_api.read(repo.pulp_href) diff --git a/pulp_deb/tests/functional/api/test_sync_optimize.py b/pulp_deb/tests/functional/api/test_sync_optimize.py new file mode 100644 index 000000000..7bc87c8d0 --- /dev/null +++ b/pulp_deb/tests/functional/api/test_sync_optimize.py @@ -0,0 +1,231 @@ +"""Tests that sync deb repositories in optimized mode.""" +from pulp_smash.pulp3.bindings import monitor_task +from pulp_smash.pulp3.utils import gen_repo +import pytest + +from pulp_deb.tests.functional.constants import ( + DEB_FIXTURE_ARCH, + DEB_FIXTURE_ARCH_UPDATE, + DEB_FIXTURE_COMPONENT, + DEB_FIXTURE_COMPONENT_UPDATE, + DEB_FIXTURE_SINGLE_DIST, + DEB_FIXTURE_URL, + DEB_FIXTURE_DISTRIBUTIONS, + DEB_FIXTURE_URL_UPDATE, + DEB_REPORT_CODE_SKIP_PACKAGE, + DEB_REPORT_CODE_SKIP_RELEASE, +) +from pulp_deb.tests.functional.utils import gen_deb_remote + +from pulpcore.client.pulp_deb import AptRepositorySyncURL + + +def test_sync_optimize_skip_unchanged_release_file( + gen_remote, + gen_repository, + get_repository_by_href, + is_latest_repo_version, + is_sync_skipped, + synchronize_and_get_task, +): + """Test whether synchronization is skipped when the ReleaseFile of a remote + has not been changed. + + 1. Create a repository and a remote. + 2. Assert that the latest RepositoryVersion is 0. + 3. Sync the repository. + 4. Assert that the latest RepositoryVersion is 1. + 5. Assert that the sync was not skipped. + 6. Sync the repository again. + 7. Assert that the latest RepositoryVersion is still 1. + 8. Assert that this time the sync was skipped. + """ + repo = gen_repository() + remote = gen_remote() + assert is_latest_repo_version(repo, 0) + task = synchronize_and_get_task(remote, repo) + repo = get_repository_by_href(repo.pulp_href) + assert is_latest_repo_version(repo, 1) + assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_RELEASE) + task_skip = synchronize_and_get_task(remote, repo) + repo = get_repository_by_href(repo.pulp_href) + assert is_latest_repo_version(repo, 1) + assert is_sync_skipped(task_skip, DEB_REPORT_CODE_SKIP_RELEASE) + + +@pytest.mark.parametrize( + "remote_params, remote_diff_params", + [ + ( + [DEB_FIXTURE_URL, DEB_FIXTURE_SINGLE_DIST, DEB_FIXTURE_COMPONENT, None], + [DEB_FIXTURE_URL, DEB_FIXTURE_SINGLE_DIST, DEB_FIXTURE_COMPONENT_UPDATE, None], + ), + ( + [DEB_FIXTURE_URL, DEB_FIXTURE_SINGLE_DIST, None, DEB_FIXTURE_ARCH], + [DEB_FIXTURE_URL, DEB_FIXTURE_SINGLE_DIST, None, DEB_FIXTURE_ARCH_UPDATE], + ), + ( + [DEB_FIXTURE_URL, DEB_FIXTURE_SINGLE_DIST, DEB_FIXTURE_COMPONENT, None], + [DEB_FIXTURE_URL_UPDATE, DEB_FIXTURE_SINGLE_DIST, DEB_FIXTURE_COMPONENT_UPDATE, None], + ), + ], +) +def test_sync_optimize_no_skip_release_file( + gen_remote, + gen_repository, + get_repository_by_href, + is_latest_repo_version, + is_sync_skipped, + remote_params, + remote_diff_params, + synchronize_and_get_task, +): + """Test whether repository synchronizations have not been skipped for certain conditions. + The following cases are tested: + + * `Sync a repository with same ReleaseFile but updated Components.`_ + * `Sync a repository with same ReleaseFile but updated Architectures.`_ + * `Sync a repository with updated ReleaseFile and updated Components.`_ + + 1. Create a repository and a remote. + 2. Assert that the latest RepositoryVersion is 0. + 3. Synchronize the repository. + 4. Assert that the latest RepositoryVersion is 1. + 5. Assert that the synchronization was not skipped. + 6. Create a new remote with different conditions. + 7. Synchronize the repository with the new remote. + 8. Asser that the latest RepositoryVersion is 2. + 9. Assert that the synchronization was not skipped. + """ + repo = gen_repository() + remote = gen_remote( + url=remote_params[0], + distributions=remote_params[1], + components=remote_params[2], + architectures=remote_params[3], + ) + assert is_latest_repo_version(repo, 0) + task = synchronize_and_get_task(remote, repo) + repo = get_repository_by_href(repo.pulp_href) + assert is_latest_repo_version(repo, 1) + assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_RELEASE) + assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_PACKAGE) + remote_diff = gen_remote( + url=remote_diff_params[0], + distributions=remote_diff_params[1], + components=remote_diff_params[2], + architectures=remote_diff_params[3], + ) + task_diff = synchronize_and_get_task(remote_diff, repo) + repo = get_repository_by_href(repo.pulp_href) + assert is_latest_repo_version(repo, 2) + assert not is_sync_skipped(task_diff, DEB_REPORT_CODE_SKIP_RELEASE) + assert not is_sync_skipped(task_diff, DEB_REPORT_CODE_SKIP_PACKAGE) + + +def test_sync_optimize_skip_unchanged_package_index( + gen_remote, + gen_repository, + get_repository_by_href, + is_latest_repo_version, + is_sync_skipped, + synchronize_and_get_task, +): + """Test whether a repository synchronization of PackageIndex is skipped when + the package has not been changed. + + 1. Create a repository and a remote. + 2. Assert that the latest RepositoryVersion is 0. + 3. Sync the repository. + 4. Assert that the latest RepositoryVersion is 1. + 5. Assert that the sync was not skipped. + 6. Create a new remote with at least one updated package and one that remains the same. + 7. Sync the repository with the new remote. + 8. Assert that the latest RepositoryVersion is 2. + 9. Asssert that at least one PackageIndex was skipped. + """ + repo = gen_repository() + remote = gen_remote(url=DEB_FIXTURE_URL, distributions=DEB_FIXTURE_SINGLE_DIST) + assert is_latest_repo_version(repo, 0) + task = synchronize_and_get_task(remote, repo) + repo = get_repository_by_href(repo.pulp_href) + assert is_latest_repo_version(repo, 1) + assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_RELEASE) + assert not is_sync_skipped(task, DEB_REPORT_CODE_SKIP_PACKAGE) + remote_diff = gen_remote(url=DEB_FIXTURE_URL_UPDATE, distributions=DEB_FIXTURE_SINGLE_DIST) + task_diff = synchronize_and_get_task(remote_diff, repo) + repo = get_repository_by_href(repo.pulp_href) + assert is_latest_repo_version(repo, 2) + assert not is_sync_skipped(task_diff, DEB_REPORT_CODE_SKIP_RELEASE) + assert is_sync_skipped(task_diff, DEB_REPORT_CODE_SKIP_PACKAGE) + + +@pytest.fixture +def is_latest_repo_version(): + """Checks if a given version number is the latest RepositoryVersion of + a given repository. + """ + + def _is_latest_repo_version(repo, version): + return repo.latest_version_href == f"{repo.pulp_href}versions/{version}/" + + return _is_latest_repo_version + + +@pytest.fixture +def is_sync_skipped(): + """Checks if a given task has skipped the sync based of a given code.""" + + def _is_sync_skipped(task, code): + for report in task.progress_reports: + if report.code == code: + return True + return False + + return _is_sync_skipped + + +@pytest.fixture +def get_repository_by_href(apt_repository_api): + """Returns the repository of a given pulp_href.""" + + def _get_repository_by_href(href): + return apt_repository_api.read(href) + + return _get_repository_by_href + + +@pytest.fixture +def synchronize_and_get_task(apt_repository_api): + """Synchronizes a given repository with a given remote and + returns the monitored task. + """ + + def _synchronize_and_get_task(remote, repo): + repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href) + sync_response = apt_repository_api.sync(repo.pulp_href, repository_sync_data) + return monitor_task(sync_response.task) + + return _synchronize_and_get_task + + +@pytest.fixture +def gen_repository(apt_repository_api, gen_object_with_cleanup): + """Generates a semi-random repository with cleanup.""" + + def _gen_repository(): + return gen_object_with_cleanup(apt_repository_api, gen_repo()) + + return _gen_repository + + +@pytest.fixture +def gen_remote(apt_remote_api, gen_object_with_cleanup): + """Generates a remote with cleanup. Also allows for parameters to be set manually.""" + + def _gen_remote(url=DEB_FIXTURE_URL, distributions=DEB_FIXTURE_DISTRIBUTIONS, **kwargs): + return gen_object_with_cleanup( + apt_remote_api, gen_deb_remote(url=url, distributions=distributions, **kwargs) + ) + + return _gen_remote diff --git a/pulp_deb/tests/functional/constants.py b/pulp_deb/tests/functional/constants.py index d53cb2daf..6854a2d5d 100644 --- a/pulp_deb/tests/functional/constants.py +++ b/pulp_deb/tests/functional/constants.py @@ -51,7 +51,14 @@ def _clean_dict(d): DEB_SINGLE_REQUEST_UPLOAD_PATH = urljoin(BASE_PATH, "deb/upload/") DEB_FIXTURE_URL = urljoin(PULP_FIXTURES_BASE_URL, "debian/") +DEB_FIXTURE_URL_UPDATE = urljoin(PULP_FIXTURES_BASE_URL, "debian_update/") DEB_FIXTURE_DISTRIBUTIONS = "ragnarok nosuite" +DEB_FIXTURE_SINGLE_DIST = "ragnarok" +DEB_FIXTURE_MULTI_DIST = "ragnarok ginnungagap" +DEB_FIXTURE_COMPONENT = "asgard" +DEB_FIXTURE_COMPONENT_UPDATE = "jotunheimr" +DEB_FIXTURE_ARCH = "ppc64" +DEB_FIXTURE_ARCH_UPDATE = "armeb" DEB_FIXTURE_SUMMARY = _clean_dict( { @@ -85,6 +92,9 @@ def _clean_dict(d): DEB_FIXTURE_PACKAGE_COUNT = DEB_FIXTURE_SUMMARY.get(DEB_PACKAGE_NAME, 0) +DEB_REPORT_CODE_SKIP_RELEASE = "sync.release_file.was_skipped" +DEB_REPORT_CODE_SKIP_PACKAGE = "sync.package_index.was_skipped" + DEB_PACKAGE_RELPATH = "pool/asgard/o/odin/odin_1.0_ppc64.deb" DEB_PACKAGE_URL = urljoin(DEB_FIXTURE_URL, DEB_PACKAGE_RELPATH) DEB_GENERIC_CONTENT_RELPATH = "dists/ragnarok/asgard/binary-armeb/Release" diff --git a/requirements.txt b/requirements.txt index 077f1f215..0de05f607 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -pulpcore>=3.20,<3.25 +pulpcore>=3.21.0.dev,<3.25 python-debian>=0.1.44,<0.2.0