Skip to content

Commit

Permalink
fixup! Add optimize mode for sync tasks
Browse files Browse the repository at this point in the history
[noissue]
  • Loading branch information
quba42 committed Aug 18, 2022
1 parent cfb896c commit b93cde7
Show file tree
Hide file tree
Showing 5 changed files with 35 additions and 19 deletions.
10 changes: 9 additions & 1 deletion pulp_deb/app/serializers/repository_serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,15 @@ class AptRepositorySyncURLSerializer(RepositorySyncURLSerializer):
"""

optimize = serializers.BooleanField(
help_text=_("Whether or not to optimize sync."), required=False, default=True
help_text=_(
"Using optimize sync, will skip the processing of metadata if the checksum has not "
"changed since the last sync. This greately improves re-sync performance in such "
"situations. If you feel the sync is missing something that has changed about the "
"remote repository you are syncing, try using optimize=False for a full re-sync. "
"Consider opening an issue on why we should not optimize in your use case."
),
required=False,
default=True,
)


Expand Down
34 changes: 21 additions & 13 deletions pulp_deb/app/tasks/synchronizing.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ def synchronize(remote_pk, repository_pk, mirror, optimize):
if not remote.url:
raise ValueError(_("A remote must have a url specified to synchronize."))

first_stage = DebFirstStage(remote, optimize, previous_repo_version)
first_stage = DebFirstStage(remote, optimize, mirror, previous_repo_version)
DebDeclarativeVersion(first_stage, repository, mirror=mirror).create()


Expand Down Expand Up @@ -501,7 +501,7 @@ class DebFirstStage(Stage):
The first stage of a pulp_deb sync pipeline.
"""

def __init__(self, remote, optimize, previous_repo_version, *args, **kwargs):
def __init__(self, remote, optimize, mirror, previous_repo_version, *args, **kwargs):
"""
The first stage of a pulp_deb sync pipeline.
Expand All @@ -517,9 +517,15 @@ def __init__(self, remote, optimize, previous_repo_version, *args, **kwargs):
self.previous_sync_info = defaultdict(dict, previous_repo_version.info)
self.sync_info = defaultdict()
self.sync_info["remote_options"] = self._gen_remote_options()
self.sync_info["sync_options"] = {
"optimize": optimize,
"mirror": mirror,
}
self.parsed_url = urlparse(remote.url)
self.unchanged_remote = (
self.sync_options_unchanged = (
self.previous_sync_info["remote_options"] == self.sync_info["remote_options"]
and self.previous_sync_info["sync_options"]["mirror"]
== self.sync_info["sync_options"]["mirror"]
)

async def run(self):
Expand Down Expand Up @@ -556,6 +562,11 @@ def _gen_remote_options(self):
"components": self.remote.components,
"architectures": self.remote.architectures,
"policy": self.remote.policy,
"sync_sources": self.remote.sync_sources,
"sync_udebs": self.remote.sync_udebs,
"sync_installer": self.remote.sync_installer,
"gpgkey": self.remote.gpgkey,
"ignore_missing_package_indices": self.remote.ignore_missing_package_indices,
}

async def _handle_distribution(self, distribution):
Expand All @@ -575,7 +586,7 @@ async def _handle_distribution(self, distribution):
release_file = await self._create_unit(release_file_dc)
if release_file is None:
return
if self.optimize and self.unchanged_remote:
if self.optimize and self.sync_options_unchanged:
previous_release_file = await _get_previous_release_file(
self.previous_repo_version, distribution
)
Expand Down Expand Up @@ -771,14 +782,6 @@ async def _handle_flat_repo(self, file_references, release_file, release):
# Await all tasks
await asyncio.gather(*pending_tasks)

def _nested_defaultdict(self, existing=None, **kwargs):
if existing is None:
existing = {}
if not isinstance(existing, dict):
return existing
existing = {key: self._nested_defaultdict(val) for key, val in existing.items()}
return defaultdict(self._nested_defaultdict, existing, **kwargs)

async def _handle_package_index(
self,
release_file,
Expand Down Expand Up @@ -845,7 +848,7 @@ async def _handle_package_index(
else:
raise NoPackageIndexFile(relative_dir=package_index_dir)

if self.optimize and self.unchanged_remote:
if self.optimize and self.sync_options_unchanged:
previous_package_index = await _get_previous_package_index(
self.previous_repo_version, relative_path
)
Expand Down Expand Up @@ -1084,6 +1087,11 @@ def _readd_previous_package_indices(previous_version, new_version, distribution)
PackageIndex.objects.filter(relative_path__contains=distribution)
)
)
new_version.add_content(
previous_version.get_content(
InstallerFileIndex.objects.filter(relative_path__contains=distribution)
)
)


@sync_to_async
Expand Down
4 changes: 2 additions & 2 deletions pulp_deb/tests/functional/api/test_pulpexport.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

from pulpcore.client.pulp_deb import (
RepositoriesAptApi,
RepositorySyncURL,
AptRepositorySyncURL,
RemotesAptApi,
)

Expand All @@ -49,7 +49,7 @@ def _setup_repositories(cls):
# give it a remote and sync it
body = gen_deb_remote()
remote = cls.remote_api.create(body)
repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href)
sync_response = cls.repo_api.sync(a_repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
# remember it
Expand Down
4 changes: 2 additions & 2 deletions pulp_deb/tests/functional/api/test_pulpimport.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
)
from pulpcore.client.pulp_deb import (
RepositoriesAptApi,
RepositorySyncURL,
AptRepositorySyncURL,
RemotesAptApi,
)

Expand Down Expand Up @@ -56,7 +56,7 @@ def _setup_repositories(cls, url=None):
body = gen_deb_remote()
remote = cls.remote_api.create(body)

repository_sync_data = RepositorySyncURL(remote=remote.pulp_href)
repository_sync_data = AptRepositorySyncURL(remote=remote.pulp_href)
sync_response = cls.repo_api.sync(export_repo.pulp_href, repository_sync_data)
monitor_task(sync_response.task)
# remember it
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
pulpcore>=3.20,<3.25
pulpcore>=3.21.0.dev,<3.25
python-debian>=0.1.44,<0.2.0

0 comments on commit b93cde7

Please sign in to comment.