Skip to content

Commit

Permalink
Optimize ACS sync stage
Browse files Browse the repository at this point in the history
Instead of performing a gigantic AND-ed OR clause query, break up the
list of remote artifacts by checksum type and perform one IN query per
type of checksum, which ought to be easily indexable.

[noissue]
  • Loading branch information
dralley committed Aug 12, 2024
1 parent 5867258 commit 7d24ec4
Showing 1 changed file with 10 additions and 13 deletions.
23 changes: 10 additions & 13 deletions pulpcore/plugin/stages/artifact_stages.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from aiofiles import os as aos
from asgiref.sync import sync_to_async
from django.db.models import Prefetch, prefetch_related_objects, Q
from django.db.models import Prefetch, prefetch_related_objects

from pulpcore.plugin.exceptions import UnsupportedDigestValidationError
from pulpcore.plugin.models import (
Expand Down Expand Up @@ -485,21 +485,18 @@ async def run(self):
getattr(d_artifact.artifact, cks_type)
)

batch_query = Q()
existing_ras_dict = dict()
for checksum_type in batch_checksums.keys():
batch_query.add(
Q(**{f"{checksum_type}__in": batch_checksums[checksum_type]}), Q.OR
existing_ras = (
RemoteArtifact.objects.acs()
.filter(**{f"{checksum_type}__in": batch_checksums[checksum_type]})
.only("url", checksum_type, "remote")
.select_related("remote")
)

existing_ras = (
RemoteArtifact.objects.acs().filter(batch_query).select_related("remote")
)
existing_ras_dict = dict()
async for ra in existing_ras.aiterator():
for c_type in Artifact.COMMON_DIGEST_FIELDS:
checksum = getattr(ra, c_type) # todo: this used to be async-protected
async for ra in existing_ras.aiterator():
checksum = getattr(ra, checksum_type)
# pick the first occurence of RA from ACS
if checksum and checksum not in existing_ras_dict:
if checksum not in existing_ras_dict:
existing_ras_dict[checksum] = {
"remote": ra.remote,
"url": ra.url,
Expand Down

0 comments on commit 7d24ec4

Please sign in to comment.