From 82212e5672111640090a30c8ba0fc6565ab65b57 Mon Sep 17 00:00:00 2001 From: Aditya R Date: Wed, 9 Feb 2022 16:57:15 +0530 Subject: [PATCH] docker, blobCache: try to reuse compressed blobs from all known compressed digests It seems we try to reuse blobs only for the specified registry, however we can have valid known compressed digests across registry as well following pr attempts to use that by doing following steps. * Move all known blobs into common data scope i.e `all` instead of storing them in seperate registry. * Increase the sample set of potential blob reuse to all known compressed digests , also involving the one which do not belong to current registry. * If a blob is found match it against the registry where we are attempting to push. If blob is already there consider it a `CACHE HIT!` and reply skipping blob, since its already there. * Remove all images `buildah rmi --all` // needed so all new blobs can be tagged again in common bucket * Remove any previous `blob-info-cache` by ```console rm /home//.local/share/containers/cache/blob-info-cache-v1.boltdb ``` ```console $ skopeo copy docker://registry.fedoraproject.org/fedora-minimal docker://quay.io/fl/test:some-tag $ buildah pull registry.fedoraproject.org/fedora-minimal $ buildah tag registry.fedoraproject.org/fedora-minimal quay.io/fl/test $ buildah push quay.io/fl/test ``` ```console Getting image source signatures Copying blob a3497ca15bbf skipped: already exists Copying config f7e02de757 done Writing manifest to image destination Storing signatures ``` Signed-off-by: Aditya R --- docker/cache.go | 5 +++-- docker/docker_image_dest.go | 11 ++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docker/cache.go b/docker/cache.go index 728d32d170..048fd18b6b 100644 --- a/docker/cache.go +++ b/docker/cache.go @@ -7,8 +7,9 @@ import ( // bicTransportScope returns a BICTransportScope appropriate for ref. func bicTransportScope(ref dockerReference) types.BICTransportScope { - // Blobs can be reused across the whole registry. - return types.BICTransportScope{Opaque: reference.Domain(ref.ref)} + // Blobs can be reused across multiple registries + // therefore bucket all blobs in same scope + return types.BICTransportScope{Opaque: "all"} } // newBICLocationReference returns a BICLocationReference appropriate for ref. diff --git a/docker/docker_image_dest.go b/docker/docker_image_dest.go index e7af8f93d5..5836b8368c 100644 --- a/docker/docker_image_dest.go +++ b/docker/docker_image_dest.go @@ -336,6 +336,7 @@ func (d *dockerImageDestination) TryReusingBlob(ctx context.Context, info types. bic := blobinfocache.FromBlobInfoCache(cache) candidates := bic.CandidateLocations2(d.ref.Transport(), bicTransportScope(d.ref), info.Digest, canSubstitute) for _, candidate := range candidates { + crossRegistry := false candidateRepo, err := parseBICLocationReference(candidate.Location) if err != nil { logrus.Debugf("Error parsing BlobInfoCache location reference: %s", err) @@ -350,7 +351,8 @@ func (d *dockerImageDestination) TryReusingBlob(ctx context.Context, info types. // Sanity checks: if reference.Domain(candidateRepo) != reference.Domain(d.ref.ref) { logrus.Debugf("... Internal error: domain %s does not match destination %s", reference.Domain(candidateRepo), reference.Domain(d.ref.ref)) - continue + // most likely this blob is on a different registry + crossRegistry = true } if candidateRepo.Name() == d.ref.ref.Name() && candidate.Digest == info.Digest { logrus.Debug("... Already tried the primary destination") @@ -361,6 +363,13 @@ func (d *dockerImageDestination) TryReusingBlob(ctx context.Context, info types. // Checking candidateRepo, and mounting from it, requires an // expanded token scope. + + if crossRegistry { + // found following blob in different registry but we need to check blob presence against the registry + // where we are planning to push, hence switch back the candidate repo to the one where we are planning to push + candidateRepo, _ = parseBICLocationReference(types.BICLocationReference{Opaque: string(d.ref.ref.Name())}) + } + extraScope := &authScope{ remoteName: reference.Path(candidateRepo), actions: "pull",