Merge pull request #1980 from giuseppe/use-computed-digest

copy: do not fail if digest mismatches

copy/single.go

@@ -20,6 +20,7 @@ import (
 	compressiontypes "github.com/containers/image/v5/pkg/compression/types"
 	"github.com/containers/image/v5/transports"
 	"github.com/containers/image/v5/types"
+	chunkedToc "github.com/containers/storage/pkg/chunked/toc"
 	digest "github.com/opencontainers/go-digest"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
@@ -694,6 +695,13 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
 			requiredCompression = ic.compressionFormat
 			originalCompression = srcInfo.CompressionAlgorithm
 		}
+
+		// Check if we have a chunked layer in storage that's based on that blob.  These layers are stored by their TOC digest.
+		tocDigest, err := chunkedToc.GetTOCDigest(srcInfo.Annotations)
+		if err != nil {
+			return types.BlobInfo{}, "", err
+		}
+
 		reused, reusedBlob, err := ic.c.dest.TryReusingBlobWithOptions(ctx, srcInfo, private.TryReusingBlobOptions{
 			Cache:               ic.c.blobInfoCache,
 			CanSubstitute:       canSubstitute,
@@ -702,6 +710,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
 			SrcRef:              srcRef,
 			RequiredCompression: requiredCompression,
 			OriginalCompression: originalCompression,
+			TOCDigest:           tocDigest,
 		})
 		if err != nil {
 			return types.BlobInfo{}, "", fmt.Errorf("trying to reuse blob %s at destination: %w", srcInfo.Digest, err)

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/BurntSushi/toml v1.3.2
 	github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01
 	github.com/containers/ocicrypt v1.1.9
-	github.com/containers/storage v1.51.1-0.20231129011200-e9f9f6605078
+	github.com/containers/storage v1.51.1-0.20231205203947-fe005407c7d5
 	github.com/cyberphone/json-canonicalization v0.0.0-20231011164504-785e29786b46
 	github.com/distribution/reference v0.5.0
 	github.com/docker/cli v24.0.7+incompatible

go.sum

@@ -51,8 +51,8 @@ github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYgle
 github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
 github.com/containers/ocicrypt v1.1.9 h1:2Csfba4jse85Raxk5HIyEk8OwZNjRvfkhEGijOjIdEM=
 github.com/containers/ocicrypt v1.1.9/go.mod h1:dTKx1918d8TDkxXvarscpNVY+lyPakPNFN4jwA9GBys=
-github.com/containers/storage v1.51.1-0.20231129011200-e9f9f6605078 h1:b9vDfBHAbKKWotUoInxAP/rUf6KmqGRZBND6lSzUcbo=
-github.com/containers/storage v1.51.1-0.20231129011200-e9f9f6605078/go.mod h1:FHXkEBvKRmsTeB1JQIFfXnSyXCp+wVrt172O2ZlSzM4=
+github.com/containers/storage v1.51.1-0.20231205203947-fe005407c7d5 h1:eiCkAt+i9BYRjR7KEKPI3iORCSABhY+spM/w8BkI2lo=
+github.com/containers/storage v1.51.1-0.20231205203947-fe005407c7d5/go.mod h1:pMhG1O3eMGlQKpuEuv7ves+K3BsK8/UJs8ctV5fEaoI=
 github.com/coreos/go-oidc/v3 v3.7.0 h1:FTdj0uexT4diYIPlF4yoFVI5MRO1r5+SEcIpEw9vC0o=
 github.com/coreos/go-oidc/v3 v3.7.0/go.mod h1:yQzSCqBnK3e6Fs5l+f5i0F8Kwf0zpH9bPEsbY00KanM=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=

internal/private/private.go

@@ -117,6 +117,7 @@ type TryReusingBlobOptions struct {
 	EmptyLayer          bool                   // True if the blob is an "empty"/"throwaway" layer, and may not necessarily be physically represented.
 	LayerIndex          *int                   // If the blob is a layer, a zero-based index of the layer within the image; nil otherwise.
 	SrcRef              reference.Named        // A reference to the source image that contains the input blob.
+	TOCDigest           *digest.Digest         // If specified, the blob can be looked up in the destination also by its TOC digest.
 }
 
 // ReusedBlob is information about a blob reused in a destination.

manifest/oci.go

@@ -9,6 +9,7 @@ import (
 	compressiontypes "github.com/containers/image/v5/pkg/compression/types"
 	"github.com/containers/image/v5/types"
 	ociencspec "github.com/containers/ocicrypt/spec"
+	chunkedToc "github.com/containers/storage/pkg/chunked/toc"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/specs-go"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
@@ -235,7 +236,7 @@ func (m *OCI1) Inspect(configGetter func(types.BlobInfo) ([]byte, error)) (*type
 }
 
 // ImageID computes an ID which can uniquely identify this image by its contents.
-func (m *OCI1) ImageID([]digest.Digest) (string, error) {
+func (m *OCI1) ImageID(diffIDs []digest.Digest) (string, error) {
 	// The way m.Config.Digest “uniquely identifies” an image is
 	// by containing RootFS.DiffIDs, which identify the layers of the image.
 	// For non-image artifacts, the we can’t expect the config to change
@@ -259,9 +260,44 @@ func (m *OCI1) ImageID([]digest.Digest) (string, error) {
 	if err := m.Config.Digest.Validate(); err != nil {
 		return "", err
 	}
+
+	// If there is any layer that is using partial content, we calculate the image ID
+	// in a different way since the diffID cannot be validated as for regular pulled images.
+	for _, layer := range m.Layers {
+		toc, err := chunkedToc.GetTOCDigest(layer.Annotations)
+		if err != nil {
+			return "", fmt.Errorf("error looking up annotation for layer %q: %w", layer.Digest, err)
+		}
+		if toc != nil {
+			return m.calculateImageIDForPartialImage(diffIDs)
+		}
+	}
+
 	return m.Config.Digest.Hex(), nil
 }
 
+func (m *OCI1) calculateImageIDForPartialImage(diffIDs []digest.Digest) (string, error) {
+	newID := digest.Canonical.Digester()
+	for i, layer := range m.Layers {
+		diffID := diffIDs[i]
+		_, err := newID.Hash().Write([]byte(diffID.Hex()))
+		if err != nil {
+			return "", fmt.Errorf("error writing diffID %q: %w", diffID, err)
+		}
+		toc, err := chunkedToc.GetTOCDigest(layer.Annotations)
+		if err != nil {
+			return "", fmt.Errorf("error looking up annotation for layer %q: %w", layer.Digest, err)
+		}
+		if toc != nil {
+			_, err = newID.Hash().Write([]byte(toc.Hex()))
+			if err != nil {
+				return "", fmt.Errorf("error writing TOC %q: %w", toc, err)
+			}
+		}
+	}
+	return newID.Digest().Hex(), nil
+}
+
 // CanChangeLayerCompression returns true if we can compress/decompress layers with mimeType in the current image
 // (and the code can handle that).
 // NOTE: Even if this returns true, the relevant format might not accept all compression algorithms; the set of accepted