From 59b6ed0aa60a781f809de9af17749d25d8f3c324 Mon Sep 17 00:00:00 2001 From: cleverhu Date: Mon, 28 Oct 2024 14:52:00 +0800 Subject: [PATCH] support copy multi arch instance Signed-off-by: cleverhu --- copy/copy.go | 6 +- copy/multiple.go | 86 ++++++++++++++++++++++-- internal/manifest/docker_schema2_list.go | 11 ++- internal/manifest/list.go | 1 + internal/manifest/oci_index.go | 12 +++- 5 files changed, 107 insertions(+), 9 deletions(-) diff --git a/copy/copy.go b/copy/copy.go index 867ba73c7..9caeb0a4b 100644 --- a/copy/copy.go +++ b/copy/copy.go @@ -92,8 +92,9 @@ type Options struct { PreserveDigests bool // manifest MIME type of image set by user. "" is default and means use the autodetection to the manifest MIME type ForceManifestMIMEType string - ImageListSelection ImageListSelection // set to either CopySystemImage (the default), CopyAllImages, or CopySpecificImages to control which instances we copy when the source reference is a list; ignored if the source reference is not a list - Instances []digest.Digest // if ImageListSelection is CopySpecificImages, copy only these instances and the list itself + ImageListSelection ImageListSelection // set to either CopySystemImage (the default), CopyAllImages, or CopySpecificImages to control which instances we copy when the source reference is a list; ignored if the source reference is not a list + ImageListPlatforms []manifest.Schema2PlatformSpec // if ImageListSelection is CopySpecificImages, copy only these target platforms + Instances []digest.Digest // if ImageListSelection is CopySpecificImages, copy only these instances and the list itself, this is auto generated by ImageListPlatforms // Give priority to pulling gzip images if multiple images are present when configured to OptionalBoolTrue, // prefers the best compression if this is configured as OptionalBoolFalse. Choose automatically (and the choice may change over time) // if this is set to OptionalBoolUndefined (which is the default behavior, and recommended for most callers). @@ -325,6 +326,7 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, if !supportsMultipleImages(c.dest) { return nil, fmt.Errorf("copying multiple images: destination transport %q does not support copying multiple images as a group", destRef.Transport().Name()) } + // Copy some or all of the images. switch c.options.ImageListSelection { case CopyAllImages: diff --git a/copy/multiple.go b/copy/multiple.go index 009a067ce..45180237e 100644 --- a/copy/multiple.go +++ b/copy/multiple.go @@ -60,8 +60,9 @@ func platformV1ToPlatformComparable(platform *imgspecv1.Platform) platformCompar } osFeatures := slices.Clone(platform.OSFeatures) sort.Strings(osFeatures) - return platformComparable{architecture: platform.Architecture, - os: platform.OS, + return platformComparable{ + architecture: platform.Architecture, + os: platform.OS, // This is strictly speaking ambiguous, fields of OSFeatures can contain a ','. Probably good enough for now. osFeatures: strings.Join(osFeatures, ","), osVersion: platform.OSVersion, @@ -98,8 +99,65 @@ func validateCompressionVariantExists(input []OptionCompressionVariant) error { return nil } +func isPlatformSupported(list internalManifest.List, platform manifest.Schema2PlatformSpec) (digest.Digest, bool) { + for _, instanceDigest := range list.Instances() { + instance, err := list.Instance(instanceDigest) + if err != nil { + return "", false + } + + if instance.ReadOnly.Platform == nil { + continue + } + + if instance.ReadOnly.Platform.OS == platform.OS && + instance.ReadOnly.Platform.Architecture == platform.Architecture { + return instanceDigest, true + } + } + + return "", false +} + +func filterInstancesByPlatforms(list internalManifest.List, platforms []manifest.Schema2PlatformSpec) ([]digest.Digest, error) { + missingPlatforms := []manifest.Schema2PlatformSpec{} + supportedInstance := []digest.Digest{} + // Check each requested platform + for _, platform := range platforms { + if digest, ok := isPlatformSupported(list, platform); !ok { + missingPlatforms = append(missingPlatforms, platform) + } else { + supportedInstance = append(supportedInstance, digest) + } + } + + if len(missingPlatforms) > 0 { + var platformStrings []string + for _, p := range missingPlatforms { + platformStr := fmt.Sprintf("%s/%s", p.OS, p.Architecture) + if p.Variant != "" { + platformStr += "/" + p.Variant + } + platformStrings = append(platformStrings, platformStr) + } + return nil, fmt.Errorf("requested platforms not found in image: %s", strings.Join(platformStrings, ", ")) + } + + if len(platforms) == 0 { + supportedInstance = list.Instances() + } + + return supportedInstance, nil +} + // prepareInstanceCopies prepares a list of instances which needs to copied to the manifest list. func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest.Digest, options *Options) ([]instanceCopy, error) { + filteredInstanceDigests, err := filterInstancesByPlatforms(list, options.ImageListPlatforms) + if err != nil { + return nil, err + } + options.Instances = filteredInstanceDigests + res := []instanceCopy{} if options.ImageListSelection == CopySpecificImages && len(options.EnsureCompressionVariantsExist) > 0 { // List can already contain compressed instance for a compression selected in `EnsureCompressionVariantsExist` @@ -109,7 +167,8 @@ func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest. // We might define the semantics and implement this in the future. return res, fmt.Errorf("EnsureCompressionVariantsExist is not implemented for CopySpecificImages") } - err := validateCompressionVariantExists(options.EnsureCompressionVariantsExist) + + err = validateCompressionVariantExists(options.EnsureCompressionVariantsExist) if err != nil { return res, err } @@ -252,7 +311,8 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte, UpdateDigest: updated.manifestDigest, UpdateSize: int64(len(updated.manifest)), UpdateCompressionAlgorithms: updated.compressionAlgorithms, - UpdateMediaType: updated.manifestMIMEType}) + UpdateMediaType: updated.manifestMIMEType, + }) case instanceCopyClone: logrus.Debugf("Replicating instance %s (%d/%d)", instance.sourceDigest, i+1, len(instanceCopyList)) c.Printf("Replicating image %s (%d/%d)\n", instance.sourceDigest, i+1, len(instanceCopyList)) @@ -260,7 +320,8 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte, updated, err := c.copySingleImage(ctx, unparsedInstance, &instanceCopyList[i].sourceDigest, copySingleImageOptions{ requireCompressionFormatMatch: true, compressionFormat: &instance.cloneCompressionVariant.Algorithm, - compressionLevel: instance.cloneCompressionVariant.Level}) + compressionLevel: instance.cloneCompressionVariant.Level, + }) if err != nil { return nil, fmt.Errorf("replicating image %d/%d from manifest list: %w", i+1, len(instanceCopyList), err) } @@ -285,6 +346,21 @@ func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte, return nil, fmt.Errorf("updating manifest list: %w", err) } + if len(c.options.Instances) > 0 { + deletedInstanceDigests := []internalManifest.ListEdit{} + for _, instanceDigest := range updatedList.Instances() { + if !slices.Contains(c.options.Instances, instanceDigest) { + deletedInstanceDigests = append(deletedInstanceDigests, internalManifest.ListEdit{ + ListOperation: internalManifest.ListOpRemove, + UpdateOldDigest: instanceDigest, + }) + } + } + if err = updatedList.EditInstances(deletedInstanceDigests); err != nil { + return nil, fmt.Errorf("updating manifest list: %w", err) + } + } + // Iterate through supported list types, preferred format first. c.Printf("Writing manifest list to image destination\n") var errs []string diff --git a/internal/manifest/docker_schema2_list.go b/internal/manifest/docker_schema2_list.go index 07922cece..7f494b438 100644 --- a/internal/manifest/docker_schema2_list.go +++ b/internal/manifest/docker_schema2_list.go @@ -82,7 +82,8 @@ func (index *Schema2ListPublic) UpdateInstances(updates []ListUpdate) error { UpdateDigest: instance.Digest, UpdateSize: instance.Size, UpdateMediaType: instance.MediaType, - ListOperation: ListOpUpdate}) + ListOperation: ListOpUpdate, + }) } return index.editInstances(editInstances) } @@ -128,6 +129,14 @@ func (index *Schema2ListPublic) editInstances(editInstances []ListEdit) error { }, schema2PlatformSpecFromOCIPlatform(*editInstance.AddPlatform), }) + case ListOpRemove: + targetIndex := slices.IndexFunc(index.Manifests, func(m Schema2ManifestDescriptor) bool { + return m.Digest == editInstance.UpdateOldDigest + }) + if targetIndex == -1 { + return fmt.Errorf("Schema2List.EditInstances: digest %s not found", editInstance.UpdateOldDigest) + } + index.Manifests = append(index.Manifests[:targetIndex], index.Manifests[targetIndex+1:]...) default: return fmt.Errorf("internal error: invalid operation: %d", editInstance.ListOperation) } diff --git a/internal/manifest/list.go b/internal/manifest/list.go index 1c614d124..895e2a1a0 100644 --- a/internal/manifest/list.go +++ b/internal/manifest/list.go @@ -83,6 +83,7 @@ const ( listOpInvalid ListOp = iota ListOpAdd ListOpUpdate + ListOpRemove ) // ListEdit includes the fields which a List's EditInstances() method will modify. diff --git a/internal/manifest/oci_index.go b/internal/manifest/oci_index.go index 6a0f88d3a..cdb232836 100644 --- a/internal/manifest/oci_index.go +++ b/internal/manifest/oci_index.go @@ -79,7 +79,8 @@ func (index *OCI1IndexPublic) UpdateInstances(updates []ListUpdate) error { UpdateDigest: instance.Digest, UpdateSize: instance.Size, UpdateMediaType: instance.MediaType, - ListOperation: ListOpUpdate}) + ListOperation: ListOpUpdate, + }) } return index.editInstances(editInstances) } @@ -166,6 +167,15 @@ func (index *OCI1IndexPublic) editInstances(editInstances []ListEdit) error { Platform: editInstance.AddPlatform, Annotations: annotations, }) + case ListOpRemove: + targetIndex := slices.IndexFunc(index.Manifests, func(m imgspecv1.Descriptor) bool { + return m.Digest == editInstance.UpdateOldDigest + }) + + if targetIndex == -1 { + return fmt.Errorf("OCI1Index.EditInstances: digest %s not found", editInstance.UpdateOldDigest) + } + index.Manifests = append(index.Manifests[:targetIndex], index.Manifests[targetIndex+1:]...) default: return fmt.Errorf("internal error: invalid operation: %d", editInstance.ListOperation) }