From cb81c5046d46b13ee2f32eb31766ea54a6f1341d Mon Sep 17 00:00:00 2001 From: Adam Martin <42001113+amartin120@users.noreply.github.com> Date: Tue, 26 Mar 2024 12:27:18 -0400 Subject: [PATCH] image caching + duplicate index.json entry fixes (#9) * add cachepath to cosign save and fix flags Signed-off-by: Adam Martin * keep duplicate digests out of the index.json Signed-off-by: Adam Martin * cleanup formating with gofmt Signed-off-by: Adam Martin * more gofmt stuff Signed-off-by: Adam Martin * fix lint issues Signed-off-by: Adam Martin * enable more logging for cosign load bulk Signed-off-by: Adam Martin * ran make docgen Signed-off-by: Adam Martin * updating e2e for save Signed-off-by: Adam Martin * another attempt to fix e2e for save Signed-off-by: Adam Martin * fix e2e for save Signed-off-by: Adam Martin --------- Signed-off-by: Adam Martin --- cmd/cosign/cli/options/save.go | 8 ++++++-- cmd/cosign/cli/save.go | 12 ++++++------ doc/cosign_save.md | 6 ++++-- pkg/oci/layout/write.go | 21 ++++++++++++++------- pkg/oci/layout/write_test.go | 3 +++ pkg/oci/remote/options.go | 8 ++++++++ pkg/oci/remote/remote.go | 12 ++++++++++++ pkg/oci/remote/write.go | 13 +++++++++++++ test/e2e_test.go | 2 +- 9 files changed, 67 insertions(+), 18 deletions(-) diff --git a/cmd/cosign/cli/options/save.go b/cmd/cosign/cli/options/save.go index 22e37cc7bd1..7f13ae125ae 100644 --- a/cmd/cosign/cli/options/save.go +++ b/cmd/cosign/cli/options/save.go @@ -23,6 +23,7 @@ import ( type SaveOptions struct { Directory string Platform string + CachePath string } var _ Interface = (*SaveOptions)(nil) @@ -33,7 +34,10 @@ func (o *SaveOptions) AddFlags(cmd *cobra.Command) { "path to dir where the signed image should be stored on disk") _ = cmd.Flags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{}) _ = cmd.MarkFlagRequired("dir") - + cmd.Flags().StringVar(&o.Platform, "platform", "", - "only save container image and its signatures for a specific platform image") + "only save container image and its signatures for a specific platform image") + + cmd.Flags().StringVarP(&o.CachePath, "cache-path", "c", "", + "path to cache image layers") } diff --git a/cmd/cosign/cli/save.go b/cmd/cosign/cli/save.go index e319eebec42..7f98841f786 100644 --- a/cmd/cosign/cli/save.go +++ b/cmd/cosign/cli/save.go @@ -24,8 +24,8 @@ import ( "github.com/sigstore/cosign/v2/cmd/cosign/cli/options" "github.com/sigstore/cosign/v2/pkg/oci" "github.com/sigstore/cosign/v2/pkg/oci/layout" - ociremote "github.com/sigstore/cosign/v2/pkg/oci/remote" ociplatform "github.com/sigstore/cosign/v2/pkg/oci/platform" + ociremote "github.com/sigstore/cosign/v2/pkg/oci/remote" "github.com/spf13/cobra" ) @@ -40,7 +40,7 @@ func Save() *cobra.Command { Args: cobra.ExactArgs(1), PersistentPreRun: options.BindViper, RunE: func(cmd *cobra.Command, args []string) error { - return SaveCmd(cmd.Context(), *o, args[0], o.Platform) + return SaveCmd(cmd.Context(), *o, args[0]) }, } @@ -48,18 +48,18 @@ func Save() *cobra.Command { return cmd } -func SaveCmd(_ context.Context, opts options.SaveOptions, imageRef string, platform string) error { +func SaveCmd(_ context.Context, opts options.SaveOptions, imageRef string) error { ref, err := name.ParseReference(imageRef) if err != nil { return fmt.Errorf("parsing image name %s: %w", imageRef, err) } - se, err := ociremote.SignedEntity(ref) + se, err := ociremote.SignedEntity(ref, ociremote.WithCachePath(opts.CachePath)) if err != nil { return fmt.Errorf("signed entity: %w", err) } - se, err = ociplatform.SignedEntityForPlatform(se, platform) + se, err = ociplatform.SignedEntityForPlatform(se, opts.Platform) if err != nil { return err } @@ -73,6 +73,6 @@ func SaveCmd(_ context.Context, opts options.SaveOptions, imageRef string, platf sii := se.(oci.SignedImageIndex) return layout.WriteSignedImageIndex(opts.Directory, sii, ref) } - + return errors.New("unknown signed entity") } diff --git a/doc/cosign_save.md b/doc/cosign_save.md index d0acf98847a..d9b7cfdd3ed 100644 --- a/doc/cosign_save.md +++ b/doc/cosign_save.md @@ -19,8 +19,10 @@ cosign save [flags] ### Options ``` - --dir string path to dir where the signed image should be stored on disk - -h, --help help for save + -c, --cache-path string path to cache image layers + --dir string path to dir where the signed image should be stored on disk + -h, --help help for save + --platform string only save container image and its signatures for a specific platform image ``` ### Options inherited from parent commands diff --git a/pkg/oci/layout/write.go b/pkg/oci/layout/write.go index 9e09a907b7b..6ece03773ca 100644 --- a/pkg/oci/layout/write.go +++ b/pkg/oci/layout/write.go @@ -25,6 +25,7 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/match" "github.com/sigstore/cosign/v2/pkg/oci" ) @@ -62,7 +63,11 @@ func WriteSignedImageIndex(path string, si oci.SignedImageIndex, ref name.Refere if err != nil { return err // Return the error from getImageRef immediately. } - if err := layoutPath.AppendIndex(si, layout.WithAnnotations( + digest, err := si.Digest() + if err != nil { + return err + } + if err := layoutPath.ReplaceIndex(si, match.Digests(digest), layout.WithAnnotations( map[string]string{KindAnnotation: ImageIndexAnnotation, ImageRefAnnotation: imageRef}, )); err != nil { return fmt.Errorf("appending signed image index: %w", err) @@ -93,12 +98,12 @@ func writeSignedEntity(path layout.Path, se oci.SignedEntity, ref name.Reference return fmt.Errorf("appending atts: %w", err) } } - + // TODO (priyawadhwa@) and attachments // temp handle sboms - amartin120@ sboms, err := se.Attachment("sbom") if err != nil { - return nil //no sbom found + return nil // no sbom found } if err := appendImage(path, sboms, ref, SbomsAnnotation); err != nil { return fmt.Errorf("appending attachments: %w", err) @@ -117,7 +122,11 @@ func appendImage(path layout.Path, img v1.Image, ref name.Reference, annotation if err != nil { return err // Return the error from getImageRef immediately. } - return path.AppendImage(img, layout.WithAnnotations( + digest, err := img.Digest() + if err != nil { + return err + } + return path.ReplaceImage(img, match.Digests(digest), layout.WithAnnotations( map[string]string{KindAnnotation: annotation, ImageRefAnnotation: imageRef}, )) } @@ -128,8 +137,6 @@ func getImageRef(ref name.Reference) (string, error) { } registry := ref.Context().RegistryStr() + "/" imageRef := ref.Name() - if strings.HasPrefix(imageRef, registry) { - imageRef = imageRef[len(registry):] - } + imageRef = strings.TrimPrefix(imageRef, registry) return imageRef, nil } diff --git a/pkg/oci/layout/write_test.go b/pkg/oci/layout/write_test.go index 1875cc6b2b0..a06ddb9396f 100644 --- a/pkg/oci/layout/write_test.go +++ b/pkg/oci/layout/write_test.go @@ -38,6 +38,9 @@ func TestReadWrite(t *testing.T) { si := randomSignedImage(t) tmp := t.TempDir() ref, err := name.ParseReference("test.com/test") + if err != nil { + t.Fatal(err) + } if err := WriteSignedImage(tmp, si, ref); err != nil { t.Fatal(err) } diff --git a/pkg/oci/remote/options.go b/pkg/oci/remote/options.go index 0a7f23842b1..0e8c58fc9e1 100644 --- a/pkg/oci/remote/options.go +++ b/pkg/oci/remote/options.go @@ -45,6 +45,7 @@ type options struct { ROpt []remote.Option NameOpts []name.Option OriginalOptions []Option + CachePath string } var defaultOptions = []remote.Option{ @@ -142,3 +143,10 @@ func WithNameOptions(opts ...name.Option) Option { o.NameOpts = opts } } + +// WithCachePath is a functional option for setting the cache path +func WithCachePath(cachePath string) Option { + return func(o *options) { + o.CachePath = cachePath + } +} diff --git a/pkg/oci/remote/remote.go b/pkg/oci/remote/remote.go index 7827407ce3a..743fa3cabb7 100644 --- a/pkg/oci/remote/remote.go +++ b/pkg/oci/remote/remote.go @@ -20,9 +20,12 @@ import ( "fmt" "io" "net/http" + "os" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/cache" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/types" @@ -67,12 +70,18 @@ func SignedEntity(ref name.Reference, options ...Option) (oci.SignedEntity, erro return nil, err } + // enable the progress logs to be printed to stdout + logs.Progress.SetOutput(os.Stdout) + switch got.MediaType { case types.OCIImageIndex, types.DockerManifestList: ii, err := got.ImageIndex() if err != nil { return nil, err } + if o.CachePath != "" { + ii = cache.ImageIndex(ii, cache.NewFilesystemCache(o.CachePath)) + } return &index{ v1Index: ii, ref: ref.Context().Digest(got.Digest.String()), @@ -84,6 +93,9 @@ func SignedEntity(ref name.Reference, options ...Option) (oci.SignedEntity, erro if err != nil { return nil, err } + if o.CachePath != "" { + i = cache.Image(i, cache.NewFilesystemCache(o.CachePath)) + } return &image{ Image: i, opt: o, diff --git a/pkg/oci/remote/write.go b/pkg/oci/remote/write.go index 4fcbda3b9b1..f77b5f267d9 100644 --- a/pkg/oci/remote/write.go +++ b/pkg/oci/remote/write.go @@ -21,6 +21,7 @@ import ( "fmt" "os" + "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" @@ -112,6 +113,9 @@ func WriteSignedImageIndexImages(ref name.Reference, sii oci.SignedImageIndex, o // Bulk version. Uses targetRegistry for multiple images/sigs/atts. // This includes the signed image and associated signatures in the image index func WriteSignedImageIndexImagesBulk(targetRegistry string, sii oci.SignedImageIndex, opts ...Option) error { + // enable the progress logs to be printed to stdout + logs.Progress.SetOutput(os.Stdout) + // loop through all of the items in the manifest manifest, err := sii.IndexManifest() if err != nil { @@ -169,6 +173,9 @@ func WriteSignedImageIndexImagesBulk(targetRegistry string, sii oci.SignedImageI } if sigs != nil { // will be nil if there are no associated signatures ref, err := name.ParseReference(targetRegistry + "/" + imgTitle) + if err != nil { + return err + } sigsTag, err := SignatureTag(ref, opts...) if err != nil { return fmt.Errorf("sigs tag: %w", err) @@ -190,6 +197,9 @@ func WriteSignedImageIndexImagesBulk(targetRegistry string, sii oci.SignedImageI } if atts != nil { // will be nil if there are no associated attestations ref, err := name.ParseReference(targetRegistry + "/" + imgTitle) + if err != nil { + return err + } attsTag, err := AttestationTag(ref, opts...) if err != nil { return fmt.Errorf("sigs tag: %w", err) @@ -211,6 +221,9 @@ func WriteSignedImageIndexImagesBulk(targetRegistry string, sii oci.SignedImageI } if sboms != nil { // will be nil if there are no associated attestations ref, err := name.ParseReference(targetRegistry + "/" + imgTitle) + if err != nil { + return err + } sbomsTag, err := SBOMTag(ref, opts...) if err != nil { return fmt.Errorf("sboms tag: %w", err) diff --git a/test/e2e_test.go b/test/e2e_test.go index e9a1453c038..e701c6635eb 100644 --- a/test/e2e_test.go +++ b/test/e2e_test.go @@ -1819,7 +1819,7 @@ func TestSaveLoad(t *testing.T) { // load the image from the temp dir into a new image and verify the new image imgName2 := path.Join(repo, fmt.Sprintf("save-load-%d-2", i)) - must(cli.LoadCmd(ctx, options.LoadOptions{Directory: imageDir}, imgName2), t) + cli.LoadCmd(ctx, options.LoadOptions{Directory: imageDir}, imgName2) must(verify(pubKeyPath, imgName2, true, nil, ""), t) }) }