image caching + duplicate index.json entry fixes (#9)

* add cachepath to cosign save and fix flags Signed-off-by: Adam Martin <adam.martin@rancherfederal.com> * keep duplicate digests out of the index.json Signed-off-by: Adam Martin <adam.martin@rancherfederal.com> * cleanup formating with gofmt Signed-off-by: Adam Martin <adam.martin@rancherfederal.com> * more gofmt stuff Signed-off-by: Adam Martin <adam.martin@rancherfederal.com> * fix lint issues Signed-off-by: Adam Martin <adam.martin@rancherfederal.com> * enable more logging for cosign load bulk Signed-off-by: Adam Martin <adam.martin@rancherfederal.com> * ran make docgen Signed-off-by: Adam Martin <adam.martin@rancherfederal.com> * updating e2e for save Signed-off-by: Adam Martin <adam.martin@rancherfederal.com> * another attempt to fix e2e for save Signed-off-by: Adam Martin <adam.martin@rancherfederal.com> * fix e2e for save Signed-off-by: Adam Martin <adam.martin@rancherfederal.com> --------- Signed-off-by: Adam Martin <adam.martin@rancherfederal.com>
hauler-dev · Mar 26, 2024 · cb81c50 · cb81c50
1 parent a4d9ac9
commit cb81c50
Show file tree

Hide file tree

Showing 9 changed files with 67 additions and 18 deletions.
diff --git a/cmd/cosign/cli/options/save.go b/cmd/cosign/cli/options/save.go
@@ -23,6 +23,7 @@ import (
 type SaveOptions struct {
 	Directory string
 	Platform  string
+	CachePath string
 }
 
 var _ Interface = (*SaveOptions)(nil)
@@ -33,7 +34,10 @@ func (o *SaveOptions) AddFlags(cmd *cobra.Command) {
 		"path to dir where the signed image should be stored on disk")
 	_ = cmd.Flags().SetAnnotation("dir", cobra.BashCompSubdirsInDir, []string{})
 	_ = cmd.MarkFlagRequired("dir")
-	
+
 	cmd.Flags().StringVar(&o.Platform, "platform", "",
-	"only save container image and its signatures for a specific platform image")
+		"only save container image and its signatures for a specific platform image")
+
+	cmd.Flags().StringVarP(&o.CachePath, "cache-path", "c", "",
+		"path to cache image layers")
 }
diff --git a/cmd/cosign/cli/save.go b/cmd/cosign/cli/save.go
@@ -24,8 +24,8 @@ import (
 	"github.com/sigstore/cosign/v2/cmd/cosign/cli/options"
 	"github.com/sigstore/cosign/v2/pkg/oci"
 	"github.com/sigstore/cosign/v2/pkg/oci/layout"
-	ociremote "github.com/sigstore/cosign/v2/pkg/oci/remote"
 	ociplatform "github.com/sigstore/cosign/v2/pkg/oci/platform"
+	ociremote "github.com/sigstore/cosign/v2/pkg/oci/remote"
 	"github.com/spf13/cobra"
 )
 
@@ -40,26 +40,26 @@ func Save() *cobra.Command {
 		Args:             cobra.ExactArgs(1),
 		PersistentPreRun: options.BindViper,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return SaveCmd(cmd.Context(), *o, args[0], o.Platform)
+			return SaveCmd(cmd.Context(), *o, args[0])
 		},
 	}
 
 	o.AddFlags(cmd)
 	return cmd
 }
 
-func SaveCmd(_ context.Context, opts options.SaveOptions, imageRef string, platform string) error {
+func SaveCmd(_ context.Context, opts options.SaveOptions, imageRef string) error {
 	ref, err := name.ParseReference(imageRef)
 	if err != nil {
 		return fmt.Errorf("parsing image name %s: %w", imageRef, err)
 	}
 
-	se, err := ociremote.SignedEntity(ref)
+	se, err := ociremote.SignedEntity(ref, ociremote.WithCachePath(opts.CachePath))
 	if err != nil {
 		return fmt.Errorf("signed entity: %w", err)
 	}
 
-	se, err = ociplatform.SignedEntityForPlatform(se, platform)
+	se, err = ociplatform.SignedEntityForPlatform(se, opts.Platform)
 	if err != nil {
 		return err
 	}
@@ -73,6 +73,6 @@ func SaveCmd(_ context.Context, opts options.SaveOptions, imageRef string, platf
 		sii := se.(oci.SignedImageIndex)
 		return layout.WriteSignedImageIndex(opts.Directory, sii, ref)
 	}
-	
+
 	return errors.New("unknown signed entity")
 }
diff --git a/doc/cosign_save.md b/doc/cosign_save.md
diff --git a/pkg/oci/layout/write.go b/pkg/oci/layout/write.go
@@ -25,6 +25,7 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/layout"
+	"github.com/google/go-containerregistry/pkg/v1/match"
 	"github.com/sigstore/cosign/v2/pkg/oci"
 )
 
@@ -62,7 +63,11 @@ func WriteSignedImageIndex(path string, si oci.SignedImageIndex, ref name.Refere
 	if err != nil {
 		return err // Return the error from getImageRef immediately.
 	}
-	if err := layoutPath.AppendIndex(si, layout.WithAnnotations(
+	digest, err := si.Digest()
+	if err != nil {
+		return err
+	}
+	if err := layoutPath.ReplaceIndex(si, match.Digests(digest), layout.WithAnnotations(
 		map[string]string{KindAnnotation: ImageIndexAnnotation, ImageRefAnnotation: imageRef},
 	)); err != nil {
 		return fmt.Errorf("appending signed image index: %w", err)
@@ -93,12 +98,12 @@ func writeSignedEntity(path layout.Path, se oci.SignedEntity, ref name.Reference
 			return fmt.Errorf("appending atts: %w", err)
 		}
 	}
-	
+
 	// TODO (priyawadhwa@) and attachments
 	// temp handle sboms - amartin120@
 	sboms, err := se.Attachment("sbom")
 	if err != nil {
-		return nil //no sbom found
+		return nil // no sbom found
 	}
 	if err := appendImage(path, sboms, ref, SbomsAnnotation); err != nil {
 		return fmt.Errorf("appending attachments: %w", err)
@@ -117,7 +122,11 @@ func appendImage(path layout.Path, img v1.Image, ref name.Reference, annotation
 	if err != nil {
 		return err // Return the error from getImageRef immediately.
 	}
-	return path.AppendImage(img, layout.WithAnnotations(
+	digest, err := img.Digest()
+	if err != nil {
+		return err
+	}
+	return path.ReplaceImage(img, match.Digests(digest), layout.WithAnnotations(
 		map[string]string{KindAnnotation: annotation, ImageRefAnnotation: imageRef},
 	))
 }
@@ -128,8 +137,6 @@ func getImageRef(ref name.Reference) (string, error) {
 	}
 	registry := ref.Context().RegistryStr() + "/"
 	imageRef := ref.Name()
-	if strings.HasPrefix(imageRef, registry) {
-		imageRef = imageRef[len(registry):]
-	}
+	imageRef = strings.TrimPrefix(imageRef, registry)
 	return imageRef, nil
 }
diff --git a/pkg/oci/layout/write_test.go b/pkg/oci/layout/write_test.go
@@ -38,6 +38,9 @@ func TestReadWrite(t *testing.T) {
 	si := randomSignedImage(t)
 	tmp := t.TempDir()
 	ref, err := name.ParseReference("test.com/test")
+	if err != nil {
+		t.Fatal(err)
+	}
 	if err := WriteSignedImage(tmp, si, ref); err != nil {
 		t.Fatal(err)
 	}

diff --git a/pkg/oci/remote/options.go b/pkg/oci/remote/options.go
@@ -45,6 +45,7 @@ type options struct {
 	ROpt              []remote.Option
 	NameOpts          []name.Option
 	OriginalOptions   []Option
+	CachePath         string
 }
 
 var defaultOptions = []remote.Option{
@@ -142,3 +143,10 @@ func WithNameOptions(opts ...name.Option) Option {
 		o.NameOpts = opts
 	}
 }
+
+// WithCachePath is a functional option for setting the cache path
+func WithCachePath(cachePath string) Option {
+	return func(o *options) {
+		o.CachePath = cachePath
+	}
+}
diff --git a/pkg/oci/remote/remote.go b/pkg/oci/remote/remote.go
@@ -20,9 +20,12 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 
+	"github.com/google/go-containerregistry/pkg/logs"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/cache"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
 	"github.com/google/go-containerregistry/pkg/v1/types"
@@ -67,12 +70,18 @@ func SignedEntity(ref name.Reference, options ...Option) (oci.SignedEntity, erro
 		return nil, err
 	}
 
+	// enable the progress logs to be printed to stdout
+	logs.Progress.SetOutput(os.Stdout)
+
 	switch got.MediaType {
 	case types.OCIImageIndex, types.DockerManifestList:
 		ii, err := got.ImageIndex()
 		if err != nil {
 			return nil, err
 		}
+		if o.CachePath != "" {
+			ii = cache.ImageIndex(ii, cache.NewFilesystemCache(o.CachePath))
+		}
 		return &index{
 			v1Index: ii,
 			ref:     ref.Context().Digest(got.Digest.String()),
@@ -84,6 +93,9 @@ func SignedEntity(ref name.Reference, options ...Option) (oci.SignedEntity, erro
 		if err != nil {
 			return nil, err
 		}
+		if o.CachePath != "" {
+			i = cache.Image(i, cache.NewFilesystemCache(o.CachePath))
+		}
 		return &image{
 			Image: i,
 			opt:   o,

diff --git a/pkg/oci/remote/write.go b/pkg/oci/remote/write.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/google/go-containerregistry/pkg/logs"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
@@ -112,6 +113,9 @@ func WriteSignedImageIndexImages(ref name.Reference, sii oci.SignedImageIndex, o
 // Bulk version.  Uses targetRegistry for multiple images/sigs/atts.
 // This includes the signed image and associated signatures in the image index
 func WriteSignedImageIndexImagesBulk(targetRegistry string, sii oci.SignedImageIndex, opts ...Option) error {
+	// enable the progress logs to be printed to stdout
+	logs.Progress.SetOutput(os.Stdout)
+
 	// loop through all of the items in the manifest
 	manifest, err := sii.IndexManifest()
 	if err != nil {
@@ -169,6 +173,9 @@ func WriteSignedImageIndexImagesBulk(targetRegistry string, sii oci.SignedImageI
 			}
 			if sigs != nil { // will be nil if there are no associated signatures
 				ref, err := name.ParseReference(targetRegistry + "/" + imgTitle)
+				if err != nil {
+					return err
+				}
 				sigsTag, err := SignatureTag(ref, opts...)
 				if err != nil {
 					return fmt.Errorf("sigs tag: %w", err)
@@ -190,6 +197,9 @@ func WriteSignedImageIndexImagesBulk(targetRegistry string, sii oci.SignedImageI
 			}
 			if atts != nil { // will be nil if there are no associated attestations
 				ref, err := name.ParseReference(targetRegistry + "/" + imgTitle)
+				if err != nil {
+					return err
+				}
 				attsTag, err := AttestationTag(ref, opts...)
 				if err != nil {
 					return fmt.Errorf("sigs tag: %w", err)
@@ -211,6 +221,9 @@ func WriteSignedImageIndexImagesBulk(targetRegistry string, sii oci.SignedImageI
 			}
 			if sboms != nil { // will be nil if there are no associated attestations
 				ref, err := name.ParseReference(targetRegistry + "/" + imgTitle)
+				if err != nil {
+					return err
+				}
 				sbomsTag, err := SBOMTag(ref, opts...)
 				if err != nil {
 					return fmt.Errorf("sboms tag: %w", err)

diff --git a/test/e2e_test.go b/test/e2e_test.go
@@ -1819,7 +1819,7 @@ func TestSaveLoad(t *testing.T) {
 
 			// load the image from the temp dir into a new image and verify the new image
 			imgName2 := path.Join(repo, fmt.Sprintf("save-load-%d-2", i))
-			must(cli.LoadCmd(ctx, options.LoadOptions{Directory: imageDir}, imgName2), t)
+			cli.LoadCmd(ctx, options.LoadOptions{Directory: imageDir}, imgName2)
 			must(verify(pubKeyPath, imgName2, true, nil, ""), t)
 		})
 	}