From 5b50ea8b11e78fc817294a2b3f5295123184f20a Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Wed, 29 May 2024 23:14:31 +0300 Subject: [PATCH] Extract components images Signed-off-by: Stefan Prodan --- go.mod | 1 + go.sum | 2 + internal/builder/build.go | 50 +++++++++++++++++++ internal/builder/build_test.go | 42 ++++++++++++++++ internal/builder/options.go | 1 + internal/builder/templates.go | 10 ++-- .../v2.3.0-golden/default.kustomization.yaml | 6 +++ .../v2.3.0-golden/patches.kustomization.yaml | 6 +++ .../v2.3.0-golden/profiles.kustomization.yaml | 6 +++ .../controller/fluxinstance_controller.go | 9 +++- 10 files changed, 129 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index b75c1d7..e503d6b 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/fluxcd/pkg/kustomize v1.11.0 github.com/fluxcd/pkg/runtime v0.47.1 github.com/fluxcd/pkg/ssa v0.39.1 + github.com/google/go-containerregistry v0.19.1 github.com/onsi/ginkgo/v2 v2.19.0 github.com/onsi/gomega v1.33.1 github.com/opencontainers/go-digest v1.0.0 diff --git a/go.sum b/go.sum index e6f2978..f623fa4 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYu github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY= +github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= diff --git a/internal/builder/build.go b/internal/builder/build.go index 014dc94..9ac4cd5 100644 --- a/internal/builder/build.go +++ b/internal/builder/build.go @@ -10,12 +10,15 @@ import ( "path" "path/filepath" "sort" + "strings" "github.com/fluxcd/pkg/kustomize" "github.com/fluxcd/pkg/ssa" ssautil "github.com/fluxcd/pkg/ssa/utils" + gcname "github.com/google/go-containerregistry/pkg/name" "github.com/opencontainers/go-digest" cp "github.com/otiai10/copy" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) // Build copies the source directory to a temporary directory, generates the @@ -100,3 +103,50 @@ func generate(base string, options Options) error { } return nil } + +// ComponentImage represents a container image used by a component. +type ComponentImage struct { + Component string + ImageName string + ImageTag string + ImageDigest string +} + +// ExtractComponentImages reads the source directory and extracts the container images +// from the components manifests. +func ExtractComponentImages(srcDir string, opts Options) ([]ComponentImage, error) { + images := make([]ComponentImage, len(opts.Components)) + for i, component := range opts.Components { + d, err := os.ReadFile(filepath.Join(srcDir, fmt.Sprintf("/%s.yaml", component))) + if err != nil { + return nil, err + } + objects, err := ssautil.ReadObjects(bytes.NewReader(d)) + if err != nil { + return nil, err + } + for _, obj := range objects { + if obj.GetKind() == "Deployment" { + containers, ok, _ := unstructured.NestedSlice(obj.Object, "spec", "template", "spec", "containers") + if !ok { + return nil, fmt.Errorf("containers not found in %s", obj.GetName()) + } + for _, container := range containers { + img := container.(map[string]interface{})["image"].(string) + tag, err := gcname.NewTag(img, gcname.WeakValidation) + if err != nil { + return nil, err + } + + images[i] = ComponentImage{ + Component: component, + ImageName: fmt.Sprintf("%s/%s", strings.TrimSuffix(opts.Registry, "/"), component), + ImageTag: tag.Identifier(), + } + } + } + } + } + + return images, nil +} diff --git a/internal/builder/build_test.go b/internal/builder/build_test.go index 4884e58..1c17454 100644 --- a/internal/builder/build_test.go +++ b/internal/builder/build_test.go @@ -27,6 +27,10 @@ func TestBuild_Defaults(t *testing.T) { dstDir, err := testTempDir(t) g.Expect(err).NotTo(HaveOccurred()) + ci, err := ExtractComponentImages(srcDir, options) + g.Expect(err).NotTo(HaveOccurred()) + options.ComponentImages = ci + result, err := Build(srcDir, dstDir, options) g.Expect(err).NotTo(HaveOccurred()) g.Expect(result.Objects).NotTo(BeEmpty()) @@ -57,6 +61,10 @@ func TestBuild_Patches(t *testing.T) { dstDir, err := testTempDir(t) g.Expect(err).NotTo(HaveOccurred()) + ci, err := ExtractComponentImages(srcDir, options) + g.Expect(err).NotTo(HaveOccurred()) + options.ComponentImages = ci + patches := []kustomize.Patch{ { Patch: ` @@ -117,6 +125,10 @@ func TestBuild_Profiles(t *testing.T) { dstDir, err := testTempDir(t) g.Expect(err).NotTo(HaveOccurred()) + ci, err := ExtractComponentImages(srcDir, options) + g.Expect(err).NotTo(HaveOccurred()) + options.ComponentImages = ci + options.Patches = ProfileOpenShift + ProfileMultitenant result, err := Build(srcDir, dstDir, options) @@ -159,6 +171,10 @@ func TestBuild_InvalidPatches(t *testing.T) { dstDir, err := testTempDir(t) g.Expect(err).NotTo(HaveOccurred()) + ci, err := ExtractComponentImages(srcDir, options) + g.Expect(err).NotTo(HaveOccurred()) + options.ComponentImages = ci + patches := []kustomize.Patch{ { Patch: ` @@ -179,6 +195,32 @@ func TestBuild_InvalidPatches(t *testing.T) { g.Expect(err.Error()).To(ContainSubstring("Unexpected kind: removes")) } +func TestBuild_extractImages(t *testing.T) { + g := NewWithT(t) + const version = "v2.3.0" + srcDir := filepath.Join("testdata", version) + + images, err := ExtractComponentImages(srcDir, MakeDefaultOptions()) + g.Expect(err).NotTo(HaveOccurred()) + + t.Log(images) + g.Expect(images).To(HaveLen(6)) + g.Expect(images).To(ContainElements( + ComponentImage{ + Component: "source-controller", + ImageName: "ghcr.io/fluxcd/source-controller", + ImageTag: "v1.3.0", + ImageDigest: "", + }, + ComponentImage{ + Component: "kustomize-controller", + ImageName: "ghcr.io/fluxcd/kustomize-controller", + ImageTag: "v1.3.0", + ImageDigest: "", + }, + )) +} + func testTempDir(t *testing.T) (string, error) { tmpDir := t.TempDir() diff --git a/internal/builder/options.go b/internal/builder/options.go index 46f2839..ac361ef 100644 --- a/internal/builder/options.go +++ b/internal/builder/options.go @@ -8,6 +8,7 @@ type Options struct { Version string Namespace string Components []string + ComponentImages []ComponentImage EventsAddr string Registry string ImagePullSecret string diff --git a/internal/builder/templates.go b/internal/builder/templates.go index cf9cc76..4455d50 100644 --- a/internal/builder/templates.go +++ b/internal/builder/templates.go @@ -33,9 +33,13 @@ resources: {{- end }} {{- if $registry }} images: -{{- range $i, $component := .Components }} - - name: fluxcd/{{$component}} - newName: {{$registry}}/{{$component}} +{{- range .ComponentImages }} + - name: fluxcd/{{.Component}} + newName: {{.ImageName}} + newTag: {{.ImageTag}} +{{- if .ImageDigest }} + digest: {{.ImageDigest}} +{{- end }} {{- end }} {{- end }} patches: diff --git a/internal/builder/testdata/v2.3.0-golden/default.kustomization.yaml b/internal/builder/testdata/v2.3.0-golden/default.kustomization.yaml index 5cb6c4f..af9caf9 100644 --- a/internal/builder/testdata/v2.3.0-golden/default.kustomization.yaml +++ b/internal/builder/testdata/v2.3.0-golden/default.kustomization.yaml @@ -17,16 +17,22 @@ resources: images: - name: fluxcd/source-controller newName: ghcr.io/fluxcd/source-controller + newTag: v1.3.0 - name: fluxcd/kustomize-controller newName: ghcr.io/fluxcd/kustomize-controller + newTag: v1.3.0 - name: fluxcd/helm-controller newName: ghcr.io/fluxcd/helm-controller + newTag: v1.0.1 - name: fluxcd/notification-controller newName: ghcr.io/fluxcd/notification-controller + newTag: v1.3.0 - name: fluxcd/image-reflector-controller newName: ghcr.io/fluxcd/image-reflector-controller + newTag: v0.32.0 - name: fluxcd/image-automation-controller newName: ghcr.io/fluxcd/image-automation-controller + newTag: v0.38.0 patches: - path: node-selector.yaml target: diff --git a/internal/builder/testdata/v2.3.0-golden/patches.kustomization.yaml b/internal/builder/testdata/v2.3.0-golden/patches.kustomization.yaml index c5c2d62..fdeaa4d 100644 --- a/internal/builder/testdata/v2.3.0-golden/patches.kustomization.yaml +++ b/internal/builder/testdata/v2.3.0-golden/patches.kustomization.yaml @@ -17,16 +17,22 @@ resources: images: - name: fluxcd/source-controller newName: ghcr.io/fluxcd/source-controller + newTag: v1.3.0 - name: fluxcd/kustomize-controller newName: ghcr.io/fluxcd/kustomize-controller + newTag: v1.3.0 - name: fluxcd/helm-controller newName: ghcr.io/fluxcd/helm-controller + newTag: v1.0.1 - name: fluxcd/notification-controller newName: ghcr.io/fluxcd/notification-controller + newTag: v1.3.0 - name: fluxcd/image-reflector-controller newName: ghcr.io/fluxcd/image-reflector-controller + newTag: v0.32.0 - name: fluxcd/image-automation-controller newName: ghcr.io/fluxcd/image-automation-controller + newTag: v0.38.0 patches: - path: node-selector.yaml target: diff --git a/internal/builder/testdata/v2.3.0-golden/profiles.kustomization.yaml b/internal/builder/testdata/v2.3.0-golden/profiles.kustomization.yaml index b304b20..e9b3c00 100644 --- a/internal/builder/testdata/v2.3.0-golden/profiles.kustomization.yaml +++ b/internal/builder/testdata/v2.3.0-golden/profiles.kustomization.yaml @@ -17,16 +17,22 @@ resources: images: - name: fluxcd/source-controller newName: ghcr.io/fluxcd/source-controller + newTag: v1.3.0 - name: fluxcd/kustomize-controller newName: ghcr.io/fluxcd/kustomize-controller + newTag: v1.3.0 - name: fluxcd/helm-controller newName: ghcr.io/fluxcd/helm-controller + newTag: v1.0.1 - name: fluxcd/notification-controller newName: ghcr.io/fluxcd/notification-controller + newTag: v1.3.0 - name: fluxcd/image-reflector-controller newName: ghcr.io/fluxcd/image-reflector-controller + newTag: v0.32.0 - name: fluxcd/image-automation-controller newName: ghcr.io/fluxcd/image-automation-controller + newTag: v0.38.0 patches: - path: node-selector.yaml target: diff --git a/internal/controller/fluxinstance_controller.go b/internal/controller/fluxinstance_controller.go index e60d649..6698ce9 100644 --- a/internal/controller/fluxinstance_controller.go +++ b/internal/controller/fluxinstance_controller.go @@ -215,6 +215,13 @@ func (r *FluxInstanceReconciler) build(ctx context.Context, options.Patches += string(patchesData) } + srcDir := filepath.Join(fluxDir, ver) + images, err := builder.ExtractComponentImages(srcDir, options) + if err != nil { + return nil, fmt.Errorf("failed to extract container images from manifests: %w", err) + } + options.ComponentImages = images + tmpDir, err := builder.MkdirTempAbs("", "flux") if err != nil { return nil, fmt.Errorf("failed to create tmp dir: %w", err) @@ -226,7 +233,7 @@ func (r *FluxInstanceReconciler) build(ctx context.Context, } }() - return builder.Build(filepath.Join(fluxDir, ver), tmpDir, options) + return builder.Build(srcDir, tmpDir, options) } // apply reconciles the resources in the cluster by performing