Merge pull request #3 from controlplaneio-fluxcd/extract-images

Extract components images

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/fluxcd/pkg/kustomize v1.11.0
 	github.com/fluxcd/pkg/runtime v0.47.1
 	github.com/fluxcd/pkg/ssa v0.39.1
+	github.com/google/go-containerregistry v0.19.1
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1
 	github.com/opencontainers/go-digest v1.0.0

go.sum

@@ -82,6 +82,8 @@ github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYu
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-containerregistry v0.19.1 h1:yMQ62Al6/V0Z7CqIrrS1iYoA5/oQCm88DeNujc7C1KY=
+github.com/google/go-containerregistry v0.19.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=

internal/builder/build.go

@@ -10,12 +10,15 @@ import (
 	"path"
 	"path/filepath"
 	"sort"
+	"strings"
 
 	"github.com/fluxcd/pkg/kustomize"
 	"github.com/fluxcd/pkg/ssa"
 	ssautil "github.com/fluxcd/pkg/ssa/utils"
+	gcname "github.com/google/go-containerregistry/pkg/name"
 	"github.com/opencontainers/go-digest"
 	cp "github.com/otiai10/copy"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
 // Build copies the source directory to a temporary directory, generates the
@@ -100,3 +103,50 @@ func generate(base string, options Options) error {
 	}
 	return nil
 }
+
+// ComponentImage represents a container image used by a component.
+type ComponentImage struct {
+	Component   string
+	ImageName   string
+	ImageTag    string
+	ImageDigest string
+}
+
+// ExtractComponentImages reads the source directory and extracts the container images
+// from the components manifests.
+func ExtractComponentImages(srcDir string, opts Options) ([]ComponentImage, error) {
+	images := make([]ComponentImage, len(opts.Components))
+	for i, component := range opts.Components {
+		d, err := os.ReadFile(filepath.Join(srcDir, fmt.Sprintf("/%s.yaml", component)))
+		if err != nil {
+			return nil, err
+		}
+		objects, err := ssautil.ReadObjects(bytes.NewReader(d))
+		if err != nil {
+			return nil, err
+		}
+		for _, obj := range objects {
+			if obj.GetKind() == "Deployment" {
+				containers, ok, _ := unstructured.NestedSlice(obj.Object, "spec", "template", "spec", "containers")
+				if !ok {
+					return nil, fmt.Errorf("containers not found in %s", obj.GetName())
+				}
+				for _, container := range containers {
+					img := container.(map[string]interface{})["image"].(string)
+					tag, err := gcname.NewTag(img, gcname.WeakValidation)
+					if err != nil {
+						return nil, err
+					}
+
+					images[i] = ComponentImage{
+						Component: component,
+						ImageName: fmt.Sprintf("%s/%s", strings.TrimSuffix(opts.Registry, "/"), component),
+						ImageTag:  tag.Identifier(),
+					}
+				}
+			}
+		}
+	}
+
+	return images, nil
+}

internal/builder/build_test.go

@@ -27,6 +27,10 @@ func TestBuild_Defaults(t *testing.T) {
 	dstDir, err := testTempDir(t)
 	g.Expect(err).NotTo(HaveOccurred())
 
+	ci, err := ExtractComponentImages(srcDir, options)
+	g.Expect(err).NotTo(HaveOccurred())
+	options.ComponentImages = ci
+
 	result, err := Build(srcDir, dstDir, options)
 	g.Expect(err).NotTo(HaveOccurred())
 	g.Expect(result.Objects).NotTo(BeEmpty())
@@ -57,6 +61,10 @@ func TestBuild_Patches(t *testing.T) {
 	dstDir, err := testTempDir(t)
 	g.Expect(err).NotTo(HaveOccurred())
 
+	ci, err := ExtractComponentImages(srcDir, options)
+	g.Expect(err).NotTo(HaveOccurred())
+	options.ComponentImages = ci
+
 	patches := []kustomize.Patch{
 		{
 			Patch: `
@@ -117,6 +125,10 @@ func TestBuild_Profiles(t *testing.T) {
 	dstDir, err := testTempDir(t)
 	g.Expect(err).NotTo(HaveOccurred())
 
+	ci, err := ExtractComponentImages(srcDir, options)
+	g.Expect(err).NotTo(HaveOccurred())
+	options.ComponentImages = ci
+
 	options.Patches = ProfileOpenShift + ProfileMultitenant
 
 	result, err := Build(srcDir, dstDir, options)
@@ -159,6 +171,10 @@ func TestBuild_InvalidPatches(t *testing.T) {
 	dstDir, err := testTempDir(t)
 	g.Expect(err).NotTo(HaveOccurred())
 
+	ci, err := ExtractComponentImages(srcDir, options)
+	g.Expect(err).NotTo(HaveOccurred())
+	options.ComponentImages = ci
+
 	patches := []kustomize.Patch{
 		{
 			Patch: `
@@ -179,6 +195,32 @@ func TestBuild_InvalidPatches(t *testing.T) {
 	g.Expect(err.Error()).To(ContainSubstring("Unexpected kind: removes"))
 }
 
+func TestBuild_extractImages(t *testing.T) {
+	g := NewWithT(t)
+	const version = "v2.3.0"
+	srcDir := filepath.Join("testdata", version)
+
+	images, err := ExtractComponentImages(srcDir, MakeDefaultOptions())
+	g.Expect(err).NotTo(HaveOccurred())
+
+	t.Log(images)
+	g.Expect(images).To(HaveLen(6))
+	g.Expect(images).To(ContainElements(
+		ComponentImage{
+			Component:   "source-controller",
+			ImageName:   "ghcr.io/fluxcd/source-controller",
+			ImageTag:    "v1.3.0",
+			ImageDigest: "",
+		},
+		ComponentImage{
+			Component:   "kustomize-controller",
+			ImageName:   "ghcr.io/fluxcd/kustomize-controller",
+			ImageTag:    "v1.3.0",
+			ImageDigest: "",
+		},
+	))
+}
+
 func testTempDir(t *testing.T) (string, error) {
 	tmpDir := t.TempDir()
 

internal/builder/options.go

@@ -8,6 +8,7 @@ type Options struct {
 	Version                string
 	Namespace              string
 	Components             []string
+	ComponentImages        []ComponentImage
 	EventsAddr             string
 	Registry               string
 	ImagePullSecret        string

internal/builder/templates.go

@@ -33,9 +33,13 @@ resources:
 {{- end }}
 {{- if $registry }}
 images:
-{{- range $i, $component := .Components }}
-  - name: fluxcd/{{$component}}
-    newName: {{$registry}}/{{$component}}
+{{- range .ComponentImages }}
+  - name: fluxcd/{{.Component}}
+    newName: {{.ImageName}}
+    newTag: {{.ImageTag}}
+{{- if .ImageDigest }}
+    digest: {{.ImageDigest}}
+{{- end }}
 {{- end }}
 {{- end }}
 patches:

internal/builder/testdata/v2.3.0-golden/default.kustomization.yaml

@@ -17,16 +17,22 @@ resources:
 images:
   - name: fluxcd/source-controller
     newName: ghcr.io/fluxcd/source-controller
+    newTag: v1.3.0
   - name: fluxcd/kustomize-controller
     newName: ghcr.io/fluxcd/kustomize-controller
+    newTag: v1.3.0
   - name: fluxcd/helm-controller
     newName: ghcr.io/fluxcd/helm-controller
+    newTag: v1.0.1
   - name: fluxcd/notification-controller
     newName: ghcr.io/fluxcd/notification-controller
+    newTag: v1.3.0
   - name: fluxcd/image-reflector-controller
     newName: ghcr.io/fluxcd/image-reflector-controller
+    newTag: v0.32.0
   - name: fluxcd/image-automation-controller
     newName: ghcr.io/fluxcd/image-automation-controller
+    newTag: v0.38.0
 patches:
 - path: node-selector.yaml
   target:

internal/builder/testdata/v2.3.0-golden/patches.kustomization.yaml

@@ -17,16 +17,22 @@ resources:
 images:
   - name: fluxcd/source-controller
     newName: ghcr.io/fluxcd/source-controller
+    newTag: v1.3.0
   - name: fluxcd/kustomize-controller
     newName: ghcr.io/fluxcd/kustomize-controller
+    newTag: v1.3.0
   - name: fluxcd/helm-controller
     newName: ghcr.io/fluxcd/helm-controller
+    newTag: v1.0.1
   - name: fluxcd/notification-controller
     newName: ghcr.io/fluxcd/notification-controller
+    newTag: v1.3.0
   - name: fluxcd/image-reflector-controller
     newName: ghcr.io/fluxcd/image-reflector-controller
+    newTag: v0.32.0
   - name: fluxcd/image-automation-controller
     newName: ghcr.io/fluxcd/image-automation-controller
+    newTag: v0.38.0
 patches:
 - path: node-selector.yaml
   target:

internal/builder/testdata/v2.3.0-golden/profiles.kustomization.yaml

@@ -17,16 +17,22 @@ resources:
 images:
   - name: fluxcd/source-controller
     newName: ghcr.io/fluxcd/source-controller
+    newTag: v1.3.0
   - name: fluxcd/kustomize-controller
     newName: ghcr.io/fluxcd/kustomize-controller
+    newTag: v1.3.0
   - name: fluxcd/helm-controller
     newName: ghcr.io/fluxcd/helm-controller
+    newTag: v1.0.1
   - name: fluxcd/notification-controller
     newName: ghcr.io/fluxcd/notification-controller
+    newTag: v1.3.0
   - name: fluxcd/image-reflector-controller
     newName: ghcr.io/fluxcd/image-reflector-controller
+    newTag: v0.32.0
   - name: fluxcd/image-automation-controller
     newName: ghcr.io/fluxcd/image-automation-controller
+    newTag: v0.38.0
 patches:
 - path: node-selector.yaml
   target:

internal/controller/fluxinstance_controller.go

@@ -215,6 +215,13 @@ func (r *FluxInstanceReconciler) build(ctx context.Context,
 		options.Patches += string(patchesData)
 	}
 
+	srcDir := filepath.Join(fluxDir, ver)
+	images, err := builder.ExtractComponentImages(srcDir, options)
+	if err != nil {
+		return nil, fmt.Errorf("failed to extract container images from manifests: %w", err)
+	}
+	options.ComponentImages = images
+
 	tmpDir, err := builder.MkdirTempAbs("", "flux")
 	if err != nil {
 		return nil, fmt.Errorf("failed to create tmp dir: %w", err)
@@ -226,7 +233,7 @@ func (r *FluxInstanceReconciler) build(ctx context.Context,
 		}
 	}()
 
-	return builder.Build(filepath.Join(fluxDir, ver), tmpDir, options)
+	return builder.Build(srcDir, tmpDir, options)
 }
 
 // apply reconciles the resources in the cluster by performing