refactor: Update helm registry initialization (#961)

This commit updates the helm registry initialization by separating out the image used for copying the charts to the PVC and using the released mindthegap image directly for serving the bundles. This is a small security enhancement by using the minimal image that mindthegap already provides. This commit also copies a statically compiled version of `cp` to a scratch container in order to us a minimal container with no package manager or shell, again this is a minor security enhancement. This change means that the bundles are copied to a subdirectory on the PVC as globbing cannot be used without a shell present, whereas recursive copying works correctly. This is a breaking change but should not affect any users at this point (e.g. not yet included in any downstream releases). Finally, this commit updates the helm values to use a more structured approach. While this is a breaking change, the Helm chart is only used to generate the clusterctl provider components YAML and as such does not have any impact on existing users.
nutanix-cloud-native · Nov 6, 2024 · a3adcb7 · a3adcb7
1 parent f0a51bb
commit a3adcb7
Show file tree

Hide file tree

Showing 17 changed files with 140 additions and 107 deletions.
diff --git a/.envrc b/.envrc
@@ -7,8 +7,8 @@ export DEVBOX_NO_ENVRC_UPDATE=1
 
 eval "$(devbox generate direnv --print-envrc --env-file .dev-envrc)"
 
-dotenv_if_exists '.envrc.local'
-dotenv_if_exists '.envrc.e2e'
+source_env_if_exists '.envrc.local'
+source_env_if_exists '.envrc.e2e'
 
 # check out https://www.jetpack.io/devbox/docs/ide_configuration/direnv/
 # for more details
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -234,14 +234,19 @@ jobs:
         name: Build Docker images
         run: devbox run -- make release-snapshot
 
+      - if: steps.list-changed.outputs.changed == 'true'
+        name: Export image tag
+        id: export-image-tag
+        run: echo test-image-tag="$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH)" >> "$GITHUB_OUTPUT"
+
       - if: steps.list-changed.outputs.changed == 'true'
         name: Sideload docker image
         run: |
           devbox run -- \
             kind load docker-image \
               --name "${KIND_CLUSTER_NAME}" \
-              "ko.local/cluster-api-runtime-extensions-nutanix:$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH)" \
-              "ghcr.io/nutanix-cloud-native/caren-helm-reg:$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH)"
+              "ko.local/cluster-api-runtime-extensions-nutanix:${{ steps.export-image-tag.outputs.test-image-tag }}" \
+              "ghcr.io/nutanix-cloud-native/caren-helm-reg:${{ steps.export-image-tag.outputs.test-image-tag }}"
 
       - if: steps.list-changed.outputs.changed == 'true'
         name: Setup Cluster API and cert-manager
@@ -253,7 +258,7 @@ jobs:
           devbox run -- \
             ct install \
               --config charts/ct-config.yaml \
-              --helm-extra-set-args "--set-string image.repository=ko.local/cluster-api-runtime-extensions-nutanix --set-string image.tag=$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH) --set-string helmRepositoryImage.tag=$(devbox run -- gojq -r .version dist/metadata.json)-$(devbox run -- go env GOARCH)"
+              --helm-extra-set-args "--set-string image.repository=ko.local/cluster-api-runtime-extensions-nutanix --set-string image.tag=${{ steps.export-image-tag.outputs.test-image-tag }} --set-string helmRepository.images.bundleInitializer.tag=${{ steps.export-image-tag.outputs.test-image-tag }}"
         env:
           KUBECONFIG: ${{ env.KIND_KUBECONFIG }}
 

diff --git a/.goreleaser.yml b/.goreleaser.yml
@@ -44,7 +44,7 @@ before:
       $(helm template {{ .ProjectName }} ./charts/{{ .ProjectName }} \
         --namespace caren-system \
         --set-string image.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }}{{ end }} \
-        --set-string helmRepositoryImage.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }} \
+        --set-string helmRepository.images.bundleInitializer.tag=v{{ trimprefix .Version "v" }}{{ if .IsSnapshot }}-{{ .Runtime.Goarch }} \
         --set-string image.repository=ko.local/{{ .ProjectName }}{{ end }} \
       )
       EOF'
@@ -103,9 +103,9 @@ dockers:
   - image_templates:
       - 'ghcr.io/nutanix-cloud-native/caren-helm-reg:v{{ trimprefix .Version "v" }}-amd64'
     use: buildx
-    dockerfile: ./hack/addons/mindthegap-helm-registry/Dockerfile
+    dockerfile: ./hack/addons/helm-chart-bundler/Dockerfile
     extra_files:
-      - hack/addons/mindthegap-helm-registry/repos.yaml
+      - hack/addons/helm-chart-bundler/repos.yaml
     build_flag_templates:
       - "--platform=linux/amd64"
       - "--pull"
@@ -119,9 +119,9 @@ dockers:
   - image_templates:
       - 'ghcr.io/nutanix-cloud-native/caren-helm-reg:v{{ trimprefix .Version "v" }}-arm64'
     use: buildx
-    dockerfile: ./hack/addons/mindthegap-helm-registry/Dockerfile
+    dockerfile: ./hack/addons/helm-chart-bundler/Dockerfile
     extra_files:
-      - hack/addons/mindthegap-helm-registry/repos.yaml
+      - hack/addons/helm-chart-bundler/repos.yaml
     build_flag_templates:
       - "--platform=linux/arm64"
       - "--pull"

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -55,7 +55,7 @@ repos:
     name: addons-repo-yaml
     entry: make template-helm-repository
     language: system
-    files: "hack/addons/mindthegap-helm-registry/repos.yaml"
+    files: "hack/addons/helm-chart-bundler/repos.yaml"
     pass_filenames: false
   - id: check-devbox-lock
     name: check-devbox-lock

diff --git a/charts/cluster-api-runtime-extensions-nutanix/README.md b/charts/cluster-api-runtime-extensions-nutanix/README.md
@@ -32,12 +32,16 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
 | deployment.replicas | int | `1` |  |
 | env | object | `{}` |  |
 | helmAddonsConfigMap | string | `"default-helm-addons-config"` |  |
-| helmRepositoryImage.pullPolicy | string | `"IfNotPresent"` |  |
-| helmRepositoryImage.repository | string | `"ghcr.io/nutanix-cloud-native/caren-helm-reg"` |  |
-| helmRepositoryImage.tag | string | `""` |  |
-| helmRepositorySecurityContext.fsGroup | int | `65534` |  |
-| helmRepositorySecurityContext.runAsGroup | int | `65534` |  |
-| helmRepositorySecurityContext.runAsUser | int | `65534` |  |
+| helmRepository.enabled | bool | `true` |  |
+| helmRepository.images.bundleInitializer.pullPolicy | string | `"IfNotPresent"` |  |
+| helmRepository.images.bundleInitializer.repository | string | `"ghcr.io/nutanix-cloud-native/caren-helm-reg"` |  |
+| helmRepository.images.bundleInitializer.tag | string | `""` |  |
+| helmRepository.images.mindthegap.pullPolicy | string | `"IfNotPresent"` |  |
+| helmRepository.images.mindthegap.repository | string | `"ghcr.io/mesosphere/mindthegap"` |  |
+| helmRepository.images.mindthegap.tag | string | `"v1.16.0"` |  |
+| helmRepository.securityContext.fsGroup | int | `65532` |  |
+| helmRepository.securityContext.runAsGroup | int | `65532` |  |
+| helmRepository.securityContext.runAsUser | int | `65532` |  |
 | hooks.ccm.aws.helmAddonStrategy.defaultValueTemplateConfigMap.create | bool | `true` |  |
 | hooks.ccm.aws.helmAddonStrategy.defaultValueTemplateConfigMap.name | string | `"default-aws-ccm-helm-values-template"` |  |
 | hooks.ccm.aws.k8sMinorVersionToCCMVersion."1.27" | string | `"v1.27.9"` |  |
@@ -99,7 +103,6 @@ A Helm chart for cluster-api-runtime-extensions-nutanix
 | resources.requests.cpu | string | `"100m"` |  |
 | resources.requests.memory | string | `"128Mi"` |  |
 | securityContext.runAsUser | int | `65532` |  |
-| selfHostedRegistry | bool | `true` |  |
 | service.annotations | object | `{}` |  |
 | service.port | int | `443` |  |
 | service.type | string | `"ClusterIP"` |  |

diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml
@@ -10,48 +10,48 @@ data:
   aws-ccm: |
     ChartName: aws-cloud-controller-manager
     ChartVersion: 0.0.8
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/cloud-provider-aws{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/cloud-provider-aws{{ end }}'
   aws-ebs-csi: |
     ChartName: aws-ebs-csi-driver
     ChartVersion: 2.35.1
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/aws-ebs-csi-driver{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/aws-ebs-csi-driver{{ end }}'
   cilium: |
     ChartName: cilium
     ChartVersion: 1.16.2
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://helm.cilium.io/{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://helm.cilium.io/{{ end }}'
   cluster-autoscaler: |
     ChartName: cluster-autoscaler
     ChartVersion: 9.40.0
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}'
   local-path-provisioner-csi: |
     ChartName: local-path-provisioner
     ChartVersion: 0.0.29
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://charts.containeroo.ch{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://charts.containeroo.ch{{ end }}'
   metallb: |
     ChartName: metallb
     ChartVersion: 0.14.8
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://metallb.github.io/metallb{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://metallb.github.io/metallb{{ end }}'
   nfd: |
     ChartName: node-feature-discovery
     ChartVersion: 0.16.4
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/node-feature-discovery/charts{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes-sigs.github.io/node-feature-discovery/charts{{ end }}'
   nutanix-ccm: |
     ChartName: nutanix-cloud-provider
     ChartVersion: 0.4.1
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://nutanix.github.io/helm/{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://nutanix.github.io/helm/{{ end }}'
   nutanix-storage-csi: |
     ChartName: nutanix-csi-storage
     ChartVersion: 3.1.0
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://nutanix.github.io/helm-releases/{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://nutanix.github.io/helm-releases/{{ end }}'
   snapshot-controller: |
     ChartName: snapshot-controller
     ChartVersion: 3.0.6
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://piraeus.io/helm-charts/{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://piraeus.io/helm-charts/{{ end }}'
   tigera-operator: |
     ChartName: tigera-operator
     ChartVersion: v3.28.2
-    RepositoryURL: {{ if .Values.selfHostedRegistry }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://docs.tigera.io/calico/charts{{ end }}
+    RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://docs.tigera.io/calico/charts{{ end }}'
 kind: ConfigMap
 metadata:
   creationTimestamp: null
-  name: {{ .Values.helmAddonsConfigMap }}
+  name: '{{ .Values.helmAddonsConfigMap }}'
diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/helm-repository.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/helm-repository.yaml
@@ -3,7 +3,7 @@
 # This file contains the manifests to run a helmRepository deployment which contains helm charts for our addons.
 # The pod is built via goreleaser with configuration from hack/addons.
 #
-{{ if .Values.selfHostedRegistry }}
+{{ if .Values.helmRepository.enabled }}
 apiVersion: cert-manager.io/v1
 kind: Issuer
 metadata:
@@ -62,8 +62,8 @@ spec:
     spec:
       initContainers:
       - name: copy-charts
-        image: "{{ .Values.helmRepositoryImage.repository }}:{{ default $.Chart.AppVersion .Values.helmRepositoryImage.tag }}"
-        command: ["/bin/sh", "-c", "cp /charts/*.tar /helm-charts"]
+        image: "{{ .Values.helmRepository.images.bundleInitializer.repository }}:{{ default $.Chart.AppVersion .Values.helmRepository.images.bundleInitializer.tag }}"
+        command: ["/bin/cp", "-r", "/charts/",  "/helm-charts/bundles/"]
         imagePullPolicy: "{{ .Values.image.pullPolicy }}"
         volumeMounts:
         - name: charts-volume
@@ -74,13 +74,12 @@ spec:
           - name: serve
             protocol: TCP
             containerPort: 5000
-        image: "{{ .Values.helmRepositoryImage.repository }}:{{ default $.Chart.AppVersion .Values.helmRepositoryImage.tag }}"
+        image: "{{ .Values.helmRepository.images.mindthegap.repository }}:{{ .Values.helmRepository.images.mindthegap.tag }}"
         imagePullPolicy: "{{ .Values.image.pullPolicy }}"
-        command: ["/usr/bin/mindthegap"]
         args:
           - serve
           - bundle
-          - --bundle=/helm-charts/helm-charts-*.tar
+          - --bundle=/helm-charts/bundles/helm-charts-*.tar
           - --listen-port=5000
           - --listen-address=0.0.0.0
           - --tls-private-key-file=/certs/tls.key
@@ -101,7 +100,7 @@ spec:
           periodSeconds: 1
       priorityClassName: {{ .Values.priorityClassName }}
       securityContext:
-        {{ with .Values.helmRepositorySecurityContext }}
+        {{ with .Values.helmRepository.securityContext }}
         {{- toYaml . | nindent 8}}
         {{- end }}
       volumes:

diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.schema.json b/charts/cluster-api-runtime-extensions-nutanix/values.schema.json
@@ -38,30 +38,57 @@
         "helmAddonsConfigMap": {
             "type": "string"
         },
-        "helmRepositoryImage": {
+        "helmRepository": {
             "properties": {
-                "pullPolicy": {
-                    "type": "string"
-                },
-                "repository": {
-                    "type": "string"
-                },
-                "tag": {
-                    "type": "string"
-                }
-            },
-            "type": "object"
-        },
-        "helmRepositorySecurityContext": {
-            "properties": {
-                "fsGroup": {
-                    "type": "integer"
+                "enabled": {
+                    "type": "boolean"
                 },
-                "runAsGroup": {
-                    "type": "integer"
+                "images": {
+                    "properties": {
+                        "bundleInitializer": {
+                            "properties": {
+                                "pullPolicy": {
+                                    "type": "string"
+                                },
+                                "repository": {
+                                    "type": "string"
+                                },
+                                "tag": {
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "mindthegap": {
+                            "properties": {
+                                "pullPolicy": {
+                                    "type": "string"
+                                },
+                                "repository": {
+                                    "type": "string"
+                                },
+                                "tag": {
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object"
+                        }
+                    },
+                    "type": "object"
                 },
-                "runAsUser": {
-                    "type": "integer"
+                "securityContext": {
+                    "properties": {
+                        "fsGroup": {
+                            "type": "integer"
+                        },
+                        "runAsGroup": {
+                            "type": "integer"
+                        },
+                        "runAsUser": {
+                            "type": "integer"
+                        }
+                    },
+                    "type": "object"
                 }
             },
             "type": "object"
@@ -575,9 +602,6 @@
             },
             "type": "object"
         },
-        "selfHostedRegistry": {
-            "type": "boolean"
-        },
         "service": {
             "properties": {
                 "annotations": {

diff --git a/charts/cluster-api-runtime-extensions-nutanix/values.yaml b/charts/cluster-api-runtime-extensions-nutanix/values.yaml
@@ -110,8 +110,6 @@ hooks:
 
 helmAddonsConfigMap: default-helm-addons-config
 
-selfHostedRegistry: true
-
 deployDefaultClusterClasses: true
 
 # The ClusterClass and the Templates it references must be in the same namespace
@@ -132,10 +130,21 @@ image:
   tag: ""
   pullPolicy: IfNotPresent
 
-helmRepositoryImage:
-  repository: ghcr.io/nutanix-cloud-native/caren-helm-reg
-  tag: ""
-  pullPolicy: IfNotPresent
+helmRepository:
+  enabled: true
+  images:
+    bundleInitializer:
+      repository: ghcr.io/nutanix-cloud-native/caren-helm-reg
+      tag: ""
+      pullPolicy: IfNotPresent
+    mindthegap:
+      repository: ghcr.io/mesosphere/mindthegap
+      tag: "v1.16.0"
+      pullPolicy: IfNotPresent
+  securityContext:
+    runAsUser: 65532
+    runAsGroup: 65532
+    fsGroup: 65532
 
 # -- Optional secrets used for pulling the container image
 imagePullSecrets: []
@@ -160,12 +169,6 @@ resources:
 securityContext:
   runAsUser: 65532
 
-# The helm-repository containers are based on an Alpine image with a different nonroot user
-helmRepositorySecurityContext:
-  runAsUser: 65534
-  runAsGroup: 65534
-  fsGroup: 65534
-
 service:
   annotations: {}
   type: ClusterIP

diff --git a/hack/addons/add-warning-helm-configmap.sh b/hack/addons/add-warning-helm-configmap.sh
@@ -8,6 +8,8 @@ readonly SCRIPT_DIR
 # shellcheck source=hack/common.sh
 source "${SCRIPT_DIR}/../common.sh"
 ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")"
+trap 'rm -rf "${ASSETS_DIR}"' EXIT
+
 mv "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml" "${ASSETS_DIR}/helm-config.yaml"
 # add warning not to edit file directly
 cat <<EOF >"${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml"
@@ -19,5 +21,3 @@ $(cat "${GIT_REPO_ROOT}/hack/license-header.yaml.txt")
 #=================================================================
 $(cat "${ASSETS_DIR}/helm-config.yaml")
 EOF
-
-sed -i s/placeholder/"{{ .Values.helmAddonsConfigMap }}"/g "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml"
diff --git a/hack/addons/generate-mindthegap-repofile.sh b/hack/addons/generate-mindthegap-repofile.sh
@@ -9,15 +9,14 @@ readonly SCRIPT_DIR
 source "${SCRIPT_DIR}/../common.sh"
 
 ASSETS_DIR="$(mktemp -d -p "${TMPDIR:-/tmp}")"
+trap 'rm -rf "${ASSETS_DIR}"' EXIT
 
 cp "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml" "${ASSETS_DIR}"
 
-# this sed line is needed because the go library is unable to parse yaml with a template string.
-sed -i s/"{{ .Values.helmAddonsConfigMap }}"/placeholder/g "${ASSETS_DIR}/helm-config.yaml"
 go run "${GIT_REPO_ROOT}/hack/tools/mindthegap-helm-reg/main.go" --input-configmap-file="${ASSETS_DIR}/helm-config.yaml" --output-file="${ASSETS_DIR}/repos.yaml"
 
 # add warning not to edit file directly
-cat <<EOF >"${GIT_REPO_ROOT}/hack/addons/mindthegap-helm-registry/repos.yaml"
+cat <<EOF >"${GIT_REPO_ROOT}/hack/addons/helm-chart-bundler/repos.yaml"
 $(cat "${GIT_REPO_ROOT}/hack/license-header.yaml.txt")
 
 #=================================================================