fix: Handle long cluster names (#845)

This commit starts fixing a bug that means addons fail to be fully deployed if the cluster name is longer than 44 characters. This is caused by the name of the HCP being over 63 characters. This name is then used in HRP labels which have a maximum length of 63 characters, so the HRPs are rejected by the API server when CAAPH applies them. The fix is to use generate name and labels on the HCP to ensure uniqueness by lookup rather than by using a deterministic name.
nutanix-cloud-native · Aug 13, 2024 · eeeb5c2 · eeeb5c2
1 parent 5d54db0
commit eeeb5c2
Show file tree

Hide file tree

Showing 49 changed files with 1,081 additions and 597 deletions.
diff --git a/.golangci.yml b/.golangci.yml
@@ -108,3 +108,17 @@ issues:
     - text: "hugeParam: holderRef is heavy"
       linters:
         - gocritic
+    # Admission request interface is defined by k8s
+    - path: pkg/webhook
+      text: "hugeParam: req is heavy"
+      linters:
+        - gocritic
+    # This is not a problem in tests
+    - path: internal/test/envtest
+      text: "hugeParam: webhookInstallOptions is heavy"
+      linters:
+        - gocritic
+    - path: internal/test/envtest
+      text: "hugeParam: input is heavy"
+      linters:
+        - gocritic
diff --git a/api/v1alpha1/constants.go b/api/v1alpha1/constants.go
@@ -32,4 +32,6 @@ const (
 	GlobalMirrorVariableName = "globalImageRegistryMirror"
 	// ImageRegistriesVariableName is the image registries patch variable name.
 	ImageRegistriesVariableName = "imageRegistries"
+
+	ClusterUUIDAnnotationKey = APIGroup + "/cluster-uuid"
 )
diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/admission-service.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/admission-service.yaml
@@ -0,0 +1,26 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    {{- with .Values.service.annotations }}
+    {{ toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+  name: {{ template "chart.name" . }}-admission
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: {{.Values.service.type}}
+  ports:
+  - name: https
+    port: {{ .Values.service.port }}
+    protocol: TCP
+    targetPort: admission
+    {{- if and .Values.service.nodePort (eq "NodePort" .Values.service.type) }}
+    nodePort: {{ .Values.service.nodePort }}
+    {{- end }}
+  selector:
+    {{- include "chart.selectorLabels" . | nindent 4 }}
diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/certificates.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/certificates.yaml
@@ -16,3 +16,19 @@ spec:
     kind: {{ .Values.certificates.issuer.kind }}
     name: {{ template "chart.issuerName" . }}
   secretName: {{ template "chart.name" . }}-runtimehooks-tls
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ template "chart.name" . }}-admission-tls
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+spec:
+  dnsNames:
+    - {{ template "chart.name" . }}-admission.{{ .Release.Namespace }}.svc
+    - {{ template "chart.name" . }}-admission.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    kind: {{ .Values.certificates.issuer.kind }}
+    name: {{ template "chart.issuerName" . }}
+  secretName: {{ template "chart.name" . }}-admission-tls
diff --git a/...tensions-nutanix/templates/cluster-autoscaler/manifests/cluster-autoscaler-configmap.yaml b/...tensions-nutanix/templates/cluster-autoscaler/manifests/cluster-autoscaler-configmap.yaml
@@ -12,41 +12,41 @@ data:
     kind: PodDisruptionBudget
     metadata:
       labels:
-        app.kubernetes.io/instance: cluster-autoscaler-tmpl-clustername-tmpl
+        app.kubernetes.io/instance: 'ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: clusterapi-cluster-autoscaler
         helm.sh/chart: cluster-autoscaler-9.37.0
-      name: cluster-autoscaler-tmpl-clustername-tmpl
-      namespace: tmpl-clusternamespace-tmpl
+      name: 'cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
+      namespace: '{{ `{{ .Cluster.Namespace }}` }}'
     spec:
       maxUnavailable: 1
       selector:
         matchLabels:
-          app.kubernetes.io/instance: cluster-autoscaler-tmpl-clustername-tmpl
+          app.kubernetes.io/instance: 'ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
           app.kubernetes.io/name: clusterapi-cluster-autoscaler
     ---
     apiVersion: v1
     automountServiceAccountToken: true
     kind: ServiceAccount
     metadata:
       labels:
-        app.kubernetes.io/instance: cluster-autoscaler-tmpl-clustername-tmpl
+        app.kubernetes.io/instance: 'ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: clusterapi-cluster-autoscaler
         helm.sh/chart: cluster-autoscaler-9.37.0
-      name: cluster-autoscaler-tmpl-clustername-tmpl
-      namespace: tmpl-clusternamespace-tmpl
+      name: 'cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
+      namespace: '{{ `{{ .Cluster.Namespace }}` }}'
     ---
     apiVersion: rbac.authorization.k8s.io/v1
     kind: Role
     metadata:
       labels:
-        app.kubernetes.io/instance: cluster-autoscaler-tmpl-clustername-tmpl
+        app.kubernetes.io/instance: 'ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: clusterapi-cluster-autoscaler
         helm.sh/chart: cluster-autoscaler-9.37.0
-      name: cluster-autoscaler-tmpl-clustername-tmpl
-      namespace: tmpl-clusternamespace-tmpl
+      name: 'cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
+      namespace: '{{ `{{ .Cluster.Namespace }}` }}'
     rules:
     - apiGroups:
       - ""
@@ -105,71 +105,71 @@ data:
     kind: RoleBinding
     metadata:
       labels:
-        app.kubernetes.io/instance: cluster-autoscaler-tmpl-clustername-tmpl
+        app.kubernetes.io/instance: 'ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: clusterapi-cluster-autoscaler
         helm.sh/chart: cluster-autoscaler-9.37.0
-      name: cluster-autoscaler-tmpl-clustername-tmpl
-      namespace: tmpl-clusternamespace-tmpl
+      name: 'cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
+      namespace: '{{ `{{ .Cluster.Namespace }}` }}'
     roleRef:
       apiGroup: rbac.authorization.k8s.io
       kind: Role
-      name: cluster-autoscaler-tmpl-clustername-tmpl
+      name: 'cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
     subjects:
     - kind: ServiceAccount
-      name: cluster-autoscaler-tmpl-clustername-tmpl
-      namespace: tmpl-clusternamespace-tmpl
+      name: 'cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
+      namespace: '{{ `{{ .Cluster.Namespace }}` }}'
     ---
     apiVersion: v1
     kind: Service
     metadata:
       labels:
-        app.kubernetes.io/instance: cluster-autoscaler-tmpl-clustername-tmpl
+        app.kubernetes.io/instance: 'ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: clusterapi-cluster-autoscaler
         helm.sh/chart: cluster-autoscaler-9.37.0
-      name: cluster-autoscaler-tmpl-clustername-tmpl
-      namespace: tmpl-clusternamespace-tmpl
+      name: 'cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
+      namespace: '{{ `{{ .Cluster.Namespace }}` }}'
     spec:
       ports:
       - name: http
         port: 8085
         protocol: TCP
         targetPort: 8085
       selector:
-        app.kubernetes.io/instance: cluster-autoscaler-tmpl-clustername-tmpl
+        app.kubernetes.io/instance: 'ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
         app.kubernetes.io/name: clusterapi-cluster-autoscaler
       type: ClusterIP
     ---
     apiVersion: apps/v1
     kind: Deployment
     metadata:
       labels:
-        app.kubernetes.io/instance: cluster-autoscaler-tmpl-clustername-tmpl
+        app.kubernetes.io/instance: 'ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
         app.kubernetes.io/managed-by: Helm
         app.kubernetes.io/name: clusterapi-cluster-autoscaler
         helm.sh/chart: cluster-autoscaler-9.37.0
-      name: cluster-autoscaler-tmpl-clustername-tmpl
-      namespace: tmpl-clusternamespace-tmpl
+      name: 'cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
+      namespace: '{{ `{{ .Cluster.Namespace }}` }}'
     spec:
       replicas: 1
       revisionHistoryLimit: 10
       selector:
         matchLabels:
-          app.kubernetes.io/instance: cluster-autoscaler-tmpl-clustername-tmpl
+          app.kubernetes.io/instance: 'ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
           app.kubernetes.io/name: clusterapi-cluster-autoscaler
       template:
         metadata:
           labels:
-            app.kubernetes.io/instance: cluster-autoscaler-tmpl-clustername-tmpl
+            app.kubernetes.io/instance: 'ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
             app.kubernetes.io/name: clusterapi-cluster-autoscaler
         spec:
           containers:
           - command:
             - ./cluster-autoscaler
             - --cloud-provider=clusterapi
-            - --namespace=tmpl-clusternamespace-tmpl
-            - --node-group-auto-discovery=clusterapi:clusterName=tmpl-clustername-tmpl,namespace=tmpl-clusternamespace-tmpl
+            - --namespace='{{ `{{ .Cluster.Namespace }}` }}'
+            - --node-group-auto-discovery=clusterapi:clusterName='{{ `{{ .Cluster.Name }}` }}',namespace='{{ `{{ .Cluster.Namespace }}` }}'
             - --kubeconfig=/cluster/kubeconfig
             - --clusterapi-cloud-config-authoritative
             - --enforce-node-group-min-size=true
@@ -201,7 +201,7 @@ data:
               readOnly: true
           dnsPolicy: ClusterFirst
           priorityClassName: system-cluster-critical
-          serviceAccountName: cluster-autoscaler-tmpl-clustername-tmpl
+          serviceAccountName: 'cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}'
           tolerations:
           - effect: NoSchedule
             key: node-role.kubernetes.io/control-plane
@@ -211,7 +211,7 @@ data:
               items:
               - key: value
                 path: kubeconfig
-              secretName: tmpl-clustername-tmpl-kubeconfig
+              secretName: '{{ `{{ .Cluster.Name }}` }}-kubeconfig'
 kind: ConfigMap
 metadata:
   creationTimestamp: null

diff --git a/...me-extensions-nutanix/templates/cluster-autoscaler/manifests/helm-addon-installation.yaml b/...me-extensions-nutanix/templates/cluster-autoscaler/manifests/helm-addon-installation.yaml
@@ -9,7 +9,7 @@ metadata:
 data:
   values.yaml: |-
     ---
-    fullnameOverride: "cluster-autoscaler-{{ `{{ .Cluster.Name }}` }}"
+    fullnameOverride: "cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}"
 
     cloudProvider: clusterapi
 

diff --git a/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml b/charts/cluster-api-runtime-extensions-nutanix/templates/deployment.yaml
@@ -44,6 +44,7 @@ spec:
         {{- range $k, $v := .Values.hooks.ccm.aws.k8sMinorVersionToCCMVersion }}
         - --ccm.aws.aws-ccm-versions={{ $k }}={{ $v }}
         {{- end }}
+        - --admission-webhook-cert-dir=/admission-certs/
         {{- range $key, $value := .Values.extraArgs }}
         - --{{ $key }}={{ $value }}
         {{- end }}
@@ -57,6 +58,9 @@ spec:
         - containerPort: 9443
           name: runtimehooks
           protocol: TCP
+        - containerPort: 9444
+          name: admission
+          protocol: TCP
         - containerPort: 8080
           name: metrics
           protocol: TCP
@@ -76,6 +80,9 @@ spec:
         - mountPath: /runtimehooks-certs
           name: runtimehooks-cert
           readOnly: true
+        - mountPath: /admission-certs
+          name: admission-cert
+          readOnly: true
         livenessProbe:
           httpGet:
             port: probes
@@ -96,3 +103,7 @@ spec:
         secret:
           defaultMode: 420
           secretName: {{ template "chart.name" . }}-runtimehooks-tls
+      - name: admission-cert
+        secret:
+          defaultMode: 420
+          secretName: {{ template "chart.name" . }}-admission-tls