feat: set default CoreDNS version (#959)

**What problem does this PR solve?**:
Automatically upgrades the CoreDNS version. This is done by always
setting `dns.imageTag` in KCP in the CoreDNS handler, based on the
mapping to the cluster's Kubernetes version.

This component is different from etcd and kube-proxy that is also
installed by kubeadm for a few different reasons.
- an etcd upgrade is handled by kubeadm 
- kube-proxy, another "addon", is [upgraded by CAPI
core](https://github.com/kubernetes-sigs/cluster-api/blob/75c986db9e38190a2313eaf6e5f97d955fa96b65/controlplane/kubeadm/internal/controllers/controller.go#L476-L480)
because its version will match the Kubernetes version

[This functiona
call](https://github.com/kubernetes-sigs/cluster-api/blob/75c986db9e38190a2313eaf6e5f97d955fa96b65/controlplane/kubeadm/internal/controllers/controller.go#L482-L485)
is misleading and [will only update the version if its set in
KCP](https://github.com/kubernetes-sigs/cluster-api/blob/6d7104deacddf540c82734ae8abaf309f2ab3b90/controlplane/kubeadm/internal/workload_cluster_coredns.go#L223-L226).

To not cause a rollout of all managed clusters by changing the defaults,
this PR introduces a new API to opt in. To enable this functionality a
client can set this new API like so for new clusters and during cluster
upgrades:
```
spec:
  topology:
    variables:
      - name: clusterConfig
        value:
          dns:
            coreDNS: {}
```

**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

---------

Co-authored-by: Jimmi Dyson <jimmidyson@gmail.com>

api/v1alpha1/clusterconfig_types.go

@@ -322,6 +322,8 @@ type DNS struct {
 
 type CoreDNS struct {
 	// Image required for overriding Kubernetes DNS image details.
+	// If the image version is not specified,
+	// the default version based on the cluster's Kubernetes version will be used.
 	// +kubebuilder:validation:Optional
 	Image *Image `json:"image,omitempty"`
 }

api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml

@@ -379,7 +379,10 @@ spec:
                       description: CoreDNS defines the CoreDNS configuration for the cluster.
                       properties:
                         image:
-                          description: Image required for overriding Kubernetes DNS image details.
+                          description: |-
+                            Image required for overriding Kubernetes DNS image details.
+                            If the image version is not specified,
+                            the default version based on the cluster's Kubernetes version will be used.
                           properties:
                             repository:
                               description: Repository is used to override the image repository to pull from.

api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml

@@ -296,7 +296,10 @@ spec:
                       description: CoreDNS defines the CoreDNS configuration for the cluster.
                       properties:
                         image:
-                          description: Image required for overriding Kubernetes DNS image details.
+                          description: |-
+                            Image required for overriding Kubernetes DNS image details.
+                            If the image version is not specified,
+                            the default version based on the cluster's Kubernetes version will be used.
                           properties:
                             repository:
                               description: Repository is used to override the image repository to pull from.

api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml

@@ -58,8 +58,10 @@ spec:
                       cluster.
                     properties:
                       image:
-                        description: Image required for overriding Kubernetes DNS
-                          image details.
+                        description: |-
+                          Image required for overriding Kubernetes DNS image details.
+                          If the image version is not specified,
+                          the default version based on the cluster's Kubernetes version will be used.
                         properties:
                           repository:
                             description: Repository is used to override the image

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

@@ -449,7 +449,10 @@ spec:
                       description: CoreDNS defines the CoreDNS configuration for the cluster.
                       properties:
                         image:
-                          description: Image required for overriding Kubernetes DNS image details.
+                          description: |-
+                            Image required for overriding Kubernetes DNS image details.
+                            If the image version is not specified,
+                            the default version based on the cluster's Kubernetes version will be used.
                           properties:
                             repository:
                               description: Repository is used to override the image repository to pull from.

docs/content/customization/generic/dns.md

@@ -15,6 +15,35 @@ If the `dns.coreDNS` property is not specified, then the customization will be s
 
 ### Example
 
+The CoreDNS version can be updated automatically. To do this, set `coreDNS` to an empty object:
+
+```yaml
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: <NAME>
+spec:
+  topology:
+    variables:
+      - name: clusterConfig
+        value:
+          dns:
+            coreDNS: {}
+```
+
+Applying this configuration will result in the following value being set,
+with the version of the CoreDNS image being set based on the cluster's Kubernetes version:
+
+- `KubeadmControlPlaneTemplate`:
+
+  - ```yaml
+    spec:
+      kubeadmConfigSpec:
+        clusterConfiguration:
+          dns:
+            imageTag: "v1.11.3"
+    ```
+
 To change the repository and tag for the container image for the CoreDNS pod, specify the following configuration:
 
 > Note do not include "coredns" in the repository, kubeadm already appends it.

examples/capi-quick-start/aws-cluster-calico-crs.yaml

@@ -50,6 +50,8 @@ spec:
                 baseOS: ${AMI_LOOKUP_BASEOS}
                 format: ${AMI_LOOKUP_FORMAT}
                 org: "${AMI_LOOKUP_ORG}"
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/aws-cluster-calico-helm-addon.yaml

@@ -44,6 +44,8 @@ spec:
                 baseOS: ${AMI_LOOKUP_BASEOS}
                 format: ${AMI_LOOKUP_FORMAT}
                 org: "${AMI_LOOKUP_ORG}"
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/aws-cluster-cilium-crs.yaml

@@ -50,6 +50,8 @@ spec:
                 baseOS: ${AMI_LOOKUP_BASEOS}
                 format: ${AMI_LOOKUP_FORMAT}
                 org: "${AMI_LOOKUP_ORG}"
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/aws-cluster-cilium-helm-addon.yaml

@@ -44,6 +44,8 @@ spec:
                 baseOS: ${AMI_LOOKUP_BASEOS}
                 format: ${AMI_LOOKUP_FORMAT}
                 org: "${AMI_LOOKUP_ORG}"
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/docker-cluster-calico-crs.yaml

@@ -48,6 +48,8 @@ spec:
               - end: 198.18.1.30
                 start: 198.18.1.21
             provider: MetalLB
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/docker-cluster-calico-helm-addon.yaml

@@ -43,6 +43,8 @@ spec:
               - end: 198.18.1.30
                 start: 198.18.1.21
             provider: MetalLB
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/docker-cluster-cilium-crs.yaml

@@ -48,6 +48,8 @@ spec:
               - end: 198.18.1.30
                 start: 198.18.1.21
             provider: MetalLB
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/docker-cluster-cilium-helm-addon.yaml

@@ -43,6 +43,8 @@ spec:
               - end: 198.18.1.30
                 start: 198.18.1.21
             provider: MetalLB
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/nutanix-cluster-calico-crs.yaml

@@ -116,6 +116,8 @@ spec:
               systemDiskSize: 40Gi
               vcpuSockets: 2
               vcpusPerSocket: 1
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/nutanix-cluster-calico-helm-addon.yaml

@@ -113,6 +113,8 @@ spec:
               systemDiskSize: 40Gi
               vcpuSockets: 2
               vcpusPerSocket: 1
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/nutanix-cluster-cilium-crs.yaml

@@ -116,6 +116,8 @@ spec:
               systemDiskSize: 40Gi
               vcpuSockets: 2
               vcpusPerSocket: 1
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

examples/capi-quick-start/nutanix-cluster-cilium-helm-addon.yaml

@@ -113,6 +113,8 @@ spec:
               systemDiskSize: 40Gi
               vcpuSockets: 2
               vcpusPerSocket: 1
+        dns:
+          coreDNS: {}
         encryptionAtRest:
           providers:
           - aescbc: {}

hack/examples/bases/aws/cluster/kustomization.yaml.tmpl

@@ -48,6 +48,9 @@ patches:
 - target:
     kind: Cluster
   path: ../../../patches/encryption.yaml
+- target:
+    kind: Cluster
+  path: ../../../patches/coredns.yaml
 
 # Delete the clusterclass-specific resources.
 - target:

hack/examples/bases/docker/cluster/kustomization.yaml.tmpl

@@ -41,6 +41,9 @@ patches:
 - target:
     kind: Cluster
   path: ../../../patches/encryption.yaml
+- target:
+    kind: Cluster
+  path: ../../../patches/coredns.yaml
 
 # Deploy ServiceLoadBalancer MetalLB
 - target:

hack/examples/bases/nutanix/cluster/kustomization.yaml.tmpl

@@ -39,6 +39,9 @@ patches:
 - target:
     kind: Cluster
   path: ../../../patches/encryption.yaml
+- target:
+    kind: Cluster
+  path: ../../../patches/coredns.yaml
 
 # Remove Additional Trust Bundle ConfigMap
 - target:

hack/examples/patches/coredns.yaml

@@ -0,0 +1,7 @@
+# Copyright 2024 Nutanix. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+- op: "add"
+  path: "/spec/topology/variables/0/value/dns"
+  value:
+    coreDNS: {}

pkg/handlers/generic/mutation/coredns/inject.go

@@ -5,6 +5,7 @@ package coredns
 
 import (
 	"context"
+	"errors"
 
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -15,6 +16,7 @@ import (
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
+	corednsversions "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/versions"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
@@ -26,6 +28,10 @@ const (
 	VariableName = "coreDNS"
 )
 
+var ErrDefaultCoreDNSVersionNotFound = errors.New(
+	"could not determine default CoreDNS version based on the Kubernetes version",
+)
+
 type coreDNSPatchHandler struct {
 	variableName      string
 	variableFieldPath []string
@@ -53,7 +59,7 @@ func (h *coreDNSPatchHandler) Mutate(
 	vars map[string]apiextensionsv1.JSON,
 	holderRef runtimehooksv1.HolderReference,
 	_ ctrlclient.ObjectKey,
-	_ mutation.ClusterGetter,
+	clusterGetter mutation.ClusterGetter,
 ) error {
 	log := ctrl.LoggerFrom(ctx).WithValues(
 		"holderRef", holderRef,
@@ -66,7 +72,7 @@ func (h *coreDNSPatchHandler) Mutate(
 	)
 	if err != nil {
 		if variables.IsNotFoundError(err) {
-			log.V(5).Info("coreDNSVar variable not defined")
+			log.V(5).Info("coreDNS variable not defined")
 			return nil
 		}
 		return err
@@ -81,34 +87,52 @@ func (h *coreDNSPatchHandler) Mutate(
 		coreDNSVar,
 	)
 
+	cluster, err := clusterGetter(ctx)
+	if err != nil {
+		log.Error(
+			err,
+			"failed to get cluster for CoreDNS mutation handler",
+		)
+		return err
+	}
+
 	return patches.MutateIfApplicable(
 		obj, vars, &holderRef, selectors.ControlPlane(), log,
 		func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
 			log.WithValues(
 				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
 				"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
-			).Info("setting CoreDNS version if needed")
+			).Info("setting CoreDNS version")
 
 			if obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration == nil {
 				obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{}
 			}
 
-			if coreDNSVar.Image == nil {
-				return nil
-			}
-
-			dns := obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration.DNS
+			dns := &obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration.DNS
 
-			if coreDNSVar.Image.Tag != "" {
-				dns.ImageTag = coreDNSVar.Image.Tag
+			// Set the CoreDNS image from the variable if it is defined.
+			if coreDNSVar.Image != nil {
+				if coreDNSVar.Image.Tag != "" {
+					dns.ImageTag = coreDNSVar.Image.Tag
+				}
+				if coreDNSVar.Image.Repository != "" {
+					dns.ImageRepository = coreDNSVar.Image.Repository
+				}
 			}
 
-			if coreDNSVar.Image.Repository != "" {
-				dns.ImageRepository = coreDNSVar.Image.Repository
+			// If the CoreDNS image tag is still not set, set the image tag to the default CoreDNS version based on the
+			// Kubernetes version.
+			if dns.ImageTag == "" {
+				defaultCoreDNSVersion, found := corednsversions.GetCoreDNSVersion(
+					cluster.Spec.Topology.Version,
+				)
+				if !found {
+					return ErrDefaultCoreDNSVersionNotFound
+				}
+
+				dns.ImageTag = defaultCoreDNSVersion
 			}
 
-			obj.Spec.Template.Spec.KubeadmConfigSpec.ClusterConfiguration.DNS = dns
-
 			return nil
 		})
 }