feat: Make containerd restart its own patch

pkg/handlers/aws/mutation/metapatch_handler_test.go

@@ -25,6 +25,7 @@ import (
 	regiontests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/region/tests"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig"
 	auditpolicytests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy/tests"
+	containerdrestarttests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdrestart/tests"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd"
 	etcdtests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd/tests"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
@@ -188,4 +189,9 @@ func TestGeneratePatches(t *testing.T) {
 		v1alpha1.AWSVariableName,
 		controlplaneloadbalancer.VariableName,
 	)
+
+	containerdrestarttests.TestGeneratePatches(
+		t,
+		metaPatchGeneratorFunc(mgr),
+	)
 }

pkg/handlers/docker/mutation/metapatch_handler_test.go

@@ -15,6 +15,7 @@ import (
 	dockerworkerconfig "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/docker/workerconfig"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/clusterconfig"
 	auditpolicytests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy/tests"
+	containerdrestarttests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdrestart/tests"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd"
 	etcdtests "github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd/tests"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
@@ -112,4 +113,9 @@ func TestGeneratePatches(t *testing.T) {
 		clusterconfig.MetaVariableName,
 		mirrors.GlobalMirrorVariableName,
 	)
+
+	containerdrestarttests.TestGeneratePatches(
+		t,
+		metaPatchGeneratorFunc(mgr),
+	)
 }

pkg/handlers/generic/mutation/containerdrestart/inject.go

@@ -0,0 +1,86 @@
+// Copyright 2023 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+package containerdrestart
+
+import (
+	"context"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
+)
+
+type globalMirrorPatchHandler struct{}
+
+func NewPatch() *globalMirrorPatchHandler {
+	return &globalMirrorPatchHandler{}
+}
+
+func (h *globalMirrorPatchHandler) Mutate(
+	ctx context.Context,
+	obj *unstructured.Unstructured,
+	vars map[string]apiextensionsv1.JSON,
+	holderRef runtimehooksv1.HolderReference,
+	clusterKey ctrlclient.ObjectKey,
+) error {
+	log := ctrl.LoggerFrom(ctx).WithValues(
+		"holderRef", holderRef,
+	)
+
+	file, command := generateContainerdRestartScript()
+
+	if err := patches.MutateIfApplicable(
+		obj, vars, &holderRef, selectors.ControlPlane(), log,
+		func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
+			).Info("adding containerd restart script to control plane kubeadm config spec")
+			obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append(
+				obj.Spec.Template.Spec.KubeadmConfigSpec.Files,
+				file,
+			)
+
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
+			).Info("adding containerd restart command to control plane kubeadm config spec")
+			obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(
+				obj.Spec.Template.Spec.KubeadmConfigSpec.PreKubeadmCommands,
+				command,
+			)
+
+			return nil
+		}); err != nil {
+		return err
+	}
+
+	if err := patches.MutateIfApplicable(
+		obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log,
+		func(obj *bootstrapv1.KubeadmConfigTemplate) error {
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
+			).Info("adding containerd restart script to worker node kubeadm config template")
+			obj.Spec.Template.Spec.Files = append(obj.Spec.Template.Spec.Files, file)
+
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
+			).Info("adding containerd restart command to worker node kubeadm config template")
+			obj.Spec.Template.Spec.PreKubeadmCommands = append(obj.Spec.Template.Spec.PreKubeadmCommands, command)
+
+			return nil
+		}); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/handlers/generic/mutation/containerdrestart/restart.go

@@ -0,0 +1,27 @@
+// Copyright 2023 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+package containerdrestart
+
+import (
+	_ "embed"
+
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+)
+
+const (
+	ContainerdRestartScriptOnRemote        = "/etc/containerd/restart.sh"
+	ContainerdRestartScriptOnRemoteCommand = "/bin/bash " + ContainerdRestartScriptOnRemote
+)
+
+//go:embed templates/containerd-restart.sh
+var containerdRestartScript []byte
+
+//nolint:gocritic // no need for named return values
+func generateContainerdRestartScript() (bootstrapv1.File, string) {
+	return bootstrapv1.File{
+			Path:        ContainerdRestartScriptOnRemote,
+			Content:     string(containerdRestartScript),
+			Permissions: "0700",
+		},
+		ContainerdRestartScriptOnRemoteCommand
+}

pkg/handlers/generic/mutation/containerdrestart/tests/generate_patches.go

@@ -0,0 +1,71 @@
+// Copyright 2023 D2iQ, Inc. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package tests
+
+import (
+	"testing"
+
+	"github.com/onsi/gomega"
+
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest"
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request"
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdrestart"
+)
+
+func TestGeneratePatches(
+	t *testing.T,
+	generatorFunc func() mutation.GeneratePatches,
+) {
+	t.Helper()
+
+	capitest.ValidateGeneratePatches(
+		t,
+		generatorFunc,
+		capitest.PatchTestDef{
+			Name:        "restart script and command added to control plane kubeadm config spec",
+			RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
+			ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/files",
+					ValueMatcher: gomega.ContainElements(
+						gomega.HaveKeyWithValue(
+							"path", containerdrestart.ContainerdRestartScriptOnRemote,
+						),
+					),
+				},
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
+					ValueMatcher: gomega.ContainElements(
+						containerdrestart.ContainerdRestartScriptOnRemoteCommand,
+					),
+				},
+			},
+		},
+		capitest.PatchTestDef{
+			Name:        "restart script and command added to worker node kubeadm config template",
+			RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
+			ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/files",
+					ValueMatcher: gomega.ContainElements(
+						gomega.HaveKeyWithValue(
+							"path", containerdrestart.ContainerdRestartScriptOnRemote,
+						),
+					),
+				},
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
+					ValueMatcher: gomega.ContainElements(
+						containerdrestart.ContainerdRestartScriptOnRemoteCommand,
+					),
+				},
+			},
+		},
+	)
+}

pkg/handlers/generic/mutation/handlers.go

@@ -9,6 +9,7 @@ import (
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/cni/calico"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy"
+	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdrestart"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
 	"github.com/d2iq-labs/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy"
@@ -28,5 +29,10 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
 		credentials.NewPatch(mgr.GetClient()),
 		mirrors.NewPatch(mgr.GetClient()),
 		calico.NewPatch(),
+
+		// Some patches may have changed containerd configuration.
+		// We must restart containerd for the configuration to take effect.
+		// Therefore, we must apply this patch last.
+		containerdrestart.NewPatch(),
 	}
 }

pkg/handlers/generic/mutation/mirrors/inject.go

@@ -191,10 +191,6 @@ func generateFilesAndCommands(
 	}
 	files = append(files, applyPatchesFile...)
 	commands = append(commands, applyPatchesCommand)
-	// generate Containerd restart script and command
-	restartFile, restartCommand := generateContainerdRestartScript()
-	files = append(files, restartFile...)
-	commands = append(commands, restartCommand)
 
 	return files, commands, err
 }

pkg/handlers/generic/mutation/mirrors/mirror.go

@@ -28,9 +28,6 @@ const (
 	containerdPatchesDirOnRemote                = "/etc/containerd/cre.d"
 	containerdApplyPatchesScriptOnRemote        = "/etc/containerd/apply-patches.sh"
 	containerdApplyPatchesScriptOnRemoteCommand = "/bin/bash " + containerdApplyPatchesScriptOnRemote
-
-	containerdRestartScriptOnRemote        = "/etc/containerd/restart.sh"
-	containerdRestartScriptOnRemoteCommand = "/bin/bash " + containerdRestartScriptOnRemote
 )
 
 var (
@@ -50,9 +47,6 @@ var (
 
 	//go:embed templates/containerd-apply-patches.sh.gotmpl
 	containerdApplyConfigPatchesScript []byte
-
-	//go:embed templates/containerd-restart.sh
-	containerdRestartScript []byte
 )
 
 type mirrorConfig struct {
@@ -226,14 +220,3 @@ func generateContainerdApplyPatchesScript() ([]cabpkv1.File, string, error) {
 		},
 	}, containerdApplyPatchesScriptOnRemoteCommand, nil
 }
-
-//nolint:gocritic // no need for named return values
-func generateContainerdRestartScript() ([]cabpkv1.File, string) {
-	return []cabpkv1.File{
-		{
-			Path:        containerdRestartScriptOnRemote,
-			Content:     string(containerdRestartScript),
-			Permissions: "0700",
-		},
-	}, containerdRestartScriptOnRemoteCommand
-}

pkg/handlers/generic/mutation/mirrors/tests/generate_patches.go

@@ -75,17 +75,13 @@ func TestGeneratePatches(
 						gomega.HaveKeyWithValue(
 							"path", "/etc/containerd/apply-patches.sh",
 						),
-						gomega.HaveKeyWithValue(
-							"path", "/etc/containerd/restart.sh",
-						),
 					),
 				},
 				{
 					Operation: "add",
 					Path:      "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
 					ValueMatcher: gomega.ContainElements(
 						"/bin/bash /etc/containerd/apply-patches.sh",
-						"/bin/bash /etc/containerd/restart.sh",
 					),
 				},
 			},
@@ -124,17 +120,13 @@ func TestGeneratePatches(
 						gomega.HaveKeyWithValue(
 							"path", "/etc/containerd/apply-patches.sh",
 						),
-						gomega.HaveKeyWithValue(
-							"path", "/etc/containerd/restart.sh",
-						),
 					),
 				},
 				{
 					Operation: "add",
 					Path:      "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands",
 					ValueMatcher: gomega.ContainElements(
 						"/bin/bash /etc/containerd/apply-patches.sh",
-						"/bin/bash /etc/containerd/restart.sh",
 					),
 				},
 			},
@@ -173,17 +165,13 @@ func TestGeneratePatches(
 						gomega.HaveKeyWithValue(
 							"path", "/etc/containerd/apply-patches.sh",
 						),
-						gomega.HaveKeyWithValue(
-							"path", "/etc/containerd/restart.sh",
-						),
 					),
 				},
 				{
 					Operation: "add",
 					Path:      "/spec/template/spec/preKubeadmCommands",
 					ValueMatcher: gomega.ContainElements(
 						"/bin/bash /etc/containerd/apply-patches.sh",
-						"/bin/bash /etc/containerd/restart.sh",
 					),
 				},
 			},
@@ -230,17 +218,13 @@ func TestGeneratePatches(
 						gomega.HaveKeyWithValue(
 							"path", "/etc/containerd/apply-patches.sh",
 						),
-						gomega.HaveKeyWithValue(
-							"path", "/etc/containerd/restart.sh",
-						),
 					),
 				},
 				{
 					Operation: "add",
 					Path:      "/spec/template/spec/preKubeadmCommands",
 					ValueMatcher: gomega.ContainElements(
 						"/bin/bash /etc/containerd/apply-patches.sh",
-						"/bin/bash /etc/containerd/restart.sh",
 					),
 				},
 			},