feat: Enable unprivileged ports sysctl in containerd config (#645)

This enabled pods to run as non-root and bind to privileged ports
as long as they have the necessary capability, `CAP_NET_BIND_SERVICE`
added.

This fixes an issue on AWS when bringing up coredns which binds to port
53 but runs as an unprivileged user.

Overall this is a net security improvement for clusters, meaning users
can stop giving too many privileged to pods - see
kubernetes/kubernetes#102612 for discussion.

Depends on #644.

pkg/handlers/generic/mutation/containerdunprivilegedports/files/unprivileged-ports-config.toml

@@ -0,0 +1,3 @@
+[plugins."io.containerd.grpc.v1.cri"]
+  enable_unprivileged_ports = true
+  enable_unprivileged_icmp = true

pkg/handlers/generic/mutation/containerdunprivilegedports/inject.go

@@ -0,0 +1,75 @@
+// Copyright 2024 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+package containerdunprivilegedports
+
+import (
+	"context"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
+)
+
+type containerdUnprivilegedPortsPatchHandler struct{}
+
+func NewPatch() *containerdUnprivilegedPortsPatchHandler {
+	return &containerdUnprivilegedPortsPatchHandler{}
+}
+
+func (h *containerdUnprivilegedPortsPatchHandler) Mutate(
+	ctx context.Context,
+	obj *unstructured.Unstructured,
+	vars map[string]apiextensionsv1.JSON,
+	holderRef runtimehooksv1.HolderReference,
+	_ ctrlclient.ObjectKey,
+	_ mutation.ClusterGetter,
+) error {
+	log := ctrl.LoggerFrom(ctx).WithValues(
+		"holderRef", holderRef,
+	)
+
+	unprivilegedPortsConfigDropIn := generateUnprivilegedPortsConfigDropIn()
+
+	if err := patches.MutateIfApplicable(
+		obj, vars, &holderRef, selectors.ControlPlane(), log,
+		func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
+			).Info("adding containerd unprivileged ports config to control plane kubeadm config spec")
+			obj.Spec.Template.Spec.KubeadmConfigSpec.Files = append(
+				obj.Spec.Template.Spec.KubeadmConfigSpec.Files,
+				unprivilegedPortsConfigDropIn,
+			)
+
+			return nil
+		}); err != nil {
+		return err
+	}
+
+	if err := patches.MutateIfApplicable(
+		obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log,
+		func(obj *bootstrapv1.KubeadmConfigTemplate) error {
+			log.WithValues(
+				"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
+				"patchedObjectName", ctrlclient.ObjectKeyFromObject(obj),
+			).Info("adding containerd unprivileged ports config to worker node kubeadm config template")
+			obj.Spec.Template.Spec.Files = append(
+				obj.Spec.Template.Spec.Files,
+				unprivilegedPortsConfigDropIn)
+
+			return nil
+		}); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/handlers/generic/mutation/containerdunprivilegedports/inject_test.go

@@ -0,0 +1,84 @@
+// Copyright 2024 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package containerdunprivilegedports
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers"
+)
+
+func TestContainerdUnprivilegedPortsPatch(t *testing.T) {
+	gomega.RegisterFailHandler(Fail)
+	RunSpecs(t, "Containerd unprivileged ports mutator suite")
+}
+
+var _ = Describe("Generate containerd unprivileged ports patches", func() {
+	// only add aws region patch
+	patchGenerator := func() mutation.GeneratePatches {
+		return mutation.NewMetaGeneratePatchesHandler("", helpers.TestEnv.Client, NewPatch()).(mutation.GeneratePatches)
+	}
+
+	testDefs := []capitest.PatchTestDef{
+		{
+			Name:        "containerd unprivileged ports config added to control plane kubeadm config spec",
+			RequestItem: request.NewKubeadmControlPlaneTemplateRequestItem(""),
+			ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/kubeadmConfigSpec/files",
+					ValueMatcher: gomega.ContainElements(
+						gomega.HaveKeyWithValue(
+							"path", unprivilegedPortsConfigDropInFileOnRemote,
+						),
+					),
+				},
+			},
+		},
+		{
+			Name: "containerd unprivileged ports config added to worker node kubeadm config template",
+			Vars: []runtimehooksv1.Variable{
+				capitest.VariableWithValue(
+					"builtin",
+					map[string]any{
+						"machineDeployment": map[string]any{
+							"class": "*",
+						},
+					},
+				),
+			},
+			RequestItem: request.NewKubeadmConfigTemplateRequestItem(""),
+			ExpectedPatchMatchers: []capitest.JSONPatchMatcher{
+				{
+					Operation: "add",
+					Path:      "/spec/template/spec/files",
+					ValueMatcher: gomega.ContainElements(
+						gomega.HaveKeyWithValue(
+							"path", unprivilegedPortsConfigDropInFileOnRemote,
+						),
+					),
+				},
+			},
+		},
+	}
+
+	// create test node for each case
+	for testIdx := range testDefs {
+		tt := testDefs[testIdx]
+		It(tt.Name, func() {
+			capitest.AssertGeneratePatches(
+				GinkgoT(),
+				patchGenerator,
+				&tt,
+			)
+		})
+	}
+})

pkg/handlers/generic/mutation/containerdunprivilegedports/unprivileged_ports.go

@@ -0,0 +1,27 @@
+// Copyright 2024 Nutanix. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+package containerdunprivilegedports
+
+import (
+	_ "embed"
+
+	cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/common"
+)
+
+var (
+	//go:embed files/unprivileged-ports-config.toml
+	unprivilegedPortsConfigDropIn             []byte
+	unprivilegedPortsConfigDropInFileOnRemote = common.ContainerdPatchPathOnRemote(
+		"unprivileged-ports-config.toml",
+	)
+)
+
+func generateUnprivilegedPortsConfigDropIn() cabpkv1.File {
+	return cabpkv1.File{
+		Path:        unprivilegedPortsConfigDropInFileOnRemote,
+		Content:     string(unprivilegedPortsConfigDropIn),
+		Permissions: "0600",
+	}
+}

pkg/handlers/generic/mutation/handlers.go

@@ -11,6 +11,7 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/auditpolicy"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdapplypatchesandrestart"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdmetrics"
+	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/containerdunprivilegedports"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/etcd"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy"
@@ -33,6 +34,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
 		calico.NewPatch(),
 		users.NewPatch(),
 		containerdmetrics.NewPatch(),
+		containerdunprivilegedports.NewPatch(),
 
 		// Some patches may have changed containerd configuration.
 		// We write the configuration changes to disk, and must run a command