From 9cf759b7aaa45149e8079a73254b763f0db75f21 Mon Sep 17 00:00:00 2001
From: Derek Su <derek.su@suse.com>
Date: Tue, 2 Apr 2024 08:47:57 +0000
Subject: [PATCH] feat(rwx): share manager pod respects the
 storageClass.allowedTopologies

In the Longhorn storage system, a share manager pod of a rwx volume is created on a random node.
The purpose of this update is to enhance the locality of an RWX volume and its share manager pod.
It accomplishes this by converting the storageClass.allowedTopologies setting into an affinity
configuration for the share manager pod. As a result, both the pod and the RWX volume respect
this affinity, ensuring their allocation to an optimal node.

Longhorn 7872

Signed-off-by: Derek Su <derek.su@suse.com>
---
 controller/share_manager_controller.go | 35 ++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/controller/share_manager_controller.go b/controller/share_manager_controller.go
index ac3631c909..eba94370a7 100644
--- a/controller/share_manager_controller.go
+++ b/controller/share_manager_controller.go
@@ -18,6 +18,7 @@ import (
 	"k8s.io/kubernetes/pkg/controller"
 
 	corev1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -743,6 +744,32 @@ func (c *ShareManagerController) syncShareManagerPod(sm *longhorn.ShareManager)
 	return nil
 }
 
+func (c *ShareManagerController) getAffinityFromStorageClass(sc *storagev1.StorageClass) *corev1.Affinity {
+	var matchLabelExpressions []corev1.NodeSelectorRequirement
+
+	for _, topology := range sc.AllowedTopologies {
+		for _, expression := range topology.MatchLabelExpressions {
+			matchLabelExpressions = append(matchLabelExpressions, corev1.NodeSelectorRequirement{
+				Key:      expression.Key,
+				Operator: corev1.NodeSelectorOpIn,
+				Values:   expression.Values,
+			})
+		}
+	}
+
+	return &corev1.Affinity{
+		NodeAffinity: &corev1.NodeAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
+				NodeSelectorTerms: []corev1.NodeSelectorTerm{
+					{
+						MatchExpressions: matchLabelExpressions,
+					},
+				},
+			},
+		},
+	}
+}
+
 func (c *ShareManagerController) getShareManagerNodeSelectorFromStorageClass(sc *storagev1.StorageClass) map[string]string {
 	value, ok := sc.Parameters["shareManagerNodeSelector"]
 	if !ok {
@@ -814,6 +841,7 @@ func (c *ShareManagerController) createShareManagerPod(sm *longhorn.ShareManager
 		return nil, err
 	}
 
+	var affinity *corev1.Affinity
 	if pv.Spec.StorageClassName != "" {
 		sc, err := c.ds.GetStorageClass(pv.Spec.StorageClassName)
 		if err != nil {
@@ -827,6 +855,8 @@ func (c *ShareManagerController) createShareManagerPod(sm *longhorn.ShareManager
 			for k, v := range nodeSelectorFromStorageClass {
 				nodeSelector[k] = v
 			}
+
+			affinity = c.getAffinityFromStorageClass(sc)
 		}
 	}
 
@@ -854,7 +884,7 @@ func (c *ShareManagerController) createShareManagerPod(sm *longhorn.ShareManager
 			string(secret.Data[csi.CryptoPBKDF]))
 	}
 
-	manifest := c.createPodManifest(sm, annotations, tolerations, imagePullPolicy, nil, registrySecret,
+	manifest := c.createPodManifest(sm, annotations, tolerations, affinity, imagePullPolicy, nil, registrySecret,
 		priorityClass, nodeSelector, fsType, mountOptions, cryptoKey, cryptoParams)
 	pod, err := c.ds.CreatePod(manifest)
 	if err != nil {
@@ -890,7 +920,7 @@ func (c *ShareManagerController) createServiceManifest(sm *longhorn.ShareManager
 }
 
 func (c *ShareManagerController) createPodManifest(sm *longhorn.ShareManager, annotations map[string]string, tolerations []corev1.Toleration,
-	pullPolicy corev1.PullPolicy, resourceReq *corev1.ResourceRequirements, registrySecret, priorityClass string,
+	affinity *corev1.Affinity, pullPolicy corev1.PullPolicy, resourceReq *corev1.ResourceRequirements, registrySecret, priorityClass string,
 	nodeSelector map[string]string, fsType string, mountOptions []string, cryptoKey string, cryptoParams *crypto.EncryptParams) *corev1.Pod {
 
 	// command args for the share-manager
@@ -914,6 +944,7 @@ func (c *ShareManagerController) createPodManifest(sm *longhorn.ShareManager, an
 			OwnerReferences: datastore.GetOwnerReferencesForShareManager(sm, true),
 		},
 		Spec: corev1.PodSpec{
+			Affinity:           affinity,
 			ServiceAccountName: c.serviceAccount,
 			Tolerations:        util.GetDistinctTolerations(tolerations),
 			NodeSelector:       nodeSelector,