support dataflow affinity (#3829)

* support dataflow affinity Signed-off-by: xliuqq <xlzq1992@gmail.com> * support databackup Signed-off-by: xliuqq <xlzq1992@gmail.com> * fix check Signed-off-by: xliuqq <xlzq1992@gmail.com> * revert dockerfile and use nodeAffinity Signed-off-by: xliuqq <xlzq1992@gmail.com> * fix conflict and use nodeAffinity Signed-off-by: xliuqq <xlzq1992@gmail.com> * fix: add apireader for get node Signed-off-by: xliuqq <xlzq1992@gmail.com> * revert apireader to cache client Signed-off-by: xliuqq <xlzq1992@gmail.com> --------- Signed-off-by: xliuqq <xlzq1992@gmail.com>
fluid-cloudnative · May 14, 2024 · 84029c5 · 84029c5
1 parent c8ccac2
commit 84029c5
Show file tree

Hide file tree

Showing 56 changed files with 3,283 additions and 118 deletions.
diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go
@@ -255,6 +255,35 @@ const (
 	DataProcessType OperationType = "DataProcess"
 )
 
+// AffinityPolicy the strategy for the affinity between Data Operation Pods.
+type AffinityPolicy string
+
+const (
+	DefaultAffinityStrategy AffinityPolicy = ""
+	RequireAffinityStrategy AffinityPolicy = "Require"
+	PreferAffinityStrategy  AffinityPolicy = "Prefer"
+)
+
+type AffinityStrategy struct {
+	// Policy one of: "", "Require", "Prefer"
+	// +optional
+	Policy AffinityPolicy `json:"policy,omitempty"`
+
+	Prefer  []Prefer  `json:"prefer,omitempty"`
+	Require []Require `json:"require,omitempty"`
+}
+
+// Prefer defines the label key and weight for generating a PreferredSchedulingTerm.
+type Prefer struct {
+	Name   string `json:"name"`
+	Weight int32  `json:"weight"`
+}
+
+// Require defines the label key for generating a NodeSelectorTerm.
+type Require struct {
+	Name string `json:"name"`
+}
+
 type OperationRef struct {
 	// API version of the referent operation
 	// +optional
@@ -272,6 +301,10 @@ type OperationRef struct {
 	// Namespace specifies the namespace of the referent operation.
 	// +optional
 	Namespace string `json:"namespace,omitempty"`
+
+	// AffinityStrategy specifies the pod affinity strategy with the referent operation.
+	// +optional
+	AffinityStrategy AffinityStrategy `json:"affinityStrategy,omitempty"`
 }
 
 type WaitingStatus struct {

diff --git a/api/v1alpha1/openapi_generated.go b/api/v1alpha1/openapi_generated.go
diff --git a/api/v1alpha1/status.go b/api/v1alpha1/status.go
@@ -151,6 +151,9 @@ type OperationStatus struct {
 	LastSuccessfulTime *metav1.Time `json:"lastSuccessfulTime,omitempty"`
 	// WaitingStatus stores information about waiting operation.
 	WaitingFor WaitingStatus `json:"waitingFor,omitempty"`
+
+	// NodeAffinity records the node affinity for operation pods
+	NodeAffinity *corev1.NodeAffinity `json:"nodeAffinity,omitempty"`
 }
 
 type RuntimePhase string

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/fluid-databackup/alluxio/templates/databackup.yaml b/charts/fluid-databackup/alluxio/templates/databackup.yaml
@@ -89,6 +89,11 @@ spec:
           mountPath: /etc/group
           readOnly: true
         {{- end}}
+  {{- if .Values.dataBackup.affinity }}
+  affinity:
+{{ toYaml .Values.dataBackup.affinity | indent 4 }}
+  {{- end }}
+
   restartPolicy: Never
   volumes:
     {{- if .Values.dataBackup.workdir }}

diff --git a/charts/fluid-databackup/alluxio/values.yaml b/charts/fluid-databackup/alluxio/values.yaml
@@ -45,6 +45,8 @@ dataBackup:
   # Description: optional image pull secrets on DataLoad pods
   imagePullSecrets: []
 
+  affinity:
+
 initUsers:
   enabled: false
   image: registry.cn-hangzhou.aliyuncs.com/fluid/init-users

diff --git a/charts/fluid-dataprocess/common/templates/job.yaml b/charts/fluid-dataprocess/common/templates/job.yaml
@@ -69,6 +69,10 @@ spec:
           {{- if .Values.dataProcess.scriptProcessor.volumeMounts }}
             {{- toYaml .Values.dataProcess.scriptProcessor.volumeMounts | nindent 12 }}
           {{- end }}
+      {{- if .Values.dataProcess.scriptProcessor.affinity }}
+      affinity:
+{{ toYaml .Values.dataProcess.scriptProcessor.affinity | indent 8  }}
+      {{- end }}
       volumes:
         - name: script-cm-vol
           configMap: