Job-like workload supported by WorkloadSpread (#1838)

* A TargetFilter is added to WorkloadSpread to make it possible to manage only a part of Pods owned by a target workload to support AI workloads like TFJob. And it also provides support for workloads without replicas. Signed-off-by: AiRanthem <zhongtianyun.zty@alibaba-inc.com> * fix some logs Signed-off-by: AiRanthem <zhongtianyun.zty@alibaba-inc.com> --------- Signed-off-by: AiRanthem <zhongtianyun.zty@alibaba-inc.com>
openkruise · Dec 27, 2024 · 0f3b58a · 0f3b58a
1 parent 2cdb760
commit 0f3b58a
Show file tree

Hide file tree

Showing 22 changed files with 1,439 additions and 286 deletions.
diff --git a/.github/workflows/e2e-1.18.yaml b/.github/workflows/e2e-1.18.yaml
@@ -557,6 +557,8 @@ jobs:
       - name: Install Kruise
         run: |
           set -ex
+          kubectl create ns kruise-system
+          kubectl apply -f test/kruise-e2e-config.yaml
           kubectl cluster-info
           IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
           NODES=$(kubectl get node | wc -l)
@@ -588,6 +590,7 @@ jobs:
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
           set +e
+          kubectl apply -f https://raw.githubusercontent.com/kubeflow/training-operator/refs/tags/v1.3.0/manifests/base/crds/kubeflow.org_tfjobs.yaml
           ./bin/ginkgo -timeout 90m -v --skip='\[apps\] (AppStatefulSetStorage|StatefulSet|PullImage|PullImages|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip='\[policy\] PodUnavailableBudget' test/e2e
           retVal=$?
           restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')

diff --git a/.github/workflows/e2e-1.24.yaml b/.github/workflows/e2e-1.24.yaml
@@ -618,6 +618,8 @@ jobs:
       - name: Install Kruise
         run: |
           set -ex
+          kubectl create ns kruise-system
+          kubectl apply -f test/kruise-e2e-config.yaml
           kubectl cluster-info
           IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
           NODES=$(kubectl get node | wc -l)
@@ -649,6 +651,7 @@ jobs:
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
           set +e
+          kubectl apply -f https://raw.githubusercontent.com/kubeflow/training-operator/refs/heads/v1.8-branch/manifests/base/crds/kubeflow.org_tfjobs.yaml
           ./bin/ginkgo -timeout 90m -v --skip='\[apps\] (AppStatefulSetStorage|StatefulSet|PullImage|PullImages|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip='\[policy\] PodUnavailableBudget' test/e2e
           retVal=$?
           restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')

diff --git a/.github/workflows/e2e-1.26.yaml b/.github/workflows/e2e-1.26.yaml
@@ -617,6 +617,8 @@ jobs:
       - name: Install Kruise
         run: |
           set -ex
+          kubectl create ns kruise-system
+          kubectl apply -f test/kruise-e2e-config.yaml
           kubectl cluster-info
           IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
           NODES=$(kubectl get node | wc -l)
@@ -648,6 +650,7 @@ jobs:
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
           set +e
+          kubectl apply -f https://raw.githubusercontent.com/kubeflow/training-operator/refs/heads/v1.8-branch/manifests/base/crds/kubeflow.org_tfjobs.yaml
           ./bin/ginkgo -timeout 90m -v --skip='\[apps\] (AppStatefulSetStorage|StatefulSet|PullImage|PullImages|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip='\[policy\] PodUnavailableBudget' test/e2e
           retVal=$?
           restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')

diff --git a/.github/workflows/e2e-1.28.yaml b/.github/workflows/e2e-1.28.yaml
@@ -700,6 +700,8 @@ jobs:
       - name: Install Kruise
         run: |
           set -ex
+          kubectl create ns kruise-system
+          kubectl apply -f test/kruise-e2e-config.yaml
           kubectl cluster-info
           IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
           NODES=$(kubectl get node | wc -l)
@@ -731,6 +733,7 @@ jobs:
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
           set +e
+          kubectl apply -f https://raw.githubusercontent.com/kubeflow/training-operator/refs/heads/v1.8-branch/manifests/base/crds/kubeflow.org_tfjobs.yaml
           ./bin/ginkgo -timeout 90m -v --skip='\[apps\] (InplaceVPA|AppStatefulSetStorage|StatefulSet|PullImage|PullImages|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip='\[policy\] PodUnavailableBudget' test/e2e
           retVal=$?
           restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')

diff --git a/Makefile b/Makefile
@@ -175,6 +175,8 @@ kube-load-image: $(tools/kind)
 # install-kruise install kruise with local build image to kube cluster.
 .PHONY: install-kruise
 install-kruise:
+	kubectl create ns kruise-system
+	kubectl apply -f test/kruise-e2e-config.yaml
 	tools/hack/install-kruise.sh $(IMG)
 
 # run-kruise-e2e-test starts to run kruise e2e tests.

diff --git a/apis/apps/v1alpha1/workloadspread_types.go b/apis/apps/v1alpha1/workloadspread_types.go
@@ -28,6 +28,11 @@ type WorkloadSpreadSpec struct {
 	// TargetReference is the target workload that WorkloadSpread want to control.
 	TargetReference *TargetReference `json:"targetRef"`
 
+	// TargetFilter allows WorkloadSpread to manage only a portion of the Pods in the TargetReference:
+	// by specifying the criteria for the Pods to be managed through a label selector,
+	// and by specifying how to obtain the total number of these selected Pods from the workload using replicasPaths.
+	TargetFilter *TargetFilter `json:"targetFilter,omitempty"`
+
 	// Subsets describes the pods distribution details between each of subsets.
 	// +patchMergeKey=name
 	// +patchStrategy=merge
@@ -48,6 +53,58 @@ type TargetReference struct {
 	Name string `json:"name"`
 }
 
+/*
+TargetFilter is an optional parameter that allows WorkloadSpread to manage only a subset of the Pods generated by the target workload.
+
+For example, suppose a WorkloadSpread points to the following Kubeflow TFJob resource:
+
+	```yaml
+	apiVersion: kubeflow.org/v1
+	kind: TFJob
+	spec:
+	  tfReplicaSpecs:
+		PS:
+		  replicas: 1
+		  ...
+		MASTER:
+		  replicas: 1
+		  ...
+		Worker:
+		  replicas: 2
+		  ...
+	```
+
+If you want to manage only the 2 Worker Pods that are generated, you need to configure the TargetFilter as follows:
+
+	```yaml
+	targetFilter:
+	  selector:
+		matchLabels:
+		  role: worker
+	  replicasPathList:
+		- spec.tfReplicaSpecs.Worker.replicas
+	```
+
+With this configuration, the PS Pods and Master Pods generated by the TFJob will not be managed by WorkloadSpread and will not be
+counted toward the total number of replicas.
+*/
+type TargetFilter struct {
+	// Selector is used to filter the Pods to be managed.
+	//
+	//+optional
+	Selector *metav1.LabelSelector `json:"selector,omitempty"`
+
+	// ReplicasPathList is a list of resource paths used to specify how to determine the total number of replicas of
+	// the target workload after filtering. If this list is not empty, WorkloadSpread will look for the corresponding
+	// values in the target resource according to each path, and treat the sum of these values as the total number of replicas after filtering.
+	//
+	// The replicas path is a dot-separated path, similar to "spec.replicas". If there are arrays, you can use numbers to denote indexes, like "subsets.1.replicas".
+	// The real values of these paths must be integers.
+	//
+	// +optional
+	ReplicasPathList []string `json:"replicasPathList,omitempty"`
+}
+
 // WorkloadSpreadScheduleStrategyType is a string enumeration type that enumerates
 // all possible schedule strategies for the WorkloadSpread controller.
 // +kubebuilder:validation:Enum=Adaptive;Fixed;""

diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/apps.kruise.io_workloadspreads.yaml b/config/crd/bases/apps.kruise.io_workloadspreads.yaml
@@ -310,6 +310,69 @@ spec:
                   - name
                   type: object
                 type: array
+              targetFilter:
+                description: |-
+                  TargetFilter allows WorkloadSpread to manage only a portion of the Pods in the TargetReference:
+                  by specifying the criteria for the Pods to be managed through a label selector,
+                  and by specifying how to obtain the total number of these selected Pods from the workload using replicasPaths.
+                properties:
+                  replicasPathList:
+                    description: |-
+                      ReplicasPathList is a list of resource paths used to specify how to determine the total number of replicas of
+                      the target workload after filtering. If this list is not empty, WorkloadSpread will look for the corresponding
+                      values in the target resource according to each path, and treat the sum of these values as the total number of replicas after filtering.
+
+
+                      The replicas path is a dot-separated path, similar to "spec.replicas". If there are arrays, you can use numbers to denote indexes, like "subsets.1.replicas".
+                      The real values of these paths must be integers.
+                    items:
+                      type: string
+                    type: array
+                  selector:
+                    description: Selector is used to filter the Pods to be managed.
+                    properties:
+                      matchExpressions:
+                        description: matchExpressions is a list of label selector
+                          requirements. The requirements are ANDed.
+                        items:
+                          description: |-
+                            A label selector requirement is a selector that contains values, a key, and an operator that
+                            relates the key and values.
+                          properties:
+                            key:
+                              description: key is the label key that the selector
+                                applies to.
+                              type: string
+                            operator:
+                              description: |-
+                                operator represents a key's relationship to a set of values.
+                                Valid operators are In, NotIn, Exists and DoesNotExist.
+                              type: string
+                            values:
+                              description: |-
+                                values is an array of string values. If the operator is In or NotIn,
+                                the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                                the values array must be empty. This array is replaced during a strategic
+                                merge patch.
+                              items:
+                                type: string
+                              type: array
+                          required:
+                          - key
+                          - operator
+                          type: object
+                        type: array
+                      matchLabels:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                          map is equivalent to an element of matchExpressions, whose key field is "key", the
+                          operator is "In", and the values array contains only "value". The requirements are ANDed.
+                        type: object
+                    type: object
+                    x-kubernetes-map-type: atomic
+                type: object
               targetRef:
                 description: TargetReference is the target workload that WorkloadSpread
                   want to control.