test: Created a new GitHub action to reuse the creation of a KFP clus…

…ter (#10946)

* Created a new GitHub action to reuse the creation of a KFP cluster

Signed-off-by: hbelmiro <helber.belmiro@gmail.com>

* Created a new GHA for deploying a pure KFP cluster

Signed-off-by: hbelmiro <helber.belmiro@gmail.com>

---------

Signed-off-by: hbelmiro <helber.belmiro@gmail.com>

.github/actions/kfp-cluster/action.yml

@@ -0,0 +1,21 @@
+name: "Set up KFP on KinD"
+description: "Step to start and configure KFP on KinD"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create k8s Kind Cluster
+      uses: container-tools/kind-action@v2
+      with:
+        cluster_name: kfp
+        kubectl_version: v1.29.2
+        version: v0.22.0
+        node_image: kindest/node:v1.29.2
+
+    - name: Build images
+      shell: bash
+      run: ./scripts/deploy/github/build-images.sh
+
+    - name: Deploy KFP
+      shell: bash
+      run: ./scripts/deploy/github/deploy-kfp.sh

.github/actions/kfp-tekton-cluster/action.yml

@@ -0,0 +1,21 @@
+name: "Set up KFP Tekton on KinD"
+description: "Step to start and configure KFP on KinD"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create k8s Kind Cluster
+      uses: container-tools/kind-action@v2
+      with:
+        cluster_name: kfp
+        kubectl_version: v1.29.2
+        version: v0.22.0
+        node_image: kindest/node:v1.29.2
+
+    - name: Build images
+      shell: bash
+      run: ./scripts/deploy/github/build-images.sh
+
+    - name: Deploy KFP
+      shell: bash
+      run: ./scripts/deploy/github/deploy-kfp-tekton.sh

.github/workflows/backend.yml

@@ -7,6 +7,7 @@ on:
   # Run tests for any PRs which change the backend code
   pull_request:
     paths:
+      - '.github/workflows/backend.yml'
       - 'go.mod'
       - 'backend/**'
       - 'scripts/deploy/github/**'
@@ -33,21 +34,12 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - name: Create k8s Kind Cluster
-        uses: container-tools/kind-action@v2
-        with:
-          cluster_name: kfp-tekton
-          kubectl_version: v1.29.2
-          version: v0.22.0
-          node_image: kindest/node:v1.29.2
-      - name: build images
-        run: ./scripts/deploy/github/build-images.sh
+      - name: Create KFP cluster
+        uses: ./.github/actions/kfp-tekton-cluster
       - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
-      - name: "deploy kfp-tekton"
-        run: ./scripts/deploy/github/deploy-kfp.sh
       - name: Install sdk
         run: |
           python3 -m venv .venv

.github/workflows/e2e-test.yaml

@@ -23,19 +23,8 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Create k8s Kind Cluster
-        uses: container-tools/kind-action@v2
-        with:
-          cluster_name: kfp
-          kubectl_version: v1.29.2
-          version: v0.22.0
-          node_image: kindest/node:v1.29.2
-
-      - name: Build images
-        run: ./scripts/deploy/github/build-images.sh
-
-      - name: Deploy KFP
-        run: ./scripts/deploy/github/deploy-kfp.sh
+      - name: Create KFP cluster
+        uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
         run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888

scripts/deploy/github/deploy-kfp-tekton.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Copyright 2024 kubeflow.org
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Remove the x if you need no print out of each command
+set -e
+
+REGISTRY="${REGISTRY:-kind-registry:5000}"
+EXIT_CODE=0
+
+C_DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$C_DIR" ]]; then C_DIR="$PWD"; fi
+source "${C_DIR}/helper-functions.sh"
+
+kubectl apply -k "manifests/kustomize/cluster-scoped-resources/"
+kubectl wait crd/applications.app.k8s.io --for condition=established --timeout=60s || EXIT_CODE=$?
+if [[ $EXIT_CODE -ne 0 ]]
+then
+  echo "Failed to deploy cluster-scoped resources."
+  exit $EXIT_CODE
+fi
+
+# Deploy manifest
+kubectl apply -k "scripts/deploy/github/manifests" || EXIT_CODE=$?
+if [[ $EXIT_CODE -ne 0 ]]
+then
+  echo "Deploy unsuccessful. Failure applying $KUSTOMIZE_DIR."
+  exit 1
+fi
+
+# Check if all pods are running - allow 20 retries (10 minutes)
+wait_for_pods kubeflow 40 30 || EXIT_CODE=$?
+if [[ $EXIT_CODE -ne 0 ]]
+then
+  echo "Deploy unsuccessful. Not all pods running."
+  exit 1
+fi
+
+echo "List Kubeflow: "
+kubectl get pod -n kubeflow
+collect_artifacts kubeflow
+
+echo "List Tekton control plane: "
+kubectl get pod -n tekton-pipelines
+collect_artifacts tekton-pipelines
+
+echo "Finished kfp-tekton deployment."
+

scripts/deploy/github/deploy-kfp.sh

@@ -32,11 +32,12 @@ then
   exit $EXIT_CODE
 fi
 
-# Deploy manifest
-kubectl apply -k "scripts/deploy/github/manifests" || EXIT_CODE=$?
+PLATFORM_AGNOSTIC_MANIFESTS="manifests/kustomize/env/platform-agnostic"
+
+kubectl apply -k "${PLATFORM_AGNOSTIC_MANIFESTS}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
-  echo "Deploy unsuccessful. Failure applying $KUSTOMIZE_DIR."
+  echo "Deploy unsuccessful. Failure applying ${PLATFORM_AGNOSTIC_MANIFESTS}."
   exit 1
 fi
 
@@ -52,9 +53,5 @@ echo "List Kubeflow: "
 kubectl get pod -n kubeflow
 collect_artifacts kubeflow
 
-echo "List Tekton control plane: "
-kubectl get pod -n tekton-pipelines
-collect_artifacts tekton-pipelines
-
-echo "Finished kfp-tekton deployment."
+echo "Finished KFP deployment."