Merge branch 'kubeflow:master' into platformspec

.github/OWNERS

@@ -0,0 +1,5 @@
+approvers:
+  - hbelmiro
+  - DharmitD
+reviewers:
+  - rimolive

.github/actions/kfp-cluster/action.yml

@@ -0,0 +1,21 @@
+name: "Set up KFP on KinD"
+description: "Step to start and configure KFP on KinD"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create k8s Kind Cluster
+      uses: container-tools/kind-action@v2
+      with:
+        cluster_name: kfp
+        kubectl_version: v1.29.2
+        version: v0.22.0
+        node_image: kindest/node:v1.29.2
+
+    - name: Build images
+      shell: bash
+      run: ./scripts/deploy/github/build-images.sh
+
+    - name: Deploy KFP
+      shell: bash
+      run: ./scripts/deploy/github/deploy-kfp.sh

.github/actions/kfp-tekton-cluster/action.yml

@@ -0,0 +1,21 @@
+name: "Set up KFP Tekton on KinD"
+description: "Step to start and configure KFP on KinD"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create k8s Kind Cluster
+      uses: container-tools/kind-action@v2
+      with:
+        cluster_name: kfp
+        kubectl_version: v1.29.2
+        version: v0.22.0
+        node_image: kindest/node:v1.29.2
+
+    - name: Build images
+      shell: bash
+      run: ./scripts/deploy/github/build-images.sh
+
+    - name: Deploy KFP
+      shell: bash
+      run: ./scripts/deploy/github/deploy-kfp-tekton.sh

.github/workflows/backend-visualization.yml

@@ -0,0 +1,26 @@
+name: KFP backend visualization tests
+
+on:
+  push:
+    branches: [ master ]
+
+  pull_request:
+    paths:
+      - '.github/workflows/backend-visualization.yml'
+      - 'backend/src/apiserver/visualization/**'
+      - 'test/presubmit-backend-visualization.sh'
+
+jobs:
+  backend-visualization-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Run tests
+        run: ./test/presubmit-backend-visualization.sh

.github/workflows/backend.yml

@@ -7,6 +7,7 @@ on:
   # Run tests for any PRs which change the backend code
   pull_request:
     paths:
+      - '.github/workflows/backend.yml'
       - 'go.mod'
       - 'backend/**'
       - 'scripts/deploy/github/**'
@@ -23,7 +24,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v4
       with:
-        go-version: 1.20.x
+        go-version: 1.21.x
     - name: Checkout code
       uses: actions/checkout@v4
     - name: "run go unit tests"
@@ -33,21 +34,12 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - name: Create k8s Kind Cluster
-        uses: container-tools/kind-action@v2
-        with:
-          cluster_name: kfp-tekton
-          kubectl_version: v1.29.2
-          version: v0.22.0
-          node_image: kindest/node:v1.29.2
-      - name: build images
-        run: ./scripts/deploy/github/build-images.sh
+      - name: Create KFP cluster
+        uses: ./.github/actions/kfp-tekton-cluster
       - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
-      - name: "deploy kfp-tekton"
-        run: ./scripts/deploy/github/deploy-kfp.sh
       - name: Install sdk
         run: |
           python3 -m venv .venv

.github/workflows/e2e-test.yaml

@@ -23,19 +23,8 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Create k8s Kind Cluster
-        uses: container-tools/kind-action@v2
-        with:
-          cluster_name: kfp
-          kubectl_version: v1.29.2
-          version: v0.22.0
-          node_image: kindest/node:v1.29.2
-
-      - name: Build images
-        run: ./scripts/deploy/github/build-images.sh
-
-      - name: Deploy KFP
-        run: ./scripts/deploy/github/deploy-kfp.sh
+      - name: Create KFP cluster
+        uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
         run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888

.github/workflows/frontend.yml

@@ -0,0 +1,35 @@
+name: Frontend Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'frontend/**'
+      - '.github/workflows/frontend.yml'
+
+jobs:
+  frontend-tests:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v3
+      with:
+        node-version: '14'
+
+    - name: Clean npm cache
+      run: npm cache clean --force
+
+    - name: Install dependencies
+      run: cd ./frontend && npm ci
+
+    - name: Run Frontend Tests
+      run: cd ./frontend && npm run test:ci
+

.github/workflows/kfp-kubernetes-execution-tests.yml

@@ -0,0 +1,72 @@
+name: k8s execution tests
+
+on:
+  push:
+    branches: [master]
+
+  pull_request:
+    paths:
+      - '.github/workflows/kfp-kubernetes-execution-tests.yml'
+      - 'sdk/python/**'
+      - 'api/v2alpha1/**'
+      - 'kubernetes_platform/**'
+
+jobs:
+  kfp-kubernetes-execution-tests:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Create KFP cluster
+        uses: ./.github/actions/kfp-cluster
+
+      - name: Forward API port
+        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: apt-get update
+        run: sudo apt-get update
+
+      - name: Install protobuf-compiler
+        run: sudo apt-get install protobuf-compiler -y
+
+      - name: Install setuptools
+        run: | 
+          pip3 install setuptools
+          pip3 freeze
+
+      - name: Install Wheel
+        run: pip3 install wheel==0.42.0
+
+      - name: Install protobuf
+        run: pip3 install protobuf==4.25.3
+
+      - name: Generate API proto files
+        working-directory: ./api
+        run: make clean python
+
+      - name: Install kfp-pipeline-spec from source
+        run: |
+          python3 -m pip install api/v2alpha1/python
+
+      - name: Generate kfp-kubernetes proto files from source
+        working-directory: ./kubernetes_platform
+        run: make clean python
+
+      - name: Install kfp-kubernetes from source
+        run: |
+          pip install -e ./kubernetes_platform/python[dev]
+
+      - name: Install requirements
+        run: pip install -r ./test/kfp-kubernetes-execution-tests/requirements.txt
+
+      - name: Run tests
+        run: |
+          export KFP_ENDPOINT="http://localhost:8888"
+          export TIMEOUT_SECONDS=2700
+          pytest ./test/kfp-kubernetes-execution-tests/sdk_execution_tests.py --asyncio-task-timeout $TIMEOUT_SECONDS

.github/workflows/kfp-sdk-runtime-tests.yml

@@ -0,0 +1,31 @@
+name: KFP Runtime Code Tests
+
+on:
+  push:
+    branches: [master]
+
+  pull_request:
+    paths:
+      - '.github/workflows/kfp-sdk-runtime-tests.yml'
+      - 'sdk/python/**'
+      - 'test/presubmit-test-kfp-runtime-code.sh'
+
+jobs:
+  kfp-runtime-tests:
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        python: ['3.8', '3.9', '3.10', '3.11', '3.12']
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Run KFP Runtime Code Tests
+        run: |
+          export PULL_NUMBER="${{ github.event.inputs.pull_number || github.event.pull_request.number }}"
+          ./test/presubmit-test-kfp-runtime-code.sh

.github/workflows/kfp-sdk-tests.yml

@@ -0,0 +1,30 @@
+name: KFP SDK Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - 'sdk/**'
+      - '.github/workflows/kfp-sdk-tests.yml'
+
+jobs:
+  sdk-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9, 3.10, 3.11, 3.12]
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+
+    - name: Run SDK Tests
+      run: |
+        ./test/presubmit-tests-sdk.sh

.github/workflows/periodic.yml

@@ -10,13 +10,11 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Create k8s Kind Cluster
-        uses: container-tools/kind-action@v2
-        with:
-          cluster_name: kfp-tekton
-          kubectl_version: v1.29.2
-          version: v0.22.0
-          node_image: kindest/node:v1.29.2
+      - name: Create KFP cluster
+        uses: ./.github/actions/kfp-cluster
+      - name: Port forward kfp apiserver
+        run: |
+          nohup kubectl port-forward --namespace kubeflow svc/ml-pipeline 8888:8888 &
       - name: Run Functional Tests
         run: |
           log_dir=$(mktemp -d)

.github/workflows/rerun-pr-jobs.yaml

@@ -0,0 +1,16 @@
+name: Re-Run PR tests
+
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  rerun_pr_tests:
+    name: rerun_pr_tests
+    if: ${{ github.event.issue.pull_request }}
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: estroz/rerun-actions@main
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        comment_id: ${{ github.event.comment.id }}