Merge branch 'master' of https://github.com/kubeflow/pipelines into t…

…ask-setters update RELEASE.md
kubeflow · Sep 11, 2024 · 6fbb291 · 6fbb291
2 parents b70aef8 + 1cded35
commit 6fbb291
Show file tree

Hide file tree

Showing 195 changed files with 3,066 additions and 1,880 deletions.
diff --git a/.cloudbuild.yaml b/.cloudbuild.yaml
@@ -170,10 +170,10 @@ steps:
   args: ['pull', 'gcr.io/cloudsql-docker/gce-proxy:1.25.0']
   id:   'pullCloudsqlProxy'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.4.16-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/argoexec:v3.4.17-license-compliance']
   id:   'pullArgoExecutor'
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.4.16-license-compliance']
+  args: ['pull', 'gcr.io/ml-pipeline/workflow-controller:v3.4.17-license-compliance']
   id:   'pullArgoWorkflowController'
 
 # V2 related images

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -2,6 +2,7 @@
 
 
 **Checklist:**
+- [ ] You have [signed off your commits](https://www.kubeflow.org/docs/about/contributing/#sign-off-your-commits)
 - [ ] The title for your pull request (PR) should follow our title convention. [Learn more about the pull request title convention used in this repository](https://github.com/kubeflow/pipelines/blob/master/CONTRIBUTING.md#pull-request-title-convention). 
 <!--
    PR titles examples:

diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -34,17 +34,17 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-      - name: Create KFP cluster
-        uses: ./.github/actions/kfp-tekton-cluster
-      - name: Set up Python 3.10
+      - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.9'
       - name: Install sdk
         run: |
           python3 -m venv .venv
           . .venv/bin/activate
           pip install -e sdk/python
+      - name: Create KFP cluster
+        uses: ./.github/actions/kfp-tekton-cluster
       - name: "flip coin test"
         run: |
           . .venv/bin/activate

diff --git a/.github/workflows/e2e-test.yaml → .github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yaml → .github/workflows/e2e-test.yml
@@ -6,7 +6,7 @@ on:
 
   pull_request:
     paths:
-      - '.github/workflows/e2e-test.yaml'
+      - '.github/workflows/e2e-test.yml'
       - 'scripts/deploy/github/**'
       - 'go.mod'
       - 'go.sum'
@@ -23,6 +23,11 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster
 
@@ -46,6 +51,11 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster
 
@@ -69,6 +79,11 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster
 
@@ -92,6 +107,11 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster
 
@@ -115,6 +135,11 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster
 
@@ -144,18 +169,25 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
         run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
+      - name: Install prerequisites
+        run: pip3 install -r ./test/sample-test/requirements.txt
+
       - name: Basic sample tests - sequential
-        run: pip3 install -r ./test/sample-test/requirements.txt && pip3 install kfp~=2.0 && python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name sequential --results-gcs-dir output
+        run:  python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name sequential --results-gcs-dir output
 
-#      Disabled while https://github.com/kubeflow/pipelines/issues/10885 is not resolved
-#      - name: Basic sample tests - exit_handler
-#        run: pip3 install -r ./test/sample-test/requirements.txt && pip3 install kfp~=2.0 && python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name exit_handler --results-gcs-dir output
+      - name: Basic sample tests - exit_handler
+        run: python3 ./test/sample-test/sample_test_launcher.py sample_test run_test --namespace kubeflow --test-name exit_handler --results-gcs-dir output
 
       - name: Collect test results
         if: always()

diff --git a/.github/workflows/gcpc-modules-tests.yml b/.github/workflows/gcpc-modules-tests.yml
@@ -0,0 +1,56 @@
+name: GCPC modules test
+
+on:
+  push:
+    branches: [master]
+
+  pull_request:
+    paths:
+      - '.github/workflows/gcpc-modules-tests.yml'
+      - 'sdk/python/**'
+
+jobs:
+  all-gcpc-tests:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: apt-get update
+        run: sudo apt-get update
+
+      - name: Install protobuf-compiler
+        run: sudo apt-get install protobuf-compiler -y
+
+      - name: Install setuptools
+        run: | 
+          pip3 install setuptools
+          pip3 freeze
+
+      - name: Install Wheel
+        run: pip3 install wheel==0.42.0
+
+      - name: Install python sdk 
+        run: pip install sdk/python
+
+      - name: Generate API proto files
+        working-directory: ./api
+        run: make clean python
+
+      - name: Install kfp-pipeline-spec from source
+        run: | 
+          python3 -m pip install api/v2alpha1/python
+
+      - name: Install google-cloud component 
+        run: pip install components/google-cloud
+
+      - name: Install Pytest
+        run: pip install $(grep 'pytest==' sdk/python/requirements-dev.txt)
+
+      - name: Run test
+        run: pytest ./test/gcpc-tests/run_all_gcpc_modules.py
diff --git a/.github/workflows/kfp-kubernetes-execution-tests.yml b/.github/workflows/kfp-kubernetes-execution-tests.yml
@@ -7,6 +7,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/kfp-kubernetes-execution-tests.yml'
+      - 'scripts/deploy/github/**'
       - 'sdk/python/**'
       - 'api/v2alpha1/**'
       - 'kubernetes_platform/**'
@@ -18,17 +19,17 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
         run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.9'
-
       - name: apt-get update
         run: sudo apt-get update
 

diff --git a/...orkflows/kfp-kubernetes-library-test.yaml → ...workflows/kfp-kubernetes-library-test.yml b/...orkflows/kfp-kubernetes-library-test.yaml → ...workflows/kfp-kubernetes-library-test.yml
@@ -5,7 +5,7 @@ on:
     branches: [master]
   pull_request:
     paths:
-      - '.github/workflows/kfp-kubernetes-library-test.yaml'
+      - '.github/workflows/kfp-kubernetes-library-test.yml'
       - 'sdk/python/**'
       - 'api/v2alpha1/**'
       - 'kubernetes_platform/**'

diff --git a/.github/workflows/kfp-samples.yml b/.github/workflows/kfp-samples.yml
@@ -6,6 +6,7 @@ on:
       - master
   pull_request:
     paths:
+      - 'scripts/deploy/github/**'
       - 'samples/**'
       - 'backend/src/v2/**'
       - '.github/workflows/kfp-samples.yml'
@@ -21,7 +22,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.9
 
     - name: Create KFP cluster
       uses: ./.github/actions/kfp-cluster

diff --git a/.github/workflows/kubeflow-pipelines-integration-v2.yml b/.github/workflows/kubeflow-pipelines-integration-v2.yml
@@ -0,0 +1,39 @@
+name: Kubeflow Pipelines V2 integration Tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - '.github/workflows/kubeflow-pipelines-integration-v2.yml'
+      - 'scripts/deploy/github/**'
+      - 'samples'
+      - 'core'
+      - 'backend'
+
+jobs:
+  integration-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+
+      - name: Create KFP cluster
+        uses: ./.github/actions/kfp-cluster
+
+      - name: Forward API port
+        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+
+      - name: Run the Integration Tests
+        run: |
+          ./backend/src/v2/test/integration-test.sh
diff --git a/.github/workflows/kubeflow-pipelines-manifests.yml b/.github/workflows/kubeflow-pipelines-manifests.yml
@@ -0,0 +1,19 @@
+name: KFP Manifests
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    paths:
+      - '.github/workflows/kubeflow-pipelines-manifests.yml'
+      - 'manifests/kustomize/**'     
+
+jobs:
+  kubeflow-pipelines-manifests:    
+    runs-on: ubuntu-latest    
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4    
+
+      - name: Run tests
+        run: ./manifests/kustomize/hack/presubmit.sh 
diff --git a/.github/workflows/periodic.yml b/.github/workflows/periodic.yml
@@ -10,6 +10,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster
       - name: Port forward kfp apiserver

diff --git a/.github/workflows/rerun-pr-jobs.yaml → .github/workflows/rerun-pr-jobs.yml b/.github/workflows/rerun-pr-jobs.yaml → .github/workflows/rerun-pr-jobs.yml
diff --git a/.github/workflows/sdk-execution.yml b/.github/workflows/sdk-execution.yml
@@ -7,6 +7,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/sdk-execution.yml'
+      - 'scripts/deploy/github/**'
       - 'sdk/python/**'
       - 'api/v2alpha1/**'
 
@@ -17,17 +18,17 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
         run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.8
-
       - name: apt-get update
         run: sudo apt-get update
 

diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml
@@ -7,6 +7,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/upgrade-test.yml'
+      - 'scripts/deploy/github/**'
       - 'backend/**'
       - 'manifests/kustomize/**'
 
@@ -17,6 +18,11 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster