Merge remote-tracking branch 'upstream/main' into issues/2826

opensearch-project · Dec 31, 2024 · 69baa3d · 69baa3d
2 parents d63d2ef + 4224f31
commit 69baa3d
Show file tree

Hide file tree

Showing 617 changed files with 22,562 additions and 6,725 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,2 +1,2 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @pjfitzgibbons @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @rupal-bq @mengweieric @vamsi-amazon @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224
+*   @pjfitzgibbons @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @rupal-bq @mengweieric @vamsi-amazon @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 @LantaoJin @noCharger
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,16 +1,18 @@
 ### Description
 [Describe what this change achieves]
-
-### Issues Resolved
-[List any issues this PR will resolve]
-
+
+### Related Issues
+Resolves #[Issue number to be closed when this PR is merged]
+<!-- List any other related issues here -->
+
 ### Check List
 - [ ] New functionality includes testing.
-  - [ ] All tests pass, including unit test, integration test and doctest
 - [ ] New functionality has been documented.
-  - [ ] New functionality has javadoc added
-  - [ ] New functionality has user manual doc added
-- [ ] Commits are signed per the DCO using --signoff 
+ - [ ] New functionality has javadoc added.
+ - [ ] New functionality has a user manual doc added.
+- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md).
+- [ ] Commits are signed per the DCO using `--signoff`.
+- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose).
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
-For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).
+For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/sql/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).
diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml
@@ -21,21 +21,21 @@ jobs:
       fail-fast: false
       matrix:
         java: [21]
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
@@ -47,7 +47,7 @@ jobs:
 
     - name: Upload test reports
       if: ${{ always() }}
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: test-reports-${{ matrix.os }}-${{ matrix.java }}
@@ -60,16 +60,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ windows-latest ]
+        os: [ windows-latest, macos-13 ]
         java: [21]
 
     runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
@@ -79,7 +79,7 @@ jobs:
 
     - name: Upload test reports
       if: ${{ always() }}
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: test-reports-${{ matrix.os }}-${{ matrix.java }}

diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml
@@ -22,21 +22,22 @@ jobs:
       matrix:
         java:
           - 21
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
@@ -48,7 +49,7 @@ jobs:
 
     - name: Upload test reports
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: test-reports-${{ matrix.entry.java }}
         path: |

diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml
@@ -29,21 +29,21 @@ jobs:
       fail-fast: false
       matrix:
         java: [21]
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
@@ -53,11 +53,6 @@ jobs:
         chown -R 1000:1000 `pwd`
         su `id -un 1000` -c "./gradlew --continue build"
 
-    - name: Run backward compatibility tests
-      run: |
-        chown -R 1000:1000 `pwd`
-        su `id -un 1000` -c "./scripts/bwctest.sh"
-
     - name: Create Artifact Path
       run: |
         mkdir -p opensearch-sql-builds
@@ -66,22 +61,22 @@ jobs:
     # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
     - name: Upload SQL Coverage Report
       if: ${{ always() }}
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       continue-on-error: true
       with:
         flags: sql-engine
         token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: Upload Artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: opensearch-sql-ubuntu-latest-${{ matrix.java }}
         path: opensearch-sql-builds
 
     - name: Upload test reports
       if: ${{ always() }}
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: test-reports-ubuntu-latest-${{ matrix.java }}
@@ -105,13 +100,14 @@ jobs:
       matrix:
         entry:
           - { os: windows-latest, java: 21, os_build_args: -x doctest -PbuildPlatform=windows }
+          - { os: macos-13, java: 21 }
     runs-on: ${{ matrix.entry.os }}
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.entry.java }}
@@ -127,22 +123,22 @@ jobs:
     # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
     - name: Upload SQL Coverage Report
       if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }}
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       continue-on-error: true
       with:
         flags: sql-engine
         token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: Upload Artifacts
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: opensearch-sql-${{ matrix.entry.os }}-${{ matrix.entry.java }}
         path: opensearch-sql-builds
 
     - name: Upload test reports
       if: ${{ always() && matrix.entry.os == 'ubuntu-latest' }}
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: test-reports-${{ matrix.entry.os }}-${{ matrix.entry.java }}
@@ -158,3 +154,49 @@ jobs:
           plugin/build/reports/**
           doctest/build/testclusters/docTestCluster-0/logs/*
           integ-test/build/testclusters/*/logs/*
+
+  bwc-tests:
+    needs: Get-CI-Image-Tag
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java: [21]
+    container:
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
+
+    steps:
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
+
+    - name: Set up JDK ${{ matrix.java }}
+      uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: ${{ matrix.java }}
+
+    - name: Run backward compatibility tests
+      run: |
+        chown -R 1000:1000 `pwd`
+        su `id -un 1000` -c "./scripts/bwctest.sh"
+
+    - name: Upload test reports
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v4
+      continue-on-error: true
+      with:
+        name: test-reports-ubuntu-latest-${{ matrix.java }}-bwc
+        path: |
+          sql/build/reports/**
+          ppl/build/reports/**
+          core/build/reports/**
+          common/build/reports/**
+          opensearch/build/reports/**
+          integ-test/build/reports/**
+          protocol/build/reports/**
+          legacy/build/reports/**
+          plugin/build/reports/**
+          doctest/build/testclusters/docTestCluster-0/logs/*
+          integ-test/build/testclusters/*/logs/*
diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml
@@ -22,21 +22,21 @@ jobs:
       matrix:
         java:
           - 21
-    env:
-      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Run start commands
+      run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
+    - uses: actions/checkout@v4
 
     - name: Set up JDK ${{ matrix.java }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
         java-version: ${{ matrix.java }}
@@ -86,7 +86,7 @@ jobs:
 
     - name: Upload test reports
       if: always()
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: test-reports-${{ matrix.entry.java }}
         path: |

diff --git a/.github/workflows/stalled.yml b/.github/workflows/stalled.yml
@@ -0,0 +1,28 @@
+name: Label Stalled PRs
+on:
+  schedule:
+    - cron: '15 15 * * *' # Run every day at 15:15 UTC / 7:15 PST / 8:15 PDT
+permissions:
+  pull-requests: write
+jobs:
+  stale:
+    if: github.repository == 'opensearch-project/sql'
+    runs-on: ubuntu-latest
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/github-app-token@v2.1.0
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Stale PRs
+        uses: actions/stale@v9
+        with:
+          repo-token: ${{ steps.github_app_token.outputs.token }}
+          stale-pr-label: 'stalled'
+          stale-pr-message: 'This PR is stalled because it has been open for 30 days with no activity.'
+          days-before-pr-stale: 30
+          days-before-issue-stale: -1
+          days-before-pr-close: -1
+          days-before-issue-close: -1
diff --git a/.gitignore b/.gitignore
@@ -49,4 +49,5 @@ gen
 .worktrees
 http-client.env.json
 /doctest/sql-cli/
+/doctest/opensearch-job-scheduler/
 .factorypath
diff --git a/DEVELOPER_GUIDE.rst b/DEVELOPER_GUIDE.rst
@@ -405,7 +405,7 @@ Sample test class:
 Doctest
 >>>>>>>
 
-Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `Doctest <./dev/Doctest.md>`_.
+Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `testing-doctest <./docs/dev/testing-doctest.md>`_.
 
 
 Backports
-Original file line number
+Diff line change
@@ Expand Up / @@ -405,7 +405,7 @@ Sample test class: @@
     Doctest
     >>>>>>>
-    Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `Doctest <./dev/Doctest.md>`_.
+    Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `testing-doctest <./docs/dev/testing-doctest.md>`_.
     Backports
@@ Expand Down @@