[ENG-31454] Batch Requests to Reduce Metastore Calls

.github/workflows/arrow-flight-tests.yml

@@ -51,7 +51,7 @@ jobs:
 
       # Run Maven tests for the target module, excluding native tests
       - name: Maven Tests
-        run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} -Dtest="*,!TestArrowFlightNativeQueries"
+        run: ./mvnw test ${MAVEN_TEST} -pl ${{ matrix.modules }} -Dtest="*,!TestArrowFlightNativeQueries*"
 
   prestocpp-linux-build-for-test:
     runs-on: ubuntu-22.04
@@ -207,7 +207,7 @@ jobs:
           mvn test \
             ${MAVEN_TEST} \
             -pl ${{ matrix.modules }} \
-            -Dtest="TestArrowFlightNativeQueries" \
+            -Dtest="TestArrowFlightNativeQueries*" \
             -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
             -DDATA_DIR=${RUNNER_TEMP} \
             -Duser.timezone=America/Bahia_Banderas \

.github/workflows/presto-release-publish.yml

@@ -19,6 +19,11 @@ on:
         type: boolean
         default: true
         required: false
+      publish_github_release:
+        description: 'Publish executable jars to github release'
+        type: boolean
+        default: true
+        required: false
       publish_image:
         description: 'Publish presto docker image'
         type: boolean
@@ -33,11 +38,6 @@ on:
         description: 'prestissimo dependency image(e.g., prestodb/presto-native-dependency:latest)'
         required: false
         default: ''
-      publish_native_dep_image:
-        description: 'Publish prestissimo dependency docker image'
-        type: boolean
-        default: true
-        required: false
       tag_image_as_latest:
         description: 'Tag the published images as latest'
         type: boolean
@@ -102,8 +102,12 @@ jobs:
   publish-maven-artifacts:
     needs: publish-release-tag
     if: |
-      (!failure() &&!cancelled()) &&
-      (github.event.inputs.publish_maven == 'true' || github.event.inputs.publish_image == 'true' || github.event.inputs.publish_docs == 'true')
+      (!failure() &&!cancelled()) && (
+        github.event.inputs.publish_maven == 'true' ||
+        github.event.inputs.publish_image == 'true' ||
+        github.event.inputs.publish_docs == 'true' ||
+        github.event.inputs.publish_github_release == 'true'
+      )
     runs-on: ubuntu-latest
     environment: release
     timeout-minutes: 60
@@ -181,18 +185,8 @@ jobs:
           </settings>
           EOL
 
-      - name: Maven build
-        run: mvn clean install -DskipTests
-
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: presto-artifacts-${{ env.RELEASE_TAG }}
-          retention-days: 1
-          path: |
-            presto-server/target/presto-server-*.tar.gz
-            presto-cli/target/presto-cli-*-executable.jar
-            presto-docs/target/presto-docs-*.zip
+      - name: Build release artifacts
+        run: ./mvnw clean install -DskipTests
 
       - name: Set up GPG key
         env:
@@ -213,28 +207,67 @@ jobs:
           GPG_PASSPHRASE: "${{ secrets.GPG_PASSPHRASE }}"
         run: |
           unset MAVEN_CONFIG
-          modules=$(find . -maxdepth 1 -type d \( ! -name . \) -exec test -f '{}/pom.xml' \; -print | sed 's|^./||')
-          for module in $modules; do
-            if [ "$module" = "presto-test-coverage" ]; then
-              echo "Skipping $module"
-              continue
-            fi
-            MODULE_PATH=$(awk -F'[<>]' '/^    <groupId>/ {print $3; exit}' "$module/pom.xml" | sed 's|\.|/|g' | sed 's|^com/facebook/presto||')
-            echo "Checking https://repo1.maven.org/maven2/com/facebook/presto${MODULE_PATH}/${module}/${RELEASE_TAG}"
-            if ! curl --silent --fail "https://repo1.maven.org/maven2/com/facebook/presto${MODULE_PATH}/${module}/${RELEASE_TAG}" > /dev/null; then
-              echo "Publishing ${module}"
-              ./mvnw -s ${{ github.workspace }}/settings.xml -V -B -U -e -T1C deploy \
-                -Dgpg.passphrase="${{ secrets.GPG_PASSPHRASE }}" \
-                -Dmaven.wagon.http.retryHandler.count=8 \
-                -DskipTests \
-                -Drelease.autoPublish=${{ env.MAVEN_AUTO_PUBLISH }} \
-                -Poss-release \
-                -Pdeploy-to-ossrh \
-                -pl "$module"
-            else
-              echo "${module} ${RELEASE_TAG} already published, skipping."
-            fi
-          done
+          ./mvnw -s ${{ github.workspace }}/settings.xml -V -B -U -e -T1C deploy \
+            -Dgpg.passphrase="${{ secrets.GPG_PASSPHRASE }}" \
+            -Dmaven.wagon.http.retryHandler.count=8 \
+            -DskipTests \
+            -DstagingProfileId="${{ secrets.MAVEN_STAGING_PROFILE_ID }}" \
+            -DkeepStagingRepositoryOnFailure=true \
+            -DkeepStagingRepositoryOnCloseRuleFailure=true \
+            -DautoReleaseAfterClose=true \
+            -DstagingProgressTimeoutMinutes=60 \
+            -DdeploymentName=presto-${{ env.RELEASE_TAG }} \
+            -Drelease.autoPublish=${{ env.MAVEN_AUTO_PUBLISH }} \
+            -DskipExecutableJar \
+            -Poss-release \
+            -Pdeploy-to-ossrh \
+            -pl '!presto-test-coverage'
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: presto-artifacts-${{ env.RELEASE_TAG }}
+          retention-days: 7
+          path: |
+            presto-server/target/presto-server-*.tar.gz
+            presto-docs/target/presto-docs-*.zip
+            presto-cli/target/presto-cli-*-executable.jar
+            presto-benchmark-driver/target/presto-benchmark-driver-*-executable.jar
+            presto-testing-server-launcher/target/presto-testing-server-launcher-*-executable.jar
+
+  publish-github-release:
+    needs: publish-maven-artifacts
+    if: (!failure() && !cancelled()) && github.event.inputs.publish_github_release == 'true'
+    runs-on: ubuntu-latest
+    environment: release
+    timeout-minutes: 150
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.RELEASE_TAG}}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: presto-artifacts-${{ env.RELEASE_TAG }}
+          path: ./
+
+      - name: Create github release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ env.RELEASE_TAG }}
+          name: Presto ${{ env.RELEASE_TAG }}
+          token: ${{ secrets.PRESTODB_CI_TOKEN }}
+          body: |
+            See the release notes at https://prestodb.io/docs/current/release/release-${{ env.RELEASE_TAG }}.html
+          files: |
+            ./presto-cli/target/presto-cli-*-executable.jar
+            ./presto-benchmark-driver/target/presto-benchmark-driver-*-executable.jar
+            ./presto-testing-server-launcher/target/presto-testing-server-launcher-*-executable.jar
 
   publish-docker-image:
     needs: publish-maven-artifacts
@@ -349,7 +382,7 @@ jobs:
           else
             echo "Building new depedency image"
             docker compose build centos-native-dependency
-            if [[ "${{ github.event.inputs.publish_native_dep_image }}" == "true" ]]; then
+            if [[ "${{ github.event.inputs.publish_native_image }}" == "true" ]]; then
               docker tag presto/prestissimo-dependency:centos9 ${{ github.repository_owner }}/presto-native-dependency:${{ env.RELEASE_TAG }}-${{ env.COMMIT_SHA }}
               docker push ${{ github.repository_owner }}/presto-native-dependency:${{ env.RELEASE_TAG }}-${{ env.COMMIT_SHA }}
 
@@ -365,7 +398,17 @@ jobs:
         working-directory: presto-native-execution
         run: |
           df -h
-          docker compose build centos-native-runtime
+          docker compose build --build-arg EXTRA_CMAKE_FLAGS="
+            -DPRESTO_ENABLE_PARQUET=ON \
+            -DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
+            -DPRESTO_ENABLE_JWT=ON \
+            -DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
+            -DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
+            -DPRESTO_ENABLE_SPATIAL=ON \
+            -DPRESTO_ENABLE_TESTING=OFF \
+            -DPRESTO_ENABLE_S3=ON" \
+            --build-arg NUM_THREADS=2 \
+            centos-native-runtime
 
       - name: Add release tag
         working-directory: presto-native-execution

.github/workflows/prestocpp-linux-build-and-unit-test.yml

@@ -312,6 +312,84 @@ jobs:
             -Duser.timezone=America/Bahia_Banderas \
             -T1C
 
+  prestocpp-linux-presto-on-spark-e2e-tests:
+    needs: prestocpp-linux-build-for-test
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        storage-format: [ "PARQUET", "DWRF" ]
+        enable-sidecar: [ "true", "false" ]
+    container:
+      image: prestodb/presto-native-dependency:0.293-20250522140509-484b00e
+    env:
+      MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
+      MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
+      MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Fix git permissions
+        # Usually actions/checkout does this but as we run in a container
+        # it doesn't work
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: presto-native-build
+          path: presto-native-execution/_build/release
+
+      # Permissions are lost when uploading. Details here: https://github.com/actions/upload-artifact/issues/38
+      - name: Restore execute permissions and library path
+        run: |
+          chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server
+          chmod +x ${GITHUB_WORKSPACE}/presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
+          # Ensure transitive dependency libboost-iostreams is found.
+          ldconfig /usr/local/lib
+
+      - name: Install OpenJDK17
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: 17.0.15
+          cache: 'maven'
+      - name: Download nodejs to maven cache
+        run: .github/bin/download_nodejs
+
+      - name: Maven install
+        env:
+          # Use different Maven options to install.
+          MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
+        run: |
+          for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-tests' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
+
+      - name: Run presto-on-spark native tests
+        run: |
+          export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
+          export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'`
+          # Convert file paths to comma separated class names
+          export TESTCLASSES=TestPrestoSparkExpressionCompiler,TestPrestoSparkNativeBitwiseFunctionQueries,TestPrestoSparkNativeTpchConnectorQueries,TestPrestoSparkNativeSimpleQueries,TestPrestoSparkSqlFunctions,TestPrestoSparkNativeTpchQueries
+          for test_file in $TESTFILES
+          do
+            tmp=${test_file##*/}
+            test_class=${tmp%%\.*}
+            export TESTCLASSES="${TESTCLASSES},$test_class"
+          done
+          export TESTCLASSES=${TESTCLASSES#,}
+          echo "TESTCLASSES = $TESTCLASSES"
+
+          mvn test \
+            ${MAVEN_TEST} \
+            -pl 'presto-native-execution' \
+            -DstorageFormat=${{ matrix.storage-format }} \
+            -DsidecarEnabled=${{ matrix.enable-sidecar }} \
+            -Dtest="${TESTCLASSES}" \
+            -DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
+            -DDATA_DIR=${RUNNER_TEMP} \
+            -Duser.timezone=America/Bahia_Banderas \
+            -T1C
+
   prestocpp-linux-presto-sidecar-tests:
     needs: [changes, prestocpp-linux-build-for-test]
     runs-on: ubuntu-22.04
@@ -370,7 +448,7 @@ jobs:
           # Use different Maven options to install.
           MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
         run: |
-          for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
+          for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-sidecar-plugin' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
 
       - name: Run presto-native sidecar tests
         if: |

.github/workflows/tests.yml

@@ -59,7 +59,7 @@ jobs:
           - ":presto-hive -P test-hive-parquet"
           - ":presto-main-base"
           - ":presto-main"
-          - ":presto-mongodb -P test-mongo-distributed-queries"
+          - ":presto-mongodb -P ci-full-tests"
           - ":presto-redis -P test-redis-integration-smoke-test"
           - ":presto-elasticsearch"
           - ":presto-orc"

CODEOWNERS

@@ -50,6 +50,7 @@
 /presto-lark-sheets @prestodb/committers
 /presto-local-file @prestodb/committers
 /presto-main @prestodb/committers
+/presto-main-base @prestodb/committers
 /presto-matching @prestodb/committers
 /presto-memory @prestodb/committers
 /presto-memory-context @prestodb/committers
@@ -75,7 +76,9 @@
 /presto-resource-group-managers @prestodb/committers
 /presto-router @prestodb/committers
 /presto-server @prestodb/committers
-/presto-session-property-managers @prestodb/committers
+/presto-session-property-managers-common @prestodb/committers
+/presto-file-session-property-manager @prestodb/committers
+/presto-db-session-property-manager @prestodb/committers
 /presto-singlestore @prestodb/committers
 /presto-spi @prestodb/committers
 /presto-sqlserver @prestodb/committers
@@ -100,25 +103,27 @@ CODEOWNERS @prestodb/team-tsc
 # Presto core
 
 # Presto analyzer and optimizer
-/presto-main/src/*/java/com/facebook/presto/sql @jaystarshot @feilong-liu @ClarenceThreepwood @prestodb/committers
+/presto-main-base/src/*/java/com/facebook/presto/sql @jaystarshot @feilong-liu @ClarenceThreepwood @prestodb/committers
 
 # Presto cost based optimizer framework
-/presto-main/src/*/java/com/facebook/presto/cost @jaystarshot @feilong-liu @ClarenceThreepwood @prestodb/committers
+/presto-main-base/src/*/java/com/facebook/presto/cost @jaystarshot @feilong-liu @ClarenceThreepwood @prestodb/committers
 
 # Testing module
 # Note: all code owners in Presto core should be included here as well
 /presto-tests @jaystarshot @feilong-liu @elharo @ClarenceThreepwood @prestodb/committers
 
 #####################################################################
 # Prestissimo module
-/presto-native-execution @prestodb/team-velox
-/presto-native-sidecar-plugin @prestodb/team-velox
-/presto-native-tests @prestodb/team-velox
+/presto-native-execution @prestodb/team-velox @prestodb/committers
+/presto-native-sidecar-plugin @pdabre12 @prestodb/team-velox @prestodb/committers
+/presto-native-tests @prestodb/team-velox @prestodb/committers
+/presto-main-base/src/main/java/com/facebook/presto/sessionpropertyproviders/NativeWorkerSessionPropertyProvider.java  @prestodb/team-velox @prestodb/committers
 /.github/workflows/prestocpp-* @prestodb/team-velox @prestodb/committers
 
 #####################################################################
 # Presto on Spark module
 /presto-spark* @shrinidhijoshi @prestodb/committers
+/presto-native-execution/*/com/facebook/presto/spark/* @shrinidhijoshi @prestodb/committers
 
 #####################################################################
 # Presto connectors and plugins
@@ -162,4 +167,3 @@ CODEOWNERS @prestodb/team-tsc
 # Presto CI and builds
 /.github @czentgr @unidevel @prestodb/committers
 /docker @czentgr @unidevel @prestodb/committers
-

README.md

@@ -59,6 +59,8 @@ Additionally, the Hive plugin must be configured with location of your Hive meta
 
     -Dhive.metastore.uri=thrift://localhost:9083
 
+To modify the loaded plugins in IntelliJ, modify the `config.properties` located in `presto-main/etc`. You can modify `plugin.bundles` with the location of the plugin pom.xml
+
 ### Additional configuration for Java 17
 
 When running with Java 17, additional `--add-opens` flags are required to allow reflective access used by certain catalogs based on which catalogs are configured.