Use TransformPipeline in SegmentMapper #29963

Workflow file for this run

.github/workflows/pinot_tests.yml at e0cd518

	#
	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.
	#

	name: Pinot Tests

	on:
	push:
	branches:
	- master
	paths-ignore:
	- "contrib/**"
	- "docs/**"
	- "docker/**"
	- "kubernetes/**"
	- "licenses/**"
	- "licenses-binary/**"
	- "**.md"
	pull_request:
	branches:
	- master
	paths-ignore:
	- "contrib/**"
	- "docs/**"
	- "docker/**"
	- "kubernetes/**"
	- "licenses/**"
	- "licenses-binary/**"
	- "**.md"

	jobs:
	linter-test:
	if: github.repository == 'apache/pinot'
	runs-on: ubuntu-latest
	name: Pinot Linter Test Set
	steps:
	- uses: actions/checkout@v4
	- name: Set up JDK 11
	uses: actions/setup-java@v4
	with:
	java-version: 11
	distribution: 'temurin'
	cache: 'maven'
	- name: Linter Test
	env:
	GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	MAVEN_OPTS: >
	-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
	-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
	-B -ntp
	-XX:+IgnoreUnrecognizedVMOptions
	--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
	run: .github/workflows/scripts/.pinot_linter.sh
	unit-test:
	if: github.repository == 'apache/pinot'
	runs-on: ubuntu-latest
	strategy:
	# Changed to false in order to improve coverage using unsafe buffers
	fail-fast: false
	matrix:
	testset: [ 1, 2 ]
	java: [ 11, 21 ]
	distribution: [ "temurin" ]
	skip_bytebuffer: [false]
	include:
	- java: 21
	testset: 1
	distribution: "temurin"
	skip_bytebuffer: true
	- java: 21
	testset: 2
	distribution: "temurin"
	skip_bytebuffer: true
	name: Pinot Unit Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with skip bytebuffers:${{matrix.skip_bytebuffer}}
	steps:
	- uses: actions/checkout@v4
	- name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
	uses: actions/setup-java@v4
	with:
	java-version: ${{ matrix.java }}
	distribution: ${{ matrix.distribution }}
	cache: 'maven'
	# Step that does that actual cache save and restore
	- uses: actions/cache@v4
	env:
	SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-
	- name: Build Project
	env:
	RUN_INTEGRATION_TESTS: false
	RUN_TEST_SET: ${{ matrix.testset }}
	GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	MAVEN_OPTS: >
	-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
	-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
	-B -ntp
	-XX:+IgnoreUnrecognizedVMOptions
	--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
	--add-opens=java.base/java.nio=ALL-UNNAMED
	--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
	--add-opens=java.base/java.lang=ALL-UNNAMED
	--add-opens=java.base/java.util=ALL-UNNAMED
	--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
	run: .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
	- name: Unit Test
	env:
	RUN_INTEGRATION_TESTS: false
	RUN_TEST_SET: ${{ matrix.testset }}
	PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }}
	GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	MAVEN_OPTS: >
	-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
	-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
	-B -ntp
	-XX:+IgnoreUnrecognizedVMOptions
	--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
	--add-opens=java.base/java.nio=ALL-UNNAMED
	--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
	--add-opens=java.base/java.lang=ALL-UNNAMED
	--add-opens=java.base/java.util=ALL-UNNAMED
	--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
	run: .github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	continue-on-error: true
	timeout-minutes: 5
	with:
	flags: unittests,unittests${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}},skip-bytebuffers-${{matrix.skip_bytebuffer}}
	name: codecov-unit-tests
	fail_ci_if_error: false
	verbose: true

	integration-test:
	if: github.repository == 'apache/pinot'
	runs-on: ubuntu-latest
	strategy:
	# Changed to false in order to improve coverage using unsafe buffers
	fail-fast: false
	matrix:
	testset: [ 1, 2 ]
	java: [ 11, 21 ]
	distribution: [ "temurin" ]
	skip_bytebuffer: [false]
	include:
	- java: 21
	testset: 1
	distribution: "temurin"
	skip_bytebuffer: true
	- java: 21
	testset: 2
	distribution: "temurin"
	skip_bytebuffer: true
	name: Pinot Integration Test Set ${{ matrix.testset }} (${{matrix.distribution}}-${{matrix.java}}) with skip bytebuffers:${{matrix.skip_bytebuffer}}
	steps:
	- uses: actions/checkout@v4
	- name: Set up JDK ${{ matrix.java }}-${{ matrix.distribution }}
	uses: actions/setup-java@v4
	with:
	java-version: ${{ matrix.java }}
	distribution: ${{ matrix.distribution }}
	cache: 'maven'
	# Step that does that actual cache save and restore
	- uses: actions/cache@v4
	env:
	SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-
	- name: Build Project
	env:
	RUN_INTEGRATION_TESTS: true
	GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	MAVEN_OPTS: >
	-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
	-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
	-B -ntp
	-XX:+IgnoreUnrecognizedVMOptions
	--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
	run: .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
	- name: Integration Test
	env:
	RUN_INTEGRATION_TESTS: true
	RUN_TEST_SET: ${{ matrix.testset }}
	PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }}
	GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	MAVEN_OPTS: >
	-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
	-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
	-B -ntp
	-XX:+IgnoreUnrecognizedVMOptions
	--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
	run: .github/workflows/scripts/pr-tests/.pinot_tests_integration.sh
	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	continue-on-error: true
	timeout-minutes: 5
	with:
	flags: integration,integration${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}},skip-bytebuffers-${{matrix.skip_bytebuffer}}
	name: codecov-integration-tests
	fail_ci_if_error: false
	verbose: true
	- name: Custom Integration Test
	if : ${{ matrix.testset == 1 }}
	env:
	RUN_INTEGRATION_TESTS: true
	RUN_TEST_SET: ${{ matrix.testset }}
	GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	MAVEN_OPTS: >
	-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
	-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
	-B -ntp
	-XX:+IgnoreUnrecognizedVMOptions
	--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
	run: .github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh
	- name: Upload coverage to Codecov
	if : ${{ matrix.testset == 1 }}
	uses: codecov/codecov-action@v4
	continue-on-error: true
	timeout-minutes: 5
	with:
	flags: integration,custom-integration${{ matrix.testset }},${{matrix.distribution}},java-${{matrix.java}}
	name: codecov-custom-integration-tests
	fail_ci_if_error: false
	verbose: true

	compatibility-verifier:
	if: github.repository == 'apache/pinot'
	runs-on: ubuntu-latest
	strategy:
	# Changed to false in order to improve coverage using unsafe buffers
	fail-fast: false
	matrix:
	test_suite: [ "compatibility-verifier/sample-test-suite" ]
	old_commit: [
	"release-1.0.0", "release-1.2.0", "master"
	]
	name: Pinot Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }}
	steps:
	- uses: actions/checkout@v4
	- name: Set up JDK 11
	uses: actions/setup-java@v4
	with:
	java-version: 11
	distribution: 'temurin'
	cache: 'maven'
	- name: Setup node
	uses: actions/setup-node@v4
	with:
	node-version: v16.15.0
	cache: 'npm'
	cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
	- name: Install npm
	run: \|
	npm install -g npm@8.5.5
	npm --version
	# Step that does that actual cache save and restore
	- uses: actions/cache@v4
	env:
	SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-
	- name: Pinot Compatibility Regression Testing
	env:
	OLD_COMMIT: ${{ matrix.old_commit }}
	WORKING_DIR: /tmp/compatibility-verifier
	TEST_SUITE: ${{ matrix.test_suite }}
	MAVEN_OPTS: >
	-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
	-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
	-B -ntp
	-XX:+IgnoreUnrecognizedVMOptions
	--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
	run: .github/workflows/scripts/.pinot_compatibility_verifier.sh

	multi-stage-compatibility-verifier:
	if: github.repository == 'apache/pinot'
	runs-on: ubuntu-latest
	strategy:
	# Changed to false in order to improve coverage using unsafe buffers
	fail-fast: false
	matrix:
	test_suite: [ "compatibility-verifier/multi-stage-query-engine-test-suite" ]
	old_commit: [
	"release-1.2.0", "master"
	]
	name: Pinot Multi-Stage Query Engine Compatibility Regression Testing against ${{ matrix.old_commit }} on ${{ matrix.test_suite }}
	steps:
	- uses: actions/checkout@v4
	- name: Set up JDK 11
	uses: actions/setup-java@v4
	with:
	java-version: 11
	distribution: 'temurin'
	cache: 'maven'
	- name: Setup node
	uses: actions/setup-node@v4
	with:
	node-version: v16.15.0
	cache: 'npm'
	cache-dependency-path: pinot-controller/src/main/resources/package-lock.json
	- name: Install npm
	run: \|
	npm install -g npm@8.5.5
	npm --version
	# Step that does that actual cache save and restore
	- uses: actions/cache@v4
	env:
	SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-
	- name: Pinot Multi-Stage Query Engine Compatibility Regression Testing
	env:
	OLD_COMMIT: ${{ matrix.old_commit }}
	WORKING_DIR: /tmp/multi-stage-compatibility-verifier
	TEST_SUITE: ${{ matrix.test_suite }}
	MAVEN_OPTS: >
	-Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25
	-Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
	-B -ntp
	-XX:+IgnoreUnrecognizedVMOptions
	--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
	--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
	run: .github/workflows/scripts/.pinot_compatibility_verifier.sh

	quickstarts:
	if: github.repository == 'apache/pinot'
	runs-on: ubuntu-latest
	strategy:
	# Changed to false in order to improve coverage using unsafe buffers
	fail-fast: false
	matrix:
	java: [ 11, 21 ]
	distribution: [ "temurin" ]
	skip_bytebuffer: [false]
	include:
	- java: 21
	distribution: "temurin"
	skip_bytebuffer: true
	name: Pinot Quickstart on JDK ${{ matrix.java }}-${{ matrix.distribution }} with skip bytebuffers:${{matrix.skip_bytebuffer}}
	steps:
	- uses: actions/checkout@v4
	- name: Set up JDK ${{ matrix.java }}
	uses: actions/setup-java@v4
	with:
	java-version: ${{ matrix.java }}
	distribution: ${{ matrix.distribution }}
	cache: 'maven'
	# Step that does that actual cache save and restore
	- uses: actions/cache@v4
	env:
	SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10
	GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
	with:
	path: ~/.m2/repository
	key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
	restore-keys: \|
	${{ runner.os }}-maven-
	- name: Quickstart on JDK ${{ matrix.java }}
	env:
	PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }}
	run: .github/workflows/scripts/.pinot_quickstart.sh

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Use TransformPipeline in SegmentMapper #29963

Workflow file

Use TransformPipeline in SegmentMapper #29963

Jobs

Run details

Workflow file for this run