Test: creating a PR #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| name: Gluten Unit Tests Suite | |
| on: | |
| pull_request | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| jobs: | |
| arrow-backend-test: | |
| runs-on: sr404-self-hosted | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK 1.8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: '8' | |
| java-package: jdk | |
| overwrite-settings: false | |
| - run: sudo swapoff -a | |
| - run: free | |
| - run: sudo apt-get update | |
| - run: sudo apt-get install -y cmake ccache build-essential | |
| - run: sudo apt-get install -y maven | |
| - run: sudo apt-get install -y libboost-all-dev libcurl4-openssl-dev | |
| - run: sudo apt-get install -y libssl-dev | |
| - run: sudo apt-get install -y libz-dev | |
| - run: sudo apt-get install -y llvm clang | |
| - name: Install Gluten Arrow-backend | |
| run: | | |
| rm -rf ~/.m2/repository/org/apache/arrow && rm -rf ~/.m2/repository/io/glutenproject/ | |
| mvn clean package -Pspark-3.2 -Pbackends-gazelle -DskipTests -Dbuild_cpp=ON -Dbuild_gazelle_cpp=ON -Dbuild_arrow=ON | |
| rm -rf /tmp/gluten_jars && mkdir -p /tmp/gluten_jars | |
| cp package/gazelle/spark32/target/gluten-spark3.2_2.12-1.0.0-SNAPSHOT-jar-with-dependencies.jar /tmp/gluten_jars/ | |
| # - name: Run unit tests | |
| # run: | | |
| # mvn test -Dcheckstyle.skip -Dexec.skip -Pbackends-gazelle -Dbuild_cpp=OFF -pl backends-gazelle -DwildcardSuites=io.glutenproject.execution.ArrowParquetWriteSuite -DfailIfNoTests=false | |
| - name: Run TPCDS q82,q85,q37,q60,q26 | |
| run: | | |
| cd /home/sparkuser/tpcds/ | |
| bash tpcds_spark.sh | |
| velox-backend-test: | |
| runs-on: velox-self-hosted | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Checkout gluten-te code | |
| run: | | |
| git clone -b main https://github.com/zhztheplayer/gluten-te.git gluten-te | |
| - name: Setup docker container | |
| run: | | |
| EXTRA_DOCKER_OPTIONS="--name velox-backend-test-$GITHUB_RUN_ID --detach" NON_INTERACTIVE=ON gluten-te/cbash.sh sleep 14400 | |
| - name: Install Gluten Velox-backend spark3.2 and Unit test | |
| run: | | |
| docker exec velox-backend-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \ | |
| mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Dcpp_tests=ON -Dcpp_benchmarks=ON \ | |
| -Dbuild_protobuf=OFF -Dbuild_velox_from_source=ON -Dbuild_arrow=ON' | |
| - name: Install Gluten Velox-backend spark3.3 and Unit test | |
| run: | | |
| docker exec velox-backend-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \ | |
| mvn clean install -Pspark-3.3 -Pbackends-velox' | |
| - name: Build spark3.2 and spark3.3 at the same time | |
| run: | | |
| docker exec velox-backend-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \ | |
| mvn clean install -Pspark-3.2 -Pspark-3.3 -DskipTests -Dcheckstyle.skip -Dscalastyle.skip=true -Pbackends-velox' | |
| - name: Checkout gluten-it code in docker | |
| run: | | |
| docker exec velox-backend-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten && \ | |
| git clone -b main https://github.com/zhztheplayer/gluten-it.git gluten-it' | |
| - name: TPC-H SF1.0 && TPC-DS SF0.1 Parquet local spark3.2 | |
| run: | | |
| docker exec velox-backend-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/gluten-it && \ | |
| mvn clean package -Pspark-3.2 -Pgluten-velox -Darrow.version=10.0.0-SNAPSHOT \ | |
| && java -Xmx5G -XX:ErrorFile=/var/log/java/hs_err_pid%p.log -cp target/gluten-it-1.0-SNAPSHOT-jar-with-dependencies.jar io.glutenproject.integration.tpc.Tpc \ | |
| --backend-type=velox --benchmark-type=h --fixed-width-as-double --disable-aqe --off-heap-size=10g -s=1.0 --cpus=4 --iterations=1 \ | |
| && java -Xmx5G -XX:ErrorFile=/var/log/java/hs_err_pid%p.log -cp target/gluten-it-1.0-SNAPSHOT-jar-with-dependencies.jar io.glutenproject.integration.tpc.Tpc \ | |
| --backend-type=velox --benchmark-type=ds --fixed-width-as-double --off-heap-size=2g -s=0.1 --cpus=4 --iterations=1' | |
| - name: TPC-H SF1.0 && TPC-DS SF0.1 Parquet local spark3.3 | |
| run: | | |
| docker exec velox-backend-test-$GITHUB_RUN_ID bash -c 'cd /opt/gluten/gluten-it && \ | |
| mvn clean package -Pspark-3.3 -Pgluten-velox -Darrow.version=10.0.0-SNAPSHOT \ | |
| && java -Xmx5G -XX:ErrorFile=/var/log/java/hs_err_pid%p.log -cp target/gluten-it-1.0-SNAPSHOT-jar-with-dependencies.jar io.glutenproject.integration.tpc.Tpc \ | |
| --backend-type=velox --benchmark-type=h --fixed-width-as-double --disable-aqe --off-heap-size=10g -s=1.0 --cpus=4 --iterations=1 --use-existing-data \ | |
| && java -Xmx5G -XX:ErrorFile=/var/log/java/hs_err_pid%p.log -cp target/gluten-it-1.0-SNAPSHOT-jar-with-dependencies.jar io.glutenproject.integration.tpc.Tpc \ | |
| --backend-type=velox --benchmark-type=ds --fixed-width-as-double --off-heap-size=2g -s=0.1 --cpus=4 --iterations=1 --use-existing-data' | |
| # - name: Run micro benchmarks | |
| # run: | | |
| # cd cpp/velox/benchmarks/ | |
| # ./generic_benchmark | |
| - name: Exit docker container | |
| if: ${{ always() }} | |
| run: | | |
| docker stop velox-backend-test-$GITHUB_RUN_ID || true | |
| ch-backend-test: | |
| runs-on: libch-self-hosted | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up JDK 1.8 | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'zulu' | |
| java-version: '8' | |
| java-package: jdk | |
| overwrite-settings: false | |
| - run: sudo swapoff -a | |
| - run: free | |
| - run: sudo apt-get update | |
| - run: sudo apt-get install -y maven | |
| - name: Install and Check ClickHouse Backend lib | |
| uses: liuneng1994/pull-request-comment-trigger@master | |
| id: ch_version | |
| with: | |
| trigger: 'test by CH\[\[(\S*)]]' | |
| - run: | | |
| wget -O /tmp/libch.so.new https://devopsnexus.kyligence.io/repository/raw-tars-hosted/io.kyligence.clickhouse/clickhouse_backend/latest/libch.so | |
| mv /tmp/libch.so.new /tmp/libch.so | |
| if: steps.ch_version.outputs.triggered == 'false' | |
| - run: | | |
| cd /home/qwe1398775315/code/ClickHouse | |
| gh pr checkout ${{ steps.ch_version.outputs.trigger_var }} --force | |
| sudo rm -f /tmp/ch_output/* | |
| sudo docker run --rm --volume=/tmp/ch_output:/output --volume=/home/qwe1398775315/code/ClickHouse:/clickhouse --volume=/home/qwe1398775315/.cache:/ccache -e ENABLE_EMBEDDED_COMPILER=ON qwe1398775315/libchbuilder:0.1.0 | |
| cp /tmp/ch_output/libch*.so /tmp/libch.so | |
| if: steps.ch_version.outputs.triggered == 'true' | |
| - name: Run Gluten + ClickHouse Backend unit tests with Spark 3.2 | |
| run: | | |
| export MAVEN_OPTS="-Xmx5g -XX:ReservedCodeCacheSize=1g" | |
| mvn clean install -Pbackends-clickhouse -Pspark-3.2 -Pspark-ut -Dtpcds.data.path=/home/changchen/tpcds-sf1-data -Dclickhouse.lib.path=/tmp/libch.so | |
| formatting-check: | |
| name: Formatting Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Run clang-format style check for C/C++ programs. | |
| uses: jidicula/clang-format-action@v3.5.1 | |
| with: | |
| clang-format-version: '10' | |
| check-path: 'gluten/cpp/src' | |
| fallback-style: 'Google' # optional |