ci.yml

name: GraphScope CI

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  push:
    branches:
      - main
    paths-ignore:
      - 'CONTRIBUTORS'
      - 'LICENSE'
      - 'NOTICE.txt'
      - '**.md'
      - '**.rst'
      - 'docs/**'
      - 'demo/**'
      - 'scripts/**'
      - 'tutorials/**'
  pull_request:
    branches:
      - main
    paths:
      - '.github/workflows/ci.yml'
      - 'Makefile'
      - 'analytical_engine/**'
      - 'charts/**'
      - 'coordinator/**'
      - 'interactive_engine/**'
      - 'k8s/**'
      - 'learning_engine/**'
      - 'proto/**'
      - 'python/**'
      - '!**.md'
      - '!**.rst'

concurrency:
  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

env:
  GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope

jobs:
  # JOB to run change detection
  changes:
    runs-on: ubuntu-20.04
    # Set job outputs to values from filter step
    outputs:
      gae-python: ${{ steps.filter.outputs.gae-python }}
      networkx: ${{ steps.filter.outputs.networkx }}
      gie-function-test: ${{ steps.filter.outputs.gie-function-test }}
    steps:
    # For push it's necessary to checkout the code
    - uses: actions/checkout@v2
    # For pull requests it's not necessary to checkout the code
    - uses: dorny/paths-filter@v2
      id: filter
      with:
        base: main  # Change detection against merge-base with main before push
        filters: |
          gae-python:
            - 'proto/**'
            - 'analytical_engine/**'
            - 'python/graphscope/analytical/**'
            - 'python/graphscope/client/**'
            - 'python/graphscope/dataset/**'
            - 'python/graphscope/deploy/**'
            - 'python/graphscope/framework/**'
            - 'python/graphscope/tests/unittest/**'
            - 'coordinator/gscoordinator/**'
            - '.github/workflows/ci.yml'
          networkx:
            - 'analytical_engine/apps/**'
            - 'analytical_engine/frame/**'
            - 'analytical_engine/core/**'
            - 'python/graphscope/nx/**'
          gie-function-test:
            - 'interactive_engine/**'
            - 'python/graphscope/interactive/**'
            - '.github/workflows/ci.yml'

    - name: Cpp Format and Lint Check
      run: |
        # install clang-format
        sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output /usr/bin/clang-format
        sudo chmod +x /usr/bin/clang-format

        # run format
        cd analytical_engine/
        find ./apps ./benchmarks ./core ./frame ./misc ./test  -name "*.h" | xargs clang-format -i --style=file
        find ./apps ./benchmarks ./core ./frame ./misc ./test  -name "*.cc" | xargs clang-format -i --style=file

        # validate format
        function prepend() { while read line; do echo "${1}${line}"; done; }

        GIT_DIFF=$(git diff --ignore-submodules)
        if [[ -n $GIT_DIFF ]]; then
            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            echo "| clang-format failures found!"
            echo "|"
            echo "$GIT_DIFF" | prepend "| "
            echo "|"
            echo "| Run: "
            echo "|"
            echo "|    make gsa_clformat"
            echo "|"
            echo "| to fix this error."
            echo "|"
            echo "| Ensure you are working with clang-format-8, which can be obtained from"
            echo "|"
            echo "|    https://github.com/muttleyxd/clang-tools-static-binaries/releases"
            echo "|"
            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            exit -1
        fi

    - name: Python Format and Lint Check
      run: |
        echo "Checking formatting for $GITHUB_REPOSITORY"
        pip3 install -r coordinator/requirements-dev.txt
        pushd python
        python3 -m isort --check --diff .
        python3 -m black --check --diff .
        python3 -m flake8 .
        popd
        pushd coordinator
        python3 -m isort --check --diff .
        python3 -m black --check --diff .
        python3 -m flake8 .

    - name: Setup Java11
      uses: actions/setup-java@v2
      with:
         distribution: 'zulu'
         java-version: '11'

    - name: Java Format and Lint Check
      run: |
        wget https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar

        files_to_format=$(git ls-files *.java)

        # run formatter in-place
        java -jar ${GITHUB_WORKSPACE}/google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $files_to_format

        # validate format
        function prepend() { while read line; do echo "${1}${line}"; done; }

        GIT_DIFF=$(git diff --ignore-submodules)
        if [[ -n $GIT_DIFF ]]; then
            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            echo "| google-java-format failures found!"
            echo "|"
            echo "$GIT_DIFF" | prepend "| "
            echo "|"
            echo "| Run: "
            echo "|"
            echo '|    java -jar google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $(git ls-files **/*.java)'
            echo "|"
            echo "| to fix this error."
            echo "|"
            echo "| Ensure you are working with google-java-format-1.13.0, which can be obtained from"
            echo "|"
            echo "|    https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar"
            echo "|"
            echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
            exit -1
        fi

  build-wheels:
    # Require the user id of the self-hosted is 1001, which may need to be
    # configured manually when a new self-hosted runner is added.
    runs-on: [self-hosted, manylinux2014]
    needs: [changes]
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true

    - uses: actions/cache@v3
      with:
        path: ~/.m2/repository
        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
        restore-keys: |
          ${{ runner.os }}-maven-

    - uses: actions/cache@v3
      with:
        path: |
          ~/.cargo/bin/
          ~/.cargo/registry/index/
          ~/.cargo/registry/cache/
          ~/.cargo/git/db/
          ~/.cache/sccache
        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}

    - uses: actions/cache@v3
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-

    - name: Install latest libgrape-lite and vineyard
      if: false
      run: |
        source ~/.bashrc

        git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
        pushd /tmp/libgrape-lite
        mkdir -p build && cd build
        cmake .. -DCMAKE_INSTALL_PREFIX=/opt/vineyard
        make -j$(nproc)
        sudo make install
        popd
        rm -fr /tmp/libgrape-lite

        git clone --single-branch --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
        pushd /tmp/v6d
        git submodule update --init
        mkdir -p build && cd build
        cmake .. -DCMAKE_INSTALL_PREFIX=/opt/vineyard \
                 -DUSE_EXTERNAL_ETCD_LIBS=OFF \
                 -DBUILD_SHARED_LIBS=ON \
                 -DBUILD_VINEYARD_TESTS=OFF
        make -j$(nproc)
        sudo make install
        popd

        rm -fr /tmp/v6d

        # copy to /usr/local, keep the same logic as gsvineyard.Dockerfile
        sudo cp -r /opt/vineyard/* /usr/local/

    - name: Build Wheels
      run: |
        source ~/.bashrc

        export SCCACHE_DIR=~/.cache/sccache
        export RUSTC_WRAPPER=/usr/local/bin/sccache
        sccache --start-server

        cd ${GITHUB_WORKSPACE}/k8s
        # build graphscope wheels
        make graphscope-manylinux2014-py3-nodocker
        # build client wheels
        make graphscope-client-manylinux2014-py3-nodocker

        sccache --show-stats

        # package
        cd ${GITHUB_WORKSPACE}
        tar -zcf client.tar.gz python/dist/wheelhouse/*.whl
        tar -zcf graphscope.tar.gz coordinator/dist/

    - name: Upload Artifact
      uses: actions/upload-artifact@v2
      with:
        name: wheel-${{ github.sha }}
        path: |
          client.tar.gz
          graphscope.tar.gz
        retention-days: 5

  mini-test:
    runs-on: ubuntu-20.04
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    needs: [build-wheels]
    steps:
      - uses: actions/checkout@v2.3.2

      - uses: actions/download-artifact@v2
        with:
          path: artifacts

      - name: Prepare Environment
        run: |
          # install graphscope-client
          cd artifacts
          tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
          pushd python/dist/wheelhouse
          for f in * ; do python3 -m pip install $f || true; done
          popd

          # install graphscope
          tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
          pushd coordinator/dist
          python3 -m pip install ./*.whl
          popd
          pushd coordinator/dist/wheelhouse
          python3 -m pip install ./*.whl
          popd

          # install tensorflow
          python3 -m pip install pytest "tensorflow<=2.5.2" --user
          # install java
          sudo apt update -y && sudo apt install openjdk-11-jdk -y

      - name: Run Minimum Test
        env:
          GS_TEST_DIR: ${{ github.workspace }}/gstest
        run: |
          git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}

          python3 -m pytest -s -v $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/minitest

      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        if: false

      - name: Upload GIE log
        if: always()
        uses: actions/upload-artifact@v2
        with:
          name: gie-log
          path: ~/.local/log/graphscope

  python-unittest:
    runs-on: ubuntu-20.04
    needs: [build-wheels, changes]
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    defaults:
      run:
        shell: bash --noprofile --norc -eo pipefail {0}
    strategy:
      matrix:
        deployment: ["standalone", "distributed"]

    steps:
    - uses: actions/checkout@v2.3.2
      if: ${{ needs.changes.outputs.gae-python == 'true' }}

    - uses: actions/download-artifact@v2
      if: ${{ needs.changes.outputs.gae-python == 'true' }}
      with:
        path: artifacts

    - name: Prepare Environment
      if: ${{ needs.changes.outputs.gae-python == 'true' }}
      run: |
        # install graphscope-client
        cd artifacts
        tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
        pushd python/dist/wheelhouse
        for f in * ; do python3 -m pip install $f || true; done
        popd

        # install graphscope
        tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
        pushd coordinator/dist
        python3 -m pip install ./*.whl
        popd
        pushd coordinator/dist/wheelhouse
        python3 -m pip install ./*.whl
        popd

        # install pytest
        python3 -m pip install pytest pytest-cov

    - name: Run Python Test
      if: ${{ needs.changes.outputs.gae-python == 'true' }}
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
        DEPLOYMENT: ${{ matrix.deployment }}
      run: |
        # download dataset
        git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}

        python3 -m pytest -s -v --cov=graphscope --cov-config=python/.coveragerc \
                                --cov-report=xml --cov-report=term --exitfirst \
                                $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/tests/unittest

    - name: Upload Coverage
      if: ${{ needs.changes.outputs.gae-python == 'true' || github.ref == 'refs/heads/main' }}
      uses: codecov/codecov-action@v2
      with:
        file: ./coverage.xml
        fail_ci_if_error: false

  networkx-basic-test:
    runs-on: ubuntu-20.04
    needs: [build-wheels, changes]
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    defaults:
      run:
        shell: bash
    strategy:
      matrix:
        deployment: ["standalone", "distributed"]
    steps:
    - uses: actions/checkout@v2.3.2
      if: ${{ needs.changes.outputs.networkx == 'true' }}

    - uses: dorny/paths-filter@v2
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      id: nx-filter
      with:
        filters: |
          basic:
            - 'analytical_engine/apps/**'
            - 'analytical_engine/frame/**'
            - 'analytical_engine/core/**'
            - 'python/graphscope/nx/__init__.py'
            - 'python/graphscope/nx/classes/**'
            - 'python/graphscope/nx/!(tests)'
            - 'python/graphscope/nx/tests/!(convert)'
          convert:
            - 'python/graphscope/nx/convert.py'
            - 'python/graphscope/nx/convert_matrix.py'
            - 'python/graphscope/nx/tests/convert/**'

    - uses: actions/download-artifact@v2
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      with:
        path: artifacts

    - name: Prepare Environment
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        # install graphscope-client
        cd artifacts
        tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
        pushd python/dist/wheelhouse
        for f in * ; do python3 -m pip install $f || true; done
        popd

        # install graphscope
        tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
        pushd coordinator/dist
        python3 -m pip install ./*.whl
        popd
        pushd coordinator/dist/wheelhouse
        python3 -m pip install ./*.whl
        popd

        # install pytest
        python3 -m pip install pytest

        # download dataset
        git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}

    - name: Setup tmate session
      uses: mxschmitt/action-tmate@v3
      if: false

    - name: Basic Test
      if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.basic == 'true' }}
      env:
        DEPLOYMENT: ${{ matrix.deployment }}
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        pip3 show networkx
        python3 -m pytest --exitfirst -s -v \
            $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests \
            --ignore=$(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests/convert

    - name: Convert Test
      if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.convert == 'true' }}
      env:
        DEPLOYMENT: ${{ matrix.deployment }}
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        pip3 show networkx
        python3 -m pytest --exitfirst -s -v \
            $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/tests/convert

  networkx-algo-and-generator-test:
    runs-on: ubuntu-20.04
    needs: [build-wheels, changes]
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    strategy:
      matrix:
        deployment: ["standalone", "distributed"]
    steps:
    - uses: actions/checkout@v2.3.2
      if: ${{ needs.changes.outputs.networkx == 'true' }}

    - uses: dorny/paths-filter@v2
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      id: nx-filter
      with:
        filters: |
          algorithm:
            - 'analytical_engine/apps/**'
            - 'analytical_engine/frame/**'
            - 'python/graphscope/nx/algorithms/builtin.py'
            - 'python/graphscope/nx/algorithms/tests/builtin/**'
          generator:
            - 'python/graphscope/nx/classes/**'
            - 'python/graphscope/nx/utils/**'
            - 'python/graphscope/nx/generators/**'
          io:
            - 'python/graphscope/nx/readwrite/**'

    - uses: actions/download-artifact@v2
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      with:
        path: artifacts

    - name: Prepare Environment
      if: ${{ needs.changes.outputs.networkx == 'true' }}
      env:
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        # install graphscope-client
        cd artifacts
        tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
        pushd python/dist/wheelhouse
        for f in * ; do python3 -m pip install $f || true; done
        popd

        # install graphscope
        tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz
        pushd coordinator/dist
        python3 -m pip install ./*.whl
        popd
        pushd coordinator/dist/wheelhouse
        python3 -m pip install ./*.whl
        popd

        # install pytest
        python3 -m pip install pytest

        # download dataset
        git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}

    - name: Algorithm test
      if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.algorithm == 'true' }}
      env:
        DEPLOYMENT: ${{ matrix.deployment }}
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        pip3 show networkx
        python3 -m pytest --exitfirst -s -v \
            $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/algorithms/tests/builtin

    - name: Generator test
      if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.generator == 'true' }}
      env:
        DEPLOYMENT: ${{ matrix.deployment }}
        GS_TEST_DIR: ${{ github.workspace }}/gstest
      run: |
        pip3 show networkx
        python3 -m pytest --exitfirst -s -v \
            $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/generators/tests

    - name: Readwrite test
      if: ${{ needs.changes.outputs.networkx == 'true' && steps.nx-filter.outputs.io == 'true' }}
      env:
        DEPLOYMENT: ${{ matrix.deployment }}
      run: |
        python3 -m pytest --exitfirst -s -v -m "not slow" \
            $(dirname $(python3 -c "import graphscope; print(graphscope.__file__)"))/nx/readwrite/tests

  gie-test:
    runs-on: [self-hosted, ubuntu2004]
    needs: [build-wheels, changes]
    if: ${{ needs.changes.outputs.gie-function-test == 'true' && github.repository == 'alibaba/GraphScope' }}
    steps:
      - uses: actions/checkout@v2.3.2

      - uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-maven-

      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - uses: actions/download-artifact@v2
        with:
          path: artifacts

      - name: Add envs to GITHUB_ENV
        run: |
          short_sha=$(git rev-parse --short HEAD)
          echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV

      - name: Prepare Environment
        run: |
          cd ${GITHUB_WORKSPACE}/artifacts
          tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
          tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz

          # build graphscope image
          cd ${GITHUB_WORKSPACE}
          make graphscope-image

          cd ${GITHUB_WORKSPACE}/python
          pip3 install -r requirements.txt
          pip3 install -r requirements-dev.txt

          # build python client proto
          python3 setup.py build_proto


      - name: Run Function Test
        run: |
          export GS_TEST_DIR=${GITHUB_WORKSPACE}/interactive_engine/tests/src/main/resources
          minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.30' \
                         --cpus='12' --memory='32000mb' --disk-size='40000mb' \
                         --mount=true --mount-string="${GS_TEST_DIR}:${GS_TEST_DIR}"
          minikube image load graphscope/graphscope:${SHORT_SHA}

          export PYTHONPATH=${GITHUB_WORKSPACE}/python:${PYTHONPATH}
          cd ${GITHUB_WORKSPACE}/interactive_engine && mvn clean install --quiet -DskipTests -Drust.compile.skip=true -P graphscope,graphscope-assembly
          cd ${GITHUB_WORKSPACE}/interactive_engine/tests
          # ./function_test.sh 8111 1 graphscope/graphscope:${SHORT_SHA}
          ./function_test.sh 8112 2 graphscope/graphscope:${SHORT_SHA}

  k8s-test:
    runs-on: [self-hosted, ubuntu2004]
    if: ${{ github.repository == 'alibaba/GraphScope' }}
    needs: [build-wheels]
    steps:
      - uses: actions/checkout@v2.3.2

      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - uses: actions/download-artifact@v2
        with:
          path: artifacts

      - name: Add envs to GITHUB_ENV
        run: |
          short_sha=$(git rev-parse --short HEAD)
          echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV

      - name: Prepare Environment
        run: |
          cd ${GITHUB_WORKSPACE}/artifacts
          tar -zxf ./wheel-${{ github.sha }}/client.tar.gz
          tar -zxf ./wheel-${{ github.sha }}/graphscope.tar.gz

          # build graphscope image
          cd ${GITHUB_WORKSPACE}
          make graphscope-image
          docker tag graphscope/graphscope:${SHORT_SHA} ${{ env.GS_IMAGE }}:${SHORT_SHA}

          cd ${GITHUB_WORKSPACE}/python
          pip3 install -r requirements.txt
          pip3 install -r requirements-dev.txt

          # build python client proto
          cd ${GITHUB_WORKSPACE}/python
          python3 setup.py build_proto

          # install mars
          python3 -m pip install pymars==0.8.0

          # install pytest
          python3 -m pip install pytest pytest-cov

      - name: Setup SSH
        run: |
          ssh-keygen -t rsa -f ~/.ssh/id_rsa -N ''
          cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
          chmod og-wx ~/.ssh/authorized_keys
          echo "StrictHostKeyChecking no" >> ~/.ssh/config
          sudo /etc/init.d/ssh start

      - name: Kubernetes Test
        env:
          GS_TEST_DIR: ${{ github.workspace }}/gstest
        run: |
          # download dataset
          git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}

          # set GS_IMAGE
          export GS_IMAGE=${{ env.GS_IMAGE }}:${SHORT_SHA}

          minikube start --base-image='registry-vpc.cn-hongkong.aliyuncs.com/graphscope/kicbase:v0.0.30' \
                         --cpus='12' --memory='32000mb' --disk-size='40000mb' \
                         --mount=true --mount-string="${GS_TEST_DIR}:${GS_TEST_DIR}"

          minikube image load ${{ env.GS_IMAGE }}:${SHORT_SHA}
          cd ${GITHUB_WORKSPACE}/python
          python3 -m pytest --ignore=./graphscope/tests/kubernetes/test_store_service.py \
                          --cov=graphscope --cov-config=.coveragerc --cov-report=xml \
                          --cov-report=term --exitfirst -s -vvv --log-cli-level=INFO \
                          ./graphscope/tests/kubernetes

      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        if: false

      - name: Upload Coverage
        uses: codecov/codecov-action@v2
        with:
          file: ./python/coverage.xml
          fail_ci_if_error: true

      - name: Helm Test
        run: |
          cd charts
          helm install graphscope --set coordinator.image.tag=${SHORT_SHA} \
                                  --set engines.image.tag=${SHORT_SHA} \
                                  --set vineyard.image.tag=${SHORT_SHA} \
                                  --set imagePullPolicy=IfNotPresent \
                                  ./graphscope
          helm test graphscope --timeout 5m0s

      - name: HDFS test
        if: false  # temporary disable the hdfs tests
        env:
          JAVA_HOME: /usr/lib/jvm/default-java
          GS_TEST_DIR: ${{ github.workspace }}/gstest
        run: |
          export GS_IMAGE=${{ env.GS_IMAGE }}:${SHORT_SHA}

          # install hadoop HDFS
          tar -zxf /home/runner/hadoop-2.10.1.tar.gz -C /tmp/
          cd ${GITHUB_WORKSPACE}/.github/workflows/hadoop_scripts
          ./prepare_hadoop.sh /tmp/hadoop-2.10.1
          export PATH=${PATH}:/tmp/hadoop-2.10.1/bin

          # upload data to HDFS
          hadoop fs -mkdir /ldbc_sample || true
          hadoop fs -chmod 777 /ldbc_sample
          hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_0_0.csv /ldbc_sample/person_0_0.csv
          hadoop fs -put ${GS_TEST_DIR}/ldbc_sample/person_knows_person_0_0.csv /ldbc_sample/person_knows_person_0_0.csv
          export HDFS_TEST_DIR=hdfs:///ldbc_sample
          export HDFS_HOST=$(hostname -I | awk '{print $1}')

          # run test
          cd ${GITHUB_WORKSPACE}/python
          python3 -m pytest -s ./graphscope/tests/kubernetes/test_demo_script.py -k test_demo_on_hdfs
          # Check the result file have successfully written to the given location
          # hdfs dfs -test -e /ldbc_sample/res.csv_0 && hdfs dfs -test -e /ldbc_sample/res.csv_1