ci.yml

name: CI
on: [push, pull_request]
jobs:
  dependencies:
    name: Dependencies
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Install Rust
        run: |
          rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
          . "${HOME}/.cargo/env"
          echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
      - name: Cache dependencies
        id: cache
        uses: actions/cache@v3
        # Environment variables do not seem to work, use ~ instead.
        with:
          path: |
            ~/.cargo
            target
          key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.toml') }}
      - name: Build dependencies
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        # We need to manually clean Haura until there is a way to build only dependencies.
        # https://github.com/rust-lang/cargo/issues/2644
        run: |
          cd betree/tests
          cargo build --tests
          cargo build --tests --release
          cargo clean --package betree-tests
          cargo clean --release --package betree-tests
          cd ..
          cargo build --tests
          cargo build --tests --release
          cargo clean --package betree_storage_stack
          cargo clean --release --package betree_storage_stack
  betree-integration:
    name: Integration Tests
    needs: dependencies
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    env:
      RUST_BACKTRACE: 1
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Install Rust
        run: |
          rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
          . "${HOME}/.cargo/env"
          echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
      - name: Cache dependencies
        id: cache
        uses: actions/cache@v3
        # Environment variables do not seem to work, use ~ instead.
        with:
          path: |
            ~/.cargo
            target
          key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.toml') }}
      - name: Check dependencies
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          exit 1
      - name: Run integration tests
        env:
          HAURA_NUM_THREAD: 1
          QUICKCHECK_TESTS: 20
        run: |
          cd betree/tests
          ./scripts/ci-test.sh
  betree-unit:
    name: Unit Tests
    needs: dependencies
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    env:
      RUST_BACKTRACE: 1
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Install Rust
        run: |
          rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
          . "${HOME}/.cargo/env"
          echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
      - name: Cache dependencies
        id: cache
        uses: actions/cache@v3
        # Environment variables do not seem to work, use ~ instead.
        with:
          path: |
            ~/.cargo
            target
          key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.toml') }}
      - name: Check dependencies
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          exit 1
      - name: Run unit tests
        env:
          HAURA_NUM_THREAD: 8
          QUICKCHECK_TESTS: 20
        run: |
          cd betree
          ./tests/scripts/ci-test.sh
  betree-msrv:
    name: MSRV Check
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Install Rust
        run: |
          rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
          . "${HOME}/.cargo/env"
          echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
      - name: Cache MSRV
        id: cache
        uses: actions/cache@v3
        # Environment variables do not seem to work, use ~ instead.
        with:
          path: |
            ~/.cargo
            target
          key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.toml') }}-msrv
      - name: Prepare JULEA
        run: |
          sudo apt update || true
          sudo apt --yes --no-install-recommends install pkgconf libglib2.0-dev libbson-dev
          git clone --depth 1 https://github.com/parcio/julea.git "${HOME}/julea"
      - name: Install MSRV
        run: |
          cargo install cargo-msrv
      - name: Verify betree
        run: |
          cd betree
          cargo msrv verify
      - name: Verify bectl
        run: |
          cd bectl
          cargo msrv verify
      - name: Verify julea-betree
        run: |
          cd julea-betree
          export JULEA_INCLUDE="${HOME}/julea/include"
          export BINDGEN_EXTRA_CLANG_ARGS="$(pkg-config --cflags glib-2.0) $(pkg-config --cflags libbson-1.0)"
          cargo msrv verify
  fio-haura:
    name: fio ioengine for Haura
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: dependencies
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Install Rust
        run: |
          rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
          . "${HOME}/.cargo/env"
          echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
      - name: Cache
        id: cache
        uses: actions/cache@v3
        # Environment variables do not seem to work, use ~ instead.
        with:
          path: |
            ~/.cargo
            target
          key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.toml') }}
      - name: Build betree
        run: |
          cd betree
          cargo build
      - name: Build fio-engine
        run: |
          cd fio-haura
          source ./env-dev.sh
          make
      - name: Test run
        run: |
          test -e ./target/debug/deps/betree_storage_stack.d
          cd fio-haura
          source ./env-dev.sh
          # The fio version provided by ubuntu is too outdated, so let's just
          # build it from the copy we already have.
          make fio
          export BETREE_CONFIG=$(realpath ./.ci/haura.json)
          ./fio --direct=1 --rw=write --bs=4M --ioengine=external:src/fio-engine-haura.o --numjobs=1 --name=iops-test-job --size=128M
  haura-benchmarks:
    name: Benchmark Compatibility Tests
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: dependencies
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          persist-credentials: false
      - name: Install Rust
        run: |
          rm --recursive --force "${HOME}/.cargo" "${HOME}/.rustup"
          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
          . "${HOME}/.cargo/env"
          echo "RUSTC_VERSION=$(rustc --version | grep --only-matching '[0-9]\+\.[0-9]\+\.[0-9]\+' | head --lines=1)" >> $GITHUB_ENV
      - name: Common Cache
        uses: actions/cache@v3
        with:
          path: |
            ~/.cargo
            target
          key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.toml') }}
      - name: Poetry and Data Cache
        id: cache
        uses: actions/cache@v3
        # Environment variables do not seem to work, use ~ instead.
        with:
          path: |
            ~/.cache/pip
            ~/.cache/pypoetry
            betree/haura-benchmarks/data
            betree/haura-benchmarks/haura-plots/poetry.lock
          key: ubuntu-22.04-rustc-${{ env.RUSTC_VERSION }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('**/pyproject.toml')}}-benchmarks
      - name: Build haura-benchmarks
        run: |
          cd betree/haura-benchmarks
          cargo build --release
      - name: Prepare scripts and configuration
        run: |
          cd betree/haura-benchmarks
          cp example_config/.ci-config.json perf-config.json
          echo "ci" >> run.sh
      - name: Run benchmark smoke test
        run: |
          cd betree/haura-benchmarks
          ./run.sh smoke-test
      - name: Prepare poetry for plots
        run: |
          sudo apt update || true
          sudo apt --yes --no-install-recommends install python3-poetry
          cd betree/haura-benchmarks/haura-plots
          poetry install
      - name: Run plots
        run: |
          cd betree/haura-benchmarks/haura-plots
          poetry run plots ../results/*/*
          rm -rf results