coverage.yml

name: Code coverage

on:
  workflow_dispatch:
  push:
    branches:
      - main
      - trigger-coverage-workflow
    paths:
      - quickwit/Cargo.toml
      - quickwit/Cargo.lock
      - quickwit/quickwit-*/**

env:
  AWS_REGION: us-east-1
  AWS_ACCESS_KEY_ID: "placeholder"
  AWS_SECRET_ACCESS_KEY: "placeholder"
  CARGO_INCREMENTAL: 0
  PUBSUB_EMULATOR_HOST: "localhost:9898"
  QW_DISABLE_TELEMETRY: 1
  QW_S3_ENDPOINT: "http://localhost:4566" # Services are exposed as localhost because we are not running coverage in a container.
  QW_S3_FORCE_PATH_STYLE_ACCESS: 1
  QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev

jobs:
  test:
    name: Coverage
    runs-on: buildjet-8vcpu-ubuntu-2204 # ubuntu-latest
    # Setting a containing will require to fix the QW_S3_ENDPOINT to http://localstack:4566
    services:
      localstack:
        image: localstack/localstack:latest
        ports:
          - "4566:4566"
          - "4571:4571"
          - "8080:8080"
        env:
          SERVICES: kinesis,s3
        options: >-
          --health-cmd "curl -k https://localhost:4566"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        credentials:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}

      postgres:
        image: postgres:latest
        ports:
          - "5432:5432"
        env:
          POSTGRES_USER: quickwit-dev
          POSTGRES_PASSWORD: quickwit-dev
          POSTGRES_DB: quickwit-metastore-dev
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        credentials:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}

      kafka-broker:
        image: confluentinc/cp-kafka:7.2.1
        ports:
          - "9092:9092"
          - "9101:9101"
        env:
          KAFKA_BROKER_ID: 1
          KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:29092,PLAINTEXT_HOST://localhost:9092
          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
          KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
          KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
          KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
          KAFKA_JMX_PORT: 9101
          KAFKA_JMX_HOSTNAME: localhost
          KAFKA_HEAP_OPTS: -Xms256M -Xmx256M
        options: >-
          --health-cmd "cub kafka-ready -b localhost:9092 1 5"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        credentials:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}

      zookeeper:
        image: confluentinc/cp-zookeeper:7.2.1
        ports:
          - "2181:2181"
        env:
          KAFKA_HEAP_OPTS: -Xms256M -Xmx256M
          ZOOKEEPER_CLIENT_PORT: 2181
          ZOOKEEPER_TICK_TIME: 2000
        options: >-
          --health-cmd "cub zk-ready localhost:2181 5"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        credentials:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}

      gcp-pubsub-emulator:
        image: thekevjames/gcloud-pubsub-emulator:7555256f2c
        ports:
          - "9898:8681"
        env:
          PUBSUB_PROJECT1: "quickwit-emulator,emulator_topic:emulator_subscription"
        credentials:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}

    steps:
      - uses: actions/checkout@v3

      - name: Install lib libsasl2
        run: |
          sudo apt update
          sudo apt install libsasl2-dev
          sudo apt install libsasl2-2

      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/git
            ~/.cargo/registry
          key: ${{ runner.os }}-cargo-test-${{ hashFiles('Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-test-${{ hashFiles('Cargo.lock') }}
            ${{ runner.os }}-cargo-test

      - name: Install awslocal
        run: pip install awscli-local

      - name: Prepare LocalStack S3
        run: ./quickwit-cli/tests/prepare_tests.sh
        working-directory: ./quickwit

      # GitHub Actions does not allow services to be started with a custom command,
      # so we are running Azurite as a container manually.
      - name: Run Azurite service
        run: DOCKER_SERVICES=azurite make docker-compose-up

      - name: Run Pulsar service
        run: DOCKER_SERVICES=pulsar make docker-compose-up

      - name: Install Rust
        run: rustup update stable

      - uses: taiki-e/install-action@v2
        with:
          tool: cargo-llvm-cov,nextest,protoc

      # We limit the number of jobs to 4 to avoid OOM errors when linking the binary.
      - name: Generate code coverage
        run: |
          cargo llvm-cov clean --workspace
          cargo llvm-cov nextest --no-report --test failpoints --features fail/failpoints --retries 2
          CARGO_BUILD_JOBS=4 cargo llvm-cov nextest --no-report --all-features --retries 2
          cargo llvm-cov report --lcov --output-path lcov.info
        working-directory: ./quickwit

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
          files: ./quickwit/lcov.info