build_and_test.yml

name: Build & Test
on:
  pull_request:
    paths-ignore:
    # ignore markdown files (CHANGELOG.md, README.md, etc.)
    - '**/*.md'
    - '.github/workflows/release.yml'
    - 'changelog/**'
    - 'kong.conf.default'
  push:
    paths-ignore:
    # ignore markdown files (CHANGELOG.md, README.md, etc.)
    - '**/*.md'
    # ignore PRs for the generated COPYRIGHT file
    - 'COPYRIGHT'
    branches:
    - master
    - release/*
    - test-please/*
  workflow_dispatch:
    inputs:
      coverage:
        description: 'Coverage enabled'
        required: false
        type: boolean
        default: false

# cancel previous runs if new commits are pushed to the PR, but run for each commit on master
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  BUILD_ROOT: ${{ github.workspace }}/bazel-bin/build
  KONG_TEST_COVERAGE: ${{ inputs.coverage == true || github.event_name == 'schedule' }}
  RUNNER_COUNT: 7

jobs:
  metadata:
    name: Metadata
    runs-on: ubuntu-22.04
    outputs:
      old-kong-version: ${{ steps.old-kong-version.outputs.ref }}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0  # `git merge-base` requires the history

    - name: Get Old Kong Version
      id: old-kong-version
      run: |
        KONG_VERSION=$(bash scripts/grep-kong-version.sh)
        major=$(echo "$KONG_VERSION" | cut -d. -f1)
        minor=$(echo "$KONG_VERSION" | cut -d. -f2)
        # if the minor version isn't 0, use the first release or starting point of the previous minor branch;
        # otherwise just leave it empty, so later the default branch or commit will be used.
        if [ "$minor" -ne 0 ]; then
          minor=$((minor - 1))
          git fetch origin master -t
          if [ $(git tag -l "$major.$minor.0") ]; then
              echo "ref=$major.$minor.0" >> $GITHUB_OUTPUT
          else
              git fetch origin release/$major.$minor.x
              COMMIT_HASH=$(git merge-base origin/master origin/release/$major.$minor.x)
              echo "ref=$COMMIT_HASH" >> $GITHUB_OUTPUT
          fi
        else
          echo "ref=" >> $GITHUB_OUTPUT
        fi

  build:
    uses: ./.github/workflows/build.yml
    with:
      relative-build-root: bazel-bin/build

  lint-and-doc-tests:
    name: Lint and Doc tests
    runs-on: ubuntu-22.04
    needs: build

    steps:
    - name: Bump max open files
      run: |
          sudo echo 'kong soft nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
          sudo echo 'kong hard nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
          sudo echo "$(whoami) soft nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf
          sudo echo "$(whoami) hard nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf

    - name: Checkout Kong source code
      uses: actions/checkout@v4

    - name: Lookup build cache
      id: cache-deps
      uses: actions/cache@v4
      with:
        path: ${{ env.BUILD_ROOT }}
        key: ${{ needs.build.outputs.cache-key }}

    - name: Check test-helpers doc generation
      run: |
          source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
          pushd ./spec && ldoc .

    - name: Check autodoc generation
      run: |
          source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
          scripts/autodoc

    - name: Lint Lua code
      run: |
          make lint

    - name: Validate rockspec file
      run: |
          source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
          scripts/validate-rockspec

    - name: Check spec file misspelling
      run: |
          scripts/check_spec_files_spelling.sh

    - name: Check labeler configuration
      run: scripts/check-labeler.pl .github/labeler.yml

  schedule:
    name: Schedule busted tests to run
    runs-on: ubuntu-22.04
    needs: build

    env:
      WORKFLOW_ID: ${{ github.run_id }}

    outputs:
      runners: ${{ steps.generate-runner-array.outputs.RUNNERS }}

    steps:
    - name: Checkout source code
      uses: actions/checkout@v4

    - name: Download runtimes file
      uses: Kong/gh-storage/download@v1
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
        repo-path: Kong/gateway-action-storage/main/.ci/runtimes.json

    - name: Schedule tests
      uses: Kong/gateway-test-scheduler/schedule@69f0c2a562ac44fc3650b8bfa62106b34094b5ce # v3
      with:
        test-suites-file: .ci/test_suites.json
        test-file-runtime-file: .ci/runtimes.json
        output-prefix: test-chunk.
        runner-count: ${{ env.RUNNER_COUNT }}
        static-mode: ${{ github.run_attempt > 1 }}

    - name: Upload schedule files
      uses: actions/upload-artifact@v4
      continue-on-error: true
      with:
        name: schedule-test-files
        path: test-chunk.*
        retention-days: 7

    - name: Generate runner array
      id: generate-runner-array
      run: |
        echo "RUNNERS=[$(seq -s "," 1 $(( "$RUNNER_COUNT" )))]" >> "$GITHUB_OUTPUT"

  busted-tests:
    name: Busted test runner ${{ matrix.runner }}
    runs-on: ubuntu-22.04
    needs: [metadata,build,schedule]

    strategy:
      fail-fast: false
      matrix:
        runner: ${{ fromJSON(needs.schedule.outputs.runners) }}

    services:
      postgres:
        image: postgres:13
        env:
          POSTGRES_USER: kong
          POSTGRES_DB: kong
          POSTGRES_HOST_AUTH_METHOD: trust
        ports:
          - 5432:5432
        options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 8

      grpcbin:
        image: kong/grpcbin
        ports:
          - 15002:9000
          - 15003:9001

      redis:
        image: redis
        ports:
          - 6379:6379
          - 6380:6380
        options: >-
          --name kong_redis

      zipkin:
        image: openzipkin/zipkin:2
        ports:
          - 9411:9411

      redis-auth:
        image: redis/redis-stack-server
        # Set health checks to wait until redis has started
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 6381:6379
        env:
          REDIS_ARGS: "--requirepass passdefault"

    steps:
    - name: Bump max open files
      run: |
          sudo echo 'kong soft nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
          sudo echo 'kong hard nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
          sudo echo "$(whoami) soft nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf
          sudo echo "$(whoami) hard nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf

    - name: Checkout Kong source code
      uses: actions/checkout@v4

    # used for plugin compatibility test
    - name: Checkout old version Kong source code
      uses: actions/checkout@v4
      with:
        path: kong-old
        # if the minor version is 0, `ref` will default to ''
        # which is same as in the previous step
        ref: ${{ needs.metadata.outputs.old-kong-version }}

    - name: Lookup build cache
      id: cache-deps
      uses: actions/cache@v4
      with:
        path: ${{ env.BUILD_ROOT }}
        key: ${{ needs.build.outputs.cache-key }}

    - name: Build WASM Test Filters
      uses: ./.github/actions/build-wasm-test-filters

    - name: Add gRPC test host names
      run: |
          echo "127.0.0.1 grpcs_1.test" | sudo tee -a /etc/hosts
          echo "127.0.0.1 grpcs_2.test" | sudo tee -a /etc/hosts

    - name: Enable SSL for Redis
      run: |
          docker cp ${{ github.workspace }} kong_redis:/workspace
          docker cp ${{ github.workspace }}/spec/fixtures/redis/docker-entrypoint.sh kong_redis:/usr/local/bin/docker-entrypoint.sh
          docker restart kong_redis
          docker logs kong_redis

    - name: Run OpenTelemetry Collector
      run: |
          mkdir -p ${{ github.workspace }}/tmp/otel
          touch ${{ github.workspace }}/tmp/otel/file_exporter.json
          sudo chmod 777 -R ${{ github.workspace }}/tmp/otel
          docker run -p 4317:4317 -p 4318:4318 -p 55679:55679 \
              -v ${{ github.workspace }}/spec/fixtures/opentelemetry/otelcol.yaml:/etc/otel-collector-config.yaml \
              -v ${{ github.workspace }}/tmp/otel:/etc/otel \
              --name opentelemetry-collector -d \
              otel/opentelemetry-collector-contrib:0.52.0 \
              --config=/etc/otel-collector-config.yaml
          sleep 2
          docker logs opentelemetry-collector

    - name: Install AWS SAM cli tool
      run: |
          curl -L -s -o /tmp/aws-sam-cli.zip https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip
          unzip -o /tmp/aws-sam-cli.zip -d /tmp/aws-sam-cli
          sudo /tmp/aws-sam-cli/install --update

    - name: Update PATH
      run: |
        echo "$BUILD_ROOT/kong-dev/bin" >> $GITHUB_PATH
        echo "$BUILD_ROOT/kong-dev/openresty/nginx/sbin" >> $GITHUB_PATH
        echo "$BUILD_ROOT/kong-dev/openresty/bin" >> $GITHUB_PATH

    - name: Debug (nginx)
      run: |
        echo nginx: $(which nginx)
        nginx -V 2>&1 | sed -re 's/ --/\n--/g'
        ldd $(which nginx)

    - name: Debug (luarocks)
      run: |
        echo luarocks: $(which luarocks)
        luarocks --version
        luarocks config

    - name: Tune up postgres max_connections
      run: |
        # arm64 runners may use more connections due to more worker cores
        psql -hlocalhost -Ukong kong -tAc 'alter system set max_connections = 5000;'

    - name: Download test schedule file
      uses: actions/download-artifact@v4
      with:
        name: schedule-test-files

    - name: Generate helper environment variables
      run: |
           echo FAILED_TEST_FILES_FILE=failed-tests.json >> $GITHUB_ENV
           echo TEST_FILE_RUNTIME_FILE=test-runtime.json >> $GITHUB_ENV
           echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/build_test/${{ matrix.runner }} >> $GITHUB_ENV

    - name: Build & install dependencies
      run: |
        make dev

    - name: Download test rerun information
      uses: actions/download-artifact@v4
      continue-on-error: true
      with:
        name: test-rerun-info-${{ matrix.runner }}

    - name: Download test runtime statistics from previous runs
      uses: actions/download-artifact@v4
      continue-on-error: true
      with:
        name: test-runtime-statistics-${{ matrix.runner }}

    - name: Run Tests
      env:
        KONG_TEST_PG_DATABASE: kong
        KONG_TEST_PG_USER: kong
        KONG_TEST_DATABASE: postgres
        KONG_SPEC_TEST_GRPCBIN_PORT: "15002"
        KONG_SPEC_TEST_GRPCBIN_SSL_PORT: "15003"
        KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json
        KONG_SPEC_TEST_OLD_VERSION_KONG_PATH: ${{ github.workspace }}/kong-old
        DD_ENV: ci
        DD_SERVICE: kong-ce-ci
        DD_CIVISIBILITY_MANUAL_API_ENABLED: 1
        DD_CIVISIBILITY_AGENTLESS_ENABLED: true
        DD_TRACE_GIT_METADATA_ENABLED: true
        DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
        SPEC_ERRLOG_CACHE_DIR: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
      uses: Kong/gateway-test-scheduler/runner@69f0c2a562ac44fc3650b8bfa62106b34094b5ce # v3
      with:
        tests-to-run-file: test-chunk.${{ matrix.runner }}.json
        failed-test-files-file: ${{ env.FAILED_TEST_FILES_FILE }}
        test-file-runtime-file: ${{ env.TEST_FILE_RUNTIME_FILE }}
        setup-venv-path: ${{ env.BUILD_ROOT }}

    - name: Upload error logs
      if: failure()
      uses: actions/upload-artifact@v4
      with:
        name: busted-test-errlogs-${{ matrix.runner }}
        path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
        retention-days: 1

    - name: Upload test rerun information
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: test-rerun-info-${{ matrix.runner }}
        path: ${{ env.FAILED_TEST_FILES_FILE }}
        retention-days: 2

    - name: Upload test runtime statistics for offline scheduling
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: test-runtime-statistics-${{ matrix.runner }}
        path: ${{ env.TEST_FILE_RUNTIME_FILE }}
        retention-days: 7

    - name: Archive coverage stats file
      uses: actions/upload-artifact@v4
      if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }}
      with:
        name: luacov-stats-out-${{ github.job }}-${{ github.run_id }}-${{ matrix.runner }}
        retention-days: 1
        path: |
          luacov.stats.out

    - name: Get kernel message
      if: failure()
      run: |
        sudo dmesg -T

  pdk-tests:
    name: PDK tests
    runs-on: ubuntu-22.04
    needs: build

    steps:
    - name: Bump max open files
      run: |
          sudo echo 'kong soft nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
          sudo echo 'kong hard nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
          sudo echo "$(whoami) soft nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf
          sudo echo "$(whoami) hard nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf

    - name: Checkout Kong source code
      uses: actions/checkout@v4

    - name: Lookup build cache
      id: cache-deps
      uses: actions/cache@v4
      with:
        path: ${{ env.BUILD_ROOT }}
        key: ${{ needs.build.outputs.cache-key }}

    - name: Install Test::Nginx
      run: |
          CPAN_DOWNLOAD=./cpanm
          mkdir -p $CPAN_DOWNLOAD
          curl -o $CPAN_DOWNLOAD/cpanm https://cpanmin.us
          chmod +x $CPAN_DOWNLOAD/cpanm

          echo "Installing CPAN dependencies..."
          $CPAN_DOWNLOAD/cpanm --notest --local-lib=$HOME/perl5 local::lib && eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib)
          $CPAN_DOWNLOAD/cpanm --notest Test::Nginx

    - name: Generate environment variables
      run: |
           echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/PDK_test >> $GITHUB_ENV

    - name: Tests
      env:
        TEST_SUITE: pdk
      run: |
          source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
          if [[ $KONG_TEST_COVERAGE = true ]]; then
            export PDK_LUACOV=1
          fi
          eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib)
          prove -I. -r t

    - name: Upload error logs
      if: failure()
      uses: actions/upload-artifact@v4
      with:
        name: PDK-test-errlogs
        path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
        retention-days: 1

    - name: Archive coverage stats file
      uses: actions/upload-artifact@v4
      if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }}
      with:
        name: luacov-stats-out-${{ github.job }}-${{ github.run_id }}
        retention-days: 1
        path: |
          luacov.stats.out

    - name: Get kernel message
      if: failure()
      run: |
        sudo dmesg -T

  cleanup-and-aggregate-stats:
    needs: [lint-and-doc-tests,pdk-tests,busted-tests]
    name: Cleanup and Luacov stats aggregator
    if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }}
    runs-on: ubuntu-22.04

    steps:
    - name: Checkout source code
      uses: actions/checkout@v4

    - name: Install requirements
      run: |
        sudo apt-get update && sudo apt-get install -y luarocks
        sudo luarocks install luacov
        sudo luarocks install luafilesystem

    # Download all archived coverage stats files
    - uses: actions/download-artifact@v4

    - name: Stats aggregation
      shell: bash
      run: |
        lua .ci/luacov-stats-aggregator.lua "luacov-stats-out-" "luacov.stats.out" ${{ github.workspace }}/
        # The following prints a report with each file sorted by coverage percentage, and the total coverage
        printf "\n\nCoverage   File\n\n"
        awk -v RS='Coverage\n-+\n' 'NR>1{print $0}' luacov.report.out | grep -vE "^-|^$" > summary.out
        cat summary.out | grep -v "^Total" | awk '{printf "%7d%%   %s\n", $4, $1}' | sort -n
        cat summary.out | grep "^Total" | awk '{printf "%7d%%   %s\n", $4, $1}'