Regression.yml

name: Regression
on:
  workflow_dispatch:
  repository_dispatch:
  push:
    branches:
      - '**'
      - '!master'
      - '!feature'
    tags:
      - '**'
    paths-ignore:
      - '**.md'
      - 'tools/**'
      - '!tools/pythonpkg/**'
      - '.github/workflows/**'
      - '!.github/workflows/Regression.yml'
      - '.github/config/uncovered_files.csv'
  pull_request:
    paths-ignore:
      - '**.md'
      - 'tools/**'
      - '!tools/pythonpkg/**'
      - '.github/workflows/**'
      - '!.github/workflows/Regression.yml'
      - '.github/config/uncovered_files.csv'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/master' || github.sha }}
  cancel-in-progress: true

env:
  GH_TOKEN: ${{ secrets.GH_TOKEN }}

jobs:
 regression-test-benchmark-runner:
  name: Regression Tests
  runs-on: ubuntu-20.04
  env:
    CC: gcc-10
    CXX: g++-10
    GEN: ninja
    BUILD_BENCHMARK: 1
    BUILD_TPCH: 1
    BUILD_TPCDS: 1
    BUILD_HTTPFS: 1
    BUILD_JEMALLOC: 1

  steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v4
      with:
        python-version: '3.7'

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }}

    - name: Build
      shell: bash
      run: |
        make
        git clone --branch ${GITHUB_BASE_REF:-master} https://github.com/duckdb/duckdb.git --depth=1
        cd duckdb
        make
        cd ..

    - name: Set up benchmarks
      shell: bash
      run: |
        cp -r benchmark duckdb/

    - name: Regression Test Micro
      if: always()
      shell: bash
      run: |
        python scripts/regression_test_runner.py --old=duckdb/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/micro.csv --verbose --threads=2

    - name: Regression Test TPCH
      if: always()
      shell: bash
      run: |
        python scripts/regression_test_runner.py --old=duckdb/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpch.csv --verbose --threads=2

    - name: Regression Test TPCDS
      if: always()
      shell: bash
      run: |
        python scripts/regression_test_runner.py --old=duckdb/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpcds.csv --verbose --threads=2

    - name: Regression Test H2OAI
      if: always()
      shell: bash
      run: |
        python scripts/regression_test_runner.py --old=duckdb/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/h2oai.csv --verbose --threads=2

    - name: Regression Test IMDB
      if: always()
      shell: bash
      run: |
        python scripts/regression_test_runner.py --old=duckdb/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/imdb.csv --verbose --threads=2

 regression-test-memory-safety:
  name: Regression Tests between safe and unsafe builds
  runs-on: ubuntu-20.04
  env:
    CC: gcc-10
    CXX: g++-10
    GEN: ninja
    BUILD_BENCHMARK: 1
    BUILD_TPCH: 1
    BUILD_TPCDS: 1
    BUILD_HTTPFS: 1
    BUILD_JEMALLOC: 1

  steps:
    - name: Checkout
      uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - name: Checkout tools repo
      uses: actions/checkout@v3
      with:
        fetch-depth: 0
        path: unsafe

    - uses: actions/setup-python@v4
      with:
        python-version: '3.7'

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }}

    - name: Build
      shell: bash
      run: |
        make

        cd unsafe
        DISABLE_MEMORY_SAFETY=1 make

    - name: Set up benchmarks
      shell: bash
      run: |
        cp -r benchmark unsafe/

    - name: Regression Test Micro
      if: always()
      shell: bash
      run: |
        python scripts/regression_test_runner.py --old=unsafe/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/micro.csv --verbose --threads=2

    - name: Regression Test TPCH
      if: always()
      shell: bash
      run: |
        python scripts/regression_test_runner.py --old=unsafe/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpch.csv --verbose --threads=2

    - name: Regression Test TPCDS
      if: always()
      shell: bash
      run: |
        python scripts/regression_test_runner.py --old=unsafe/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/tpcds.csv --verbose --threads=2

    - name: Regression Test H2OAI
      if: always()
      shell: bash
      run: |
        python scripts/regression_test_runner.py --old=unsafe/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/h2oai.csv --verbose --threads=2

    - name: Regression Test IMDB
      if: always()
      shell: bash
      run: |
        python scripts/regression_test_runner.py --old=unsafe/build/release/benchmark/benchmark_runner --new=build/release/benchmark/benchmark_runner --benchmarks=.github/regression/imdb.csv --verbose --threads=2

 regression-test-storage:
  name: Storage Size Regression Test
  runs-on: ubuntu-20.04
  env:
    CC: gcc-10
    CXX: g++-10
    GEN: ninja
    BUILD_TPCH: 1
    BUILD_TPCDS: 1

  steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v4
      with:
        python-version: '3.7'

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install requests

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }}

    - name: Build
      shell: bash
      run: |
        make
        git clone --branch ${GITHUB_BASE_REF:-master} https://github.com/duckdb/duckdb.git --depth=1
        cd duckdb
        make
        cd ..

    - name: Regression Test
      shell: bash
      run: |
        python scripts/regression_test_storage_size.py --old=duckdb/build/release/duckdb --new=build/release/duckdb

 regression-test-python:
  name: Regression Test (Python Client)
  runs-on: ubuntu-20.04
  env:
    CC: gcc-10
    CXX: g++-10
    GEN: ninja

  steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v4
      with:
        python-version: '3.7'

    - name: Install
      shell: bash
      run: |
        sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build
        pip install numpy pytest pandas mypy psutil "pyarrow<=11.0.0"

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }}

    - name: Build Current Version
      shell: bash
      run: |
        cd tools/pythonpkg
        python setup.py install --user
        cd ../..

    - name: Run New Version
      shell: bash
      run: |
        python scripts/regression_test_python.py --threads=2 --out-file=new.csv

    - name: Cleanup New Version
      shell: bash
      run: |
        cd tools/pythonpkg
        ./clean.sh
        cd ../..

    - name: Build Current
      shell: bash
      run: |
        git clone --branch ${GITHUB_BASE_REF:-master} https://github.com/duckdb/duckdb.git --depth=1
        cd duckdb/tools/pythonpkg
        python setup.py install --user
        cd ../../..

    - name: Run Current Version
      shell: bash
      run: |
        python scripts/regression_test_python.py --threads=2 --out-file=current.csv

    - name: Regression Test
      shell: bash
      run: |
        cp -r benchmark duckdb/
        python scripts/regression_check.py --old=current.csv --new=new.csv

 regression-test-plan-cost:
  name: Regression Test Join Order Plan Cost
  runs-on: ubuntu-20.04
  env:
    CC: gcc-10
    CXX: g++-10
    GEN: ninja
    BUILD_TPCH: 1
    BUILD_HTTPFS: 1

  steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0

    - uses: actions/setup-python@v4
      with:
        python-version: '3.7'

    - name: Install
      shell: bash
      run: sudo apt-get update -y -qq && sudo apt-get install -y -qq ninja-build && pip install tqdm

    - name: Setup Ccache
      uses: hendrikmuhs/ccache-action@main
      with:
        key: ${{ github.job }}
        save: ${{ github.ref == 'refs/heads/master' || github.repository != 'duckdb/duckdb' }}

    - name: Build
      shell: bash
      run: |
        make
        git clone --branch ${GITHUB_BASE_REF:-master} https://github.com/duckdb/duckdb.git --depth=1
        cd duckdb
        make
        cd ..

    - name: Set up benchmarks
      shell: bash
      run: |
        cp -r benchmark duckdb/

    - name: Regression Test IMDB
      continue-on-error: true
      shell: bash
      run: |
        python scripts/plan_cost_runner.py --old=duckdb/build/release/duckdb --new=build/release/duckdb --dir=benchmark/imdb_plan_cost

    - name: Regression Test TPCH
      continue-on-error: true
      shell: bash
      run: |
        python scripts/plan_cost_runner.py --old=duckdb/build/release/duckdb --new=build/release/duckdb --dir=benchmark/tpch_plan_cost