weekly.yml

# This workflow runs once a week and tests a variety of configurations,
# dependencies and other specific or expensive checkes (sanitizers).
# It also contains the Houdini Cache jobs which update the CI cache
# with new Houdini releases.

name: Weekly

on:
  schedule:
    # run this workflow Sunday 00:00 UTC
    - cron:  '0 0 * * 0'
  workflow_dispatch:
    inputs:
      type:
        description: 'The type of CI to run (all, houdini, latest, extra, ax, blosc, abi)'
        required: true
        default: 'all'

# Allow subsequent pushes to the same PR or REF to cancel any previous jobs.
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  #############################################################################
  ################################## Houdini ##################################
  #############################################################################

  # Check that valid github secrets have been set for the ability to
  # download Houdini and cache it. The secrets are used in download_houdini.py
  checksecret:
    name: Verify Houdini Secrets
    runs-on: ubuntu-latest
    outputs:
      HOUDINI_SECRETS: ${{ steps.check.outputs.HOUDINI_SECRETS }}
    steps:
      - id: check
        env:
            HOUDINI_CLIENT_ID: ${{ secrets.HOUDINI_CLIENT_ID }}
            HOUDINI_SECRET_KEY: ${{ secrets.HOUDINI_SECRET_KEY }}
        run: echo "HOUDINI_SECRETS=${{ env.HOUDINI_CLIENT_ID != '' && env.HOUDINI_SECRET_KEY != '' }}" >> $GITHUB_OUTPUT
      - name: Skip Next Jobs
        if: steps.check.outputs.HOUDINI_SECRETS != 'true'
        run: echo "HOUDINI_CLIENT_ID and HOUDINI_SECRET_KEY GitHub Action Secrets needs to be set to install Houdini builds"
      # Explicitly error on the ASWF repo, we expect this secret to always exist
      - name: Error ASWF
        if: steps.check.outputs.HOUDINI_SECRETS != 'true' && github.repository_owner == 'AcademySoftwareFoundation'
        run: exit 1

  # download the latest production version of Houdini X, strip out headers,
  # libraries and binaries required for building OpenVDB and put it into
  # the GitHub Actions cache
  linux_houdini:
    needs: [checksecret]
    if: |
      (needs.checksecret.outputs.HOUDINI_SECRETS == 'true') &&
      (github.event_name != 'workflow_dispatch' ||
       github.event.inputs.type == 'all' ||
       github.event.inputs.type == 'houdini')
    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: linux-houdini:${{ matrix.config.hou_hash }}
    env:
      CXX: clang++
      HOUDINI_CLIENT_ID: ${{ secrets.HOUDINI_CLIENT_ID }}
      HOUDINI_SECRET_KEY: ${{ secrets.HOUDINI_SECRET_KEY }}
    strategy:
      matrix:
        config:
          - { houdini_version: '19.5', platform: 'linux_x86_64_gcc9.3',  hou_hash: '19_5' }
          - { houdini_version: '20.0', platform: 'linux_x86_64_gcc9.3',  hou_hash: '20_0-oldabi' }
          - { houdini_version: '20.0', platform: 'linux_x86_64_gcc11.2', hou_hash: '20_0-newabi' }
      fail-fast: false
    container:
      image: aswf/ci-base:2023
    steps:
    - uses: actions/checkout@v3
    # We bumped from the 2021 CI image to 2023 here to fix some OpenSSL issues
    # with the Houdini download script. In so doing we broke some of the caching
    # between this job and the jobs in houdini.yml which _don't_ use the 2023
    # image yet. The issue is that the cache action will use zstd if it's
    # available to zip the cache and this causes it to be inserted with a unique
    # hash which images without zstd (i.e. the 2021/2022 images don't have
    # access to). For now, uninstall zstd here instead of installing it
    # everywhere and ask the LF to add zstd to the older base images.
    - name: remove zstd
      run: yum -y remove zstd
    - name: timestamp
      id: timestamp
      run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
    - name: download_houdini
      run: ./ci/download_houdini.sh ${{ matrix.config.houdini_version }} ${{ matrix.config.platform }} --prod
    - name: install_houdini
      run: |
        mkdir $HOME/houdini_install
        cp hou/hou.tar.gz $HOME/houdini_install/hou.tar.gz
        cd $HOME/houdini_install && tar -xzf hou.tar.gz && cd -
    - name: write_houdini_cache
      uses: actions/cache/save@v3
      with:
        path: hou
        key: vdb-v5-houdini${{ matrix.config.hou_hash }}-${{ steps.timestamp.outputs.timestamp }}

  macos_houdini:
    needs: [checksecret]
    if: |
      (needs.checksecret.outputs.HOUDINI_SECRETS == 'true') &&
      (github.event_name != 'workflow_dispatch' ||
       github.event.inputs.type == 'all' ||
       github.event.inputs.type == 'houdini')
    # Note that macos-14 (current macos-latest) switches to M1. We could instead test
    # the arm build here instead of the x86 one.
    runs-on: macos-latest
    name: macos-houdini-20
    env:
      HOUDINI_CLIENT_ID: ${{ secrets.HOUDINI_CLIENT_ID }}
      HOUDINI_SECRET_KEY: ${{ secrets.HOUDINI_SECRET_KEY }}
    steps:
    - uses: actions/checkout@v3
    - name: timestamp
      id: timestamp
      run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
    - name: download_houdini
      run: ./ci/download_houdini.sh 20.0 macosx_x86_64_clang12.0_11 --prod
    - name: install_houdini
      run: |
        mkdir $HOME/houdini_install
        cp hou/hou.tar.gz $HOME/houdini_install/hou.tar.gz
        cd $HOME/houdini_install && tar -xzf hou.tar.gz && cd -
    - name: write_houdini_cache
      uses: actions/cache/save@v3
      with:
        path: hou
        key: vdb-v5-houdini-macos-${{ steps.timestamp.outputs.timestamp }}

  #############################################################################
  ########################### Core Library Extras #############################
  #############################################################################

  # Extra configuration tests for the OpenVDB Core library. These test a
  # variety of options with newer compilers.
  linux-extra:
    if: |
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'extra'
    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: linux-extra:${{ matrix.config.name }}
    container:
      # @note  we specifically use clang15.0 (not clang15) here as the newest
      #  versions of the clang15.X containers have some issues with the GLFW
      #  installation
      image: aswf/ci-openvdb:2023-clang15.0
    env:
      CXX: clang++
    strategy:
      matrix:
        config:
          - { name: 'all',   build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_BLOSC=ON  -DUSE_ZLIB=ON  -DUSE_EXR=ON  -DUSE_PNG=ON'  }
          - { name: 'lite',  build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF -DOPENVDB_USE_DELAYED_LOADING=OFF' }
          - { name: 'half',  build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_BLOSC=OFF -DUSE_IMATH_HALF=ON' }
          - { name: 'sse',   build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DOPENVDB_SIMD=SSE42' }
          - { name: 'avx',   build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DOPENVDB_SIMD=AVX' }
          - { name: 'numpy', build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_NUMPY=ON -DOPENVDB_PYTHON_WRAP_ALL_GRID_TYPES=ON' }
          - { name: 'asan',  build: 'asan',    components: 'core,test,axcore,axtest',                        cmake: '-DNANOVDB_USE_OPENVDB=ON -DOPENVDB_AX_STATIC=OFF -DOPENVDB_CORE_STATIC=OFF -DUSE_BLOSC=OFF' } # We never called blosc_destroy(), so disable blosc to silence these errors
          - { name: 'ubsan', build: 'ubsan',   components: 'core,test,axcore,axtest',                        cmake: '' }
          - { name: 'c++20', build: 'Release', components: 'core,test,axcore,axtest',                        cmake: '-DCMAKE_CXX_STANDARD=20' }
          - { name: 'conf',  build: 'Release', components: 'core,python,bin,view,render,test,axcore,axtest', cmake: '-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON' }
      fail-fast: false
    steps:
    - uses: actions/checkout@v3
    - name: pybind11
      #if: contains(container.image, '2023') == false
      run: ./ci/install_pybind11.sh 2.10.0
    - name: build
      run: >
        ./ci/build.sh -v
        --build-type=${{ matrix.config.build }}
        --components="${{ matrix.config.components }}"
        --cargs=\"-DOPENVDB_CXX_STRICT=ON ${{ matrix.config.cmake }}\"
    - name: test
      run: cd build && ctest -V

  # Test latest dependencies, latest compilers and options
  latest:
    if: |
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'latest'
    runs-on: ${{ matrix.config.runson }}
    env:
      CXX: ${{ matrix.config.cxx }}
    strategy:
      matrix:
        config:
          - { runson: ubuntu-latest, cxx: g++,     cmake: '' }
          # Disable the clang job for now. See https://github.com/actions/runner-images/issues/8659
          # - { runson: ubuntu-latest, cxx: clang++, cmake: '' }
          # @todo gcc on macos
          - { runson: macos-latest,  cxx: '',      cmake: '-D CMAKE_CXX_COMPILER=/opt/homebrew/opt/llvm@15/bin/clang++' }
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: install_deps
        run: |
          if [ "$RUNNER_OS" == "Linux" ]; then
            sudo apt-get -q install -y libboost-dev libboost-iostreams-dev libtbb-dev libblosc-dev llvm-dev libgtest-dev libcppunit-dev pybind11-dev
          elif [ "$RUNNER_OS" == "macOS" ]; then
            ./ci/install_macos.sh 15
          else
            echo "$RUNNER_OS not supported"; exit 1
          fi
      - name: build
        run: >
          ./ci/build.sh -v
          --build-type=Release
          --components=\"core,axcore,python,bin,render,test,axtest,axbin\"
          --cargs=\"-DCMAKE_CXX_STANDARD=20 -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install ${{ matrix.config.cmake }}\"
      - name: test
        run: cd build && ctest -V

  #############################################################################
  ############################ AX Library Extras ##############################
  #############################################################################

  linux-ax:
    if: |
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'ax'
    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: >
      linux-ax:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}-${{ matrix.config.build }}
    container:
      image: aswf/ci-openvdb:${{ matrix.config.image }}
    env:
      CXX: ${{ matrix.config.cxx }}
    strategy:
      matrix:
        config:
          # Unified
          - { image: '2022-clang14', cxx: 'clang++', build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
          - { image: '2022-clang14', cxx: 'g++',     build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
          - { image: '2022-clang13', cxx: 'clang++', build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
          - { image: '2022-clang13', cxx: 'g++',     build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
          # Standalone
          - { image: '2021-clang10', cxx: 'clang++', build: 'Release', components: 'core', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON' }
          - { image: '2022-clang11', cxx: 'clang++', build: 'Debug',   components: 'core', cmake: '' }
          - { image: '2022-clang11', cxx: 'clang++', build: 'Release', components: 'core', cmake: '' }
          - { image: '2022-clang11', cxx: 'g++',     build: 'Release', components: 'core', cmake: '' }
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: pybind11
        #f: contains(matrix.config.image, '2023') == false
        run: ./ci/install_pybind11.sh 2.10.0
      - name: build
        run: >
          ./ci/build.sh -v
          --build-type=${{ matrix.config.build }}
          --components=${{ matrix.config.components }}
          --cargs=\"
          ${{ matrix.config.cmake }}
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
          -DOPENVDB_CXX_STRICT=ON
          \"
      - name: clean
        if: matrix.config.components == 'core'
        run: rm -rf build
      - name: build
        if: matrix.config.components == 'core'
        run: >
          ./ci/build.sh -v
          --build-type=${{ matrix.config.build }}
          --components="bin,axcore,axbin,axtest,python"
          --cargs=\"
          ${{ matrix.config.cmake }}
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
          -DOPENVDB_CXX_STRICT=ON
          \"
      - name: test
        run: cd build && ctest -V
      - name: test_doxygen_examples
        run: ./ci/extract_test_examples.sh

  macos-ax:
    if: |
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'ax'
    runs-on: macos-13
    name: macos-cxx:${{ matrix.config.cxx }}-llvm:${{ matrix.config.llvm }}-${{ matrix.config.build }}
    env:
      CXX: ${{ matrix.config.cxx }}
    strategy:
      matrix:
        config:
          - { cxx: 'clang++', build: 'Release', llvm: '14' }
          - { cxx: 'clang++', build: 'Release', llvm: '15' }
          #- { cxx: 'clang++', build: 'Release', llvm: '16' } - not supported yet
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: install_deps
        run: ./ci/install_macos.sh ${{ matrix.config.llvm }}
      - name: build
        run: >
          ./ci/build.sh -v
          --build-type=${{ matrix.config.build }}
          --components="core,python,bin,axcore,axbin,axtest"
          --cargs=\"
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
          -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install
          -DLLVM_DIR=/opt/homebrew/opt/llvm@${{ matrix.config.llvm }}/lib/cmake/llvm
          \"
      - name: test
        run: cd build && ctest -V
      - name: test_doxygen_examples
        run: ./ci/extract_test_examples.sh

  windows-ax:
    if: |
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'ax'
    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'windows-2022-8c-32g-300h') || 'windows-latest' }}
    name: windows-vc:${{ matrix.config.vc }}-type:${{ matrix.config.build }}
    env:
      VCPKG_DEFAULT_TRIPLET: ${{ matrix.config.vc }}
      # Export this with '' avoid bash treating \ as escape
      VDB_INSTALL_PREFIX: '${{ github.workspace }}\\install'
    strategy:
      matrix:
        config:
          # static build of blosc from vcpkg does not build internal sources.
          # USE_STATIC_DEPENDENCIES is required for IlmBase/OpenEXR defines and
          # Boost as both shared and static libs are installed.
          # @todo  We don't currently run the axtests with shared builds of ax
          # due to symbol issues using LLVM as a static lib (which is the only
          # option on Windows).
          - { vc: 'x64-windows',        crt: 'MD',  components: 'core,bin,axcore,axbin,python', build: 'Release', cmake: '-DOPENVDB_CORE_STATIC=OFF -DOPENVDB_AX_STATIC=OFF' }
          - { vc: 'x64-windows-static', crt: 'MT',  components: 'core,bin,axcore,axbin,axtest', build: 'Release', cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
          - { vc: 'x64-windows-static', crt: 'MTd', components: 'core,bin,axcore,axbin,axtest', build: 'Debug',   cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
      fail-fast: false
    steps:
    - uses: actions/checkout@v3
    - name: llvm
      run: ./ci/install_llvm_windows.sh ${{ matrix.config.crt }}
    - name: install
      run: ./ci/install_windows.sh
    - name: build
      run: >
        ./ci/build.sh -v
        --config=${{ matrix.config.build }}
        --components="${{ matrix.config.components }}"
        --cargs=\'
        -A x64 -G \"Visual Studio 17 2022\"
        -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET}
        -DCMAKE_TOOLCHAIN_FILE=\"${VCPKG_INSTALLATION_ROOT}\\scripts\\buildsystems\\vcpkg.cmake\"
        -DMSVC_COMPRESS_PDB=ON
        -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
        -DUSE_EXPLICIT_INSTANTIATION=OFF
        -DLLVM_DIR=\"${HOME}\\llvm_install\\lib\\cmake\\llvm\"
        -DCMAKE_INSTALL_PREFIX=\"${VDB_INSTALL_PREFIX}\"
        ${{ matrix.config.cmake }}
        \'
    - name: runtime_path
      shell: pwsh
      run: |
        # note: system path must be modified in a previous step to it's use
        echo "$Env:VCPKG_INSTALLATION_ROOT\installed\${{ matrix.config.vc }}\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
        echo "$Env:VDB_INSTALL_PREFIX\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
    - name: test
      run: cd build && ctest -V -C ${{ matrix.config.build }}

  #############################################################################
  ################################## Blosc ####################################
  #############################################################################

  linux-blosc:
    if: |
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'blosc'
    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: linux-blosc:${{ matrix.blosc }}
    container:
      image: aswf/ci-base:2023
    strategy:
      matrix:
        blosc: ['1.18.0','1.19.0','1.20.0','1.21.0']
      fail-fast: false
    steps:
    - uses: actions/checkout@v3
    - name: install_blosc
      run: sudo ./ci/install_blosc.sh ${{ matrix.blosc }}
    - name: build
      run: >
        sudo ./ci/build.sh -v
        --build-type=Release
        --components=\"core,test\"
    - name: test
      run: cd build && sudo ctest -V

  #############################################################################
  ################################## ABI ######################################
  #############################################################################

  linux-abi-checker:
    if: |
      github.event_name == 'workflow_dispatch' &&
      (github.event.inputs.type == 'all' ||
       github.event.inputs.type == 'abi')
    runs-on: ubuntu-22.04
    env:
      # The 'abicheck' build type sets these, but older versions of the library
      # may not have this build type. See OpenVDBCXX.cmake
      CXXFLAGS: "-gdwarf-4 -g3 -ggdb -Og"
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
        fetch-tags: true
    # Compute the latest major version - that is used as our baseline
    # note: For CI forks, make sure you have your tags synced
    - name: get_major_version
      run: |
        LATEST_VERSION_TAG=$(git tag --merged | sort --version-sort | tail -n1)
        echo "Computed latest VDB tag: ${LATEST_VERSION_TAG}"
        VDB_MAJOR_VERSION=$(echo ${LATEST_VERSION_TAG} | cut -f1 -d '.' | tr -d -c 0-9)
        echo "Using major version: ${VDB_MAJOR_VERSION}"
        echo "VDB_MAJOR_VERSION=${VDB_MAJOR_VERSION}" >> "$GITHUB_ENV"
    - name: install_deps
      run: sudo apt-get -q install -y libboost-iostreams-dev libtbb-dev libblosc-dev elfutils
    - name: install_abi_checker
      run: sudo apt-get -q install -y abi-dumper abi-compliance-checker
    - name: build_new
      run: >
        ./ci/build.sh -v
        --build-dir=build_new
        --build-type=abicheck
        --target=openvdb_shared
        --components=\"core\"
        --cargs=\'-DUSE_EXPLICIT_INSTANTIATION=OFF -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON\'
    - name: checkout_baseline
      run: git checkout v${VDB_MAJOR_VERSION}.0.0
    - name: build_old
      run: >
        ./ci/build.sh -v
        --build-dir=build_old
        --build-type=abicheck
        --target=openvdb_shared
        --components=\"core\"
        --cargs=\'-DUSE_EXPLICIT_INSTANTIATION=OFF -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON\'
    - name: abi_dump
      run: |
        abi-dumper build_new/openvdb/openvdb/libopenvdb.so -o ABI-NEW.dump -lver 1
        abi-dumper build_old/openvdb/openvdb/libopenvdb.so -o ABI-OLD.dump -lver 2
      # Replace the version namespace in the latest ABI dump with the baseline
      # version we're comparing against. We should probably instead build the
      # latest with the baseline version number but no CMake/defines allow us to
      # do this.
    - name: replace_symbols
      run: sed -i -E 's/openvdb([^v]*)v[0-9]*_[0-9]/openvdb\1v'${VDB_MAJOR_VERSION}'_0/g' ABI-NEW.dump
    - name: abi_check
      # -strict treats warnings as errors
      # -extended checks all member data
      # we check everything _not_ in openvdb::**::internal namespace
      run: >
        abi-compliance-checker -l OPENVDB
        -old ABI-OLD.dump
        -new ABI-NEW.dump
        -skip-internal-symbols "\d(openvdb.*internal)"
        -skip-internal-types "(openvdb.*internal)::"
        -strict
        -extended
    - name: upload_report
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: abi_report
        path: ./compat_reports/OPENVDB/2_to_1/compat_report.html
        retention-days: 5