From 31a192154b13457d5882dc8d81c2d64eeeb04785 Mon Sep 17 00:00:00 2001 From: Tom Birdsong Date: Mon, 21 Nov 2022 20:51:34 -0500 Subject: [PATCH] ENH: Bump for ITK v5.3.0 Includes CI and build script updates for v5.3.0 build procedure. --- .github/workflows/build-test-package.yml | 70 ++++++++++++------- .github/workflows/test-gpu.yml | 4 +- setup.py | 4 +- ...ss-manylinux-build-module-wheels-opencl.sh | 21 ++++-- .../dockcross-manylinux-download-cache.sh | 55 ++++++++++++--- 5 files changed, 110 insertions(+), 44 deletions(-) diff --git a/.github/workflows/build-test-package.yml b/.github/workflows/build-test-package.yml index ba3dbc8..cd35348 100644 --- a/.github/workflows/build-test-package.yml +++ b/.github/workflows/build-test-package.yml @@ -3,8 +3,9 @@ name: Build, test, package on: [push,pull_request] env: - itk-git-tag: "171fb2ba33a87041f99328a2f26612ff33aa9cc8" - itk-wheel-tag: "v5.3rc04.post3" + itk-git-tag: "v5.3.0" + itk-wheel-tag: "v5.3.0" + itk-python-package-tag: "03391ad738438661fff40bfe37a7cfabd171b9b1" opencl-icd-loader-git-tag: "v2021.04.29" opencl-headers-git-tag: "v2021.04.29" vkfft-backend: 3 @@ -17,13 +18,13 @@ jobs: strategy: max-parallel: 3 matrix: - os: [ubuntu-20.04, windows-2019, macos-11] + os: [ubuntu-20.04, windows-2022, macos-11] include: - os: ubuntu-20.04 c-compiler: "gcc" cxx-compiler: "g++" cmake-build-type: "MinSizeRel" - - os: windows-2019 + - os: windows-2022 c-compiler: "cl.exe" cxx-compiler: "cl.exe" cmake-build-type: "Release" @@ -46,7 +47,7 @@ jobs: python -m pip install ninja - name: Get specific version of CMake, Ninja - uses: lukka/get-cmake@v3.18.3 + uses: lukka/get-cmake@v3.22.2 - name: Download OpenCL-SDK if: matrix.os == 'macos-11' @@ -73,7 +74,7 @@ jobs: shell: bash - name: Install pocl - if: matrix.os != 'windows-2019' + if: matrix.os != 'windows-2022' run: | sudo conda config --add channels conda-forge sudo conda config --set channel_priority strict @@ -98,7 +99,7 @@ jobs: shell: bash - name: Build OpenCL-ICD-Loader - if: matrix.os != 'windows-2019' + if: matrix.os != 'windows-2022' run: | cd .. mkdir OpenCL-ICD-Loader-build @@ -107,18 +108,18 @@ jobs: sudo cmake --build . --target install - name: Build OpenCL-ICD-Loader - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-2022' run: | cd .. mkdir OpenCL-ICD-Loader-build cd OpenCL-ICD-Loader-build - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" cmake -DCMAKE_C_COMPILER:FILEPATH="${{ matrix.c-compiler }}" -DBUILD_SHARED_LIBS:BOOL=ON -DCMAKE_CXX_COMPILER="${{ matrix.cxx-compiler }}" -DCMAKE_BUILD_TYPE:STRING=${{ matrix.cmake-build-type }} -DBUILD_TESTING:BOOL=OFF -GNinja ../OpenCL-ICD-Loader cmake --build . --target install shell: cmd - name: Build ITK - if: matrix.os != 'windows-2019' + if: matrix.os != 'windows-2022' run: | cd .. mkdir ITK-build @@ -127,12 +128,12 @@ jobs: ninja - name: Build ITK - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-2022' run: | cd .. mkdir ITK-build cd ITK-build - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" cmake -DCMAKE_C_COMPILER:FILEPATH="${{ matrix.c-compiler }}" -DBUILD_SHARED_LIBS:BOOL=ON -DCMAKE_CXX_COMPILER="${{ matrix.cxx-compiler }}" -DCMAKE_BUILD_TYPE:STRING=${{ matrix.cmake-build-type }} -DBUILD_TESTING:BOOL=OFF -GNinja ../ITK ninja shell: cmd @@ -196,23 +197,23 @@ jobs: shell: bash - name: Build and test - if: matrix.os != 'windows-2019' + if: matrix.os != 'windows-2022' run: | ctest --output-on-failure -j 2 -V -S dashboard.cmake -R "VkFFTBackend" - name: Build and test - if: matrix.os == 'windows-2019' + if: matrix.os == 'windows-2022' run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" ctest --output-on-failure -j 2 -V -S dashboard.cmake -R "VkFFTBackend" shell: cmd build-windows-opencl-python-packages: - runs-on: windows-2019 + runs-on: windows-2022 strategy: max-parallel: 2 matrix: - python-version-minor: ["7", "8", "9", "10"] + python-version-minor: ["7", "8", "9", "10", "11"] include: - c-compiler: "cl.exe" cxx-compiler: "cl.exe" @@ -220,7 +221,7 @@ jobs: steps: - name: Get specific version of CMake, Ninja - uses: lukka/get-cmake@v3.18.3 + uses: lukka/get-cmake@v3.22.2 - uses: actions/checkout@v2 with: @@ -233,6 +234,7 @@ jobs: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/scikit-build/scikit-ci-addons/master/windows/install-python.ps1')) - name: 'Fetch build dependencies' + shell: bash run: | mv im ../../ cd ../../im @@ -242,7 +244,14 @@ jobs: 7z x doxygen-1.8.11.windows.bin.zip -o/c/P/doxygen -aoa -r curl -L "https://data.kitware.com/api/v1/file/5bbf87ba8d777f06b91f27d6/download/grep-win.zip" -o "grep-win.zip" 7z x grep-win.zip -o/c/P/grep -aoa -r - shell: bash + + echo "Updating ITKPythonPackage build scripts to ${{ env.itk-python-package-tag }}" + pushd /c/P/IPP + git remote add InsightSoftwareConsortium https://github.com/InsightSoftwareConsortium/ITKPythonPackage.git --tags + git fetch InsightSoftwareConsortium + git checkout ${{ env.itk-python-package-tag }} + git status + popd - name: Download OpenCL-ICD-Loader run: | @@ -265,7 +274,7 @@ jobs: cd .. mkdir OpenCL-ICD-Loader-build cd OpenCL-ICD-Loader-build - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" cmake -DCMAKE_C_COMPILER:FILEPATH="${{ matrix.c-compiler }}" -DBUILD_SHARED_LIBS:BOOL=ON -DCMAKE_CXX_COMPILER="${{ matrix.cxx-compiler }}" -DCMAKE_BUILD_TYPE:STRING=${{ matrix.cmake-build-type }} -DBUILD_TESTING:BOOL=OFF -GNinja ../OpenCL-ICD-Loader cmake --build . --target install shell: cmd @@ -273,7 +282,7 @@ jobs: - name: 'Build 🐍 Python 📦 package' run: | cd ../../im - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" set PATH=C:\P\grep;%PATH% set CC=cl.exe set CXX=cl.exe @@ -291,7 +300,7 @@ jobs: strategy: max-parallel: 2 matrix: - python-version: ["37", "38", "39", "310"] + python-version: ["37", "38", "39", "310", "311"] steps: - uses: actions/checkout@v2 @@ -305,11 +314,19 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" df -h + - name: 'Fetch build dependencies' + shell: bash + run: | + sudo apt-get install zstd + unzstd --version + - name: 'Build 🐍 Python 📦 package' run: | export ITK_PACKAGE_VERSION=${{ env.itk-wheel-tag }} - export TARBALL_SPECIALIZATION="-manylinux_2_28" - ./wrapping/dockcross-manylinux-download-cache.sh + export ITKPYTHONPACKAGE_TAG=${{ env.itk-python-package-tag }} + export MANYLINUX_VERSION="_2_28" + echo "Building for manylinux specialization ${MANYLINUX_VERSION}" + ./wrapping/dockcross-manylinux-download-cache.sh cp${{ matrix.python-version }} ./wrapping/dockcross-manylinux-build-module-wheels-opencl.sh cp${{ matrix.python-version }} - name: Publish Python package as GitHub Artifact @@ -331,16 +348,17 @@ jobs: sudo xcode-select -s "/Applications/Xcode_13.2.1.app" - name: Get specific version of CMake, Ninja - uses: lukka/get-cmake@v3.18.3 + uses: lukka/get-cmake@v3.22.2 - name: 'Fetch build script' run: | - curl -L https://raw.githubusercontent.com/InsightSoftwareConsortium/ITKPythonPackage/master/scripts/macpython-download-cache-and-build-module-wheels.sh -O + curl -L https://raw.githubusercontent.com/InsightSoftwareConsortium/ITKPythonPackage/${{ env.itk-python-package-tag }}/scripts/macpython-download-cache-and-build-module-wheels.sh -O chmod u+x macpython-download-cache-and-build-module-wheels.sh - name: 'Build 🐍 Python 📦 package' run: | export ITK_PACKAGE_VERSION=${{ env.itk-wheel-tag }} + export ITKPYTHONPACKAGE_TAG=${{ env.itk-python-package-tag }} export MACOSX_DEPLOYMENT_TARGET=10.9 ./macpython-download-cache-and-build-module-wheels.sh diff --git a/.github/workflows/test-gpu.yml b/.github/workflows/test-gpu.yml index 0b4ecad..476a001 100644 --- a/.github/workflows/test-gpu.yml +++ b/.github/workflows/test-gpu.yml @@ -3,7 +3,7 @@ name: Test GPU on: [pull_request] env: - itk-git-tag: "171fb2ba33a87041f99328a2f26612ff33aa9cc8" + itk-git-tag: "abf5fa10522a36bc51f42f20f426a622f42ed90d" jobs: build-test-gpu: @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@v2 - name: Get specific version of CMake, Ninja - uses: lukka/get-cmake@v3.18.3 + uses: lukka/get-cmake@v3.22.2 - name: Download OpenCL-ICD-Loader run: | diff --git a/setup.py b/setup.py index 3c05ff3..27598e5 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ setup( name="itk-vkfft", - version="0.1.9", + version="0.2.0", author="Insight Software Consortium", author_email="itk+community@discourse.itk.org", packages=["itk"], @@ -42,5 +42,5 @@ license="Apache", keywords="ITK InsightToolkit", url=r"https://itk.org/", - install_requires=[r"itk>=5.3rc04.post3"], + install_requires=[r"itk>=5.3.0"], ) diff --git a/wrapping/dockcross-manylinux-build-module-wheels-opencl.sh b/wrapping/dockcross-manylinux-build-module-wheels-opencl.sh index 9425652..b50597a 100755 --- a/wrapping/dockcross-manylinux-build-module-wheels-opencl.sh +++ b/wrapping/dockcross-manylinux-build-module-wheels-opencl.sh @@ -7,11 +7,14 @@ # For example, # # scripts/dockcross-manylinux-build-module-wheels.sh cp39 +# +# Forked from +# https://github.com/InsightSoftwareConsortium/ITKPythonPackage/blob/master/scripts/dockcross-manylinux-build-module-wheels.sh # Generate dockcross scripts -MANYLINUX_VERSION="_2_28" -IMAGE_TAG=20220715-9ce3707 +MANYLINUX_VERSION=${MANYLINUX_VERSION:=_2_28} +IMAGE_TAG=${IMAGE_TAG:=20221108-102ebcc} OPENCL_ICD_LOADER_TAG=v2021.04.29 OPENCL_HEADERS_TAG=v2021.04.29 @@ -20,6 +23,12 @@ chmod u+x /tmp/dockcross-manylinux-x64 script_dir=$(cd $(dirname $0) || exit 1; pwd) +mkdir -p $(pwd)/tools +chmod 777 $(pwd)/tools +# Build wheels +mkdir -p dist + +# Build OpenCL-ICD-Loader before ITKVkFFTBackend if ! test -d ./OpenCL-ICD-Loader; then git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader pushd OpenCL-ICD-Loader @@ -40,8 +49,12 @@ if ! test -d ./OpenCL-ICD-Loader; then fi # Build wheels -mkdir -p dist -DOCKER_ARGS="-v $(pwd)/dist:/work/dist/ -v $script_dir/../ITKPythonPackage:/ITKPythonPackage -v $(pwd)/tools:/tools -v $(pwd)/OpenCL-ICD-Loader/inc/CL:/usr/include/CL -v $(pwd)/OpenCL-ICD-Loader-build/libOpenCL.so.1.2:/usr/lib64/libOpenCL.so.1 -v $(pwd)/OpenCL-ICD-Loader-build/libOpenCL.so.1.2:/usr/lib64/libOpenCL.so" +DOCKER_ARGS="-v $(pwd)/dist:/work/dist/ -v $script_dir/../ITKPythonPackage:/ITKPythonPackage -v $(pwd)/tools:/tools" +DOCKER_ARGS+=" -v $(pwd)/OpenCL-ICD-Loader/inc/CL:/usr/include/CL" +DOCKER_ARGS+=" -v $(pwd)/OpenCL-ICD-Loader-build/libOpenCL.so.1.2:/usr/lib64/libOpenCL.so.1" +DOCKER_ARGS+=" -v $(pwd)/OpenCL-ICD-Loader-build/libOpenCL.so.1.2:/usr/lib64/libOpenCL.so" +DOCKER_ARGS+=" -e MANYLINUX_VERSION" + /tmp/dockcross-manylinux-x64 \ -a "$DOCKER_ARGS" \ "/ITKPythonPackage/scripts/internal/manylinux-build-module-wheels.sh" "$@" diff --git a/wrapping/dockcross-manylinux-download-cache.sh b/wrapping/dockcross-manylinux-download-cache.sh index 842e636..0949a9f 100755 --- a/wrapping/dockcross-manylinux-download-cache.sh +++ b/wrapping/dockcross-manylinux-download-cache.sh @@ -2,20 +2,55 @@ # This module should be pulled and run from an ITKModule root directory to generate the Linux python wheels of this module, # it is used by the circle.yml file contained in ITKModuleTemplate: https://github.com/InsightSoftwareConsortium/ITKModuleTemplate +# +# Forked from +# https://github.com/InsightSoftwareConsortium/ITKPythonPackage/blob/master/scripts/dockcross-manylinux-download-cache-and-build-module-wheels.sh -curl https://data.kitware.com/api/v1/file/592dd8068d777f16d01e1a92/download -o zstd-1.2.0-linux.tar.gz -gunzip -d zstd-1.2.0-linux.tar.gz -tar xf zstd-1.2.0-linux.tar +# ----------------------------------------------------------------------- +# Verifies that unzstd binary is available to decompress ITK build archives. +unzstd_exe=`(which unzstd)` + +if [[ -z ${unzstd_exe} ]]; then + echo "ERROR: can not find required binary 'unzstd' " + exit 255 +fi + +# Expect unzstd > v1.3.2, see discussion in `dockcross-manylinux-build-tarball.sh` +${unzstd_exe} --version + +# ----------------------------------------------------------------------- +# Fetch build archive + +TARBALL_SPECIALIZATION="-manylinux${MANYLINUX_VERSION:=_2_28}" TARBALL_NAME="ITKPythonBuilds-linux${TARBALL_SPECIALIZATION}.tar" -curl -L https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.3rc04.post2}/${TARBALL_NAME}.zst -O -./zstd-1.2.0-linux/bin/unzstd ${TARBALL_NAME}.zst -o ${TARBALL_NAME} -echo "Extracting all files" -tar xf ${TARBALL_NAME} +curl -L https://github.com/InsightSoftwareConsortium/ITKPythonBuilds/releases/download/${ITK_PACKAGE_VERSION:=v5.3.0}/${TARBALL_NAME}.zst -O + +${unzstd_exe} --long=31 ${TARBALL_NAME}.zst -o ${TARBALL_NAME} +if [ "$#" -lt 1 ]; then + echo "Extracting all files"; + tar xf ${TARBALL_NAME} +else + echo "Extracting files relevant for: $1"; + tar xf ${TARBALL_NAME} ITKPythonPackage/scripts/ + tar xf ${TARBALL_NAME} ITKPythonPackage/ITK-source/ + tar xf ${TARBALL_NAME} ITKPythonPackage/oneTBB-prefix/ + tar xf ${TARBALL_NAME} --wildcards ITKPythonPackage/ITK-$1* +fi rm ${TARBALL_NAME} -mkdir tools -curl https://data.kitware.com/api/v1/file/5c0aa4b18d777f2179dd0a71/download -o doxygen-1.8.11.linux.bin.tar.gz -tar -xvzf doxygen-1.8.11.linux.bin.tar.gz -C tools +# Optional: Update build scripts +if [[ -n ${ITKPYTHONPACKAGE_TAG} ]]; then + echo "Updating build scripts to ${ITKPYTHONPACKAGE_ORG:=InsightSoftwareConsortium}/ITKPythonPackage@${ITKPYTHONPACKAGE_TAG}" + git clone "https://github.com/${ITKPYTHONPACKAGE_ORG}/ITKPythonPackage.git" "IPP-tmp" + pushd IPP-tmp/ + git checkout "${ITKPYTHONPACKAGE_TAG}" + git status + popd + + rm -rf ITKPythonPackage/scripts/ + cp -r IPP-tmp/scripts ITKPythonPackage/ + rm -rf IPP-tmp/ +fi cp -a ITKPythonPackage/oneTBB-prefix ./