diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml new file mode 100644 index 0000000000..12761e78a9 --- /dev/null +++ b/.github/workflows/build-wheel-linux.yml @@ -0,0 +1,48 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +name: Build Python Wheel for Linux + +on: + # Trigger the workflow manually + workflow_dispatch: ~ + + # Allow to be called from another workflow + workflow_call: ~ + + # TODO automation trigger + +jobs: + build: + name: Build manylinux_2_28 + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? + container: + image: eccr.ecmwf.int/wheelmaker/2_28:latest + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} + steps: + # TODO convert this to be matrix-friendly. Note it's a bit tricky since + # we'd ideally not reexecute the compile step multiple times, but it + # (non-essentially) depends on a matrix-based step + # NOTE we dont use action checkout because it doesnt cleanup after itself correctly + - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj + - run: cd /proj && /buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && python_wrapper/pre-compile.sh + - run: cd /proj && /buildscripts/compile.sh ./python_wrapper/buildconfig + - run: cd /proj && PYTHONPATH=/buildscripts /buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && python_wrapper/post-build.sh + - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/upload-pypi.sh ./python_wrapper/buildconfig + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # NOTE temporary thing until all the mess gets cleared + - run: rm -rf ./* ./.git ./.github diff --git a/.gitignore b/.gitignore index 8791994fc8..91ac37b0f2 100644 --- a/.gitignore +++ b/.gitignore @@ -75,3 +75,5 @@ data/bufr/*test build/ *.back trash + +*swp diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig new file mode 100644 index 0000000000..2c105b759c --- /dev/null +++ b/python_wrapper/buildconfig @@ -0,0 +1,23 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh +# NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation +# TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent + +NAME="eccodes" + +CMAKE_PARAMS1="-DENABLE_PNG=1 -DENABLE_JPG=1 -DENABLE_AEC=1 -DENABLE_JPG_LIBJASPER=0 -DENABLE_JPG_LIBOPENJPEG=1 -DOPENJPEG_DIR=/tmp/openjpeg/target" +CMAKE_PARAMS2="-DENABLE_NETCDF=0 -DENABLE_FORTRAN=0" +CMAKE_PARAMS3="-DENABLE_EXAMPLES=0 -DENABLE_PYTHON=0 -DENABLE_BUILD_TOOLS=0 -DENABLE_INSTALL_ECCODES_DEFINITIONS=0 -DENABLE_INSTALL_ECCODES_SAMPLES=0" +CMAKE_PARAMS4="-DENABLE_ECCODES_THREADS=1 -DENABLE_MEMFS=1" +CMAKE_PARAMS="$CMAKE_PARAMS1 $CMAKE_PARAMS2 $CMAKE_PARAMS3 $CMAKE_PARAMS4" + +PYPROJECT_DIR="python_wrapper" +# TODO add eckit once the dependency is in place +DEPENDENCIES='[]' diff --git a/python_wrapper/post-build.sh b/python_wrapper/post-build.sh new file mode 100755 index 0000000000..a984e19cc3 --- /dev/null +++ b/python_wrapper/post-build.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +: +# NOTE auditwheel is problematic since it changes libnames -- all is well from +# the pov # of this very package's libs, but subsequent packages compiled with +# this as a dependency end up not working +# auditwheel repair -w /tmp/eccodes/auditwheel /tmp/eccodes/build/wheel/*whl diff --git a/python_wrapper/pre-compile.sh b/python_wrapper/pre-compile.sh new file mode 100755 index 0000000000..3cb5b0c32a --- /dev/null +++ b/python_wrapper/pre-compile.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# the procedure for adding a new ext dependency to be bundled in here: +# - add git checkout, compile, etc +# - ensure the version ends up in python_wrapper/src/versions.txt +# - ensure the licence ends up in python_wrapper/src/copying/, and fname is referenced in copying/list.json +# - ensure the .so ends up in target/lib64/ with the expected libname +# - validate that the resulting wheel contains all the above +# additionally, make sure this script is aligned with /buildscripts/compile.sh and /buildscripts/wheel-linux.sh, +# in particular when it comes to install targets and package data, etc + +mkdir -p python_wrapper/src/copying +mkdir -p /tmp/eccodes/target/eccodes/lib64/ + +## yum-available prereqs +for p in libaec-devel libpng-devel gobject-introspection-devel +do + yum install -y $p + # TODO improve + yum install $p 2>&1 > tmp + cat tmp + v=$(grep 'already installed' < tmp | awk '{print $2;}' | sed 's/\\d://') + echo "yum $p $v" >> python_wrapper/src/versions.txt +done +rm tmp + + +## buildable prereqs +GIT_OPENJPEG=https://github.com/uclouvain/openjpeg +OPENJPEG_VERSION=v2.5.2 + +git clone --branch $OPENJPEG_VERSION --depth=1 $GIT_OPENJPEG /src/openjpeg + +mkdir -p /tmp/openjpeg/build +cd /tmp/openjpeg/build +cmake /src/openjpeg/ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/tmp/openjpeg/target +cmake --build . --target install + +cd - +cd /src/openjpeg && echo "$(git remote -v | head -1 | awk '{print $2;}') $(git rev-parse HEAD)" > /tmp/openjpeg/version.txt +cd - +cat /tmp/openjpeg/version.txt >> python_wrapper/src/versions.txt + +## licenses +wget https://raw.githubusercontent.com/MathisRosenhauer/libaec/master/LICENSE.txt -O python_wrapper/src/copying/libaec.txt +wget https://raw.githubusercontent.com/uclouvain/openjpeg/master/LICENSE -O python_wrapper/src/copying/libopenjpeg.txt +wget https://raw.githubusercontent.com/glennrp/libpng/libpng16/LICENSE -O python_wrapper/src/copying/libpng.txt +cp LICENSE python_wrapper/src/copying/libeccodes.txt +echo '{"libeccodes": {"path": "copying/libeccodes.txt", "home": "https://github.com/ecmwf/eccodes"}, "libaec": {"path": "copying/libaec.txt", "home": "https://github.com/MathisRosenhauer/libaec"}, "home": "https://github.com/uclouvain/openjpeg"}, "libpng": {"path": "copying/libpng.txt", "home": "https://github.com/glennrp/libpng"}}' > python_wrapper/src/copying/list.json + +echo "$(git remote -v | head -1 | awk '{print $2;}') $(git rev-parse HEAD)" >> python_wrapper/src/versions.txt + +cp /tmp/openjpeg/target/lib64/libopenjp2.so.7 /tmp/eccodes/target/eccodes/lib64/libopenjp2.so.7 +cp /lib64/libaec.so.0 /tmp/eccodes/target/eccodes/lib64/libaec.so.0 +cp /lib64/libpng16.so.16 /tmp/eccodes/target/eccodes/lib64/libpng16.so.16 + diff --git a/python_wrapper/setup.cfg b/python_wrapper/setup.cfg new file mode 100644 index 0000000000..87a7fe1a0d --- /dev/null +++ b/python_wrapper/setup.cfg @@ -0,0 +1,5 @@ +[metadata] +description = "eccodeslib" +long_description = file: README.md +long_description_content_type = text/markdown +author = file: AUTHORS diff --git a/python_wrapper/setup.py b/python_wrapper/setup.py new file mode 100644 index 0000000000..c64ad6d789 --- /dev/null +++ b/python_wrapper/setup.py @@ -0,0 +1,2 @@ +from setup_utils import plain_setup +plain_setup()