From a7b7189c621b57a15b0ea4ccf597f5d1c2b2cc63 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Thu, 12 Dec 2024 16:26:57 +0100 Subject: [PATCH 1/3] prototype of so-only wheel --- .github/workflows/build-wheel-linux.yml | 46 +++++++++++++++++++++++++ python_wrapper/buildconfig | 25 ++++++++++++++ python_wrapper/setup.cfg | 5 +++ python_wrapper/setup.py | 2 ++ 4 files changed, 78 insertions(+) create mode 100644 .github/workflows/build-wheel-linux.yml create mode 100644 python_wrapper/buildconfig create mode 100644 python_wrapper/setup.cfg create mode 100644 python_wrapper/setup.py diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml new file mode 100644 index 000000000..92fce3000 --- /dev/null +++ b/.github/workflows/build-wheel-linux.yml @@ -0,0 +1,46 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +name: Build Python Wheel for Linux + +on: + # Trigger the workflow manually + workflow_dispatch: ~ + + # Allow to be called from another workflow + workflow_call: ~ + + # TODO automation trigger + +jobs: + build: + name: Build manylinux_2_28 + runs-on: [self-hosted, Linux, platform-builder-Rocky-8.6] + # TODO which manylinux do we want to build for? 2014? 2_28? 2_34? Matrix? + container: + image: eccr.ecmwf.int/wheelmaker/2_28:latest + credentials: + username: ${{ secrets.ECMWF_DOCKER_REGISTRY_USERNAME }} + password: ${{ secrets.ECMWF_DOCKER_REGISTRY_ACCESS_TOKEN }} + steps: + # TODO convert this to be matrix-friendly. Note it's a bit tricky since + # we'd ideally not reexecute the compile step multiple times, but it + # (non-essentially) depends on a matrix-based step + # NOTE we dont use action checkout because it doesnt cleanup after itself correctly + - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj + - run: cd /proj && /buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/compile.sh ./python_wrapper/buildconfig + - run: cd /proj && PYTHONPATH=/buildscripts /buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /tmp/build/wheel/*whl + - run: cd /proj && /buildscripts/upload-pypi.sh /tmp/build/wheel/*whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + # NOTE temporary thing until all the mess gets cleared + - run: rm -rf ./* ./.git ./.github diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig new file mode 100644 index 000000000..1abdbff8f --- /dev/null +++ b/python_wrapper/buildconfig @@ -0,0 +1,25 @@ +# (C) Copyright 2024- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh +# NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation +# TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent + +NAME="eccodes" + +# TODO solve the libpng, libjpg, libaec problem +CMAKE_PARAMS1="-DENABLE_PNG=0 -DENABLE_JPG=0 -DENABLE_AEC=0 -DENABLE_JPG_LIBJASPER=0 -DENABLE_JPG_LIBOPENJPEG=0" + # -DOPENJPEG_DIR=$TOPDIR/install \ +CMAKE_PARAMS2="-DENABLE_NETCDF=0" +CMAKE_PARAMS3="-DENABLE_EXAMPLES=0 -DENABLE_PYTHON=0 -DENABLE_BUILD_TOOLS=0 -DENABLE_INSTALL_ECCODES_DEFINITIONS=0 -DENABLE_INSTALL_ECCODES_SAMPLES=0" +CMAKE_PARAMS4="-DENABLE_ECCODES_THREADS=1 -DENABLE_MEMFS=1" +CMAKE_PARAMS="$CMAKE_PARAMS1 $CMAKE_PARAMS2 $CMAKE_PARAMS3 $CMAKE_PARAMS4" + +PYPROJECT_DIR="python_wrapper" +# TODO add eckit once the dependency is in place +DEPENDENCIES='[]' diff --git a/python_wrapper/setup.cfg b/python_wrapper/setup.cfg new file mode 100644 index 000000000..87a7fe1a0 --- /dev/null +++ b/python_wrapper/setup.cfg @@ -0,0 +1,5 @@ +[metadata] +description = "eccodeslib" +long_description = file: README.md +long_description_content_type = text/markdown +author = file: AUTHORS diff --git a/python_wrapper/setup.py b/python_wrapper/setup.py new file mode 100644 index 000000000..c64ad6d78 --- /dev/null +++ b/python_wrapper/setup.py @@ -0,0 +1,2 @@ +from setup_utils import plain_setup +plain_setup() From f3ad39418dc0ff8add144d7bacc30965d635aa52 Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Tue, 21 Jan 2025 15:19:12 +0100 Subject: [PATCH 2/3] Support for prereqs --- .github/workflows/build-wheel-linux.yml | 6 ++-- python_wrapper/.pre-compile.sh.swp | Bin 0 -> 12288 bytes python_wrapper/buildconfig | 3 +- python_wrapper/post-build.sh | 6 ++++ python_wrapper/pre-compile.sh | 41 ++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 python_wrapper/.pre-compile.sh.swp create mode 100755 python_wrapper/post-build.sh create mode 100755 python_wrapper/pre-compile.sh diff --git a/.github/workflows/build-wheel-linux.yml b/.github/workflows/build-wheel-linux.yml index 92fce3000..12761e78a 100644 --- a/.github/workflows/build-wheel-linux.yml +++ b/.github/workflows/build-wheel-linux.yml @@ -35,10 +35,12 @@ jobs: # NOTE we dont use action checkout because it doesnt cleanup after itself correctly - run: git clone --depth=1 --branch="${GITHUB_REF#refs/heads/}" https://github.com/$GITHUB_REPOSITORY /proj - run: cd /proj && /buildscripts/prepare_deps.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && python_wrapper/pre-compile.sh - run: cd /proj && /buildscripts/compile.sh ./python_wrapper/buildconfig - run: cd /proj && PYTHONPATH=/buildscripts /buildscripts/wheel-linux.sh ./python_wrapper/buildconfig 3.11 - - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 /tmp/build/wheel/*whl - - run: cd /proj && /buildscripts/upload-pypi.sh /tmp/build/wheel/*whl + - run: cd /proj && python_wrapper/post-build.sh + - run: cd /proj && /buildscripts/test-wheel.sh ./python_wrapper/buildconfig 3.11 + - run: cd /proj && /buildscripts/upload-pypi.sh ./python_wrapper/buildconfig env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} diff --git a/python_wrapper/.pre-compile.sh.swp b/python_wrapper/.pre-compile.sh.swp new file mode 100644 index 0000000000000000000000000000000000000000..256f92720654cfc98bf99141ea25fd62640d9ba5 GIT binary patch literal 12288 zcmeI2OK%)S5XZ|o@k&4-?&bB$izUxaY(x|V){@B%$p*jV7eoqddS}}^W6z`M?j6T6 zh7(7~jSCkpoH%ge#ElOC5(sgHZ-4|Jfa=*9$Gf)iBEcchBmF&(?yj!-b$7K^?#|Wi z)-qnN&jVf00sQdQ!`}9l=B0^`CLl|9H8bBYxca&uC+P*1sQ822psS?sa^Fw8T)E;v z2WisYmrMvQT_L&Sr?ChlURS}z7Hthnz!DgXKws}T*Dr6qJU@5U)Ss$M;tS7jjn&jv zvIHyvOTZGa1S|ndz!I5FFoJ}0$F?~>L@KR*p{ zpY+>P0N;?jCjpwI+~3ELw{_VPummgtOTZHNzY^enkYc6!QZLjfd7NsFPKNJez*!fa zIeKFIdssUZGE6j9=U+Xl&0twQa|Dx$<85!TIa9%f1spj;s3iZlJMi6%OBJSxs_O?D zAn&hsSc$?;aj>y+0}m^L>3|+8eku;aq~|^=Hq9!tSP9aYn;L=ELfvp(A`Vb<&Gll99n z$;WEH$2FF?CEKqLkoOgrewt{W=$!trSA}zP*W9v0XZ3(AYhe zBlCB{JZCzL^ldESg%KAM5T>Ru3Vog^t^jMc(9{Rwb$0YH(GBuhLH`r$|($?%$KgTpx(1_s(2uby zu$rBn??eskR4hnCZ`kqj7R2$ z(nj&pOFO&V*A8ugGF5^eX}=ki8F*yYVU&@hVSvYz;nb#hoDu2l@1)zIRNi59#QsnA C@60&> literal 0 HcmV?d00001 diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig index 1abdbff8f..6a7698913 100644 --- a/python_wrapper/buildconfig +++ b/python_wrapper/buildconfig @@ -13,8 +13,7 @@ NAME="eccodes" # TODO solve the libpng, libjpg, libaec problem -CMAKE_PARAMS1="-DENABLE_PNG=0 -DENABLE_JPG=0 -DENABLE_AEC=0 -DENABLE_JPG_LIBJASPER=0 -DENABLE_JPG_LIBOPENJPEG=0" - # -DOPENJPEG_DIR=$TOPDIR/install \ +CMAKE_PARAMS1="-DENABLE_PNG=1 -DENABLE_JPG=1 -DENABLE_AEC=1 -DENABLE_JPG_LIBJASPER=0 -DENABLE_JPG_LIBOPENJPEG=1 -DOPENJPEG_DIR=/tmp/openjpeg/target" CMAKE_PARAMS2="-DENABLE_NETCDF=0" CMAKE_PARAMS3="-DENABLE_EXAMPLES=0 -DENABLE_PYTHON=0 -DENABLE_BUILD_TOOLS=0 -DENABLE_INSTALL_ECCODES_DEFINITIONS=0 -DENABLE_INSTALL_ECCODES_SAMPLES=0" CMAKE_PARAMS4="-DENABLE_ECCODES_THREADS=1 -DENABLE_MEMFS=1" diff --git a/python_wrapper/post-build.sh b/python_wrapper/post-build.sh new file mode 100755 index 000000000..acca90cac --- /dev/null +++ b/python_wrapper/post-build.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +mkdir /tmp/eccodes/auditwheel +auditwheel repair -w /tmp/eccodes/auditwheel /tmp/eccodes/build/wheel/*whl +rm /tmp/eccodes/build/wheel/*whl +mv /tmp/eccodes/auditwheel/*whl /tmp/eccodes/build/wheel/ diff --git a/python_wrapper/pre-compile.sh b/python_wrapper/pre-compile.sh new file mode 100755 index 000000000..8ab9b80ca --- /dev/null +++ b/python_wrapper/pre-compile.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +mkdir -p python_wrapper/src/copying + +## yum-available prereqs +for p in libaec-devel libpng-devel gobject-introspection-devel +do + yum install -y $p + # TODO improve + yum install $p 2>&1 > tmp + cat tmp + v=$(grep 'already installed' < tmp | awk '{print $2;}' | sed 's/\\d://') + echo "yum $p $v" >> python_wrapper/src/versions.txt +done +rm tmp + + +## buildable prereqs +GIT_OPENJPEG=https://github.com/uclouvain/openjpeg +OPENJPEG_VERSION=v2.5.2 + +git clone --branch $OPENJPEG_VERSION --depth=1 $GIT_OPENJPEG /src/openjpeg + +mkdir -p /tmp/openjpeg/build +cd /tmp/openjpeg/build +cmake /src/openjpeg/ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=/tmp/openjpeg/target +cmake --build . --target install + +cd - +cd /src/openjpeg && echo "$(git remote -v | head -1 | awk '{print $2;}') $(git rev-parse HEAD)" > /tmp/openjpeg/version.txt +cd - + +## licenses +wget https://raw.githubusercontent.com/MathisRosenhauer/libaec/master/LICENSE.txt -O python_wrapper/src/copying/libaec.txt +wget https://raw.githubusercontent.com/uclouvain/openjpeg/master/LICENSE -O python_wrapper/src/copying/libopenjpeg.txt +wget https://raw.githubusercontent.com/glennrp/libpng/libpng16/LICENSE -O python_wrapper/src/copying/libpng.txt +cp LICENSE python_wrapper/src/copying/libeccodes.txt +echo '{"libeccodes": {"path": "copying/libeccodes.txt", "home": "https://github.com/ecmwf/eccodes"}, "libaec": {"path": "copying/libaec.txt", "home": "https://github.com/MathisRosenhauer/libaec"}, "home": "https://github.com/uclouvain/openjpeg"}, "libpng": {"path": "copying/libpng.txt", "home": "https://github.com/glennrp/libpng"}}' > python_wrapper/src/copying/list.json + +cat /tmp/openjpeg/version.txt >> python_wrapper/src/versions.txt +echo "$(git remote -v | head -1 | awk '{print $2;}') $(git rev-parse HEAD)" >> python_wrapper/src/versions.txt From c0bc10b28aa85b89de44546b59e60484f395599b Mon Sep 17 00:00:00 2001 From: Vojta Tuma Date: Tue, 21 Jan 2025 17:03:23 +0100 Subject: [PATCH 3/3] No fortran --- python_wrapper/buildconfig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python_wrapper/buildconfig b/python_wrapper/buildconfig index 6a7698913..2c105b759 100644 --- a/python_wrapper/buildconfig +++ b/python_wrapper/buildconfig @@ -12,9 +12,8 @@ NAME="eccodes" -# TODO solve the libpng, libjpg, libaec problem CMAKE_PARAMS1="-DENABLE_PNG=1 -DENABLE_JPG=1 -DENABLE_AEC=1 -DENABLE_JPG_LIBJASPER=0 -DENABLE_JPG_LIBOPENJPEG=1 -DOPENJPEG_DIR=/tmp/openjpeg/target" -CMAKE_PARAMS2="-DENABLE_NETCDF=0" +CMAKE_PARAMS2="-DENABLE_NETCDF=0 -DENABLE_FORTRAN=0" CMAKE_PARAMS3="-DENABLE_EXAMPLES=0 -DENABLE_PYTHON=0 -DENABLE_BUILD_TOOLS=0 -DENABLE_INSTALL_ECCODES_DEFINITIONS=0 -DENABLE_INSTALL_ECCODES_SAMPLES=0" CMAKE_PARAMS4="-DENABLE_ECCODES_THREADS=1 -DENABLE_MEMFS=1" CMAKE_PARAMS="$CMAKE_PARAMS1 $CMAKE_PARAMS2 $CMAKE_PARAMS3 $CMAKE_PARAMS4"