From f6147560d69194affd4e74eb45fdda1cdcd145a2 Mon Sep 17 00:00:00 2001 From: q-posev Date: Sat, 17 Aug 2024 16:44:57 +0200 Subject: [PATCH 1/5] Upgrade manylinux wheels builder for new container envs --- python/build_manylinux_wheels.sh | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/python/build_manylinux_wheels.sh b/python/build_manylinux_wheels.sh index 1da66ea1..a45d9605 100755 --- a/python/build_manylinux_wheels.sh +++ b/python/build_manylinux_wheels.sh @@ -46,22 +46,9 @@ function build_wheel_for_py() PYVERSION=${1} # derive manylinux glibc tag from the PLAT env variable provided to docker run - # this is needed to avoid building wheel for 2010_x86_64 with CPython 3.10 - # because NumPy does not have wheels for it MANYLINUX_TAG=${PLAT:9:4} - if [[ ${PYVERSION} -eq 310 ]] && [[ ${MANYLINUX_TAG} -eq 2010 ]]; then - echo "Skip build of the wheel for CPython 3.10 on manylinux2010_x86_64" - return - fi - - # python versions <= 3.7 required additional "m" in the platform tag, e.g. cp37-cp37m - if [[ ${PYVERSION} -eq 36 ]] || [[ ${PYVERSION} -eq 37 ]]; then - PYM="m" - else - PYM="" - fi - CPYTHON="cp${PYVERSION}-cp${PYVERSION}${PYM}" + CPYTHON="cp${PYVERSION}-cp${PYVERSION}" # create and activate a virtual environment based on CPython version ${PYVERSION} /opt/python/${CPYTHON}/bin/python3 -m venv --clear trexio-manylinux-py${PYVERSION} @@ -88,7 +75,7 @@ function build_wheel_for_py() pytest -v test/test_api.py # cleaning - rm -rf -- dist/ build/ trexio.egg-info/ + rm -rf -- dist/ build/ trexio.egg-info/ #rm -- test_file_py.h5 unsafe_test_file_py.h5 # deactivate the current environment @@ -103,7 +90,7 @@ function build_wheel_for_py() # build wheels for all versions of CPython in this container -for CPYVERSION in 37 38 39 310 +for CPYVERSION in 38 39 310 311 312 313 do build_wheel_for_py ${CPYVERSION} done From ec502ea2b467e7a81d06be49b5b810b87cb7a7d5 Mon Sep 17 00:00:00 2001 From: q-posev Date: Sat, 17 Aug 2024 16:46:03 +0200 Subject: [PATCH 2/5] [wheel build] test new Docker images and Py 3.10+ wheels on Test PyPI --- .github/workflows/build-wheels.yml | 34 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 7cda44ab..43e4fab9 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -4,7 +4,7 @@ on: # Run this workflow after the TREXIO CI completed workflow_run: workflows: [ "TREXIO CI" ] - branches: [ master ] + #branches: [ master ] types: - completed @@ -16,7 +16,7 @@ jobs: get_commit_message: if: ${{ github.event.workflow_run.conclusion == 'success' }} name: Get commit message - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 outputs: message: ${{ steps.commit_message.outputs.message }} steps: @@ -39,10 +39,10 @@ jobs: if: >- contains(needs.get_commit_message.outputs.message, '[wheel build]') || (github.repository == 'TREX-CoE/trexio' && startsWith(github.ref, 'refs/tags/v')) - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: - manylinux_tag: [2014_x86_64, 2_24_x86_64] + manylinux_tag: [2014_x86_64, 2_28_x86_64] steps: - name: Checkout the branch @@ -51,7 +51,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: Install build dependencies run: python -m pip install -U setuptools @@ -98,7 +98,7 @@ jobs: publish_wheels: name: Publish all wheels on PyPI needs: [build_linux_wheels] - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Checkout the branch @@ -107,7 +107,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.10' - name: Install build dependencies run: python -m pip install -U setuptools twine @@ -135,15 +135,15 @@ jobs: ls -sh -w 1 working-directory: dist - #- name: Publish distribution 📦 to Test PyPI - # uses: pypa/gh-action-pypi-publish@master - # with: - # password: ${{ secrets.TEST_PYPI_API_TOKEN }} - # repository_url: https://test.pypi.org/legacy/ - #verbose: true - - - name: Publish distribution 📦 to PyPI + - name: Publish distribution 📦 to Test PyPI uses: pypa/gh-action-pypi-publish@master with: - password: ${{ secrets.PYPI_API_TOKEN }} - #if: startsWith(github.ref, 'refs/tags') + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + #verbose: true + + #- name: Publish distribution 📦 to PyPI + # uses: pypa/gh-action-pypi-publish@master + # with: + # password: ${{ secrets.PYPI_API_TOKEN }} + # #if: startsWith(github.ref, 'refs/tags') From 834e83170767b20af1f7eca3f1dd65b11fc3ef4f Mon Sep 17 00:00:00 2001 From: q-posev Date: Sat, 17 Aug 2024 17:09:53 +0200 Subject: [PATCH 3/5] [wheel build] test new Docker images and Py 3.10+ wheels on Test PyPI --- .github/workflows/build-wheels.yml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 43e4fab9..f213175c 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -1,12 +1,12 @@ # Controls when the workflow will run -on: - # Run this workflow after the TREXIO CI completed - workflow_run: - workflows: [ "TREXIO CI" ] - #branches: [ master ] - types: - - completed +#on: +# # Run this workflow after the TREXIO CI completed +# workflow_run: +# workflows: [ "TREXIO CI" ] +# branches: [ master ] +# types: +# - completed # Workflow to build and publish wheels. # in the get_commit_message job: Include [wheel build] in your commit message to trigger this build. @@ -14,7 +14,7 @@ name: Build CPython wheels jobs: get_commit_message: - if: ${{ github.event.workflow_run.conclusion == 'success' }} + #if: ${{ github.event.workflow_run.conclusion == 'success' }} name: Get commit message runs-on: ubuntu-22.04 outputs: @@ -30,14 +30,19 @@ jobs: run: | set -xe COMMIT_MSG=$(git log --no-merges -1 --oneline) - echo "::set-output name=message::$COMMIT_MSG" + RUN="0" + if [[ "$COMMIT_MSG" == *"[wheel build]"* ]]; then + RUN="1" + fi + echo "message=$RUN" >> $GITHUB_OUTPUT + echo github.ref ${{ github.ref }} build_linux_wheels: name: Build Linux wheels for different versions of CPython on manylinux_x86_64 needs: get_commit_message if: >- - contains(needs.get_commit_message.outputs.message, '[wheel build]') || + contains(needs.get_commit_message.outputs.message, '1') || (github.repository == 'TREX-CoE/trexio' && startsWith(github.ref, 'refs/tags/v')) runs-on: ubuntu-22.04 strategy: From 93f813696c674b7079bba447b726310a141adcf7 Mon Sep 17 00:00:00 2001 From: q-posev Date: Sat, 17 Aug 2024 17:16:47 +0200 Subject: [PATCH 4/5] [wheel build] Fix CI --- .github/workflows/actions.yml | 11 ++++++++--- .github/workflows/build-wheels.yml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 4678b3e2..49daa0d6 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -20,7 +20,7 @@ jobs: message: ${{ steps.commit_message.outputs.message }} steps: - name: Checkout the repo - uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 + uses: actions/checkout@v3 # Gets the correct commit message for pull request with: ref: ${{ github.event.pull_request.head.sha }} @@ -29,7 +29,12 @@ jobs: run: | set -xe COMMIT_MSG=$(git log --no-merges -1 --oneline) - echo "::set-output name=message::$COMMIT_MSG" + RUN="0" + if [[ "$COMMIT_MSG" == *"[wheel build]"* ]]; then + RUN="1" + fi + echo "message=$RUN" >> $GITHUB_OUTPUT + echo github.ref ${{ github.ref }} trexio_ubuntu: name: x86 Ubuntu 20.04 @@ -97,7 +102,7 @@ jobs: - name: publish Python API distribution as an artifact if: >- - contains(needs.get_commit_message.outputs.message, '[wheel build]') || + contains(needs.get_commit_message.outputs.message, '1') || github.event_name == 'release' uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 with: diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index f213175c..a76b866d 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -1,12 +1,12 @@ # Controls when the workflow will run -#on: -# # Run this workflow after the TREXIO CI completed -# workflow_run: -# workflows: [ "TREXIO CI" ] -# branches: [ master ] -# types: -# - completed +on: + # Run this workflow after the TREXIO CI completed + workflow_run: + workflows: [ "TREXIO CI" ] + branches: [ test-new-wheels ] + types: + - completed # Workflow to build and publish wheels. # in the get_commit_message job: Include [wheel build] in your commit message to trigger this build. @@ -14,7 +14,7 @@ name: Build CPython wheels jobs: get_commit_message: - #if: ${{ github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event.workflow_run.conclusion == 'success' }} name: Get commit message runs-on: ubuntu-22.04 outputs: From 576c6f7a8f6dfd1dfa168151c91af61dd33a61f6 Mon Sep 17 00:00:00 2001 From: q-posev Date: Sat, 17 Aug 2024 18:18:03 +0200 Subject: [PATCH 5/5] [wheel build] One more test --- .github/workflows/actions.yml | 10 +++------- .github/workflows/build-wheels.yml | 27 +++++++++++---------------- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 49daa0d6..7f38ebc1 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -20,7 +20,7 @@ jobs: message: ${{ steps.commit_message.outputs.message }} steps: - name: Checkout the repo - uses: actions/checkout@v3 + uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # Gets the correct commit message for pull request with: ref: ${{ github.event.pull_request.head.sha }} @@ -29,11 +29,7 @@ jobs: run: | set -xe COMMIT_MSG=$(git log --no-merges -1 --oneline) - RUN="0" - if [[ "$COMMIT_MSG" == *"[wheel build]"* ]]; then - RUN="1" - fi - echo "message=$RUN" >> $GITHUB_OUTPUT + echo "message=$COMMIT_MSG" >> $GITHUB_OUTPUT echo github.ref ${{ github.ref }} trexio_ubuntu: @@ -102,7 +98,7 @@ jobs: - name: publish Python API distribution as an artifact if: >- - contains(needs.get_commit_message.outputs.message, '1') || + contains(needs.get_commit_message.outputs.message, '[wheel build]') || github.event_name == 'release' uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 with: diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index a76b866d..8e391373 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -21,7 +21,7 @@ jobs: message: ${{ steps.commit_message.outputs.message }} steps: - name: Checkout the repo - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Gets the correct commit message for pull request with: ref: ${{ github.event.pull_request.head.sha }} @@ -30,19 +30,14 @@ jobs: run: | set -xe COMMIT_MSG=$(git log --no-merges -1 --oneline) - RUN="0" - if [[ "$COMMIT_MSG" == *"[wheel build]"* ]]; then - RUN="1" - fi - echo "message=$RUN" >> $GITHUB_OUTPUT + echo "message=$COMMIT_MSG" >> $GITHUB_OUTPUT echo github.ref ${{ github.ref }} - build_linux_wheels: name: Build Linux wheels for different versions of CPython on manylinux_x86_64 needs: get_commit_message if: >- - contains(needs.get_commit_message.outputs.message, '1') || + contains(needs.get_commit_message.outputs.message, '[wheel build]') || (github.repository == 'TREX-CoE/trexio' && startsWith(github.ref, 'refs/tags/v')) runs-on: ubuntu-22.04 strategy: @@ -51,10 +46,10 @@ jobs: steps: - name: Checkout the branch - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f with: python-version: '3.10' @@ -70,7 +65,7 @@ jobs: # Conventional download-artifact action does not work for artifact produced in a different workflow, # which is the case here (TREXIO CI produced the Python API distribution tarball) - name: Download the Python API distribution tarball - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v3 with: # Specify the name of the workflow file which uploaded the tarball workflow: actions.yml @@ -94,7 +89,7 @@ jobs: working-directory: python - name: Upload produced wheels as artifacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a with: name: pytrexio-manylinux-${{ matrix.manylinux_tag }} path: ./python/wheelhouse/*.whl @@ -107,10 +102,10 @@ jobs: steps: - name: Checkout the branch - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f with: python-version: '3.10' @@ -118,12 +113,12 @@ jobs: run: python -m pip install -U setuptools twine - name: Download the build artifacts (wheels) of this workflow - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v4 with: path: dist - name: Download the Python API distribution tarball - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@v3 with: workflow: actions.yml workflow_conclusion: success