diff --git a/.github/workflows/cibuildwheel.yaml b/.github/workflows/cibuildwheel.yaml index b3ec82cfa..c62b840af 100644 --- a/.github/workflows/cibuildwheel.yaml +++ b/.github/workflows/cibuildwheel.yaml @@ -1,5 +1,5 @@ # Build wheels using cibuildwheel (https://cibuildwheel.pypa.io/) -name: Build wheels +name: build-wheels on: # Run when a release has been created @@ -11,6 +11,7 @@ on: jobs: build-sdist: + # NOTE(vytas): We actually build sdist and pure-Python wheel. name: sdist runs-on: ubuntu-latest @@ -25,16 +26,31 @@ jobs: with: python-version: "3.12" - - name: Build sdist + - name: Build sdist and pure-Python wheel + env: + FALCON_DISABLE_CYTHON: "Y" + run: | + pip install --upgrade pip + pip install --upgrade build + python -m build + + - name: Check built artifacts + run: | + tools/check_dist.py ${{ github.event_name == 'release' && format('-r {0}', github.ref) || '' }} + + - name: Test sdist + run: | + tools/test_dist.py dist/*.tar.gz + + - name: Test pure-Python wheel run: | - pip install build - python -m build --sdist + tools/test_dist.py dist/*.whl - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: cibw-sdist - path: dist/*.tar.gz + path: dist/falcon-* build-wheels: name: ${{ matrix.python }}-${{ matrix.platform.name }} @@ -105,37 +121,96 @@ jobs: uses: actions/upload-artifact@v4 with: name: cibw-wheel-${{ matrix.python }}-${{ matrix.platform.name }} - path: wheelhouse/*.whl + path: wheelhouse/falcon-*.whl + + publish-sdist: + name: publish-sdist + needs: + - build-sdist + - build-wheels + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + pattern: cibw-sdist + path: dist + merge-multiple: true + + - name: Check collected artifacts + run: | + tools/check_dist.py ${{ github.event_name == 'release' && format('-r {0}', github.ref) || '' }} + + - name: Upload sdist to release + uses: AButler/upload-release-assets@v2.0 + if: github.event_name == 'release' + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + files: 'dist/*.tar.gz' + + - name: Publish sdist and pure-Python wheel to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + if: github.event_name == 'workflow_dispatch' + with: + password: ${{ secrets.TEST_PYPI_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + + - name: Publish sdist and pure-Python wheel to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + if: github.event_name == 'release' + with: + password: ${{ secrets.PYPI_TOKEN }} publish-wheels: - name: publish + name: publish-wheels needs: - build-sdist - build-wheels + - publish-sdist runs-on: ubuntu-latest steps: - - name: Download packages + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Download artifacts uses: actions/download-artifact@v4 with: - pattern: cibw-* + pattern: cibw-wheel-* path: dist merge-multiple: true - name: Check collected artifacts - # TODO(vytas): Run a script to perform version sanity checks instead. - run: ls -l dist/ + run: | + tools/check_dist.py ${{ github.event_name == 'release' && format('-r {0}', github.ref) || '' }} - - name: Publish artifacts to TestPyPI + - name: Publish binary wheels to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 if: github.event_name == 'workflow_dispatch' with: password: ${{ secrets.TEST_PYPI_TOKEN }} repository-url: https://test.pypi.org/legacy/ - # TODO(vytas): Enable this nuclear option once happy with other tests. - # - name: Publish artifacts to PyPI - # uses: pypa/gh-action-pypi-publish@release/v1 - # if: github.event_name == 'release' - # with: - # password: ${{ secrets.PYPI_TOKEN }} + - name: Publish binary wheels to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + if: github.event_name == 'release' + with: + password: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/create-wheels.yaml b/.github/workflows/create-wheels.yaml index 6caf8c4df..23ee6d6f4 100644 --- a/.github/workflows/create-wheels.yaml +++ b/.github/workflows/create-wheels.yaml @@ -1,9 +1,8 @@ name: Create wheel on: - # run when a release has been created - release: - types: [created] + # TODO(vytas): Phase out this workflow in favour of cibuildwheel.yaml. + workflow_dispatch: env: # set this so the falcon test uses the installed version and not the local one diff --git a/docs/conf.py b/docs/conf.py index 2a3098c25..e2390c14a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,6 +61,7 @@ 'sphinx_tabs.tabs', 'sphinx_tabs.tabs', # Falcon-specific extensions + 'ext.cibuildwheel', 'ext.doorway', 'ext.private_args', 'ext.rfc', diff --git a/docs/ext/cibuildwheel.py b/docs/ext/cibuildwheel.py new file mode 100644 index 000000000..e45e11b2d --- /dev/null +++ b/docs/ext/cibuildwheel.py @@ -0,0 +1,115 @@ +# Copyright 2024 by Vytautas Liuolia. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Binary wheels table extension for Sphinx. + +This extension parses a GitHub Actions workflow for building binary wheels, and +summarizes the build onfiguration in a stylish table. +""" + +import itertools +import pathlib + +import sphinx.util.docutils +import yaml + +FALCON_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent + +_CHECKBOX = '\u2705' +_CPYTHON_PLATFORMS = { + 'manylinux_x86_64': '**Linux Intel** ``manylinux`` 64-bit', + 'musllinux_x86_64': '**Linux Intel** ``musllinux`` 64-bit', + 'manylinux_i686': '**Linux Intel** ``manylinux`` 32-bit', + 'musllinux_i686': '**Linux Intel** ``musllinux`` 32-bit', + 'manylinux_aarch64': '**Linux ARM** ``manylinux`` 64-bit', + 'musllinux_aarch64': '**Linux ARM** ``musllinux`` 64-bit', + 'manylinux_ppc64le': '**Linux PowerPC** ``manylinux`` 64-bit', + 'musllinux_ppc64le': '**Linux PowerPC** ``musllinux`` 64-bit', + 'manylinux_s390x': '**Linux IBM Z** ``manylinux``', + 'musllinux_s390x': '**Linux IBM Z** ``musllinux``', + 'macosx_x86_64': '**macOS Intel**', + 'macosx_arm64': '**macOS Apple Silicon**', + 'win32': '**Windows** 32-bit', + 'win_amd64': '**Windows** 64-bit', + 'win_arm64': '**Windows ARM** 64-bit', +} + + +class WheelsDirective(sphinx.util.docutils.SphinxDirective): + """Directive to tabulate build info from a YAML workflow.""" + + required_arguments = 1 + + @classmethod + def _emit_table(cls, data): + columns = len(data[0]) + assert all( + len(row) == columns for row in data + ), 'All rows must have the same number of columns' + # NOTE(vytas): +2 is padding inside cell borders. + width = max(len(cell) for cell in itertools.chain(*data)) + 2 + hline = ('+' + '-' * width) * columns + '+\n' + output = [hline] + + for row in data: + for cell in row: + # NOTE(vytas): Emojis take two spaces... + padded_width = width - 1 if cell == _CHECKBOX else width + output.append('|' + cell.center(padded_width)) + output.append('|\n') + + header_line = row == data[0] + output.append(hline.replace('-', '=') if header_line else hline) + + return ''.join(output) + + def run(self): + workflow_path = pathlib.Path(self.arguments[0]) + if not workflow_path.is_absolute(): + workflow_path = FALCON_ROOT / workflow_path + with open(workflow_path) as fp: + workflow = yaml.safe_load(fp) + + matrix = workflow['jobs']['build-wheels']['strategy']['matrix'] + platforms = matrix['platform'] + include = matrix['include'] + assert not matrix.get('exclude'), 'TODO: exclude is not supported yet' + supported = set( + itertools.product( + [platform['name'] for platform in platforms], matrix['python'] + ) + ) + supported.update((item['platform']['name'], item['python']) for item in include) + cpythons = sorted({cp for _, cp in supported}, key=lambda val: (len(val), val)) + + header = ['Platform / CPython version'] + table = [header + [cp.replace('cp3', '3.') for cp in cpythons]] + table.extend( + [description] + + [(_CHECKBOX if (name, cp) in supported else '') for cp in cpythons] + for name, description in _CPYTHON_PLATFORMS.items() + ) + + return self.parse_text_to_nodes(self._emit_table(table)) + + +def setup(app): + app.add_directive('wheels', WheelsDirective) + + return { + 'version': '0.1', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/user/install.rst b/docs/user/install.rst index ef4986802..d7838d259 100644 --- a/docs/user/install.rst +++ b/docs/user/install.rst @@ -39,9 +39,13 @@ Or, to install the latest beta or release candidate, if any: In order to provide an extra speed boost, Falcon can compile itself with Cython. Wheels containing pre-compiled binaries are available from PyPI for -several common platforms. However, if a wheel for your platform of choice is not -available, you can choose to stick with the source distribution, or use the -instructions below to cythonize Falcon for your environment. +several common platforms (see :ref:`binary_wheels` below for the complete list +of the platforms that we target, or simply check +`Falcon files on PyPI `__). + +However, even if a wheel for your platform of choice is not available, you can +choose to stick with the source distribution, or use the instructions below to +cythonize Falcon for your environment. The following commands tell pip to install Cython, and then to invoke Falcon's ``setup.py``, which will in turn detect the presence of Cython @@ -64,7 +68,8 @@ pass `-v` to pip in order to echo the compilation commands: $ pip install -v --no-build-isolation --no-binary :all: falcon -**Installing on OS X** +Installing on OS X +^^^^^^^^^^^^^^^^^^ Xcode Command Line Tools are required to compile Cython. Install them with this command: @@ -87,6 +92,28 @@ these issues by setting additional Clang C compiler flags as follows: $ export CFLAGS="-Qunused-arguments -Wno-unused-function" +.. _binary_wheels: + +Binary Wheels +^^^^^^^^^^^^^ + +Binary Falcon wheels for are automatically built for many CPython platforms, +courtesy of `cibuildwheel `__. + +The following table summarizes the wheel availability on different combinations +of CPython versions vs CPython platforms: + +.. wheels:: .github/workflows/cibuildwheel.yaml + +.. note:: + The `free-threaded build + `__ + mode is not enabled for our wheels at this time. + +While we believe that the above configuration covers the most common +development and deployment scenarios, :ref:`let us known ` if you are +interested in any builds that are currently missing from our selection! + Dependencies ------------ diff --git a/requirements/docs b/requirements/docs index e59d178cf..888447d50 100644 --- a/requirements/docs +++ b/requirements/docs @@ -4,6 +4,7 @@ jinja2 markupsafe pygments pygments-style-github +PyYAML sphinx sphinx_rtd_theme sphinx-tabs diff --git a/setup.cfg b/setup.cfg index 785692f7e..830760163 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,7 +77,7 @@ console_scripts = [egg_info] # TODO replace -tag_build = dev1 +tag_build = dev2 [aliases] test=pytest diff --git a/tools/check_dist.py b/tools/check_dist.py new file mode 100755 index 000000000..b4a78ffc2 --- /dev/null +++ b/tools/check_dist.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python + +import argparse +import pathlib +import sys + +HERE = pathlib.Path(__file__).resolve().parent +DIST = HERE.parent / 'dist' + + +def check_dist(dist, git_ref): + sdist = None + versions = set() + wheels = [] + + if git_ref: + git_ref = git_ref.split('/')[-1].lower() + + for path in dist.iterdir(): + if not path.is_file(): + continue + + if path.name.endswith('.tar.gz'): + assert sdist is None, f'sdist already exists: {sdist}' + sdist = path.name + + elif path.name.endswith('.whl'): + wheels.append(path.name) + + else: + sys.stderr.write(f'Unexpected file found in dist: {path.name}\n') + sys.exit(1) + + package, _, _ = path.stem.partition('.tar') + falcon, version, *_ = package.split('-') + assert falcon == 'falcon', 'Unexpected package name: {path.name}' + versions.add(version) + + if git_ref and version != git_ref: + sys.stderr.write( + f'Unexpected version: {path.name} ({version} != {git_ref})\n' + ) + sys.exit(1) + + if not versions: + sys.stderr.write('No artifacts collected!\n') + sys.exit(1) + if len(versions) > 1: + sys.stderr.write(f'Multiple versions found: {tuple(versions)}!\n') + sys.exit(1) + version = versions.pop() + + wheel_list = ' None\n' + if wheels: + wheel_list = ''.join(f' {wheel}\n' for wheel in sorted(wheels)) + + print(f'[{dist}]\n') + print(f'sdist found:\n {sdist}\n') + print(f'wheels found:\n{wheel_list}') + print(f'version identified:\n {version}\n') + + +def main(): + description = 'Check artifacts (sdist, wheels) inside dist dir.' + + parser = argparse.ArgumentParser(description=description) + parser.add_argument( + '-d', + '--dist-dir', + default=str(DIST), + help='dist directory to check (default: %(default)s)', + ) + parser.add_argument( + '-r', + '--git-ref', + help='check version against git branch/tag ref (e.g. $GITHUB_REF)', + ) + + args = parser.parse_args() + check_dist(pathlib.Path(args.dist_dir).resolve(), args.git_ref) + + +if __name__ == '__main__': + main() diff --git a/tools/test_dist.py b/tools/test_dist.py new file mode 100755 index 000000000..a0dc967b5 --- /dev/null +++ b/tools/test_dist.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +import argparse +import logging +import pathlib +import subprocess +import sys +import tempfile + +logging.basicConfig( + format='[test_dist.py] %(asctime)s [%(levelname)s] %(message)s', level=logging.INFO +) + +FALCON_ROOT = pathlib.Path(__file__).resolve().parent.parent +REQUIREMENTS = FALCON_ROOT / 'requirements' / 'cibwtest' +TESTS = FALCON_ROOT / 'tests' + + +def test_package(package): + with tempfile.TemporaryDirectory() as tmpdir: + venv = pathlib.Path(tmpdir) / 'venv' + subprocess.check_call((sys.executable, '-m', 'venv', venv)) + logging.info(f'Created a temporary venv in {venv}.') + + subprocess.check_call((venv / 'bin' / 'pip', 'install', '--upgrade', 'pip')) + subprocess.check_call((venv / 'bin' / 'pip', 'install', '-r', REQUIREMENTS)) + logging.info(f'Installed test requirements in {venv}.') + subprocess.check_call( + (venv / 'bin' / 'pip', 'install', package), + ) + logging.info(f'Installed {package} into {venv}.') + + subprocess.check_call((venv / 'bin' / 'pytest', TESTS), cwd=venv) + logging.info(f'{package} passes tests.') + + +def main(): + description = 'Test Falcon packages (sdist or generic wheel).' + parser = argparse.ArgumentParser(description=description) + parser.add_argument( + 'package', metavar='PACKAGE', nargs='+', help='sdist/wheel(s) to test' + ) + args = parser.parse_args() + + for package in args.package: + test_package(package) + + +if __name__ == '__main__': + main()