From ad897bb843b06bb7d54c5913396ac54f736e4e7c Mon Sep 17 00:00:00 2001 From: Anja Kefala Date: Tue, 4 Jun 2024 19:00:05 -0700 Subject: [PATCH] GH-37929: [Python] begin moving static settings to pyproject.toml (#41041) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Rationale for this change To migrate Arrow to modern Python packaging standards, see [PEP-517](https://peps.python.org/pep-0517/) and [PEP-518](https://peps.python.org/pep-0518/). * GitHub Issue: #37929 This PR focuses on migrating the static settings, the metadata and version, to pyproject.toml. Future PRs will migrate more of the build process to pyproject.toml. Lead-authored-by: anjakefala Co-authored-by: Raúl Cumplido Co-authored-by: Joris Van den Bossche Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/dev.yml | 2 +- ci/conda_env_python.txt | 4 +- dev/release/01-prepare-test.rb | 6 +- dev/release/post-11-bump-versions-test.rb | 6 +- dev/release/utils-prepare.sh | 10 ++- .../python-minimal-build/github.linux.yml | 2 +- .../examples/minimal_build/Dockerfile.ubuntu | 3 +- python/examples/minimal_build/build_conda.sh | 9 +- python/examples/minimal_build/build_venv.sh | 11 +-- python/pyproject.toml | 62 +++++++++++++- python/requirements-build.txt | 4 +- python/setup.py | 85 ------------------- 12 files changed, 89 insertions(+), 115 deletions(-) diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 6111d1d2e5fe3..1ea12b0a4d23d 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -120,7 +120,7 @@ jobs: shell: bash run: | gem install test-unit - pip install "cython>=0.29.31" setuptools six pytest jira + pip install "cython>=0.29.31" setuptools six pytest jira setuptools-scm - name: Run Release Test env: ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/ci/conda_env_python.txt b/ci/conda_env_python.txt index 4366e30010389..bf915493de302 100644 --- a/ci/conda_env_python.txt +++ b/ci/conda_env_python.txt @@ -26,5 +26,5 @@ numpy>=1.16.6 pytest pytest-faulthandler s3fs>=2023.10.0 -setuptools -setuptools_scm +setuptools>=64 +setuptools_scm>=8 diff --git a/dev/release/01-prepare-test.rb b/dev/release/01-prepare-test.rb index bf6cfede15c81..fbd0b2996077c 100644 --- a/dev/release/01-prepare-test.rb +++ b/dev/release/01-prepare-test.rb @@ -254,10 +254,10 @@ def test_version_pre_tag ], }, { - path: "python/setup.py", + path: "python/pyproject.toml", hunks: [ - ["-default_version = '#{@snapshot_version}'", - "+default_version = '#{@release_version}'"], + ["-fallback_version = '#{@release_version}a0'", + "+fallback_version = '#{@release_version}'"], ], }, { diff --git a/dev/release/post-11-bump-versions-test.rb b/dev/release/post-11-bump-versions-test.rb index 966c723f70adf..8ad404ef33202 100644 --- a/dev/release/post-11-bump-versions-test.rb +++ b/dev/release/post-11-bump-versions-test.rb @@ -179,10 +179,10 @@ def test_version_post_tag ], }, { - path: "python/setup.py", + path: "python/pyproject.toml", hunks: [ - ["-default_version = '#{@snapshot_version}'", - "+default_version = '#{@next_snapshot_version}'"], + ["-fallback_version = '#{@release_version}a0'", + "+fallback_version = '#{@next_version}a0'"], ], }, { diff --git a/dev/release/utils-prepare.sh b/dev/release/utils-prepare.sh index dfe9b052b09fa..c255e728a335b 100644 --- a/dev/release/utils-prepare.sh +++ b/dev/release/utils-prepare.sh @@ -26,10 +26,12 @@ update_versions() { release) local version=${base_version} local r_version=${base_version} + local python_version=${base_version} ;; snapshot) local version=${next_version}-SNAPSHOT local r_version=${base_version}.9000 + local python_version=${next_version}a0 ;; esac local major_version=${version%%.*} @@ -126,10 +128,10 @@ update_versions() { pushd "${ARROW_DIR}/python" sed -i.bak -E -e \ - "s/^default_version = '.+'/default_version = '${version}'/" \ - setup.py - rm -f setup.py.bak - git add setup.py + "s/^fallback_version = '.+'/fallback_version = '${python_version}'/" \ + pyproject.toml + rm -f pyproject.toml.bak + git add pyproject.toml sed -i.bak -E -e \ "s/^set\(PYARROW_VERSION \".+\"\)/set(PYARROW_VERSION \"${version}\")/" \ CMakeLists.txt diff --git a/dev/tasks/python-minimal-build/github.linux.yml b/dev/tasks/python-minimal-build/github.linux.yml index e776312b93f95..d97968b86b362 100644 --- a/dev/tasks/python-minimal-build/github.linux.yml +++ b/dev/tasks/python-minimal-build/github.linux.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest {{ macros.github_set_env(env) }} steps: - {{ macros.github_checkout_arrow(submodules=false)|indent }} + {{ macros.github_checkout_arrow(fetch_depth=0, submodules=false)|indent }} - name: Run minimal build example run: | diff --git a/python/examples/minimal_build/Dockerfile.ubuntu b/python/examples/minimal_build/Dockerfile.ubuntu index ebea4b045e592..07cd69c082461 100644 --- a/python/examples/minimal_build/Dockerfile.ubuntu +++ b/python/examples/minimal_build/Dockerfile.ubuntu @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu:focal +FROM ubuntu:jammy ENV DEBIAN_FRONTEND=noninteractive @@ -32,6 +32,7 @@ RUN apt-get update -y -q && \ python3-dev \ python3-pip \ python3-venv \ + tzdata \ && \ apt-get clean && rm -rf /var/lib/apt/lists* diff --git a/python/examples/minimal_build/build_conda.sh b/python/examples/minimal_build/build_conda.sh index 72c3a5f9ea2cd..e90c800ae2eb1 100755 --- a/python/examples/minimal_build/build_conda.sh +++ b/python/examples/minimal_build/build_conda.sh @@ -97,9 +97,8 @@ export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH} export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -# You can run either "develop" or "build_ext --inplace". Your pick - -# python setup.py build_ext --inplace -python setup.py develop +# Use the same command that we use on python_build.sh +python -m pip install --no-deps --no-build-isolation -vv . +popd -py.test pyarrow +pytest -vv -r s ${PYTEST_ARGS} --pyargs pyarrow diff --git a/python/examples/minimal_build/build_venv.sh b/python/examples/minimal_build/build_venv.sh index 3bd641d0e72c9..f462c4e9b9d0a 100755 --- a/python/examples/minimal_build/build_venv.sh +++ b/python/examples/minimal_build/build_venv.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -set -e +set -ex #---------------------------------------------------------------------- # Change this to whatever makes sense for your system @@ -35,6 +35,7 @@ source $WORKDIR/venv/bin/activate git config --global --add safe.directory $ARROW_ROOT pip install -r $ARROW_ROOT/python/requirements-build.txt +pip install wheel #---------------------------------------------------------------------- # Build C++ library @@ -68,11 +69,11 @@ export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH} export PYARROW_BUILD_TYPE=Debug export PYARROW_CMAKE_GENERATOR=Ninja -# You can run either "develop" or "build_ext --inplace". Your pick +# Use the same command that we use on python_build.sh +python -m pip install --no-deps --no-build-isolation -vv . -# python setup.py build_ext --inplace -python setup.py develop +popd pip install -r $ARROW_ROOT/python/requirements-test.txt -py.test pyarrow +pytest -vv -r s ${PYTEST_ARGS} --pyargs pyarrow diff --git a/python/pyproject.toml b/python/pyproject.toml index 1588e690a7247..f72c3a91eb436 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -24,7 +24,63 @@ requires = [ # continue using oldest-support-numpy. "oldest-supported-numpy>=0.14; python_version<'3.9'", "numpy>=1.25; python_version>='3.9'", - "setuptools_scm", - "setuptools >= 40.1.0", - "wheel" + # configuring setuptools_scm in pyproject.toml requires + # versions released after 2022 + "setuptools_scm[toml]>=8", + "setuptools>=64", ] +build-backend = "setuptools.build_meta" + +[project] +name = "pyarrow" +dynamic = ["version"] +requires-python = ">=3.8" +dependencies = [ + "numpy >= 1.16.6" +] +description = "Python library for Apache Arrow" +readme = {file = "README.md", content-type = "text/markdown"} +license = {text = "Apache Software License"} +classifiers = [ + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Langauge :: Python :: 3.12', +] +maintainers = [ + {name = "Apache Arrow Developers", email = "dev@arrow.apache.org"} +] + +[project.urls] +Homepage = "https://arrow.apache.org/" +Documentation = "https://arrow.apache.org/docs/python" +Repository = "https://github.com/apache/arrow" +Issues = "https://github.com/apache/arrow/issues" + +[project.optional-dependencies] +test = [ + 'pytest', + 'hypothesis', + 'cffi', + 'pytz', + 'pandas' +] + +[tool.setuptools] +zip-safe=false +include-package-data=true + +[tool.setuptools.packages.find] +where = ["."] + +[tool.setuptools.package-data] +pyarrow = ["*.pxd", "*.pyx", "includes/*.pxd"] + +[tool.setuptools_scm] +root = '..' +version_file = 'pyarrow/_generated_version.py' +version_scheme = 'guess-next-dev' +git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' +fallback_version = '17.0.0a0' diff --git a/python/requirements-build.txt b/python/requirements-build.txt index 87dcc148ad161..c150c842a0cc6 100644 --- a/python/requirements-build.txt +++ b/python/requirements-build.txt @@ -1,5 +1,5 @@ cython>=0.29.31 oldest-supported-numpy>=0.14; python_version<'3.9' numpy>=1.25; python_version>='3.9' -setuptools_scm -setuptools>=38.6.0 +setuptools_scm>=8 +setuptools>=64 diff --git a/python/setup.py b/python/setup.py index ed2b7961e5fbb..b738b2f77290e 100755 --- a/python/setup.py +++ b/python/setup.py @@ -352,61 +352,11 @@ def get_outputs(self): for name in self.get_names()] -# If the event of not running from a git clone (e.g. from a git archive -# or a Python sdist), see if we can set the version number ourselves -default_version = '17.0.0-SNAPSHOT' -if (not os.path.exists('../.git') and - not os.environ.get('SETUPTOOLS_SCM_PRETEND_VERSION')): - os.environ['SETUPTOOLS_SCM_PRETEND_VERSION'] = \ - default_version.replace('-SNAPSHOT', 'a0') - - -# See https://github.com/pypa/setuptools_scm#configuration-parameters -scm_version_write_to_prefix = os.environ.get( - 'SETUPTOOLS_SCM_VERSION_WRITE_TO_PREFIX', setup_dir) - - -def parse_git(root, **kwargs): - """ - Parse function for setuptools_scm that ignores tags for non-C++ - subprojects, e.g. apache-arrow-js-XXX tags. - """ - from setuptools_scm.git import parse - kwargs['describe_command'] =\ - 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"' - return parse(root, **kwargs) - - -def guess_next_dev_version(version): - if version.exact: - return version.format_with('{tag}') - else: - def guess_next_version(tag_version): - return default_version.replace('-SNAPSHOT', '') - return version.format_next_version(guess_next_version) - - -with open('README.md') as f: - long_description = f.read() - - class BinaryDistribution(Distribution): def has_ext_modules(foo): return True -install_requires = ( - 'numpy >= 1.16.6', -) - - -# Only include pytest-runner in setup_requires if we're invoking tests -if {'pytest', 'test', 'ptr'}.intersection(sys.argv): - setup_requires = ['pytest-runner'] -else: - setup_requires = [] - - if strtobool(os.environ.get('PYARROW_INSTALL_TESTS', '1')): packages = find_namespace_packages(include=['pyarrow*']) exclude_package_data = {} @@ -420,11 +370,7 @@ def has_ext_modules(foo): setup( - name='pyarrow', packages=packages, - zip_safe=False, - package_data={'pyarrow': ['*.pxd', '*.pyx', 'includes/*.pxd']}, - include_package_data=True, exclude_package_data=exclude_package_data, distclass=BinaryDistribution, # Dummy extension to trigger build_ext @@ -432,35 +378,4 @@ def has_ext_modules(foo): cmdclass={ 'build_ext': build_ext }, - use_scm_version={ - 'root': os.path.dirname(setup_dir), - 'parse': parse_git, - 'write_to': os.path.join(scm_version_write_to_prefix, - 'pyarrow/_generated_version.py'), - 'version_scheme': guess_next_dev_version - }, - setup_requires=['setuptools_scm', 'cython >= 0.29.31'] + setup_requires, - install_requires=install_requires, - tests_require=['pytest', 'pandas', 'hypothesis'], - python_requires='>=3.8', - description='Python library for Apache Arrow', - long_description=long_description, - long_description_content_type='text/markdown', - classifiers=[ - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', - ], - license='Apache License, Version 2.0', - maintainer='Apache Arrow Developers', - maintainer_email='dev@arrow.apache.org', - test_suite='pyarrow.tests', - url='https://arrow.apache.org/', - project_urls={ - 'Documentation': 'https://arrow.apache.org/docs/python', - 'Source': 'https://github.com/apache/arrow', - }, )