apacheGH-37929: [Python] begin moving static settings to pyproject.to…

…ml (apache#41041)

### Rationale for this change

To migrate Arrow to modern Python packaging standards, see [PEP-517](https://peps.python.org/pep-0517/) and [PEP-518](https://peps.python.org/pep-0518/). 
* GitHub Issue: apache#37929

This PR focuses on migrating the static settings, the metadata and version, to pyproject.toml. Future PRs will migrate more of the build process to pyproject.toml.

Lead-authored-by: anjakefala <anja.kefala@gmail.com>
Co-authored-by: Raúl Cumplido <raulcumplido@gmail.com>
Co-authored-by: Joris Van den Bossche <jorisvandenbossche@gmail.com>
Signed-off-by: Jacob Wujciak-Jens <jacob@wujciak.de>

.github/workflows/dev.yml

@@ -120,7 +120,7 @@ jobs:
         shell: bash
         run: |
           gem install test-unit
-          pip install "cython>=0.29.31" setuptools six pytest jira
+          pip install "cython>=0.29.31" setuptools six pytest jira setuptools-scm
       - name: Run Release Test
         env:
           ARROW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}

ci/conda_env_python.txt

@@ -26,5 +26,5 @@ numpy>=1.16.6
 pytest
 pytest-faulthandler
 s3fs>=2023.10.0
-setuptools
-setuptools_scm
+setuptools>=64
+setuptools_scm>=8

dev/release/01-prepare-test.rb

@@ -254,10 +254,10 @@ def test_version_pre_tag
         ],
       },
       {
-        path: "python/setup.py",
+        path: "python/pyproject.toml",
         hunks: [
-          ["-default_version = '#{@snapshot_version}'",
-           "+default_version = '#{@release_version}'"],
+          ["-fallback_version = '#{@release_version}a0'",
+           "+fallback_version = '#{@release_version}'"],
         ],
       },
       {

dev/release/post-11-bump-versions-test.rb

@@ -179,10 +179,10 @@ def test_version_post_tag
         ],
       },
       {
-        path: "python/setup.py",
+        path: "python/pyproject.toml",
         hunks: [
-          ["-default_version = '#{@snapshot_version}'",
-           "+default_version = '#{@next_snapshot_version}'"],
+          ["-fallback_version = '#{@release_version}a0'",
+           "+fallback_version = '#{@next_version}a0'"],
         ],
       },
       {

dev/release/utils-prepare.sh

@@ -26,10 +26,12 @@ update_versions() {
     release)
       local version=${base_version}
       local r_version=${base_version}
+      local python_version=${base_version}
       ;;
     snapshot)
       local version=${next_version}-SNAPSHOT
       local r_version=${base_version}.9000
+      local python_version=${next_version}a0
       ;;
   esac
   local major_version=${version%%.*}
@@ -126,10 +128,10 @@ update_versions() {
 
   pushd "${ARROW_DIR}/python"
   sed -i.bak -E -e \
-    "s/^default_version = '.+'/default_version = '${version}'/" \
-    setup.py
-  rm -f setup.py.bak
-  git add setup.py
+    "s/^fallback_version = '.+'/fallback_version = '${python_version}'/" \
+    pyproject.toml
+  rm -f pyproject.toml.bak
+  git add pyproject.toml
   sed -i.bak -E -e \
     "s/^set\(PYARROW_VERSION \".+\"\)/set(PYARROW_VERSION \"${version}\")/" \
     CMakeLists.txt

dev/tasks/python-minimal-build/github.linux.yml

@@ -26,7 +26,7 @@ jobs:
     runs-on: ubuntu-latest
 {{ macros.github_set_env(env) }}
     steps:
-      {{ macros.github_checkout_arrow(submodules=false)|indent }}
+      {{ macros.github_checkout_arrow(fetch_depth=0, submodules=false)|indent }}
 
       - name: Run minimal build example
         run: |

python/examples/minimal_build/Dockerfile.ubuntu

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM ubuntu:focal
+FROM ubuntu:jammy
 
 ENV DEBIAN_FRONTEND=noninteractive
 
@@ -32,6 +32,7 @@ RUN apt-get update -y -q && \
       python3-dev \
       python3-pip \
       python3-venv \
+      tzdata \
       && \
       apt-get clean && rm -rf /var/lib/apt/lists*
 

python/examples/minimal_build/build_conda.sh

@@ -97,9 +97,8 @@ export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH}
 export PYARROW_BUILD_TYPE=Debug
 export PYARROW_CMAKE_GENERATOR=Ninja
 
-# You can run either "develop" or "build_ext --inplace". Your pick
-
-# python setup.py build_ext --inplace
-python setup.py develop
+# Use the same command that we use on python_build.sh
+python -m pip install --no-deps --no-build-isolation -vv .
+popd
 
-py.test pyarrow
+pytest -vv -r s ${PYTEST_ARGS} --pyargs pyarrow

python/examples/minimal_build/build_venv.sh

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-set -e
+set -ex
 
 #----------------------------------------------------------------------
 # Change this to whatever makes sense for your system
@@ -35,6 +35,7 @@ source $WORKDIR/venv/bin/activate
 git config --global --add safe.directory $ARROW_ROOT
 
 pip install -r $ARROW_ROOT/python/requirements-build.txt
+pip install wheel
 
 #----------------------------------------------------------------------
 # Build C++ library
@@ -68,11 +69,11 @@ export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH}
 export PYARROW_BUILD_TYPE=Debug
 export PYARROW_CMAKE_GENERATOR=Ninja
 
-# You can run either "develop" or "build_ext --inplace". Your pick
+# Use the same command that we use on python_build.sh
+python -m pip install --no-deps --no-build-isolation -vv .
 
-# python setup.py build_ext --inplace
-python setup.py develop
+popd
 
 pip install -r $ARROW_ROOT/python/requirements-test.txt
 
-py.test pyarrow
+pytest -vv -r s ${PYTEST_ARGS} --pyargs pyarrow

python/pyproject.toml

@@ -24,7 +24,63 @@ requires = [
     # continue using oldest-support-numpy.
     "oldest-supported-numpy>=0.14; python_version<'3.9'",
     "numpy>=1.25; python_version>='3.9'",
-    "setuptools_scm",
-    "setuptools >= 40.1.0",
-    "wheel"
+    # configuring setuptools_scm in pyproject.toml requires
+    # versions released after 2022
+    "setuptools_scm[toml]>=8",
+    "setuptools>=64",
 ]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pyarrow"
+dynamic = ["version"]
+requires-python = ">=3.8"
+dependencies = [
+    "numpy >= 1.16.6"
+]
+description = "Python library for Apache Arrow"
+readme = {file = "README.md", content-type = "text/markdown"}
+license = {text = "Apache Software License"}
+classifiers  = [
+    'License :: OSI Approved :: Apache Software License',
+    'Programming Language :: Python :: 3.8',
+    'Programming Language :: Python :: 3.9',
+    'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11',
+    'Programming Langauge :: Python :: 3.12',
+]
+maintainers = [
+    {name = "Apache Arrow Developers", email = "dev@arrow.apache.org"}
+]
+
+[project.urls]
+Homepage = "https://arrow.apache.org/"
+Documentation = "https://arrow.apache.org/docs/python"
+Repository = "https://github.com/apache/arrow"
+Issues = "https://github.com/apache/arrow/issues"
+
+[project.optional-dependencies]
+test = [
+    'pytest',
+    'hypothesis',
+    'cffi',
+    'pytz',
+    'pandas'
+]
+
+[tool.setuptools]
+zip-safe=false
+include-package-data=true
+
+[tool.setuptools.packages.find]
+where = ["."]
+
+[tool.setuptools.package-data]
+pyarrow = ["*.pxd", "*.pyx", "includes/*.pxd"]
+
+[tool.setuptools_scm]
+root = '..'
+version_file = 'pyarrow/_generated_version.py'
+version_scheme = 'guess-next-dev'
+git_describe_command = 'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"'
+fallback_version = '17.0.0a0'

python/requirements-build.txt

@@ -1,5 +1,5 @@
 cython>=0.29.31
 oldest-supported-numpy>=0.14; python_version<'3.9'
 numpy>=1.25; python_version>='3.9'
-setuptools_scm
-setuptools>=38.6.0
+setuptools_scm>=8
+setuptools>=64

python/setup.py

@@ -352,61 +352,11 @@ def get_outputs(self):
                 for name in self.get_names()]
 
 
-# If the event of not running from a git clone (e.g. from a git archive
-# or a Python sdist), see if we can set the version number ourselves
-default_version = '17.0.0-SNAPSHOT'
-if (not os.path.exists('../.git') and
-        not os.environ.get('SETUPTOOLS_SCM_PRETEND_VERSION')):
-    os.environ['SETUPTOOLS_SCM_PRETEND_VERSION'] = \
-        default_version.replace('-SNAPSHOT', 'a0')
-
-
-# See https://github.com/pypa/setuptools_scm#configuration-parameters
-scm_version_write_to_prefix = os.environ.get(
-    'SETUPTOOLS_SCM_VERSION_WRITE_TO_PREFIX', setup_dir)
-
-
-def parse_git(root, **kwargs):
-    """
-    Parse function for setuptools_scm that ignores tags for non-C++
-    subprojects, e.g. apache-arrow-js-XXX tags.
-    """
-    from setuptools_scm.git import parse
-    kwargs['describe_command'] =\
-        'git describe --dirty --tags --long --match "apache-arrow-[0-9]*.*"'
-    return parse(root, **kwargs)
-
-
-def guess_next_dev_version(version):
-    if version.exact:
-        return version.format_with('{tag}')
-    else:
-        def guess_next_version(tag_version):
-            return default_version.replace('-SNAPSHOT', '')
-        return version.format_next_version(guess_next_version)
-
-
-with open('README.md') as f:
-    long_description = f.read()
-
-
 class BinaryDistribution(Distribution):
     def has_ext_modules(foo):
         return True
 
 
-install_requires = (
-    'numpy >= 1.16.6',
-)
-
-
-# Only include pytest-runner in setup_requires if we're invoking tests
-if {'pytest', 'test', 'ptr'}.intersection(sys.argv):
-    setup_requires = ['pytest-runner']
-else:
-    setup_requires = []
-
-
 if strtobool(os.environ.get('PYARROW_INSTALL_TESTS', '1')):
     packages = find_namespace_packages(include=['pyarrow*'])
     exclude_package_data = {}
@@ -420,47 +370,12 @@ def has_ext_modules(foo):
 
 
 setup(
-    name='pyarrow',
     packages=packages,
-    zip_safe=False,
-    package_data={'pyarrow': ['*.pxd', '*.pyx', 'includes/*.pxd']},
-    include_package_data=True,
     exclude_package_data=exclude_package_data,
     distclass=BinaryDistribution,
     # Dummy extension to trigger build_ext
     ext_modules=[Extension('__dummy__', sources=[])],
     cmdclass={
         'build_ext': build_ext
     },
-    use_scm_version={
-        'root': os.path.dirname(setup_dir),
-        'parse': parse_git,
-        'write_to': os.path.join(scm_version_write_to_prefix,
-                                 'pyarrow/_generated_version.py'),
-        'version_scheme': guess_next_dev_version
-    },
-    setup_requires=['setuptools_scm', 'cython >= 0.29.31'] + setup_requires,
-    install_requires=install_requires,
-    tests_require=['pytest', 'pandas', 'hypothesis'],
-    python_requires='>=3.8',
-    description='Python library for Apache Arrow',
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    classifiers=[
-        'License :: OSI Approved :: Apache Software License',
-        'Programming Language :: Python :: 3.8',
-        'Programming Language :: Python :: 3.9',
-        'Programming Language :: Python :: 3.10',
-        'Programming Language :: Python :: 3.11',
-        'Programming Language :: Python :: 3.12',
-    ],
-    license='Apache License, Version 2.0',
-    maintainer='Apache Arrow Developers',
-    maintainer_email='dev@arrow.apache.org',
-    test_suite='pyarrow.tests',
-    url='https://arrow.apache.org/',
-    project_urls={
-        'Documentation': 'https://arrow.apache.org/docs/python',
-        'Source': 'https://github.com/apache/arrow',
-    },
 )