Merge branch 'main' into micol-altomare-replace-ensure-clean-store

Author: micol-altomare

.github/workflows/unit-tests.yml

@@ -216,7 +216,7 @@ jobs:
   Linux-32-bit:
     runs-on: ubuntu-24.04
     container:
-      image: quay.io/pypa/manylinux2014_i686
+      image: quay.io/pypa/manylinux_2_28_i686
       options: --platform linux/386
     steps:
       - name: Checkout pandas Repo

.gitignore

@@ -21,6 +21,7 @@
 .noseids
 .ipynb_checkpoints
 .tags
+tags
 .cache/
 .vscode/
 

doc/source/whatsnew/v3.0.0.rst

@@ -233,6 +233,9 @@ Other enhancements
 - Support reading Stata 102-format (Stata 1) dta files (:issue:`58978`)
 - Support reading Stata 110-format (Stata 7) dta files (:issue:`47176`)
 - Switched wheel upload to **PyPI Trusted Publishing** (OIDC) for release-tag pushes in ``wheels.yml``. (:issue:`61718`)
+- Added a new :meth:`DataFrame.from_arrow` method to import any Arrow-compatible
+  tabular data object into a pandas :class:`DataFrame` through the
+  `Arrow PyCapsule Protocol <https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html>`__ (:issue:`59631`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.notable_bug_fixes:
@@ -1243,6 +1246,7 @@ Groupby/resample/rolling
 - Bug in :meth:`.DataFrameGroupBy.groups` and :meth:`.SeriesGroupby.groups` that would not respect groupby argument ``dropna`` (:issue:`55919`)
 - Bug in :meth:`.DataFrameGroupBy.median` where nat values gave an incorrect result. (:issue:`57926`)
 - Bug in :meth:`.DataFrameGroupBy.quantile` when ``interpolation="nearest"`` is inconsistent with :meth:`DataFrame.quantile` (:issue:`47942`)
+- Bug in :meth:`.DataFrameGroupBy.sum` and :meth:`.SeriesGroupby.groups` returning ``NaN`` on overflow. These methods now returns ``inf`` or ``-inf`` on overflow. (:issue:`60303`)
 - Bug in :meth:`.DataFrameGroupBy` reductions where non-Boolean values were allowed for the ``numeric_only`` argument; passing a non-Boolean value will now raise (:issue:`62778`)
 - Bug in :meth:`.Resampler.interpolate` on a :class:`DataFrame` with non-uniform sampling and/or indices not aligning with the resulting resampled index would result in wrong interpolation (:issue:`21351`)
 - Bug in :meth:`.Series.rolling` when used with a :class:`.BaseIndexer` subclass and computing min/max (:issue:`46726`)

pandas/_libs/groupby.pyx

@@ -5,6 +5,7 @@ from cython cimport (
 )
 from libc.math cimport (
     NAN,
+    isfinite,
     sqrt,
 )
 from libc.stdlib cimport (
@@ -778,9 +779,9 @@ def group_sum(
                 if not isna_entry:
                     nobs[lab, j] += 1
 
-                    if sum_t is object:
+                    if sum_t is object or sum_t is int64_t or sum_t is uint64_t:
                         # NB: this does not use 'compensation' like the non-object
-                        #  track does.
+                        #  and non-integer track does.
                         if nobs[lab, j] == 1:
                             # i.e. we haven't added anything yet; avoid TypeError
                             #  if e.g. val is a str and sumx[lab, j] is 0
@@ -793,13 +794,29 @@ def group_sum(
                         y = val - compensation[lab, j]
                         t = sumx[lab, j] + y
                         compensation[lab, j] = t - sumx[lab, j] - y
-                        if compensation[lab, j] != compensation[lab, j]:
-                            # GH#53606
+
+                        # Handle float overflow
+                        if (
+                            sum_t is float32_t or sum_t is float64_t
+                        ) and not isfinite(compensation[lab, j]):
+                            # GH#53606; GH#60303
                             # If val is +/- infinity compensation is NaN
                             # which would lead to results being NaN instead
                             # of +/- infinity. We cannot use util.is_nan
                             # because of no gil
                             compensation[lab, j] = 0
+
+                        # Handle complex overflow
+                        if (
+                            sum_t is complex64_t or sum_t is complex128_t
+                        ) and not isfinite(compensation[lab, j].real):
+                            compensation[lab, j].real = 0
+
+                        if (
+                            sum_t is complex64_t or sum_t is complex128_t
+                        ) and not isfinite(compensation[lab, j].imag):
+                            compensation[lab, j].imag = 0
+
                         sumx[lab, j] = t
                 elif not skipna:
                     if uses_mask:

pandas/_libs/internals.pyi

@@ -94,7 +94,3 @@ class BlockValuesRefs:
     def add_reference(self, blk: Block) -> None: ...
     def add_index_reference(self, index: Index) -> None: ...
     def has_reference(self) -> bool: ...
-
-class SetitemMixin:
-    def __setitem__(self, key, value) -> None: ...
-    def __delitem__(self, key) -> None: ...

pandas/_libs/internals.pyx

@@ -1,9 +1,6 @@
 from collections import defaultdict
-import sys
-import warnings
 
 cimport cython
-from cpython cimport PY_VERSION_HEX
 from cpython.object cimport PyObject
 from cpython.pyport cimport PY_SSIZE_T_MAX
 from cpython.slice cimport PySlice_GetIndicesEx
@@ -23,9 +20,6 @@ from numpy cimport (
 cnp.import_array()
 
 from pandas._libs.algos import ensure_int64
-from pandas.compat import CHAINED_WARNING_DISABLED
-from pandas.errors import ChainedAssignmentError
-from pandas.errors.cow import _chained_assignment_msg
 
 from pandas._libs.util cimport (
     is_array,
@@ -1002,47 +996,3 @@ cdef class BlockValuesRefs:
                 return self._has_reference_maybe_locked()
         ELSE:
             return self._has_reference_maybe_locked()
-
-
-cdef extern from "Python.h":
-    """
-    // python version < 3.14
-    #if PY_VERSION_HEX < 0x030E0000
-    // This function is unused and is declared to avoid a build warning
-    int __Pyx_PyUnstable_Object_IsUniqueReferencedTemporary(PyObject *ref) {
-        return Py_REFCNT(ref) == 1;
-    }
-    #else
-    #define __Pyx_PyUnstable_Object_IsUniqueReferencedTemporary \
-        PyUnstable_Object_IsUniqueReferencedTemporary
-    #endif
-    """
-    int PyUnstable_Object_IsUniqueReferencedTemporary\
-        "__Pyx_PyUnstable_Object_IsUniqueReferencedTemporary"(object o) except -1
-
-
-# Python version compatibility for PyUnstable_Object_IsUniqueReferencedTemporary
-cdef inline bint _is_unique_referenced_temporary(object obj) except -1:
-    if PY_VERSION_HEX >= 0x030E0000:
-        # Python 3.14+ has PyUnstable_Object_IsUniqueReferencedTemporary
-        return PyUnstable_Object_IsUniqueReferencedTemporary(obj)
-    else:
-        # Fallback for older Python versions using sys.getrefcount
-        return sys.getrefcount(obj) <= 1
-
-
-cdef class SetitemMixin:
-    # class used in DataFrame and Series for checking for chained assignment
-
-    def __setitem__(self, key, value) -> None:
-        cdef bint is_unique = 0
-        if not CHAINED_WARNING_DISABLED:
-            is_unique = _is_unique_referenced_temporary(self)
-            if is_unique:
-                warnings.warn(
-                    _chained_assignment_msg, ChainedAssignmentError, stacklevel=1
-                )
-        self._setitem(key, value)
-
-    def __delitem__(self, key) -> None:
-        self._delitem(key)

pandas/_libs/tslibs/timedeltas.pyx

@@ -83,6 +83,7 @@ from pandas._libs.tslibs.offsets import Day
 
 from pandas._libs.tslibs.util cimport (
     is_array,
+    is_bool_object,
     is_float_object,
     is_integer_object,
 )
@@ -2311,6 +2312,13 @@ class Timedelta(_Timedelta):
                 return self.__mul__(item)
             return other * self.to_timedelta64()
 
+        elif is_bool_object(other):
+            # GH#62316
+            raise TypeError(
+                "Cannot multiply Timedelta by bool. "
+                "Explicitly cast to integer instead."
+            )
+
         return NotImplemented
 
     __rmul__ = __mul__

pandas/_testing/contexts.py

@@ -12,10 +12,7 @@
 )
 import uuid
 
-from pandas.compat import (
-    CHAINED_WARNING_DISABLED,
-    CHAINED_WARNING_DISABLED_INPLACE_METHOD,
-)
+from pandas.compat import CHAINED_WARNING_DISABLED
 from pandas.errors import ChainedAssignmentError
 
 from pandas.io.common import get_handle
@@ -163,18 +160,10 @@ def with_csv_dialect(name: str, **kwargs) -> Generator[None]:
         csv.unregister_dialect(name)
 
 
-def raises_chained_assignment_error(
-    extra_warnings=(), extra_match=(), inplace_method=False
-):
+def raises_chained_assignment_error(extra_warnings=(), extra_match=()):
     from pandas._testing import assert_produces_warning
 
-    WARNING_DISABLED = (
-        CHAINED_WARNING_DISABLED_INPLACE_METHOD
-        if inplace_method
-        else CHAINED_WARNING_DISABLED
-    )
-
-    if WARNING_DISABLED:
+    if CHAINED_WARNING_DISABLED:
         if not extra_warnings:
             from contextlib import nullcontext
 

pandas/_typing.py

@@ -533,4 +533,44 @@ def closed(self) -> bool:
 
 SliceType: TypeAlias = Hashable | None
 
+
+# Arrow PyCapsule Interface
+# from https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html#protocol-typehints
+
+
+class ArrowArrayExportable(Protocol):
+    """
+    An object with an ``__arrow_c_array__`` method.
+
+    This method indicates the object is an Arrow-compatible object implementing
+    the `Arrow PyCapsule Protocol`_ (exposing the `Arrow C Data Interface`_ in
+    Python), enabling zero-copy Arrow data interchange across libraries.
+
+    .. _Arrow PyCapsule Protocol: https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html
+    .. _Arrow C Data Interface: https://arrow.apache.org/docs/format/CDataInterface.html
+
+    """
+
+    def __arrow_c_array__(
+        self, requested_schema: object | None = None
+    ) -> tuple[object, object]: ...
+
+
+class ArrowStreamExportable(Protocol):
+    """
+    An object with an ``__arrow_c_stream__`` method.
+
+    This method indicates the object is an Arrow-compatible object implementing
+    the `Arrow PyCapsule Protocol`_ (exposing the `Arrow C Data Interface`_
+    for streams in Python), enabling zero-copy Arrow data interchange across
+    libraries.
+
+    .. _Arrow PyCapsule Protocol: https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html
+    .. _Arrow C Stream Interface: https://arrow.apache.org/docs/format/CStreamInterface.html
+
+    """
+
+    def __arrow_c_stream__(self, requested_schema: object | None = None) -> object: ...
+
+
 __all__ = ["type_t"]

pandas/compat/__init__.py

@@ -17,7 +17,6 @@
 
 from pandas.compat._constants import (
     CHAINED_WARNING_DISABLED,
-    CHAINED_WARNING_DISABLED_INPLACE_METHOD,
     IS64,
     ISMUSL,
     PY312,
@@ -154,7 +153,6 @@ def is_ci_environment() -> bool:
 
 __all__ = [
     "CHAINED_WARNING_DISABLED",
-    "CHAINED_WARNING_DISABLED_INPLACE_METHOD",
     "HAS_PYARROW",
     "IS64",
     "ISMUSL",