From 2ae56c2f9443dbe36fc49173c8b1cb06ee9f5c9b Mon Sep 17 00:00:00 2001 From: heoh Date: Wed, 18 Jun 2025 01:02:34 +0900 Subject: [PATCH 1/4] BUG: Fix GroupBy aggregate coersion of outputs inconsistency for pyarrow dtypes (#61636) --- pandas/core/arrays/arrow/array.py | 13 +++++++++++++ pandas/core/arrays/string_arrow.py | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py index 653a900fbfe45..9321e58b06e3d 100644 --- a/pandas/core/arrays/arrow/array.py +++ b/pandas/core/arrays/arrow/array.py @@ -189,6 +189,7 @@ def floordiv_compat( ArrayLike, AxisInt, Dtype, + DtypeObj, FillnaOptions, InterpolateOptions, Iterator, @@ -308,6 +309,18 @@ def __init__(self, values: pa.Array | pa.ChunkedArray) -> None: ) self._dtype = ArrowDtype(self._pa_array.type) + @classmethod + def _from_scalars(cls, scalars, dtype: DtypeObj) -> Self: + try: + pa_array = cls._from_sequence(scalars, dtype=dtype) + except pa.ArrowNotImplementedError: + # _from_scalars should only raise ValueError or TypeError. + raise ValueError + + if lib.infer_dtype(scalars, skipna=True) != lib.infer_dtype(pa_array, skipna=True): + raise ValueError + return pa_array + @classmethod def _from_sequence( cls, scalars, *, dtype: Dtype | None = None, copy: bool = False diff --git a/pandas/core/arrays/string_arrow.py b/pandas/core/arrays/string_arrow.py index 9046d83dcc09f..a91b16482963a 100644 --- a/pandas/core/arrays/string_arrow.py +++ b/pandas/core/arrays/string_arrow.py @@ -53,6 +53,7 @@ from pandas._typing import ( ArrayLike, Dtype, + DtypeObj, NpDtype, Scalar, npt, @@ -190,6 +191,12 @@ def __len__(self) -> int: """ return len(self._pa_array) + @classmethod + def _from_scalars(cls, scalars, dtype: DtypeObj) -> Self: + if lib.infer_dtype(scalars, skipna=True) not in ["string", "empty"]: + raise ValueError + return cls._from_sequence(scalars, dtype=dtype) + @classmethod def _from_sequence( cls, scalars, *, dtype: Dtype | None = None, copy: bool = False From 3c9cf09c10983e9dcf74f40a39254f4d058d4a55 Mon Sep 17 00:00:00 2001 From: heoh Date: Tue, 24 Jun 2025 03:38:51 +0900 Subject: [PATCH 2/4] Reformat code style --- pandas/core/arrays/arrow/array.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pandas/core/arrays/arrow/array.py b/pandas/core/arrays/arrow/array.py index 9321e58b06e3d..f341d13903613 100644 --- a/pandas/core/arrays/arrow/array.py +++ b/pandas/core/arrays/arrow/array.py @@ -311,13 +311,15 @@ def __init__(self, values: pa.Array | pa.ChunkedArray) -> None: @classmethod def _from_scalars(cls, scalars, dtype: DtypeObj) -> Self: + inferred_dtype = lib.infer_dtype(scalars, skipna=True) try: pa_array = cls._from_sequence(scalars, dtype=dtype) - except pa.ArrowNotImplementedError: + except pa.ArrowNotImplementedError as err: # _from_scalars should only raise ValueError or TypeError. - raise ValueError + raise ValueError from err - if lib.infer_dtype(scalars, skipna=True) != lib.infer_dtype(pa_array, skipna=True): + same_dtype = lib.infer_dtype(pa_array, skipna=True) == inferred_dtype + if not same_dtype: raise ValueError return pa_array From c2a216d803772f5d57528c47c18420a71b1bef68 Mon Sep 17 00:00:00 2001 From: heoh Date: Tue, 24 Jun 2025 03:39:09 +0900 Subject: [PATCH 3/4] Add test code --- pandas/tests/extension/test_arrow.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pandas/tests/extension/test_arrow.py b/pandas/tests/extension/test_arrow.py index e9d014a0eb29d..b56962dc1a28d 100644 --- a/pandas/tests/extension/test_arrow.py +++ b/pandas/tests/extension/test_arrow.py @@ -3251,6 +3251,27 @@ def test_groupby_count_return_arrow_dtype(data_missing): tm.assert_frame_equal(result, expected) +@pytest.mark.parametrize( + "func, func_dtype", + [ + [lambda x: x.to_dict(), "object"], + [lambda x: 1, "int64"], + [lambda x: "s", ArrowDtype(pa.string())], + ], +) +def test_groupby_aggregate_coersion(func, func_dtype): + # GH 61636 + df = pd.DataFrame( + { + "b": pd.array([0, 1]), + "c": pd.array(["X", "Y"], dtype=ArrowDtype(pa.string())), + }, + index=pd.Index(["A", "B"], name="a"), + ) + result = df.groupby("b").agg(func) + assert result["c"].dtype == func_dtype + + def test_fixed_size_list(): # GH#55000 ser = pd.Series( From 306d862defc45fb9a92688007b8b21646a4fdd62 Mon Sep 17 00:00:00 2001 From: heoh Date: Tue, 24 Jun 2025 03:51:36 +0900 Subject: [PATCH 4/4] Update whatsnew --- doc/source/whatsnew/v3.0.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v3.0.0.rst b/doc/source/whatsnew/v3.0.0.rst index f91d40c4d9ea9..5341f338e6249 100644 --- a/doc/source/whatsnew/v3.0.0.rst +++ b/doc/source/whatsnew/v3.0.0.rst @@ -1064,6 +1064,7 @@ Groupby/resample/rolling - Bug in :meth:`DataFrame.resample` changing index type to :class:`MultiIndex` when the dataframe is empty and using an upsample method (:issue:`55572`) - Bug in :meth:`DataFrameGroupBy.agg` and :meth:`SeriesGroupBy.agg` that was returning numpy dtype values when input values are pyarrow dtype values, instead of returning pyarrow dtype values. (:issue:`53030`) - Bug in :meth:`DataFrameGroupBy.agg` that raises ``AttributeError`` when there is dictionary input and duplicated columns, instead of returning a DataFrame with the aggregation of all duplicate columns. (:issue:`55041`) +- Bug in :meth:`DataFrameGroupBy.agg` when applied to columns with :class:`ArrowDtype`, where pandas attempted to cast the result back to the original dtype (:issue:`61636`) - Bug in :meth:`DataFrameGroupBy.agg` where applying a user-defined function to an empty DataFrame returned a Series instead of an empty DataFrame. (:issue:`61503`) - Bug in :meth:`DataFrameGroupBy.apply` and :meth:`SeriesGroupBy.apply` for empty data frame with ``group_keys=False`` still creating output index using group keys. (:issue:`60471`) - Bug in :meth:`DataFrameGroupBy.apply` that was returning a completely empty DataFrame when all return values of ``func`` were ``None`` instead of returning an empty DataFrame with the original columns and dtypes. (:issue:`57775`)