Merge remote-tracking branch 'upstream/main'

Author: shivamvishal

doc/source/whatsnew/v3.0.0.rst

@@ -933,6 +933,7 @@ Bug fixes
 Categorical
 ^^^^^^^^^^^
 - Bug in :func:`Series.apply` where ``nan`` was ignored for :class:`CategoricalDtype` (:issue:`59938`)
+- Bug in :func:`testing.assert_index_equal` raising ``TypeError`` instead of ``AssertionError`` for incomparable ``CategoricalIndex`` when ``check_categorical=True`` and ``exact=False`` (:issue:`61935`)
 - Bug in :meth:`Categorical.astype` where ``copy=False`` would still trigger a copy of the codes (:issue:`62000`)
 - Bug in :meth:`DataFrame.pivot` and :meth:`DataFrame.set_index` raising an ``ArrowNotImplementedError`` for columns with pyarrow dictionary dtype (:issue:`53051`)
 - Bug in :meth:`Series.convert_dtypes` with ``dtype_backend="pyarrow"`` where empty :class:`CategoricalDtype` :class:`Series` raised an error or got converted to ``null[pyarrow]`` (:issue:`59934`)

pandas/_testing/asserters.py

@@ -325,7 +325,17 @@ def _check_types(left, right, obj: str = "Index") -> None:
     # skip exact index checking when `check_categorical` is False
     elif check_exact and check_categorical:
         if not left.equals(right):
-            mismatch = left._values != right._values
+            # _values compare can raise TypeError (non-comparable
+            # categoricals (GH#61935)
+            try:
+                mismatch = left._values != right._values
+            except TypeError:
+                raise_assert_detail(
+                    obj,
+                    "types are not comparable (non-matching categorical categories)",
+                    left,
+                    right,
+                )
 
             if not isinstance(mismatch, np.ndarray):
                 mismatch = cast("ExtensionArray", mismatch).fillna(True)

pandas/tests/frame/test_arithmetic.py

@@ -11,6 +11,8 @@
 import numpy as np
 import pytest
 
+from pandas.compat._optional import import_optional_dependency
+
 import pandas as pd
 from pandas import (
     DataFrame,
@@ -24,6 +26,7 @@
     _check_mixed_float,
     _check_mixed_int,
 )
+from pandas.util.version import Version
 
 
 @pytest.fixture
@@ -1114,6 +1117,8 @@ def test_binop_other(self, op, value, dtype, switch_numexpr_min_elements):
             (operator.mod, "complex128"),
         }
 
+        ne = import_optional_dependency("numexpr", errors="ignore")
+        ne_warns_on_op = ne is not None and Version(ne.__version__) < Version("2.13.1")
         if (op, dtype) in invalid:
             warn = None
             if (dtype == "<M8[ns]" and op == operator.add) or (
@@ -1142,7 +1147,11 @@ def test_binop_other(self, op, value, dtype, switch_numexpr_min_elements):
 
         elif (op, dtype) in skip:
             if op in [operator.add, operator.mul]:
-                if expr.USE_NUMEXPR and switch_numexpr_min_elements == 0:
+                if (
+                    expr.USE_NUMEXPR
+                    and switch_numexpr_min_elements == 0
+                    and ne_warns_on_op
+                ):
                     warn = UserWarning
                 else:
                     warn = None

pandas/tests/io/excel/test_style.py

@@ -1,5 +1,4 @@
 import contextlib
-import time
 import uuid
 
 import numpy as np
@@ -324,18 +323,10 @@ def test_styler_to_s3(s3_bucket_public, s3so):
     target_file = f"{uuid.uuid4()}.xlsx"
     df = DataFrame({"x": [1, 2, 3], "y": [2, 4, 6]})
     styler = df.style.set_sticky(axis="index")
-    styler.to_excel(f"s3://{mock_bucket_name}/{target_file}", storage_options=s3so)
-    timeout = 5
-    while True:
-        if target_file in (obj.key for obj in s3_bucket_public.objects.all()):
-            break
-        time.sleep(0.1)
-        timeout -= 0.1
-        assert timeout > 0, "Timed out waiting for file to appear on moto"
-        result = read_excel(
-            f"s3://{mock_bucket_name}/{target_file}", index_col=0, storage_options=s3so
-        )
-        tm.assert_frame_equal(result, df)
+    uri = f"s3://{mock_bucket_name}/{target_file}"
+    styler.to_excel(uri, storage_options=s3so)
+    result = read_excel(uri, index_col=0, storage_options=s3so)
+    tm.assert_frame_equal(result, df)
 
 
 @pytest.mark.parametrize("merge_cells", [True, False, "columns"])

pandas/tests/io/json/test_pandas.py

@@ -7,7 +7,6 @@
 import json
 import os
 import sys
-import time
 import uuid
 
 import numpy as np
@@ -2016,14 +2015,10 @@ def test_to_s3(self, s3_bucket_public, s3so):
         mock_bucket_name = s3_bucket_public.name
         target_file = f"{uuid.uuid4()}.json"
         df = DataFrame({"x": [1, 2, 3], "y": [2, 4, 6]})
-        df.to_json(f"s3://{mock_bucket_name}/{target_file}", storage_options=s3so)
-        timeout = 5
-        while True:
-            if target_file in (obj.key for obj in s3_bucket_public.objects.all()):
-                break
-            time.sleep(0.1)
-            timeout -= 0.1
-            assert timeout > 0, "Timed out waiting for file to appear on moto"
+        uri = f"s3://{mock_bucket_name}/{target_file}"
+        df.to_json(uri, storage_options=s3so)
+        result = read_json(uri, storage_options=s3so)
+        tm.assert_frame_equal(result, df)
 
     def test_json_pandas_nulls(self, nulls_fixture):
         # GH 31615

pandas/tests/io/test_compression.py

@@ -5,7 +5,6 @@
 import sys
 import tarfile
 import textwrap
-import time
 import zipfile
 
 import numpy as np
@@ -193,7 +192,6 @@ def test_gzip_reproducibility_file_name(temp_file):
     # test for filename
     path = temp_file
     df.to_csv(path, compression=compression_options)
-    time.sleep(0.1)
     output = path.read_bytes()
     df.to_csv(path, compression=compression_options)
     assert output == path.read_bytes()
@@ -216,7 +214,6 @@ def test_gzip_reproducibility_file_object():
     buffer = io.BytesIO()
     df.to_csv(buffer, compression=compression_options, mode="wb")
     output = buffer.getvalue()
-    time.sleep(0.1)
     buffer = io.BytesIO()
     df.to_csv(buffer, compression=compression_options, mode="wb")
     assert output == buffer.getvalue()

pandas/tests/series/test_arithmetic.py

@@ -10,6 +10,7 @@
 import pytest
 
 from pandas._libs import lib
+from pandas.compat._optional import import_optional_dependency
 
 import pandas as pd
 from pandas import (
@@ -25,7 +26,7 @@
 import pandas._testing as tm
 from pandas.core import ops
 from pandas.core.computation import expressions as expr
-from pandas.core.computation.check import NUMEXPR_INSTALLED
+from pandas.util.version import Version
 
 
 @pytest.fixture(autouse=True, params=[0, 1000000], ids=["numexpr", "python"])
@@ -348,9 +349,12 @@ def test_add_list_to_masked_array(self, val, dtype):
 
     def test_add_list_to_masked_array_boolean(self, request):
         # GH#22962
+        ne = import_optional_dependency("numexpr", errors="ignore")
         warning = (
             UserWarning
-            if request.node.callspec.id == "numexpr" and NUMEXPR_INSTALLED
+            if request.node.callspec.id == "numexpr"
+            and ne
+            and Version(ne.__version__) < Version("2.13.1")
             else None
         )
         ser = Series([True, None, False], dtype="boolean")

pandas/tests/test_expressions.py

@@ -4,10 +4,13 @@
 import numpy as np
 import pytest
 
+from pandas.compat._optional import import_optional_dependency
+
 from pandas import option_context
 import pandas._testing as tm
 from pandas.core.api import DataFrame
 from pandas.core.computation import expressions as expr
+from pandas.util.version import Version
 
 
 @pytest.fixture
@@ -340,35 +343,43 @@ def test_bool_ops_warn_on_arithmetic(self, op_str, opname, monkeypatch):
             return
 
         msg = "operator is not supported by numexpr"
+        ne = import_optional_dependency("numexpr", errors="ignore")
+        warning = (
+            UserWarning
+            if ne
+            and op_str in {"+", "*"}
+            and Version(ne.__version__) < Version("2.13.1")
+            else None
+        )
         with monkeypatch.context() as m:
             m.setattr(expr, "_MIN_ELEMENTS", 5)
             with option_context("compute.use_numexpr", True):
-                with tm.assert_produces_warning(UserWarning, match=msg):
+                with tm.assert_produces_warning(warning, match=msg):
                     r = f(df, df)
                     e = fe(df, df)
                     tm.assert_frame_equal(r, e)
 
-                with tm.assert_produces_warning(UserWarning, match=msg):
+                with tm.assert_produces_warning(warning, match=msg):
                     r = f(df.a, df.b)
                     e = fe(df.a, df.b)
                     tm.assert_series_equal(r, e)
 
-                with tm.assert_produces_warning(UserWarning, match=msg):
+                with tm.assert_produces_warning(warning, match=msg):
                     r = f(df.a, True)
                     e = fe(df.a, True)
                     tm.assert_series_equal(r, e)
 
-                with tm.assert_produces_warning(UserWarning, match=msg):
+                with tm.assert_produces_warning(warning, match=msg):
                     r = f(False, df.a)
                     e = fe(False, df.a)
                     tm.assert_series_equal(r, e)
 
-                with tm.assert_produces_warning(UserWarning, match=msg):
+                with tm.assert_produces_warning(warning, match=msg):
                     r = f(False, df)
                     e = fe(False, df)
                     tm.assert_frame_equal(r, e)
 
-                with tm.assert_produces_warning(UserWarning, match=msg):
+                with tm.assert_produces_warning(warning, match=msg):
                     r = f(df, True)
                     e = fe(df, True)
                     tm.assert_frame_equal(r, e)

pandas/tests/util/test_assert_index_equal.py

@@ -317,3 +317,11 @@ def test_assert_multi_index_dtype_check_categorical(check_categorical):
             tm.assert_index_equal(idx1, idx2, check_categorical=check_categorical)
     else:
         tm.assert_index_equal(idx1, idx2, check_categorical=check_categorical)
+
+
+def test_assert_index_equal_categorical_incomparable_categories():
+    # GH#61935
+    left = Index([1, 2, 3], name="a", dtype="category")
+    right = Index([1, 2, 6], name="a", dtype="category")
+    with pytest.raises(AssertionError, match="types are not comparable"):
+        tm.assert_index_equal(left, right, check_categorical=True, exact=False)