From a509bb31ff7f745db8b8979a336fa02e839536ea Mon Sep 17 00:00:00 2001 From: nethum529 Date: Sun, 30 Aug 2026 18:22:37 -0500 Subject: [PATCH 1/2] fix(compose): support object dtype in ColumnTransformer + SimpleImputer String columns have no device representation, so ColumnTransformer and SimpleImputer failed on object dtype input from cuDF and pandas. Keep the imputer path on host memory for object dtype, stack the host results correctly, and let convert_arrays return an object output for the host output types. Closes #6183 Signed-off-by: nethum529 --- .../preprocessing/_column_transformer.py | 7 + .../sklearn/preprocessing/_imputation.py | 15 ++- python/cuml/cuml/internals/outputs.py | 73 ++++++++++ .../cuml/cuml/thirdparty_adapters/adapters.py | 29 +++- python/cuml/tests/test_adapters.py | 27 +++- python/cuml/tests/test_compose.py | 127 ++++++++++++++++++ python/cuml/tests/test_reflection.py | 66 +++++++++ 7 files changed, 335 insertions(+), 9 deletions(-) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py index f4cb94bdf1..8976a19735 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py @@ -37,6 +37,7 @@ import cuml from cuml.internals.global_settings import _global_settings_data from cuml.internals.mixins import DeprecatedGetFeatureNamesMixin +from cuml.internals.outputs import _is_object_dtype from cuml.internals.validation import ( check_is_fitted, check_features, @@ -1014,6 +1015,12 @@ def _hstack(self, Xs): return cu_sparse.hstack(converted_Xs).tocsr() else: Xs = [f.toarray() if issparse(f) else f for f in Xs] + if any(_is_object_dtype(X) for X in Xs): + # Object dtype (e.g. string columns from a categorical + # SimpleImputer) has no device representation - cupy has no + # way to store it. Stack on host instead. + Xs = [X.get() if isinstance(X, np.ndarray) else X for X in Xs] + return cpu_np.hstack(Xs) return np.hstack(Xs) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py index 7fef551763..32005b054d 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py @@ -29,7 +29,7 @@ StringInputTagMixin, _ensure_transformer_tags, ) -from cuml.internals.outputs import mlfunc, ReflectedAttr +from cuml.internals.outputs import _is_object_dtype, mlfunc, ReflectedAttr from cuml.internals.validation import ( check_is_fitted, check_inputs, @@ -144,7 +144,6 @@ def _concatenate_indicator(self, X_imputed, X_indicator): if not self.add_indicator: return X_imputed - hstack = sparse.hstack if sparse.issparse(X_imputed) else np.hstack if X_indicator is None: raise ValueError( "Data from the missing indicator are not provided. Call " @@ -152,7 +151,17 @@ def _concatenate_indicator(self, X_imputed, X_indicator): "implementation." ) - return hstack((X_imputed, X_indicator)) + if sparse.issparse(X_imputed): + return sparse.hstack((X_imputed, X_indicator)) + + if _is_object_dtype(X_imputed) or _is_object_dtype(X_indicator): + arrays = [ + array.get() if isinstance(array, np.ndarray) else array + for array in (X_imputed, X_indicator) + ] + return cpu_np.hstack(arrays) + + return np.hstack((X_imputed, X_indicator)) def __sklearn_tags__(self): tags = super().__sklearn_tags__() diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index 69a4b44251..821a8dd48d 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -284,6 +284,28 @@ def infer_output_type(array, array_like="numpy"): return None +def _is_object_dtype(res): + """Check for NumPy object dtype on an array-like or dataframe-like.""" + dtype = getattr(res, "dtype", None) + if dtype is not None: + # Array-like or Series: a single (possibly extension) dtype. Only a + # plain numpy object dtype has no device representation; extension + # dtypes are a separate, unsupported case handled elsewhere. + return isinstance(dtype, np.dtype) and dtype.kind == "O" + + dtypes = getattr(res, "dtypes", None) + if dtypes is not None: + # DataFrame-like: dtypes is per-column. + try: + return any( + isinstance(dt, np.dtype) and dt.kind == "O" for dt in dtypes + ) + except TypeError: + return False + + return False + + class ArrayIndexPair: """An array paired with an aligned index. @@ -521,6 +543,57 @@ def convert_arrays( return pd.Series(obj.flatten(), index=index) return pd.DataFrame(obj, index=index) return pd.Series(obj, index=index) + elif _is_object_dtype(obj): + # NumPy object arrays have no device representation. Preserve + # host arrays for "array" and internal "cuml" outputs, or wrap + # them in a dataframe-like type. + if output_type in ("array", "cuml"): + return obj + elif output_type in ("cupy", "numba"): + raise TypeError( + f"{output_type=!r} doesn't support outputs of dtype " + f"object and shape {obj.shape}" + ) + + if hasattr(index, "to_pandas"): + index = index.to_pandas() + if output_type == "series": + if obj.ndim == 2: + if obj.shape[1] == 1: + obj = obj.flatten() + else: + raise ValueError( + "Only single dimensional arrays can be transformed to" + " Series." + ) + elif obj.ndim == 0: + obj = obj[None] + elif output_type == "dataframe": + if obj.ndim == 1: + obj = obj[:, None] + elif obj.ndim == 0: + obj = obj[None, None] + + if obj.ndim == 2: + if ( + one_col_2d_as_series + and obj.shape[1] == 1 + and output_type != "dataframe" + ): + host_df = pd.Series(obj.flatten(), index=index) + else: + host_df = pd.DataFrame(obj, index=index) + else: + host_df = pd.Series(obj, index=index) + + try: + return cudf.from_pandas(host_df) + except (TypeError, ValueError, NotImplementedError): + if isinstance(host_df, pd.DataFrame): + # cudf cannot represent every mixed object DataFrame. + # Preserve the host DataFrame instead of losing its data. + return host_df + raise else: # Other output types use device memory, coerce to cupy and take # cupy code path. diff --git a/python/cuml/cuml/thirdparty_adapters/adapters.py b/python/cuml/cuml/thirdparty_adapters/adapters.py index c799301495..a8ce02287b 100644 --- a/python/cuml/cuml/thirdparty_adapters/adapters.py +++ b/python/cuml/cuml/thirdparty_adapters/adapters.py @@ -1,15 +1,34 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # import cupy as cp import numpy as np +def _is_na_sentinel(value): + """Return whether a scalar missing-value sentinel is NA-like.""" + if isinstance(value, str): + return value == "NaN" + return pd.isna(value) + + def _get_mask(X, value_to_mask): """Compute the boolean mask X == missing_values.""" - if value_to_mask == "NaN" or cp.isnan(value_to_mask): - return cp.isnan(X) + if isinstance(value_to_mask, str) and value_to_mask == "NaN": + if isinstance(X, cp.ndarray): + return cp.isnan(X) + return pd.isna(X) + # NaN-like sentinels (np.nan, None, pd.NA, pd.NaT, ...) require an + # NA-aware comparison: a plain `==` against e.g. pd.NA propagates + # instead of returning a boolean mask, and `cp.isnan` doesn't accept + # non-numeric scalars like pd.NA in the first place. + if _is_na_sentinel(value_to_mask): + if isinstance(X, cp.ndarray): + return cp.isnan(X) + # Host (e.g. object dtype) arrays can't use isnan - fall back to an + # NA-aware elementwise check that also recognizes None/pd.NA. + return pd.isna(X) else: return X == value_to_mask @@ -20,7 +39,7 @@ def _masked_column_median(arr, masked_value): mask = _get_mask(arr, masked_value) if arr.size == 0: return cp.full(arr.shape[1], cp.nan) - if not cp.isnan(masked_value): + if not _is_na_sentinel(masked_value): arr_sorted = arr.copy() # If nan is not the missing value, any column with nans should # have a median of nan @@ -59,7 +78,7 @@ def _masked_column_mean(arr, masked_value): count_missing_values = mask.sum(axis=0) n_elems = arr.shape[0] - count_missing_values mean = cp.nansum(arr, axis=0) - if not cp.isnan(masked_value): + if not _is_na_sentinel(masked_value): mean -= count_missing_values * masked_value mean /= n_elems return mean diff --git a/python/cuml/tests/test_adapters.py b/python/cuml/tests/test_adapters.py index b59542c473..5442bf6f14 100644 --- a/python/cuml/tests/test_adapters.py +++ b/python/cuml/tests/test_adapters.py @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -8,6 +8,7 @@ import cupy as cp import cupyx as cpx import numpy as np +import pandas as pd import pytest from scipy import stats from sklearn.utils._mask import _get_mask as sk_get_mask @@ -110,6 +111,30 @@ def test_get_mask(failure_logger, mask_dataset): assert_allclose(cu_mask, sk_mask) +def test_get_mask_nan_string_on_host_object_array(): + X = np.array([1.0, np.nan, "present"], dtype=object) + + np.testing.assert_array_equal( + cu_get_mask(X, value_to_mask="NaN"), [False, True, False] + ) + + +@pytest.mark.parametrize( + ("function", "expected"), + [ + (_masked_column_mean, [3.0, 5.0]), + (_masked_column_median, [3.0, 5.0]), + ], +) +@pytest.mark.parametrize("missing_value", [pd.NA, "NaN"]) +def test_masked_column_numeric_na_sentinel(function, expected, missing_value): + X = cp.array([[1.0, cp.nan], [3.0, 4.0], [5.0, 6.0]]) + + result = function(X, missing_value) + + np.testing.assert_allclose(result.get(), expected) + + def test_masked_column_median(failure_logger, mask_dataset): mask_value, X_np, X = mask_dataset median = _masked_column_median(X, mask_value).get() diff --git a/python/cuml/tests/test_compose.py b/python/cuml/tests/test_compose.py index 665ad12b8a..36f98cda4e 100644 --- a/python/cuml/tests/test_compose.py +++ b/python/cuml/tests/test_compose.py @@ -383,3 +383,130 @@ def test_column_transform_properly_handles_sub_output_type(): ] ).fit(df) transformer.transform(df) + + +def test_column_transformer_simple_imputer_categorical_cudf(): + """Regression test for https://github.com/rapidsai/cuml/issues/6183 + + ColumnTransformer + SimpleImputer on a native cuDF DataFrame with + categorical/string columns used to raise (KeyError / AttributeError on + ``.dtype``, later ``ValueError: Unsupported dtype object`` after + unrelated refactors). impute-then-transform is one of the most common + sklearn pipeline shapes, so this must work end to end. + """ + df = cudf.DataFrame( + { + "num1": [1.0, np.nan, 3.0], + "num2": [4.0, 5.0, np.nan], + "cat1": ["a", None, "c"], + "cat2": ["x", "y", None], + } + ) + df_np = df.to_pandas() + + num_cols = ["num1", "num2"] + cat_cols = ["cat1"] + mode_cols = ["cat2"] + + cu_transformer = cuColumnTransformer( + transformers=[ + ( + "num", + cuSimpleImputer(strategy="constant", fill_value=0), + num_cols, + ), + ( + "cat", + cuSimpleImputer( + strategy="constant", + fill_value="missing", + missing_values=pd.NA, + ), + cat_cols, + ), + ( + "mod", + cuSimpleImputer( + strategy="most_frequent", missing_values=pd.NA + ), + mode_cols, + ), + ] + ) + cu_result = cu_transformer.fit_transform(df) + + sk_transformer = skColumnTransformer( + transformers=[ + ( + "num", + skSimpleImputer(strategy="constant", fill_value=0), + num_cols, + ), + ( + "cat", + skSimpleImputer( + strategy="constant", + fill_value="missing", + missing_values=pd.NA, + ), + cat_cols, + ), + ( + "mod", + skSimpleImputer( + strategy="most_frequent", missing_values=pd.NA + ), + mode_cols, + ), + ] + ) + sk_result = sk_transformer.fit_transform(df_np) + + np.testing.assert_array_equal(np.asarray(cu_result), sk_result) + + +def test_simple_imputer_add_indicator_object_cudf(): + """Regression: SimpleImputer(add_indicator=True) on string/object columns + must stack the imputed (host object) data with the indicator mask on host + instead of routing the host array through cupy.hstack (issue #6183 follow-up). + """ + df = cudf.DataFrame( + { + "cat1": ["a", None, "c", "a"], + "cat2": ["x", "y", None, "x"], + } + ) + df_np = df.to_pandas() + cu_imp = cuSimpleImputer( + strategy="most_frequent", missing_values=pd.NA, add_indicator=True + ) + sk_imp = skSimpleImputer( + strategy="most_frequent", missing_values=pd.NA, add_indicator=True + ) + + cu_result = cu_imp.fit_transform(df) + sk_result = sk_imp.fit_transform(df_np) + + np.testing.assert_array_equal(np.asarray(cu_result), np.asarray(sk_result)) + + +def test_simple_imputer_add_indicator_clone_params(): + imputer = cuSimpleImputer(add_indicator=True, missing_values=pd.NA) + + cloned = sk_clone(imputer) + + params = cloned.get_params() + assert params["add_indicator"] is True + assert params["missing_values"] is pd.NA + + +def test_is_object_dtype_handles_series_and_extension_dtypes(): + from cuml.internals.outputs import _is_object_dtype + + assert _is_object_dtype(pd.Series(["a", "b"])) is True + assert _is_object_dtype(pd.Series([1, 2, 3])) is False + assert _is_object_dtype(pd.Series(pd.Categorical(["a", "b"]))) is False + assert _is_object_dtype(pd.Series(["a"], dtype="string")) is False + assert _is_object_dtype(pd.DataFrame({"a": ["x"], "b": [1]})) is True + assert _is_object_dtype(np.array(["a", "b"], dtype=object)) is True + assert _is_object_dtype(np.array([1, 2, 3])) is False diff --git a/python/cuml/tests/test_reflection.py b/python/cuml/tests/test_reflection.py index 84725699f2..c74fb8949d 100644 --- a/python/cuml/tests/test_reflection.py +++ b/python/cuml/tests/test_reflection.py @@ -393,6 +393,72 @@ def test_convert_arrays_dataframe_with_index( cudf.testing.assert_series_equal(cudf.Series(res), sol) +@pytest.mark.parametrize( + ("output_type", "expected_type"), + [ + ("pandas", pd.DataFrame), + ("cudf", cudf.DataFrame), + ], +) +def test_convert_arrays_object_array_dataframe_output_with_index( + output_type, expected_type +): + arr = np.array([["a", "x"], ["b", "y"]], dtype=object) + index = pd.Index(["first", "second"]) + + result = convert_arrays(arr, output_type, index=index) + + assert isinstance(result, expected_type) + cudf.testing.assert_frame_equal( + cudf.DataFrame(result), + cudf.from_pandas(pd.DataFrame(arr, index=index)), + ) + + +@pytest.mark.parametrize( + ("output_type", "expected_type"), + [ + ("series", cudf.Series), + ("dataframe", cudf.DataFrame), + ], +) +def test_convert_arrays_object_array_explicit_dataframe_outputs( + output_type, expected_type +): + arr = np.array(["a", "b"], dtype=object) + index = pd.Index(["first", "second"]) + + result = convert_arrays(arr, output_type, index=index) + + assert isinstance(result, expected_type) + if output_type == "series": + cudf.testing.assert_series_equal( + result, cudf.from_pandas(pd.Series(arr, index=index)) + ) + else: + cudf.testing.assert_frame_equal( + result, cudf.from_pandas(pd.DataFrame(arr, index=index)) + ) + + +@pytest.mark.parametrize("output_type", ["cupy", "numba"]) +def test_convert_arrays_object_array_device_output_error(output_type): + arr = np.array(["a", "b"], dtype=object) + + with pytest.raises( + TypeError, + match=f"output_type={output_type!r} doesn't support outputs of dtype", + ): + convert_arrays(arr, output_type) + + +@pytest.mark.parametrize("output_type", ["array", "cuml"]) +def test_convert_arrays_object_array_array_output(output_type): + arr = np.array(["a", "b"], dtype=object) + + assert convert_arrays(arr, output_type) is arr + + @pytest.mark.parametrize( "construct", [ From 71e0f94ed29afed99c5543d407f6e7c015dffbe4 Mon Sep 17 00:00:00 2001 From: nethum529 Date: Sun, 30 Aug 2026 18:22:37 -0500 Subject: [PATCH 2/2] fix(compose): address review feedback on object dtype support Answers the review comments on #8317. - Give pd.NA, None and np.nan distinct sentinel masks, instead of letting each one select every NA-like value. - Store NaN when a column has no observed values, so SimpleImputer drops it the same way scikit-learn does. - Keep the host fallback inside the imputer path. check_array no longer downgrades an explicit mem_type="device" request. - Raise a clear TypeError when an object layout cannot be produced in the requested output_type, instead of returning a different type. Also recognise the pandas 3 str dtype, the pandas StringDtype and Arrow string dtypes in the object check, dispatch the imputer between host and device arrays, add ReflectedAttr.get_raw for internal reads of a stored value, and copy a read-only object buffer before imputing in place. The three sentinels cannot stay distinct once pandas materialises a column on the host, because pandas stores pd.NA and None as NaN there. The imputer therefore follows scikit-learn: pd.NA imputes and None raises. Raw numpy object arrays keep the distinct masks. Signed-off-by: nethum529 --- .../preprocessing/_column_transformer.py | 6 +- .../sklearn/preprocessing/_imputation.py | 67 +++++++--- python/cuml/cuml/internals/outputs.py | 82 ++++++------ .../cuml/cuml/thirdparty_adapters/adapters.py | 72 ++++++----- python/cuml/tests/test_adapters.py | 24 +++- python/cuml/tests/test_compose.py | 117 +++++++++++++++++- python/cuml/tests/test_reflection.py | 50 ++++---- 7 files changed, 296 insertions(+), 122 deletions(-) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py index 8976a19735..7be6d321a6 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py @@ -301,7 +301,8 @@ def _list_indexing(X, key, key_dtype): def _transform_one(transformer, X, y, weight, **fit_params): - with cuml.using_output_type("cupy"): + output_type = "numpy" if _is_object_dtype(X) else "cupy" + with cuml.using_output_type(output_type): res = transformer.transform(X) # if we have a weight for this transformer, multiply output @@ -323,7 +324,8 @@ def _fit_transform_one(transformer, be multiplied by ``weight``. """ with _print_elapsed_time(message_clsname, message): - with cuml.using_output_type("cupy"): + output_type = "numpy" if _is_object_dtype(X) else "cupy" + with cuml.using_output_type(output_type): transformer.accept_sparse = True if hasattr(transformer, 'fit_transform'): res = transformer.fit_transform(X, y, **fit_params) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py index 32005b054d..cd12440120 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py @@ -20,6 +20,7 @@ import cupy as np import numpy as cpu_np +import pandas as pd from cupyx.scipy import sparse import cuml @@ -29,11 +30,11 @@ StringInputTagMixin, _ensure_transformer_tags, ) -from cuml.internals.outputs import _is_object_dtype, mlfunc, ReflectedAttr +from cuml.internals.outputs import ReflectedAttr, _is_object_dtype, mlfunc from cuml.internals.validation import ( - check_is_fitted, - check_inputs, check_input_features, + check_inputs, + check_is_fitted, ) from ....thirdparty_adapters import ( @@ -273,10 +274,12 @@ def __init__(self, *, missing_values=np.nan, strategy="mean", @classmethod def _get_param_names(cls): return super()._get_param_names() + [ + "missing_values", "strategy", "fill_value", "verbose", - "copy" + "copy", + "add_indicator", ] def _validate_input(self, X, in_fit): @@ -297,6 +300,7 @@ def _validate_input(self, X, in_fit): ensure_all_finite = "allow-nan" try: + mem_type = "host" if _is_object_dtype(X) else "device" X = check_inputs( self, X, @@ -304,6 +308,7 @@ def _validate_input(self, X, in_fit): dtype=dtype, ensure_all_finite=ensure_all_finite, copy=self.copy, + mem_type=mem_type, reset=in_fit, ) except ValueError as ve: @@ -314,6 +319,18 @@ def _validate_input(self, X, in_fit): else: raise ve + # Object nulls reach this host path as NaN. Restore pd.NA for its + # identity-based path, but reject NaN when None was requested, + # matching scikit-learn's validation semantics. + if _is_object_dtype(X): + if hasattr(X, "flags") and not X.flags.writeable: + X = X.copy() + nan_mask = _get_mask(X, np.nan) + if self.missing_values is pd.NA: + X[nan_mask] = pd.NA + elif self.missing_values is None and nan_mask.any(): + raise ValueError("Input contains NaN") + _check_inputs_dtype(X, self.missing_values) if X.dtype.kind not in ("i", "u", "f", "O"): raise ValueError("SimpleImputer does not support data with dtype " @@ -436,7 +453,8 @@ def _dense_fit(self, X, strategy, missing_values, fill_value): # Constant elif strategy == "constant": - return np.full(X.shape[1], fill_value, dtype=X.dtype) + xp = np.get_array_module(X) + return xp.full(X.shape[1], fill_value, dtype=X.dtype) @mlfunc def transform(self, X): @@ -452,24 +470,27 @@ def transform(self, X): X = self._validate_input(X, in_fit=False) X_indicator = super()._transform_indicator(X) - statistics = self.statistics_ + # Use the stored value for internal computation. ColumnTransformer + # may request cupy output, which cannot reflect object statistics. + statistics = type(self).statistics_.get_raw(self) if X.shape[1] != statistics.shape[0]: raise ValueError("X has %d features per sample, expected %d" - % (X.shape[1], self.statistics_.shape[0])) + % (X.shape[1], statistics.shape[0])) # Delete the invalid columns if strategy is not constant if self.strategy == "constant": valid_statistics = statistics else: + xp = np.get_array_module(statistics) # same as np.isnan but also works for object dtypes invalid_mask = _get_mask(statistics, np.nan) - valid_mask = np.logical_not(invalid_mask) + valid_mask = xp.logical_not(invalid_mask) valid_statistics = statistics[valid_mask] - valid_statistics_indexes = np.flatnonzero(valid_mask) + valid_statistics_indexes = xp.flatnonzero(valid_mask) if invalid_mask.any(): - missing = np.arange(X.shape[1])[invalid_mask] + missing = xp.arange(X.shape[1])[invalid_mask] if self.verbose: warnings.warn("Deleting features without " "observed values: %s" % missing) @@ -494,9 +515,10 @@ def transform(self, X): if self.strategy == "constant": X[mask] = valid_statistics[0] else: - for i, vi in enumerate(valid_statistics_indexes): - feature_idxs = np.flatnonzero(mask[:, vi]) - X[feature_idxs, vi] = valid_statistics[i] + xp = np.get_array_module(mask) + for i in range(valid_statistics.shape[0]): + feature_idxs = xp.flatnonzero(mask[:, i]) + X[feature_idxs, i] = valid_statistics[i] X = super()._concatenate_indicator(X, X_indicator) return X @@ -517,7 +539,11 @@ def get_feature_names_out(self, input_features=None): """ check_is_fitted(self) input_features = check_input_features(self, input_features) - non_missing_mask = np.logical_not(_get_mask(self.statistics_, np.nan)).get() + statistics = type(self).statistics_.get_raw(self) + xp = np.get_array_module(statistics) + non_missing_mask = xp.logical_not(_get_mask(statistics, np.nan)) + if isinstance(non_missing_mask, np.ndarray): + non_missing_mask = non_missing_mask.get() names = input_features[non_missing_mask] if self.add_indicator: indicator_names = self.indicator_.get_feature_names_out(input_features) @@ -666,9 +692,11 @@ def _get_missing_features_info(self, X): imputer_mask = sparse.csc_matrix(imputer_mask) if self.features == 'all': - features_indices = np.arange(X.shape[1]) + xp = np.get_array_module(imputer_mask) + features_indices = xp.arange(X.shape[1]) else: - features_indices = np.flatnonzero(n_missing) + xp = np.get_array_module(n_missing) + features_indices = xp.flatnonzero(n_missing) return imputer_mask, features_indices @@ -677,11 +705,13 @@ def _validate_input(self, X, in_fit): ensure_all_finite = True else: ensure_all_finite = "allow-nan" + mem_type = "host" if _is_object_dtype(X) else "device" X = check_inputs( self, X, accept_sparse=('csc', 'csr'), ensure_all_finite=ensure_all_finite, + mem_type=mem_type, reset=in_fit, ) _check_inputs_dtype(X, self.missing_values) @@ -811,10 +841,13 @@ def get_feature_names_out(self, input_features=None): check_is_fitted(self) input_features = check_input_features(self, input_features) prefix = self.__class__.__name__.lower() + features = type(self).features_.get_raw(self) + if isinstance(features, np.ndarray): + features = features.get() return cpu_np.asarray( [ f"{prefix}_{feature_name}" - for feature_name in input_features[self.features_.get()] + for feature_name in input_features[features] ], dtype=object, ) diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index 821a8dd48d..44066e9d08 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -285,21 +285,24 @@ def infer_output_type(array, array_like="numpy"): def _is_object_dtype(res): - """Check for NumPy object dtype on an array-like or dataframe-like.""" + """Check for object or string dtype on an array/dataframe-like.""" + + def is_object_or_string(dtype): + try: + if np.dtype(dtype).kind == "O": + return True + except TypeError: + pass + return pd.api.types.is_string_dtype(dtype) + dtype = getattr(res, "dtype", None) if dtype is not None: - # Array-like or Series: a single (possibly extension) dtype. Only a - # plain numpy object dtype has no device representation; extension - # dtypes are a separate, unsupported case handled elsewhere. - return isinstance(dtype, np.dtype) and dtype.kind == "O" + return is_object_or_string(dtype) dtypes = getattr(res, "dtypes", None) if dtypes is not None: - # DataFrame-like: dtypes is per-column. try: - return any( - isinstance(dt, np.dtype) and dt.kind == "O" for dt in dtypes - ) + return any(is_object_or_string(dt) for dt in dtypes) except TypeError: return False @@ -545,11 +548,11 @@ def convert_arrays( return pd.Series(obj, index=index) elif _is_object_dtype(obj): # NumPy object arrays have no device representation. Preserve - # host arrays for "array" and internal "cuml" outputs, or wrap - # them in a dataframe-like type. - if output_type in ("array", "cuml"): + # host arrays for internal outputs, or wrap them in a + # dataframe-like type. + if output_type == "cuml": return obj - elif output_type in ("cupy", "numba"): + elif output_type == "cupy": raise TypeError( f"{output_type=!r} doesn't support outputs of dtype " f"object and shape {obj.shape}" @@ -557,43 +560,26 @@ def convert_arrays( if hasattr(index, "to_pandas"): index = index.to_pandas() - if output_type == "series": - if obj.ndim == 2: - if obj.shape[1] == 1: - obj = obj.flatten() - else: - raise ValueError( - "Only single dimensional arrays can be transformed to" - " Series." - ) - elif obj.ndim == 0: - obj = obj[None] - elif output_type == "dataframe": - if obj.ndim == 1: - obj = obj[:, None] - elif obj.ndim == 0: - obj = obj[None, None] if obj.ndim == 2: - if ( - one_col_2d_as_series - and obj.shape[1] == 1 - and output_type != "dataframe" - ): - host_df = pd.Series(obj.flatten(), index=index) + if one_col_2d_as_series and obj.shape[1] == 1: + host_df = pd.Series( + obj.flatten(), index=index + ).infer_objects() else: - host_df = pd.DataFrame(obj, index=index) + host_df = pd.DataFrame(obj, index=index).infer_objects() else: - host_df = pd.Series(obj, index=index) + host_df = pd.Series(obj, index=index).infer_objects() try: return cudf.from_pandas(host_df) - except (TypeError, ValueError, NotImplementedError): - if isinstance(host_df, pd.DataFrame): - # cudf cannot represent every mixed object DataFrame. - # Preserve the host DataFrame instead of losing its data. - return host_df - raise + except (TypeError, ValueError, NotImplementedError) as exc: + raise TypeError( + "Cannot convert an object-dtype output with shape " + f"{obj.shape} to output_type='cudf'. Use " + "output_type='pandas' or output_type='numpy' for this " + "object layout." + ) from exc else: # Other output types use device memory, coerce to cupy and take # cupy code path. @@ -712,6 +698,16 @@ def __reduce__(self): def __set_name__(self, owner, name): self.name = name + def get_raw(self, instance): + """Return the value so internal work can dispatch on its stored type.""" + cache = instance.__dict__.get(self.name) + if cache is None: + raise AttributeError( + f"{type(instance).__name__!r} object has no attribute " + f"{self.name!r}" + ) + return cache.value + def __get__(self, instance, owner): if instance is None: return self diff --git a/python/cuml/cuml/thirdparty_adapters/adapters.py b/python/cuml/cuml/thirdparty_adapters/adapters.py index a8ce02287b..fc2d4375bb 100644 --- a/python/cuml/cuml/thirdparty_adapters/adapters.py +++ b/python/cuml/cuml/thirdparty_adapters/adapters.py @@ -4,33 +4,40 @@ # import cupy as cp import numpy as np +import pandas as pd -def _is_na_sentinel(value): - """Return whether a scalar missing-value sentinel is NA-like.""" - if isinstance(value, str): - return value == "NaN" - return pd.isna(value) +def _is_nan(value): + """Return whether a scalar is a floating-point NaN.""" + return isinstance(value, (float, np.floating)) and np.isnan(value) + + +def _is_nan_sentinel(value): + """Return whether a scalar selects floating-point NaN values.""" + return (isinstance(value, str) and value == "NaN") or _is_nan(value) + + +def _get_host_mask(X, predicate): + return np.fromiter( + (predicate(value) for value in X.flat), dtype=bool, count=X.size + ).reshape(X.shape) def _get_mask(X, value_to_mask): """Compute the boolean mask X == missing_values.""" - if isinstance(value_to_mask, str) and value_to_mask == "NaN": + if value_to_mask is pd.NA: if isinstance(X, cp.ndarray): - return cp.isnan(X) - return pd.isna(X) - # NaN-like sentinels (np.nan, None, pd.NA, pd.NaT, ...) require an - # NA-aware comparison: a plain `==` against e.g. pd.NA propagates - # instead of returning a boolean mask, and `cp.isnan` doesn't accept - # non-numeric scalars like pd.NA in the first place. - if _is_na_sentinel(value_to_mask): + return cp.zeros(X.shape, dtype=bool) + return _get_host_mask(X, lambda value: value is pd.NA) + if value_to_mask is None: + if isinstance(X, cp.ndarray): + return cp.zeros(X.shape, dtype=bool) + return _get_host_mask(X, lambda value: value is None) + if _is_nan_sentinel(value_to_mask): if isinstance(X, cp.ndarray): return cp.isnan(X) - # Host (e.g. object dtype) arrays can't use isnan - fall back to an - # NA-aware elementwise check that also recognizes None/pd.NA. - return pd.isna(X) - else: - return X == value_to_mask + return _get_host_mask(X, _is_nan) + return X == value_to_mask def _masked_column_median(arr, masked_value): @@ -39,7 +46,7 @@ def _masked_column_median(arr, masked_value): mask = _get_mask(arr, masked_value) if arr.size == 0: return cp.full(arr.shape[1], cp.nan) - if not _is_na_sentinel(masked_value): + if not _is_nan_sentinel(masked_value): arr_sorted = arr.copy() # If nan is not the missing value, any column with nans should # have a median of nan @@ -78,7 +85,7 @@ def _masked_column_mean(arr, masked_value): count_missing_values = mask.sum(axis=0) n_elems = arr.shape[0] - count_missing_values mean = cp.nansum(arr, axis=0) - if not _is_na_sentinel(masked_value): + if not _is_nan_sentinel(masked_value): mean -= count_missing_values * masked_value mean /= n_elems return mean @@ -87,18 +94,27 @@ def _masked_column_mean(arr, masked_value): def _masked_column_mode(arr, masked_value): """Determine the most frequently appearing element in each column in the 2D array arr, ignoring any instances of masked_value""" + xp = cp.get_array_module(arr) mask = _get_mask(arr, masked_value) + if arr.dtype.kind == "O": + na_mask = pd.isna(arr) + # Never pass NA-like values into object sorting. The strict sentinel + # mask is still used later to select values for imputation. + count_mask = xp.logical_or(mask, na_mask) + else: + count_mask = mask n_features = arr.shape[1] - most_frequent = np.empty(n_features, dtype=arr.dtype) + result_dtype = object if arr.dtype.kind == "O" else arr.dtype + most_frequent = np.empty(n_features, dtype=result_dtype) for i in range(n_features): - feature_mask_idxs = cp.where(~mask[:, i])[0] - values, counts = cp.unique( + feature_mask_idxs = xp.where(~count_mask[:, i])[0] + values, counts = xp.unique( arr[feature_mask_idxs, i], return_counts=True ) - count_max = counts.max() - if count_max > 0: - value = values[counts == count_max].min() + if counts.size == 0: + value = xp.nan else: - value = cp.nan + count_max = counts.max() + value = values[counts == count_max].min() most_frequent[i] = value - return cp.array(most_frequent) + return xp.array(most_frequent) diff --git a/python/cuml/tests/test_adapters.py b/python/cuml/tests/test_adapters.py index 5442bf6f14..14fba82ebe 100644 --- a/python/cuml/tests/test_adapters.py +++ b/python/cuml/tests/test_adapters.py @@ -119,6 +119,20 @@ def test_get_mask_nan_string_on_host_object_array(): ) +def test_get_mask_distinguishes_missing_sentinels(): + X = np.array([None, np.nan, pd.NA, "present"], dtype=object) + masks = { + None: cu_get_mask(X, None), + "nan": cu_get_mask(X, np.nan), + "pd.NA": cu_get_mask(X, pd.NA), + } + + np.testing.assert_array_equal(masks[None], [True, False, False, False]) + np.testing.assert_array_equal(masks["nan"], [False, True, False, False]) + np.testing.assert_array_equal(masks["pd.NA"], [False, False, True, False]) + np.testing.assert_array_equal(sum(masks.values()), [1, 1, 1, 0]) + + @pytest.mark.parametrize( ("function", "expected"), [ @@ -126,7 +140,7 @@ def test_get_mask_nan_string_on_host_object_array(): (_masked_column_median, [3.0, 5.0]), ], ) -@pytest.mark.parametrize("missing_value", [pd.NA, "NaN"]) +@pytest.mark.parametrize("missing_value", [np.nan, "NaN"]) def test_masked_column_numeric_na_sentinel(function, expected, missing_value): X = cp.array([[1.0, cp.nan], [3.0, 4.0], [5.0, 6.0]]) @@ -166,3 +180,11 @@ def test_masked_column_mode(failure_logger, mask_dataset): column_mask = mask[:, i] column_mode = stats.mode(X_np[:, i][column_mask], keepdims=True)[0][0] assert column_mode == mode[i] + + +def test_masked_column_mode_numeric_nan_regression_guard(): + X = np.array([[0.0], [np.nan], [np.nan], [1.0]]) + + result = _masked_column_mode(X, 0) + + assert np.isnan(result[0]) diff --git a/python/cuml/tests/test_compose.py b/python/cuml/tests/test_compose.py index 36f98cda4e..2a063c93e4 100644 --- a/python/cuml/tests/test_compose.py +++ b/python/cuml/tests/test_compose.py @@ -13,6 +13,7 @@ from sklearn.compose import ( make_column_transformer as sk_make_column_transformer, ) +from sklearn.impute import SimpleImputer as skSimpleImputer from sklearn.preprocessing import Normalizer as skNormalizer from sklearn.preprocessing import OneHotEncoder as skOneHotEncoder from sklearn.preprocessing import PolynomialFeatures as skPolynomialFeatures @@ -24,6 +25,7 @@ from cuml.preprocessing import Normalizer as cuNormalizer from cuml.preprocessing import OneHotEncoder as cuOneHotEncoder from cuml.preprocessing import PolynomialFeatures as cuPolynomialFeatures +from cuml.preprocessing import SimpleImputer as cuSimpleImputer from cuml.preprocessing import StandardScaler as cuStandardScaler from cuml.testing.test_preproc_utils import ( # noqa: F401 assert_allclose, @@ -462,7 +464,7 @@ def test_column_transformer_simple_imputer_categorical_cudf(): ) sk_result = sk_transformer.fit_transform(df_np) - np.testing.assert_array_equal(np.asarray(cu_result), sk_result) + np.testing.assert_array_equal(cu_result.to_numpy(), sk_result) def test_simple_imputer_add_indicator_object_cudf(): @@ -487,7 +489,25 @@ def test_simple_imputer_add_indicator_object_cudf(): cu_result = cu_imp.fit_transform(df) sk_result = sk_imp.fit_transform(df_np) - np.testing.assert_array_equal(np.asarray(cu_result), np.asarray(sk_result)) + np.testing.assert_array_equal(cu_result.to_numpy(), np.asarray(sk_result)) + + +def test_simple_imputer_drops_fully_missing_object_column_cudf(): + df = cudf.DataFrame( + { + "all_missing": [None, None, None], + "observed": ["a", "b", "a"], + } + ) + df_np = df.to_pandas() + cu_imp = cuSimpleImputer(strategy="most_frequent", missing_values=pd.NA) + sk_imp = skSimpleImputer(strategy="most_frequent", missing_values=pd.NA) + + cu_result = cu_imp.fit_transform(df) + sk_result = sk_imp.fit_transform(df_np) + + assert cu_result.shape == (3, 1) + np.testing.assert_array_equal(cu_result.to_numpy(), sk_result) def test_simple_imputer_add_indicator_clone_params(): @@ -500,13 +520,104 @@ def test_simple_imputer_add_indicator_clone_params(): assert params["missing_values"] is pd.NA +@pytest.mark.parametrize("input_type", ["pandas", "numpy"]) +@pytest.mark.parametrize("missing_values", [pd.NA, None, np.nan]) +def test_simple_imputer_missing_sentinel_parity(input_type, missing_values): + if input_type == "pandas": + X = pd.DataFrame({"cat": ["x", missing_values, "x", "y"]}) + else: + X = np.array([["x"], [np.nan], ["x"], ["y"]], dtype=object) + + cu_imputer = cuSimpleImputer( + strategy="most_frequent", missing_values=missing_values + ) + sk_imputer = skSimpleImputer( + strategy="most_frequent", missing_values=missing_values + ) + + if missing_values is None: + with pytest.raises(ValueError) as sk_error: + sk_imputer.fit_transform(X) + with pytest.raises(ValueError) as cu_error: + cu_imputer.fit_transform(X) + + assert str(cu_error.value) == str(sk_error.value) + else: + cu_result = cu_imputer.fit_transform(X) + sk_result = sk_imputer.fit_transform(X) + + np.testing.assert_array_equal(np.asarray(cu_result), sk_result) + + +@pytest.mark.parametrize("input_type", ["pandas", "readonly-numpy"]) +@pytest.mark.parametrize("copy", [False, True]) +@pytest.mark.parametrize("missing_values", [pd.NA, np.nan]) +def test_simple_imputer_copy_object_parity(input_type, copy, missing_values): + def make_input(): + if input_type == "pandas": + return pd.DataFrame( + { + "cat": pd.Series( + ["x", missing_values, "x", "y"], dtype=object + ) + } + ) + + X = np.array([["x"], [np.nan], ["x"], ["y"]], dtype=object) + X.flags.writeable = False + return X + + cu_imputer = cuSimpleImputer( + strategy="most_frequent", missing_values=missing_values, copy=copy + ) + sk_imputer = skSimpleImputer( + strategy="most_frequent", missing_values=missing_values, copy=copy + ) + + cu_result = cu_imputer.fit_transform(make_input()) + sk_result = sk_imputer.fit_transform(make_input()) + + np.testing.assert_array_equal(np.asarray(cu_result), sk_result) + + +@pytest.mark.parametrize("add_indicator", [False, True]) +def test_simple_imputer_string_get_feature_names_out(add_indicator): + df = pd.DataFrame( + { + "cat": pd.Series(["x", pd.NA, "x"], dtype=object), + "other": pd.Series(["a", "b", "a"], dtype=object), + } + ) + cu_imputer = cuSimpleImputer( + strategy="most_frequent", + missing_values=pd.NA, + add_indicator=add_indicator, + ).fit(df) + sk_imputer = skSimpleImputer( + strategy="most_frequent", + missing_values=pd.NA, + add_indicator=add_indicator, + ).fit(df) + + np.testing.assert_array_equal( + cu_imputer.get_feature_names_out(), + sk_imputer.get_feature_names_out(), + ) + + def test_is_object_dtype_handles_series_and_extension_dtypes(): from cuml.internals.outputs import _is_object_dtype + pa = pytest.importorskip("pyarrow") + assert _is_object_dtype(pd.Series(["a", "b"])) is True assert _is_object_dtype(pd.Series([1, 2, 3])) is False assert _is_object_dtype(pd.Series(pd.Categorical(["a", "b"]))) is False - assert _is_object_dtype(pd.Series(["a"], dtype="string")) is False + assert _is_object_dtype(pd.Series(["a"], dtype="string")) is True + assert ( + _is_object_dtype(pd.Series(["a"], dtype=pd.ArrowDtype(pa.string()))) + is True + ) assert _is_object_dtype(pd.DataFrame({"a": ["x"], "b": [1]})) is True assert _is_object_dtype(np.array(["a", "b"], dtype=object)) is True assert _is_object_dtype(np.array([1, 2, 3])) is False diff --git a/python/cuml/tests/test_reflection.py b/python/cuml/tests/test_reflection.py index c74fb8949d..c53948c520 100644 --- a/python/cuml/tests/test_reflection.py +++ b/python/cuml/tests/test_reflection.py @@ -113,6 +113,16 @@ def check_descriptor(self): assert_output_type(self.X_, "numpy") +def test_reflected_attr_get_raw(): + estimator = DummyEstimator() + value = cp.arange(3) + estimator.X_ = value + + with cuml.using_output_type("numpy"): + assert isinstance(estimator.X_, np.ndarray) + assert DummyEstimator.X_.get_raw(estimator) is value + + @mlfunc def returns_cupy(X): return cp.asarray(X) @@ -415,33 +425,7 @@ def test_convert_arrays_object_array_dataframe_output_with_index( ) -@pytest.mark.parametrize( - ("output_type", "expected_type"), - [ - ("series", cudf.Series), - ("dataframe", cudf.DataFrame), - ], -) -def test_convert_arrays_object_array_explicit_dataframe_outputs( - output_type, expected_type -): - arr = np.array(["a", "b"], dtype=object) - index = pd.Index(["first", "second"]) - - result = convert_arrays(arr, output_type, index=index) - - assert isinstance(result, expected_type) - if output_type == "series": - cudf.testing.assert_series_equal( - result, cudf.from_pandas(pd.Series(arr, index=index)) - ) - else: - cudf.testing.assert_frame_equal( - result, cudf.from_pandas(pd.DataFrame(arr, index=index)) - ) - - -@pytest.mark.parametrize("output_type", ["cupy", "numba"]) +@pytest.mark.parametrize("output_type", ["cupy"]) def test_convert_arrays_object_array_device_output_error(output_type): arr = np.array(["a", "b"], dtype=object) @@ -452,13 +436,23 @@ def test_convert_arrays_object_array_device_output_error(output_type): convert_arrays(arr, output_type) -@pytest.mark.parametrize("output_type", ["array", "cuml"]) +@pytest.mark.parametrize("output_type", ["cuml"]) def test_convert_arrays_object_array_array_output(output_type): arr = np.array(["a", "b"], dtype=object) assert convert_arrays(arr, output_type) is arr +def test_convert_arrays_unsupported_object_layout_cudf_error(): + arr = np.array([[object()], [object()]], dtype=object) + + with pytest.raises( + TypeError, + match="Use output_type='pandas' or output_type='numpy'", + ): + convert_arrays(arr, "cudf") + + @pytest.mark.parametrize( "construct", [