From 7b6eef3e526ab7239a3727ca7aff7c0122351c66 Mon Sep 17 00:00:00 2001 From: Arthur Lacote Date: Fri, 7 Aug 2026 15:53:20 +0200 Subject: [PATCH 1/2] transform cache --- sklearn/preprocessing/_encoders.py | 9 ++++++- sklearn/utils/_encode.py | 39 ++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/sklearn/preprocessing/_encoders.py b/sklearn/preprocessing/_encoders.py index 60d8a91aaa1a0..0188d5661970e 100644 --- a/sklearn/preprocessing/_encoders.py +++ b/sklearn/preprocessing/_encoders.py @@ -187,6 +187,11 @@ def _fit( self.categories_.append(cats) + # Lazily populated by `_transform` (not here) with, per feature. + # This is used to make repeated small-batch `transform` calls + # (e.g. online prediction pipeline) fast. + self._transform_cache = [{} for _ in range(n_features)] + output = {"n_samples": n_samples} if return_counts: output["category_counts"] = category_counts @@ -223,7 +228,9 @@ def _transform( columns_with_unknown = [] for i in range(n_features): Xi = X_list[i] - X_int[:, i] = _encode(Xi, uniques=self.categories_[i]) + X_int[:, i] = _encode( + Xi, uniques=self.categories_[i], cache=self._transform_cache[i] + ) X_mask[:, i] = X_int[:, i] != -1 if not np.all(X_mask[:, i]): diff --git a/sklearn/utils/_encode.py b/sklearn/utils/_encode.py index 48f2848a85078..a23b02d7a63f7 100644 --- a/sklearn/utils/_encode.py +++ b/sklearn/utils/_encode.py @@ -178,13 +178,18 @@ def __missing__(self, key): return -1 -def _map_to_integer(values, uniques): +def _map_to_integer(values, uniques, cache=None): """Map values based on their position in uniques. Values not present in `uniques` are encoded as -1. """ xp, _ = get_namespace(values, uniques) - table = _nandict({val: i for i, val in enumerate(uniques)}) + if cache is None: + table = _nandict({val: i for i, val in enumerate(uniques)}) + elif "table" in cache: + table = cache["table"] + else: + table = cache["table"] = _nandict({val: i for i, val in enumerate(uniques)}) return xp.asarray([table[v] for v in values], device=array_device(values)) @@ -209,16 +214,26 @@ def _unique_pandas(values, *, return_inverse, return_counts): return ret[0] if len(ret) == 1 else ret -def _encode_pandas(values, uniques): +def _encode_pandas(values, uniques, cache=None): """Fast pandas equivalent of `_map_to_integer`. Values not present in `uniques` are encoded as -1. - As in `_unique_pandas`, plain `object` dtype Series never reach this function. + As in `_unique_pandas`, plain `object` dtype Series never reach this + function. + + `cache`, if given, is used the same way as in `_map_to_integer`, but to + avoid rebuilding the `pandas.Index` (used for `get_indexer`) on every + call instead of a plain dict. """ import pandas as pd - index = pd.Index(uniques) + if cache is None: + index = pd.Index(uniques) + elif "index" in cache: + index = cache["index"] + else: + index = cache["index"] = pd.Index(uniques) return np.asarray(index.get_indexer(values)) @@ -280,7 +295,7 @@ def _encode_labels(values, *, uniques): return encoded -def _encode(values, *, uniques, return_diff=False): +def _encode(values, *, uniques, return_diff=False, cache=None): """Encode values into [0, n_uniques - 1]. Uses pure python method for object dtype, and numpy method for @@ -302,6 +317,14 @@ def _encode(values, *, uniques, return_diff=False): return_diff : bool, default=False If True, also return the unique values in `values` that are not present in `uniques`. + cache : dict, default=None + Mutable dict used to cache, across repeated calls with the same `uniques`, + the lookup table built from `uniques` (see `_map_to_integer`/`_encode_pandas`). + Only used for object dtype arrays and pandas Series, where building that table + is O(len(uniques)) and would otherwise dominate the cost of encoding + small batches (e.g. repeated single-row calls to `transform`). + Callers are responsible for invalidating (e.g. replacing with a + fresh dict) the cache whenever `uniques` changes. Returns ------- @@ -312,7 +335,7 @@ def _encode(values, *, uniques, return_diff=False): returned if ``return_diff=True``. """ if is_pandas_df_or_series(values): - encoded = _encode_pandas(values, uniques) + encoded = _encode_pandas(values, uniques, cache=cache) if return_diff: diff = _unique(values[encoded == -1]) return encoded, diff @@ -320,7 +343,7 @@ def _encode(values, *, uniques, return_diff=False): xp, _ = get_namespace(values, uniques) if not xp.isdtype(values.dtype, "numeric"): - encoded = _map_to_integer(values, uniques) + encoded = _map_to_integer(values, uniques, cache=cache) else: encoded = xp.searchsorted(uniques, values) if size(uniques): From bced1ef7d2c5ad8f5a2a0936017d23758c62bf31 Mon Sep 17 00:00:00 2001 From: Arthur Lacote Date: Mon, 10 Aug 2026 11:21:12 +0200 Subject: [PATCH 2/2] attempt index optim --- sklearn/utils/_encode.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sklearn/utils/_encode.py b/sklearn/utils/_encode.py index 48310d5cbf8c8..90269dad2615c 100644 --- a/sklearn/utils/_encode.py +++ b/sklearn/utils/_encode.py @@ -234,12 +234,13 @@ def _encode_pandas(values, uniques, cache=None): """ import pandas as pd + dtype = None if isinstance(values.dtype, pd.CategoricalDtype) else "string" if cache is None: - index = pd.Index(uniques) + index = pd.Index(uniques, dtype=dtype) elif "index" in cache: index = cache["index"] else: - index = cache["index"] = pd.Index(uniques) + index = cache["index"] = pd.Index(uniques, dtype=dtype) encoded = np.asarray(index.get_indexer(values)) if ( uniques.size