diff --git a/cpp/include/nvtext/normalize.hpp b/cpp/include/nvtext/normalize.hpp index b24a473e7b4c..782e7b2d5f46 100644 --- a/cpp/include/nvtext/normalize.hpp +++ b/cpp/include/nvtext/normalize.hpp @@ -49,6 +49,46 @@ std::unique_ptr normalize_spaces( cuda::stream_ref stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +/** + * @brief Flags providing more control of nvtext::normalize_characters output + * + * Flags may be combined with bitwise OR: + * @code{.cpp} + * normalize_flags::STRIP_ACCENTS | normalize_flags::PAD_PUNCTUATION + * @endcode + * + * Note: If the `character_normalizer` object is created with `do_lower_case = true`, + * accent stripping is already implied by the lower-casing transform and + * `STRIP_ACCENTS` has no additional effect. + */ +enum class normalize_flags : uint32_t { + NONE = 0, + STRIP_ACCENTS = 1 << 0, ///< Remove diacritics from accented characters + PAD_PUNCTUATION = 1 << 1, ///< Add spaces around punctuation and CJK characters +}; + +/** + * @brief Combine two normalize_flags values + * @param a First flags value + * @param b Second flags value + * @return Combined flags + */ +inline normalize_flags operator|(normalize_flags a, normalize_flags b) +{ + return static_cast(static_cast(a) | static_cast(b)); +} + +/** + * @brief Used for checking if one or more flags are set + * @param a First flags value + * @param b Second flags value + * @return Masked combination of flags + */ +inline normalize_flags operator&(normalize_flags a, normalize_flags b) +{ + return static_cast(static_cast(a) & static_cast(b)); +} + /** * @brief Normalizer object to be used with nvtext::normalize_characters * @@ -73,6 +113,10 @@ std::unique_ptr normalize_spaces( * However, if the accented character is already lower-case, then only the * accent is removed. * + * Also with `do_lower_case = true`, accent stripping is already implied by + * the lower-casing transform. Passing `normalize_flags::STRIP_ACCENTS` to + * nvtext::normalize_characters has no additional effect in that case. + * * If `special_tokens` are included the padding after `[` and before `]` is not * inserted if the characters between them match one of the given tokens. * Also, the `special_tokens` are expected to include the `[]` characters @@ -159,5 +203,40 @@ std::unique_ptr normalize_characters( cuda::stream_ref stream = cudf::get_default_stream(), rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); +/** + * @brief Normalizes the text in input strings column with explicit behavioral flags. + * + * @see nvtext::character_normalizer for details on the normalizer behavior. + * @see nvtext::normalize_flags for the available flags. + * + * @code{.pseudo} + * cn = create_character_normalizer(false) + * s = ["Héllo", "ÑOÑO", "á"] // a + combining acute + * flags = normalize_flags::STRIP_ACCENTS | normalize_flags::PAD_PUNCTUATION + * s1 = normalize_characters(s, cn, flags) + * s1 is now ["Hello", "NONO", "a"] + * @endcode + * + * A null input element at row `i` produces a corresponding null entry + * for row `i` in the output column. + * + * If the `normalizer` object was created with `do_lower_case = true`, accent stripping is already + * implied by the lower-casing transform. Passing `normalize_flags::STRIP_ACCENTS` here no + * additional effect. + * + * @param input The input strings to normalize + * @param normalizer Normalizer to use for this function + * @param flags Bitmask of nvtext::normalize_flags values + * @param stream CUDA stream used for device memory operations and kernel launches + * @param mr Memory resource to allocate any returned objects + * @return Normalized strings column + */ +std::unique_ptr normalize_characters( + cudf::strings_column_view const& input, + character_normalizer const& normalizer, + normalize_flags flags, + cuda::stream_ref stream = cudf::get_default_stream(), + rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref()); + /** @} */ // end of group } // namespace CUDF_EXPORT nvtext diff --git a/cpp/src/text/normalize.cu b/cpp/src/text/normalize.cu index 110fd12b1306..f9c6955c46a0 100644 --- a/cpp/src/text/normalize.cu +++ b/cpp/src/text/normalize.cu @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -312,6 +313,11 @@ CUDF_KERNEL void special_tokens_kernel(uint32_t* d_normalized, } } +// Highest codepoint that maps to a plain ASCII result via get_first_cp +constexpr uint32_t ASCII_MAX_CODEPOINT = 0x7Fu; +// The char_flags table covers exactly the Basic Multilingual Plane +constexpr uint32_t BMP_CODEPOINT_LIMIT = 0x10000u; + /** * @brief The normalizer kernel * @@ -328,12 +334,16 @@ CUDF_KERNEL void special_tokens_kernel(uint32_t* d_normalized, * @param do_lower_case True if the normalization includes lower-casing characters * @param d_output The output of the normalization (UTF-8 encoded) */ -CUDF_KERNEL void data_normalizer_kernel(char const* d_chars, - int64_t total_bytes, - codepoint_metadata_type const* cp_metadata, - aux_codepoint_data_type const* aux_table, - bool do_lower_case, - uint32_t* d_output) +CUDF_KERNEL void data_normalizer_kernel( + char const* d_chars, + int64_t total_bytes, + codepoint_metadata_type const* cp_metadata, + aux_codepoint_data_type const* aux_table, + bool do_lower_case, + bool strip_accents, + bool pad_punctuation, + cudf::strings::detail::character_flags_table_type const* char_flags, + uint32_t* d_output) { uint32_t replacement[MAX_NEW_CHARS] = {0}; @@ -347,11 +357,25 @@ CUDF_KERNEL void data_normalizer_kernel(char const* d_chars, }(); auto const metadata = cp_metadata[cp]; - if (!should_remove_cp(metadata, do_lower_case)) { + if (!should_remove_cp(metadata, do_lower_case, strip_accents)) { int8_t num_new_chars = 1; // retrieve the normalized value for cp - auto const new_cp = do_lower_case || always_replace(metadata) ? get_first_cp(metadata) : cp; - replacement[0] = new_cp == 0 ? cp : new_cp; + uint32_t new_cp; + if (do_lower_case || always_replace(metadata)) { + new_cp = get_first_cp(metadata); + } else if (strip_accents) { + // Use the de-accented ASCII result when available; re-uppercase if needed + auto const mapped = get_first_cp(metadata); + if (mapped != 0 && mapped <= ASCII_MAX_CODEPOINT) { + auto const flag = cp < BMP_CODEPOINT_LIMIT ? char_flags[cp] : uint8_t{0}; + new_cp = cudf::strings::detail::IS_UPPER(flag) ? (mapped - 'a' + 'A') : mapped; + } else { + new_cp = 0; + } + } else { + new_cp = 0; + } + replacement[0] = new_cp == 0 ? cp : new_cp; if (do_lower_case && is_multi_char_transform(metadata)) { auto const next_cps = aux_table[cp]; @@ -360,7 +384,7 @@ CUDF_KERNEL void data_normalizer_kernel(char const* d_chars, num_new_chars = 2 + (replacement[2] != 0); } - if (should_add_spaces(metadata, do_lower_case) && (num_new_chars == 1)) { + if (should_add_spaces(metadata, do_lower_case, pad_punctuation) && (num_new_chars == 1)) { replacement[1] = replacement[0]; replacement[0] = SPACE_CODE_POINT; // add spaces around the new codepoint replacement[2] = SPACE_CODE_POINT; @@ -491,6 +515,8 @@ Iterator remove_safe(Iterator first, Iterator last, T const& value, cuda::stream std::unique_ptr normalize_characters(cudf::strings_column_view const& input, character_normalizer const& normalizer, + bool strip_accents, + bool pad_punctuation, cuda::stream_ref stream, rmm::device_async_resource_ref mr) { @@ -509,6 +535,11 @@ std::unique_ptr normalize_characters(cudf::strings_column_view con auto const& parameters = normalizer._impl; + // char_flags only needed when stripping accents without lowercasing (re-uppercase logic) + auto const char_flags = (strip_accents && !parameters->do_lower_case) + ? cudf::strings::detail::get_character_flags_table(stream) + : nullptr; + auto d_normalized = rmm::device_uvector(max_new_char_total, stream); data_normalizer_kernel<<>>( d_input_chars, @@ -516,6 +547,9 @@ std::unique_ptr normalize_characters(cudf::strings_column_view con parameters->cp_metadata.data(), parameters->aux_table.data(), parameters->do_lower_case, + strip_accents, + pad_punctuation, + char_flags, d_normalized.data()); CUDF_CUDA_TRY(cudaGetLastError()); @@ -560,7 +594,19 @@ std::unique_ptr normalize_characters(cudf::strings_column_view con rmm::device_async_resource_ref mr) { CUDF_FUNC_RANGE(); - return detail::normalize_characters(input, normalizer, stream, mr); + return detail::normalize_characters(input, normalizer, false, true, stream, mr); +} + +std::unique_ptr normalize_characters(cudf::strings_column_view const& input, + character_normalizer const& normalizer, + normalize_flags flags, + cuda::stream_ref stream, + rmm::device_async_resource_ref mr) +{ + CUDF_FUNC_RANGE(); + auto const strip = static_cast(flags & normalize_flags::STRIP_ACCENTS); + auto const tokenize = static_cast(flags & normalize_flags::PAD_PUNCTUATION); + return detail::normalize_characters(input, normalizer, strip, tokenize, stream, mr); } } // namespace nvtext diff --git a/cpp/src/text/normalize.cuh b/cpp/src/text/normalize.cuh index 603a0217505d..6c094667a3df 100644 --- a/cpp/src/text/normalize.cuh +++ b/cpp/src/text/normalize.cuh @@ -54,19 +54,26 @@ __device__ constexpr uint32_t extract_token_cat(uint32_t metadata) /** * @brief Return true if category of metadata value specifies the character should be replaced. */ -__device__ constexpr bool should_remove_cp(uint32_t metadata, bool lower_case) +__device__ constexpr bool should_remove_cp(uint32_t metadata, bool lower_case, bool strip_accents) { auto const cat = extract_token_cat(metadata); - return (cat == TOKEN_CAT_REMOVE_CHAR) || (lower_case && (cat == TOKEN_CAT_REMOVE_CHAR_IF_LOWER)); + return (cat == TOKEN_CAT_REMOVE_CHAR) || + ((lower_case || strip_accents) && (cat == TOKEN_CAT_REMOVE_CHAR_IF_LOWER)); } /** * @brief Return true if category of metadata value specifies the character should be padded. + * + * @param lower_case Controls padding of the 4 chars with category TOKEN_CAT_ADD_SPACE_IF_LOWER + * @param pad_punctuation Controls padding of punctuation and CJK (TOKEN_CAT_ADD_SPACE) */ -__device__ constexpr bool should_add_spaces(uint32_t metadata, bool lower_case) +__device__ constexpr bool should_add_spaces(uint32_t metadata, + bool lower_case, + bool pad_punctuation) { auto const cat = extract_token_cat(metadata); - return (cat == TOKEN_CAT_ADD_SPACE) || (lower_case && (cat == TOKEN_CAT_ADD_SPACE_IF_LOWER)); + return (pad_punctuation && (cat == TOKEN_CAT_ADD_SPACE)) || + (lower_case && (cat == TOKEN_CAT_ADD_SPACE_IF_LOWER)); } /** diff --git a/cpp/tests/text/normalize_tests.cpp b/cpp/tests/text/normalize_tests.cpp index af3fb23dbf18..f1db8bfa936c 100644 --- a/cpp/tests/text/normalize_tests.cpp +++ b/cpp/tests/text/normalize_tests.cpp @@ -194,6 +194,56 @@ TEST_F(TextNormalizeTest, SpecialTokens) CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); } +TEST_F(TextNormalizeTest, NormalizeCharactersStripAccents) +{ + // Tests normalize_flags::STRIP_ACCENTS via the flags overload of normalize_characters. + // Row 2: NFD form — plain 'a' followed by U+0301 (combining acute accent) followed by 'b'. + // Constructed via char casts to avoid source-encoding ambiguity with precomposed á (U+00E1). + char const u0301[] = {(char)0xcc, (char)0x81, 0}; // UTF-8 for U+0301 + std::string const nfd_str = std::string("a") + u0301 + "b"; + auto input = cudf::test::strings_column_wrapper({ + "éàïü", // lowercase precomposed NFC accented + "ÉÀÏÜ", // uppercase precomposed NFC accented + nfd_str, // NFD: a + U+0301 (combining acute accent) + b + "ACENU", // no accents + }); + auto sv = cudf::strings_column_view(input); + + // strip_accents=true, do_lower_case=false: de-accent while preserving case + auto normalizer = nvtext::create_character_normalizer(false); + auto flags = nvtext::normalize_flags::STRIP_ACCENTS | nvtext::normalize_flags::PAD_PUNCTUATION; + auto results = nvtext::normalize_characters(sv, *normalizer, flags); + auto expected = cudf::test::strings_column_wrapper({"eaiu", "EAIU", "ab", "ACENU"}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + + // strip_accents=true, do_lower_case=true: same result as do_lower_case=true alone + normalizer = nvtext::create_character_normalizer(true); + results = nvtext::normalize_characters(sv, *normalizer, flags); + expected = cudf::test::strings_column_wrapper({"eaiu", "eaiu", "ab", "acenu"}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); +} + +TEST_F(TextNormalizeTest, NormalizeCharactersNoTokenizePunctuation) +{ + // Tests normalize_flags::NONE / pad_punctuation=false. + // Punctuation and CJK characters should not receive padding spaces. + // Whitespace normalization (tab→space via ALWAYS_REPLACE) is unaffected. + auto input = cudf::test::strings_column_wrapper({"P^NP", "$41.07", "[a,b]", "丏丟", "éè\tâ"}); + auto sv = cudf::strings_column_view(input); + + // no pad_punctuation, do_lower_case=false: chars pass through unchanged + auto normalizer = nvtext::create_character_normalizer(false); + auto results = nvtext::normalize_characters(sv, *normalizer, nvtext::normalize_flags::NONE); + auto expected = cudf::test::strings_column_wrapper({"P^NP", "$41.07", "[a,b]", "丏丟", "éè â"}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); + + // no pad_punctuation, do_lower_case=true: lowercase + de-accent, no punct padding + normalizer = nvtext::create_character_normalizer(true); + results = nvtext::normalize_characters(sv, *normalizer, nvtext::normalize_flags::NONE); + expected = cudf::test::strings_column_wrapper({"p^np", "$41.07", "[a,b]", "丏丟", "ee a"}); + CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected); +} + TEST_F(TextNormalizeTest, NormalizeSlicedColumn) { cudf::test::strings_column_wrapper strings( diff --git a/python/cudf/cudf/core/character_normalizer.py b/python/cudf/cudf/core/character_normalizer.py index e159eafdc5e4..ea118fbc2c82 100644 --- a/python/cudf/cudf/core/character_normalizer.py +++ b/python/cudf/cudf/core/character_normalizer.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations @@ -34,18 +34,24 @@ def __init__( do_lower, special_tokens._column.plc_column ) - def normalize(self, text: Series) -> Series: + def normalize(self, text: Series, flags: int | None = None) -> Series: """ Parameters ---------- text : cudf.Series The strings to be normalized. + flags : int, optional + Bitmask of ``pylibcudf.nvtext.normalize.NormalizeFlags`` values. + Default is ``NormalizeFlags.PAD_PUNCTUATION``. Returns ------- cudf.Series Normalized strings """ - result = text._column.normalize_characters(self.normalizer) + NormalizeFlags = plc.nvtext.normalize.NormalizeFlags + if flags is None: + flags = int(NormalizeFlags.PAD_PUNCTUATION) + result = text._column.normalize_characters(self.normalizer, flags) return Series._from_column(result) diff --git a/python/cudf/cudf/core/column/string.py b/python/cudf/cudf/core/column/string.py index 1770ca203b2a..8bd369914d2a 100644 --- a/python/cudf/cudf/core/column/string.py +++ b/python/cudf/cudf/core/column/string.py @@ -861,12 +861,15 @@ def normalize_spaces(self) -> Self: ) def normalize_characters( - self, normalizer: plc.nvtext.normalize.CharacterNormalizer + self, + normalizer: plc.nvtext.normalize.CharacterNormalizer, + flags: int = int(plc.nvtext.normalize.NormalizeFlags.PAD_PUNCTUATION), ) -> Self: with self.access(mode="read", scope="internal"): plc_column = plc.nvtext.normalize.normalize_characters( self.plc_column, normalizer, + flags, ) return cast( "Self", diff --git a/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt b/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt index 0a999dc704a6..ee8316e64518 100644 --- a/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt @@ -1,11 +1,11 @@ # ============================================================================= # cmake-format: off -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= -set(cython_sources stemmer.pyx) +set(cython_sources normalize.pyx stemmer.pyx) set(linked_libraries cudf::cudf) diff --git a/python/pylibcudf/pylibcudf/libcudf/nvtext/normalize.pxd b/python/pylibcudf/pylibcudf/libcudf/nvtext/normalize.pxd index 0184c1d87854..85a2a6a0ac30 100644 --- a/python/pylibcudf/pylibcudf/libcudf/nvtext/normalize.pxd +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/normalize.pxd @@ -1,5 +1,6 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from libc.stdint cimport uint32_t from libcpp cimport bool from libcpp.memory cimport unique_ptr from pylibcudf.exception_handler cimport libcudf_exception_handler @@ -27,9 +28,17 @@ cdef extern from "nvtext/normalize.hpp" namespace "nvtext" nogil: device_async_resource_ref mr ) except +libcudf_exception_handler + cpdef enum class normalize_flags: + NONE + STRIP_ACCENTS + PAD_PUNCTUATION + cdef unique_ptr[column] normalize_characters( const column_view & strings, const character_normalizer & normalizer, + normalize_flags flags, cudaStream_t stream, device_async_resource_ref mr ) except +libcudf_exception_handler + +ctypedef uint32_t underlying_type_t_normalize_flags diff --git a/python/pylibcudf/pylibcudf/libcudf/nvtext/normalize.pyx b/python/pylibcudf/pylibcudf/libcudf/nvtext/normalize.pyx new file mode 100644 index 000000000000..8eca3cc68ad4 --- /dev/null +++ b/python/pylibcudf/pylibcudf/libcudf/nvtext/normalize.pyx @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 diff --git a/python/pylibcudf/pylibcudf/nvtext/normalize.pxd b/python/pylibcudf/pylibcudf/nvtext/normalize.pxd index 30e459f75a51..056f5e86541c 100644 --- a/python/pylibcudf/pylibcudf/nvtext/normalize.pxd +++ b/python/pylibcudf/pylibcudf/nvtext/normalize.pxd @@ -1,10 +1,13 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from libcpp cimport bool from libcpp.memory cimport unique_ptr from pylibcudf.column cimport Column -from pylibcudf.libcudf.nvtext.normalize cimport character_normalizer +from pylibcudf.libcudf.nvtext.normalize cimport ( + character_normalizer, + underlying_type_t_normalize_flags, +) from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource cdef class CharacterNormalizer: @@ -15,8 +18,9 @@ cpdef Column normalize_spaces( ) cpdef Column normalize_characters( - Column input, - CharacterNormalizer normalizer, - object stream = *, - DeviceMemoryResource mr=* + Column input, + CharacterNormalizer normalizer, + underlying_type_t_normalize_flags flags = *, + object stream = *, + DeviceMemoryResource mr=* ) diff --git a/python/pylibcudf/pylibcudf/nvtext/normalize.pyi b/python/pylibcudf/pylibcudf/nvtext/normalize.pyi index 0fbd2e7e725b..f20b8dc8da8f 100644 --- a/python/pylibcudf/pylibcudf/nvtext/normalize.pyi +++ b/python/pylibcudf/pylibcudf/nvtext/normalize.pyi @@ -1,11 +1,18 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from enum import IntFlag + from rmm.pylibrmm.memory_resource import DeviceMemoryResource from pylibcudf.column import Column from pylibcudf.utils import CudaStreamLike +class NormalizeFlags(IntFlag): + NONE = 0 + STRIP_ACCENTS = 1 + PAD_PUNCTUATION = 2 + class CharacterNormalizer: def __init__( self, @@ -23,6 +30,7 @@ def normalize_spaces( def normalize_characters( input: Column, normalizer: CharacterNormalizer, + flags: int = ..., stream: CudaStreamLike | None = None, mr: DeviceMemoryResource | None = None, ) -> Column: ... diff --git a/python/pylibcudf/pylibcudf/nvtext/normalize.pyx b/python/pylibcudf/pylibcudf/nvtext/normalize.pyx index ec083549ccee..6f53d9330869 100644 --- a/python/pylibcudf/pylibcudf/nvtext/normalize.pyx +++ b/python/pylibcudf/pylibcudf/nvtext/normalize.pyx @@ -9,6 +9,10 @@ from pylibcudf.column cimport Column from pylibcudf.libcudf.column.column cimport column from pylibcudf.libcudf.column.column_view cimport column_view from pylibcudf.libcudf.nvtext cimport normalize as cpp_normalize +from pylibcudf.libcudf.nvtext.normalize cimport ( + normalize_flags, + underlying_type_t_normalize_flags, +) from pylibcudf.utils cimport _get_stream, _get_memory_resource from typing import TYPE_CHECKING @@ -18,8 +22,11 @@ from rmm.pylibrmm.memory_resource cimport DeviceMemoryResource from rmm.pylibrmm.stream cimport Stream from cuda.bindings.cyruntime cimport cudaStream_t +from pylibcudf.libcudf.nvtext.normalize import normalize_flags as NormalizeFlags # no-cython-lint + __all__ = [ - "CharacterNormalizer" + "CharacterNormalizer", + "NormalizeFlags", "normalize_characters", "normalize_spaces", ] @@ -90,6 +97,7 @@ cpdef Column normalize_spaces( cpdef Column normalize_characters( Column input, CharacterNormalizer normalizer, + underlying_type_t_normalize_flags flags = normalize_flags.PAD_PUNCTUATION, object stream: CudaStreamLike | None = None, DeviceMemoryResource mr=None, ): @@ -104,6 +112,9 @@ cpdef Column normalize_characters( Input strings normalizer : CharacterNormalizer Normalizer object used for modifying the input column text + flags : NormalizeFlags, optional + Bitmask of :py:class:`NormalizeFlags` values controlling normalization + behavior. Default is ``NormalizeFlags.PAD_PUNCTUATION``. stream : Stream | None CUDA stream on which to perform the operation. @@ -118,12 +129,16 @@ cpdef Column normalize_characters( mr = _get_memory_resource(mr) cdef column_view c_input = input.view() + cdef normalize_flags c_flags = flags with nogil: c_result = cpp_normalize.normalize_characters( c_input, dereference(normalizer.c_obj.get()), + c_flags, _cs, mr.get_mr() ) return Column.from_libcudf(move(c_result), _stream, mr) + +NormalizeFlags.__str__ = NormalizeFlags.__repr__ diff --git a/python/pylibcudf/tests/test_nvtext_normalize.py b/python/pylibcudf/tests/test_nvtext_normalize.py index ff8321c9e53c..c4b28a26a8a5 100644 --- a/python/pylibcudf/tests/test_nvtext_normalize.py +++ b/python/pylibcudf/tests/test_nvtext_normalize.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 import pyarrow as pa @@ -62,6 +62,46 @@ def test_normalizer(norm_chars_input_data, do_lower): assert_column_eq(expect, got) +@pytest.mark.parametrize( + "flags,do_lower,expected", + [ + ( + plc.nvtext.normalize.NormalizeFlags.STRIP_ACCENTS + | plc.nvtext.normalize.NormalizeFlags.PAD_PUNCTUATION, + False, + [ + "eaio eaio", + "ACENU", + "ACENU", + " $ 24 . 08", + " [ a , bb ] ", + " [ PAD ] ", + ], + ), + ( + plc.nvtext.normalize.NormalizeFlags.NONE, + False, + ["éâîô eaio", "ĂĆĖÑÜ", "ACENU", "$24.08", "[a,bb]", "[PAD]"], + ), + ( + plc.nvtext.normalize.NormalizeFlags.NONE, + True, + ["eaio eaio", "acenu", "acenu", "$24.08", "[a,bb]", "[pad]"], + ), + ], +) +def test_normalizer_flags(norm_chars_input_data, flags, do_lower, expected): + got = plc.nvtext.normalize.normalize_characters( + plc.Column.from_arrow(norm_chars_input_data), + plc.nvtext.normalize.CharacterNormalizer( + do_lower, + plc.column_factories.make_empty_column(plc.types.TypeId.STRING), + ), + int(flags), + ) + assert_column_eq(pa.array(expected), got) + + @pytest.mark.parametrize("do_lower", [True, False]) def test_normalizer_with_special_tokens(norm_chars_input_data, do_lower): special_tokens = pa.array(["[PAD]"])