Add nvtext::normalize_characters with strip-accents and pad-punctuation options

cpp/include/nvtext/normalize.hpp

@@ -49,6 +49,46 @@ std::unique_ptr<cudf::column> normalize_spaces(
   cuda::stream_ref stream           = cudf::get_default_stream(),
   rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
 
+/**
+ * @brief Flags providing more control of nvtext::normalize_characters output
+ *
+ * Flags may be combined with bitwise OR:
+ * @code{.cpp}
+ * normalize_flags::STRIP_ACCENTS | normalize_flags::PAD_PUNCTUATION
+ * @endcode
+ *
+ * Note: If the `character_normalizer` object is created with `do_lower_case = true`,
+ * accent stripping is already implied by the lower-casing transform and
+ * `STRIP_ACCENTS` has no additional effect.
+ */
+enum class normalize_flags : uint32_t {
+  NONE            = 0,
+  STRIP_ACCENTS   = 1 << 0,  ///< Remove diacritics from accented characters
+  PAD_PUNCTUATION = 1 << 1,  ///< Add spaces around punctuation and CJK characters
+};
+
+/**
+ * @brief Combine two normalize_flags values
+ * @param a First flags value
+ * @param b Second flags value
+ * @return Combined flags
+ */
+inline normalize_flags operator|(normalize_flags a, normalize_flags b)
+{
+  return static_cast<normalize_flags>(static_cast<uint32_t>(a) | static_cast<uint32_t>(b));
+}
+
+/**
+ * @brief Used for checking if one or more flags are set
+ * @param a First flags value
+ * @param b Second flags value
+ * @return Masked combination of flags
+ */
+inline normalize_flags operator&(normalize_flags a, normalize_flags b)
+{
+  return static_cast<normalize_flags>(static_cast<uint32_t>(a) & static_cast<uint32_t>(b));
+}
+
 /**
  * @brief Normalizer object to be used with nvtext::normalize_characters
  *
@@ -73,6 +113,10 @@ std::unique_ptr<cudf::column> normalize_spaces(
  * However, if the accented character is already lower-case, then only the
  * accent is removed.
  *
+ * Also with `do_lower_case = true`, accent stripping is already implied by
+ * the lower-casing transform. Passing `normalize_flags::STRIP_ACCENTS` to
+ * nvtext::normalize_characters has no additional effect in that case.
+ *
  * If `special_tokens` are included the padding after `[` and before `]` is not
  * inserted if the characters between them match one of the given tokens.
  * Also, the `special_tokens` are expected to include the `[]` characters
@@ -159,5 +203,40 @@ std::unique_ptr<cudf::column> normalize_characters(
   cuda::stream_ref stream           = cudf::get_default_stream(),
   rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
 
+/**
+ * @brief Normalizes the text in input strings column with explicit behavioral flags.
+ *
+ * @see nvtext::character_normalizer for details on the normalizer behavior.
+ * @see nvtext::normalize_flags for the available flags.
+ *
+ * @code{.pseudo}
+ * cn = create_character_normalizer(false)
+ * s = ["Héllo", "ÑOÑO", "á"]   // a + combining acute
+ * flags = normalize_flags::STRIP_ACCENTS | normalize_flags::PAD_PUNCTUATION
+ * s1 = normalize_characters(s, cn, flags)
+ * s1 is now ["Hello", "NONO", "a"]
+ * @endcode
+ *
+ * A null input element at row `i` produces a corresponding null entry
+ * for row `i` in the output column.
+ *
+ * If the `normalizer` object was created with `do_lower_case = true`, accent stripping is already
+ * implied by the lower-casing transform. Passing `normalize_flags::STRIP_ACCENTS` here no
+ * additional effect.
+ *
+ * @param input The input strings to normalize
+ * @param normalizer Normalizer to use for this function
+ * @param flags Bitmask of nvtext::normalize_flags values
+ * @param stream CUDA stream used for device memory operations and kernel launches
+ * @param mr Memory resource to allocate any returned objects
+ * @return Normalized strings column
+ */
+std::unique_ptr<cudf::column> normalize_characters(
+  cudf::strings_column_view const& input,
+  character_normalizer const& normalizer,
+  normalize_flags flags,
+  cuda::stream_ref stream           = cudf::get_default_stream(),
+  rmm::device_async_resource_ref mr = cudf::get_current_device_resource_ref());
+
 /** @} */  // end of group
 }  // namespace CUDF_EXPORT nvtext

cpp/src/text/normalize.cu

@@ -19,6 +19,7 @@
 #include <cudf/detail/utilities/integer_utils.hpp>
 #include <cudf/sorting.hpp>
 #include <cudf/strings/case.hpp>
+#include <cudf/strings/detail/char_tables.hpp>
 #include <cudf/strings/detail/strings_children.cuh>
 #include <cudf/strings/detail/strings_column_factories.cuh>
 #include <cudf/strings/detail/utilities.cuh>
@@ -312,6 +313,11 @@ CUDF_KERNEL void special_tokens_kernel(uint32_t* d_normalized,
   }
 }
 
+// Highest codepoint that maps to a plain ASCII result via get_first_cp
+constexpr uint32_t ASCII_MAX_CODEPOINT = 0x7Fu;
+// The char_flags table covers exactly the Basic Multilingual Plane
+constexpr uint32_t BMP_CODEPOINT_LIMIT = 0x10000u;
+
 /**
  * @brief The normalizer kernel
  *
@@ -328,12 +334,16 @@ CUDF_KERNEL void special_tokens_kernel(uint32_t* d_normalized,
  * @param do_lower_case True if the normalization includes lower-casing characters
  * @param d_output The output of the normalization (UTF-8 encoded)
  */
-CUDF_KERNEL void data_normalizer_kernel(char const* d_chars,
-                                        int64_t total_bytes,
-                                        codepoint_metadata_type const* cp_metadata,
-                                        aux_codepoint_data_type const* aux_table,
-                                        bool do_lower_case,
-                                        uint32_t* d_output)
+CUDF_KERNEL void data_normalizer_kernel(
+  char const* d_chars,
+  int64_t total_bytes,
+  codepoint_metadata_type const* cp_metadata,
+  aux_codepoint_data_type const* aux_table,
+  bool do_lower_case,
+  bool strip_accents,
+  bool pad_punctuation,
+  cudf::strings::detail::character_flags_table_type const* char_flags,
+  uint32_t* d_output)
 {
   uint32_t replacement[MAX_NEW_CHARS] = {0};
 
@@ -347,11 +357,25 @@ CUDF_KERNEL void data_normalizer_kernel(char const* d_chars,
     }();
     auto const metadata = cp_metadata[cp];
 
-    if (!should_remove_cp(metadata, do_lower_case)) {
+    if (!should_remove_cp(metadata, do_lower_case, strip_accents)) {
       int8_t num_new_chars = 1;
       // retrieve the normalized value for cp
-      auto const new_cp = do_lower_case || always_replace(metadata) ? get_first_cp(metadata) : cp;
-      replacement[0]    = new_cp == 0 ? cp : new_cp;
+      uint32_t new_cp;
+      if (do_lower_case || always_replace(metadata)) {
+        new_cp = get_first_cp(metadata);
+      } else if (strip_accents) {
+        // Use the de-accented ASCII result when available; re-uppercase if needed
+        auto const mapped = get_first_cp(metadata);
+        if (mapped != 0 && mapped <= ASCII_MAX_CODEPOINT) {
+          auto const flag = cp < BMP_CODEPOINT_LIMIT ? char_flags[cp] : uint8_t{0};
+          new_cp          = cudf::strings::detail::IS_UPPER(flag) ? (mapped - 'a' + 'A') : mapped;
+        } else {
+          new_cp = 0;
+        }
+      } else {
+        new_cp = 0;
+      }
+      replacement[0] = new_cp == 0 ? cp : new_cp;
 
       if (do_lower_case && is_multi_char_transform(metadata)) {
         auto const next_cps = aux_table[cp];
@@ -360,7 +384,7 @@ CUDF_KERNEL void data_normalizer_kernel(char const* d_chars,
         num_new_chars       = 2 + (replacement[2] != 0);
       }
 
-      if (should_add_spaces(metadata, do_lower_case) && (num_new_chars == 1)) {
+      if (should_add_spaces(metadata, do_lower_case, pad_punctuation) && (num_new_chars == 1)) {
         replacement[1] = replacement[0];
         replacement[0] = SPACE_CODE_POINT;  // add spaces around the new codepoint
         replacement[2] = SPACE_CODE_POINT;
@@ -491,6 +515,8 @@ Iterator remove_safe(Iterator first, Iterator last, T const& value, cuda::stream
 
 std::unique_ptr<cudf::column> normalize_characters(cudf::strings_column_view const& input,
                                                    character_normalizer const& normalizer,
+                                                   bool strip_accents,
+                                                   bool pad_punctuation,
                                                    cuda::stream_ref stream,
                                                    rmm::device_async_resource_ref mr)
 {
@@ -509,13 +535,21 @@ std::unique_ptr<cudf::column> normalize_characters(cudf::strings_column_view con
 
   auto const& parameters = normalizer._impl;
 
+  // char_flags only needed when stripping accents without lowercasing (re-uppercase logic)
+  auto const char_flags = (strip_accents && !parameters->do_lower_case)
+                            ? cudf::strings::detail::get_character_flags_table(stream)
+                            : nullptr;
+
   auto d_normalized = rmm::device_uvector<uint32_t>(max_new_char_total, stream);
   data_normalizer_kernel<<<grid.num_blocks, grid.num_threads_per_block, 0, stream.get()>>>(
     d_input_chars,
     chars_size,
     parameters->cp_metadata.data(),
     parameters->aux_table.data(),
     parameters->do_lower_case,
+    strip_accents,
+    pad_punctuation,
+    char_flags,
     d_normalized.data());
   CUDF_CUDA_TRY(cudaGetLastError());
 
@@ -560,7 +594,19 @@ std::unique_ptr<cudf::column> normalize_characters(cudf::strings_column_view con
                                                    rmm::device_async_resource_ref mr)
 {
   CUDF_FUNC_RANGE();
-  return detail::normalize_characters(input, normalizer, stream, mr);
+  return detail::normalize_characters(input, normalizer, false, true, stream, mr);
+}
+
+std::unique_ptr<cudf::column> normalize_characters(cudf::strings_column_view const& input,
+                                                   character_normalizer const& normalizer,
+                                                   normalize_flags flags,
+                                                   cuda::stream_ref stream,
+                                                   rmm::device_async_resource_ref mr)
+{
+  CUDF_FUNC_RANGE();
+  auto const strip    = static_cast<bool>(flags & normalize_flags::STRIP_ACCENTS);
+  auto const tokenize = static_cast<bool>(flags & normalize_flags::PAD_PUNCTUATION);
+  return detail::normalize_characters(input, normalizer, strip, tokenize, stream, mr);
 }
 
 }  // namespace nvtext

cpp/src/text/normalize.cuh

@@ -54,19 +54,26 @@ __device__ constexpr uint32_t extract_token_cat(uint32_t metadata)
 /**
  * @brief Return true if category of metadata value specifies the character should be replaced.
  */
-__device__ constexpr bool should_remove_cp(uint32_t metadata, bool lower_case)
+__device__ constexpr bool should_remove_cp(uint32_t metadata, bool lower_case, bool strip_accents)
 {
   auto const cat = extract_token_cat(metadata);
-  return (cat == TOKEN_CAT_REMOVE_CHAR) || (lower_case && (cat == TOKEN_CAT_REMOVE_CHAR_IF_LOWER));
+  return (cat == TOKEN_CAT_REMOVE_CHAR) ||
+         ((lower_case || strip_accents) && (cat == TOKEN_CAT_REMOVE_CHAR_IF_LOWER));
 }
 
 /**
  * @brief Return true if category of metadata value specifies the character should be padded.
+ *
+ * @param lower_case Controls padding of the 4 chars with category TOKEN_CAT_ADD_SPACE_IF_LOWER
+ * @param pad_punctuation Controls padding of punctuation and CJK (TOKEN_CAT_ADD_SPACE)
  */
-__device__ constexpr bool should_add_spaces(uint32_t metadata, bool lower_case)
+__device__ constexpr bool should_add_spaces(uint32_t metadata,
+                                            bool lower_case,
+                                            bool pad_punctuation)
 {
   auto const cat = extract_token_cat(metadata);
-  return (cat == TOKEN_CAT_ADD_SPACE) || (lower_case && (cat == TOKEN_CAT_ADD_SPACE_IF_LOWER));
+  return (pad_punctuation && (cat == TOKEN_CAT_ADD_SPACE)) ||
+         (lower_case && (cat == TOKEN_CAT_ADD_SPACE_IF_LOWER));
 }
 
 /**

cpp/tests/text/normalize_tests.cpp

@@ -194,6 +194,56 @@ TEST_F(TextNormalizeTest, SpecialTokens)
   CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
 }
 
+TEST_F(TextNormalizeTest, NormalizeCharactersStripAccents)
+{
+  // Tests normalize_flags::STRIP_ACCENTS via the flags overload of normalize_characters.
+  // Row 2: NFD form — plain 'a' followed by U+0301 (combining acute accent) followed by 'b'.
+  // Constructed via char casts to avoid source-encoding ambiguity with precomposed á (U+00E1).
+  char const u0301[]        = {(char)0xcc, (char)0x81, 0};  // UTF-8 for U+0301
+  std::string const nfd_str = std::string("a") + u0301 + "b";
+  auto input                = cudf::test::strings_column_wrapper({
+    "éàïü",   // lowercase precomposed NFC accented
+    "ÉÀÏÜ",   // uppercase precomposed NFC accented
+    nfd_str,  // NFD: a + U+0301 (combining acute accent) + b
+    "ACENU",  // no accents
+  });
+  auto sv                   = cudf::strings_column_view(input);
+
+  // strip_accents=true, do_lower_case=false: de-accent while preserving case
+  auto normalizer = nvtext::create_character_normalizer(false);
+  auto flags    = nvtext::normalize_flags::STRIP_ACCENTS | nvtext::normalize_flags::PAD_PUNCTUATION;
+  auto results  = nvtext::normalize_characters(sv, *normalizer, flags);
+  auto expected = cudf::test::strings_column_wrapper({"eaiu", "EAIU", "ab", "ACENU"});
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
+
+  // strip_accents=true, do_lower_case=true: same result as do_lower_case=true alone
+  normalizer = nvtext::create_character_normalizer(true);
+  results    = nvtext::normalize_characters(sv, *normalizer, flags);
+  expected   = cudf::test::strings_column_wrapper({"eaiu", "eaiu", "ab", "acenu"});
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
+}
+
+TEST_F(TextNormalizeTest, NormalizeCharactersNoTokenizePunctuation)
+{
+  // Tests normalize_flags::NONE / pad_punctuation=false.
+  // Punctuation and CJK characters should not receive padding spaces.
+  // Whitespace normalization (tab→space via ALWAYS_REPLACE) is unaffected.
+  auto input = cudf::test::strings_column_wrapper({"P^NP", "$41.07", "[a,b]", "丏丟", "éè\tâ"});
+  auto sv    = cudf::strings_column_view(input);
+
+  // no pad_punctuation, do_lower_case=false: chars pass through unchanged
+  auto normalizer = nvtext::create_character_normalizer(false);
+  auto results    = nvtext::normalize_characters(sv, *normalizer, nvtext::normalize_flags::NONE);
+  auto expected   = cudf::test::strings_column_wrapper({"P^NP", "$41.07", "[a,b]", "丏丟", "éè â"});
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
+
+  // no pad_punctuation, do_lower_case=true: lowercase + de-accent, no punct padding
+  normalizer = nvtext::create_character_normalizer(true);
+  results    = nvtext::normalize_characters(sv, *normalizer, nvtext::normalize_flags::NONE);
+  expected   = cudf::test::strings_column_wrapper({"p^np", "$41.07", "[a,b]", "丏丟", "ee a"});
+  CUDF_TEST_EXPECT_COLUMNS_EQUAL(*results, expected);
+}
+
 TEST_F(TextNormalizeTest, NormalizeSlicedColumn)
 {
   cudf::test::strings_column_wrapper strings(

python/cudf/cudf/core/character_normalizer.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION.
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
 from __future__ import annotations
@@ -34,18 +34,24 @@ def __init__(
             do_lower, special_tokens._column.plc_column
         )
 
-    def normalize(self, text: Series) -> Series:
+    def normalize(self, text: Series, flags: int | None = None) -> Series:
         """
         Parameters
         ----------
         text : cudf.Series
             The strings to be normalized.
+        flags : int, optional
+            Bitmask of ``pylibcudf.nvtext.normalize.NormalizeFlags`` values.
+            Default is ``NormalizeFlags.PAD_PUNCTUATION``.
 
         Returns
         -------
         cudf.Series
             Normalized strings
         """
-        result = text._column.normalize_characters(self.normalizer)
+        NormalizeFlags = plc.nvtext.normalize.NormalizeFlags
+        if flags is None:
+            flags = int(NormalizeFlags.PAD_PUNCTUATION)
+        result = text._column.normalize_characters(self.normalizer, flags)
 
         return Series._from_column(result)

python/cudf/cudf/core/column/string.py

@@ -861,12 +861,15 @@ def normalize_spaces(self) -> Self:
         )
 
     def normalize_characters(
-        self, normalizer: plc.nvtext.normalize.CharacterNormalizer
+        self,
+        normalizer: plc.nvtext.normalize.CharacterNormalizer,
+        flags: int = int(plc.nvtext.normalize.NormalizeFlags.PAD_PUNCTUATION),
     ) -> Self:
         with self.access(mode="read", scope="internal"):
             plc_column = plc.nvtext.normalize.normalize_characters(
                 self.plc_column,
                 normalizer,
+                flags,
             )
             return cast(
                 "Self",

python/pylibcudf/pylibcudf/libcudf/nvtext/CMakeLists.txt

@@ -1,11 +1,11 @@
 # =============================================================================
 # cmake-format: off
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION.
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 # cmake-format: on
 # =============================================================================
 
-set(cython_sources stemmer.pyx)
+set(cython_sources normalize.pyx stemmer.pyx)
 
 set(linked_libraries cudf::cudf)