From 3422cc930c395e35e54c73d2d26eb3b56a2a39e7 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:44:27 +0900 Subject: [PATCH 01/10] Implement N-gram indexer --- include/iris/ngram.hpp | 163 +++++++++++++++++++++++++++++++++++++++++ iris.natvis | 15 ++++ test/CMakeLists.txt | 1 + test/ngram.cpp | 124 +++++++++++++++++++++++++++++++ 4 files changed, 303 insertions(+) create mode 100644 include/iris/ngram.hpp create mode 100644 test/ngram.cpp diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp new file mode 100644 index 0000000..f5ecadb --- /dev/null +++ b/include/iris/ngram.hpp @@ -0,0 +1,163 @@ +#ifndef IRIS_NGRAM_HPP +#define IRIS_NGRAM_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace iris { + +enum struct ngram_document_id : unsigned {}; + +} // iris + +template +struct std::formatter + : std::formatter, CharT> +{ + using base_type = std::formatter, CharT>; + + template + Ctx::iterator format(iris::ngram_document_id doc_id, Ctx& ctx) const + { + return base_type::format(std::to_underlying(doc_id), ctx); + } +}; + +namespace iris { + +struct ngram_occurrence +{ + ngram_document_id doc_id; + int pos; + + [[nodiscard]] constexpr bool operator==(ngram_occurrence const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_occurrence const&) const noexcept = default; +}; + +template +struct ngram +{ + // TODO: optimize for N=1 + std::array chars; + + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; +}; + +inline namespace ngram_literals { + +[[nodiscard]] constexpr ngram_document_id operator ""_doc_id(unsigned long long id) noexcept +{ + return ngram_document_id{static_cast>(id)}; +} + +[[nodiscard]] constexpr auto operator ""_2gram(char32_t const* str, std::size_t len) noexcept +{ + assert(len == 2); + return ngram<2, char32_t>{str[0], str[1]}; +} + +} // ngram_literals + + +namespace detail { + +template +struct ngram_index +{ + void append(ngram ng, ngram_document_id doc_id, int pos) + { + occs[ng].emplace_back(doc_id, pos); + } + + std::flat_map, std::vector> occs; +}; + +template +struct ngram_index_storage +{ + ngram_index<1, CharT> uni_idx; + ngram_index<2, CharT> bi_idx; +}; + +} // detail + +template +class ngram_database +{ +public: + [[nodiscard]] + ngram_document_id add_document(std::basic_string_view const doc_text) + { + ngram_document_id const doc_id{max_doc_id_}; + max_doc_id_ = ngram_document_id{std::to_underlying(max_doc_id_) + 1u}; + + for (std::size_t i = 0; i < doc_text.size(); ++i) { + store_.uni_idx.append(ngram<1, CharT>{doc_text[i]}, doc_id, int(i)); + } + + auto const do_ngram = [&](detail::ngram_index& idx) { + if (doc_text.size() < N) return; + + ngram ng; + std::size_t i = 0; + for (; i < N; ++i) { // TODO: loop unroll + ng.chars[i] = doc_text[i]; + } + idx.append(ng, doc_id, 0); + + for (; i < doc_text.size(); ++i) { + std::ranges::shift_left(ng.chars, 1); + ng.chars[N - 1] = doc_text[i]; + idx.append(ng, doc_id, int(i - N + 1)); + } + }; + do_ngram(get_index<2>()); + + return doc_id; + } + + template + [[nodiscard]] + std::vector const* get_occurrences(ngram ng) const noexcept + { + auto const& idx = get_index(); + auto const it = idx.occs.find(ng); + if (it == idx.occs.end()) return nullptr; + return &it->second; + } + +private: + template + [[nodiscard]] auto& get_index(this auto& self) noexcept + { + if constexpr (N == 1) { + return self.store_.uni_idx; + } else if constexpr (N == 2) { + return self.store_.bi_idx; + } else { + static_assert(false, "unhandled N"); + } + } + + ngram_document_id max_doc_id_{0_doc_id}; + detail::ngram_index_storage store_; +}; + + +} // iris + +#endif diff --git a/iris.natvis b/iris.natvis index b928d79..76f86a6 100644 --- a/iris.natvis +++ b/iris.natvis @@ -252,4 +252,19 @@ (int)index_ + + + + + {chars._Elems,na1} + + + {chars._Elems,na2} + + + {chars._Elems,na3} + + + {chars._Elems,na4} + diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 66a345c..8b7f62e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -184,6 +184,7 @@ if(PROJECT_IS_TOP_LEVEL) colorize_format preprocess string_algo + ngram ) foreach(test_name IN LISTS IRIS_TEST_IRIS_TESTS) diff --git a/test/ngram.cpp b/test/ngram.cpp new file mode 100644 index 0000000..e722d9d --- /dev/null +++ b/test/ngram.cpp @@ -0,0 +1,124 @@ +#include "iris_test.hpp" + +#include + +#include +#include +#include + +#ifdef _MSC_VER +# include +#endif + +namespace iris { + +inline std::ostream& operator<<(std::ostream& os, iris::ngram_occurrence const& occ) +{ + return os << std::format("{}:{}", occ.doc_id, occ.pos); +} + +} // iris + +// -------------------------------------------------- + +using namespace iris::ngram_literals; +using iris::ngram_occurrence; + +[[nodiscard]] +constexpr auto make_occurrences(std::initializer_list occs) +{ + return std::vector{occs}; +} + +#define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ + std::vector const* occs = nullptr; \ + CHECK((occs = ngram_db.get_occurrences(U ## ng_str ## _2gram))); \ + if (occs) { \ + CHECK(*occs == make_occurrences({__VA_ARGS__})); \ + } \ + } while (false); + +TEST_CASE("ngram") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + // https://gihyo.jp/dev/serial/01/make-findspot/0005 + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は良い天気です。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("は良", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("良い", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("い天", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("天気", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("気で", {0_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 7}); + IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 8}); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は大雨です。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("は大", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("大雨", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("雨で", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 6}); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日の", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("の東", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("東海", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("海地", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("地方", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("方は", {0_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("は大", {0_doc_id, 7}); + IRIS_CHECK_OCCURRENCE("大雨", {0_doc_id, 8}); + IRIS_CHECK_OCCURRENCE("雨で", {0_doc_id, 9}); + IRIS_CHECK_OCCURRENCE("でし", {0_doc_id, 10}); + IRIS_CHECK_OCCURRENCE("しょ", {0_doc_id, 11}); + IRIS_CHECK_OCCURRENCE("ょう", {0_doc_id, 12}); + IRIS_CHECK_OCCURRENCE("う。", {0_doc_id, 13}); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は良い天気です。"); + (void)ngram_db.add_document(U"今日は大雨です。"); + (void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}, {1_doc_id, 0}, {2_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}, {1_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("は良", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("良い", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("い天", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("天気", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("気で", {0_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 7}, {1_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 8}, {1_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("は大", {1_doc_id, 2}, {2_doc_id, 7}); + IRIS_CHECK_OCCURRENCE("大雨", {1_doc_id, 3}, {2_doc_id, 8}); + IRIS_CHECK_OCCURRENCE("雨で", {1_doc_id, 4}, {2_doc_id, 9}); + IRIS_CHECK_OCCURRENCE("日の", {2_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("の東", {2_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("東海", {2_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("海地", {2_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("地方", {2_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("方は", {2_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("でし", {2_doc_id, 10}); + IRIS_CHECK_OCCURRENCE("しょ", {2_doc_id, 11}); + IRIS_CHECK_OCCURRENCE("ょう", {2_doc_id, 12}); + IRIS_CHECK_OCCURRENCE("う。", {2_doc_id, 13}); + } +} From b02faea09887b6c2c1710aae791629f73c01cd6b Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:55:28 +0900 Subject: [PATCH 02/10] std::ranges::shift_left IS NOT IMPLEMENTED --- include/iris/ngram.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index f5ecadb..c6b9d0d 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -120,7 +120,7 @@ class ngram_database idx.append(ng, doc_id, 0); for (; i < doc_text.size(); ++i) { - std::ranges::shift_left(ng.chars, 1); + std::shift_left(ng.chars.begin(), ng.chars.end(), 1); ng.chars[N - 1] = doc_text[i]; idx.append(ng, doc_id, int(i - N + 1)); } From ae5d511078437abf33296dabd4286faa26262558 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:41:31 +0900 Subject: [PATCH 03/10] Add more basic tests --- include/iris/ngram.hpp | 44 +++++++++++++++++++++++++++++++++++++++--- test/ngram.cpp | 44 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index c6b9d0d..88c7ab3 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -29,7 +29,7 @@ struct std::formatter { using base_type = std::formatter, CharT>; - template + template Ctx::iterator format(iris::ngram_document_id doc_id, Ctx& ctx) const { return base_type::format(std::to_underlying(doc_id), ctx); @@ -38,6 +38,12 @@ struct std::formatter namespace iris { +namespace detail { + +inline constexpr std::size_t N_GRAM_MAX_OPTIMIZED_N = 2; + +} // detail + struct ngram_occurrence { ngram_document_id doc_id; @@ -50,9 +56,28 @@ struct ngram_occurrence template struct ngram { + static_assert(N >= 1); + // TODO: optimize for N=1 std::array chars; + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + assert(chars[Len - 1] == static_cast(0)); + + if constexpr (N == 1) { + return ngram{chars[0]}; + } else if constexpr (N == 2) { + return ngram{chars[0], chars[1]}; + } else { + static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); + ngram ng; + std::ranges::copy_n(chars, Len - 1, ng.chars.begin()); + return ng; + } + } + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; }; @@ -70,8 +95,20 @@ inline namespace ngram_literals { return ngram<2, char32_t>{str[0], str[1]}; } +[[nodiscard]] constexpr auto operator ""_1gram(char32_t const* str, std::size_t len) noexcept +{ + assert(len == 1); + return ngram<1, char32_t>{str[0]}; +} + } // ngram_literals +template +[[nodiscard]] ngram to_ngram(CharT const (&chars)[N]) noexcept +{ + return ngram::from_c_array(chars); +} + namespace detail { @@ -114,7 +151,7 @@ class ngram_database ngram ng; std::size_t i = 0; - for (; i < N; ++i) { // TODO: loop unroll + for (; i < N; ++i) { ng.chars[i] = doc_text[i]; } idx.append(ng, doc_id, 0); @@ -126,6 +163,7 @@ class ngram_database } }; do_ngram(get_index<2>()); + static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); return doc_id; } @@ -149,7 +187,7 @@ class ngram_database } else if constexpr (N == 2) { return self.store_.bi_idx; } else { - static_assert(false, "unhandled N"); + static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); } } diff --git a/test/ngram.cpp b/test/ngram.cpp index e722d9d..0c51e41 100644 --- a/test/ngram.cpp +++ b/test/ngram.cpp @@ -30,15 +30,55 @@ constexpr auto make_occurrences(std::initializer_list occs) return std::vector{occs}; } +#define IRIS_CHECK_NO_OCCURRENCE(ng_str) do { \ + CHECK(!ngram_db.get_occurrences(iris::to_ngram(U ## ng_str))); \ + } while (false); + #define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ std::vector const* occs = nullptr; \ - CHECK((occs = ngram_db.get_occurrences(U ## ng_str ## _2gram))); \ + CHECK((occs = ngram_db.get_occurrences(iris::to_ngram(U ## ng_str)))); \ if (occs) { \ CHECK(*occs == make_occurrences({__VA_ARGS__})); \ } \ } while (false); -TEST_CASE("ngram") +TEST_CASE("ngram (minimal input)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U""); + IRIS_CHECK_NO_OCCURRENCE("今"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今"); + IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("無"); + IRIS_CHECK_NO_OCCURRENCE("今日"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今"); + IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("無"); + IRIS_CHECK_NO_OCCURRENCE("今日"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日"); + IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日", {0_doc_id, 1}); + IRIS_CHECK_NO_OCCURRENCE("無"); + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("今無"); + } +} + +TEST_CASE("ngram (realistic input)") { #ifdef _MSC_VER SetConsoleOutputCP(CP_UTF8); From 287c11eca250c2bda886743b3ddfc7dba583c325 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:47:38 +0900 Subject: [PATCH 04/10] Organize formatters --- include/iris/ngram.hpp | 34 +++++++++++++++++++++++----------- test/ngram.cpp | 6 +++--- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index 88c7ab3..61fee8f 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -1,7 +1,8 @@ #ifndef IRIS_NGRAM_HPP #define IRIS_NGRAM_HPP -#include +#include + #include #include #include @@ -21,6 +22,15 @@ namespace iris { enum struct ngram_document_id : unsigned {}; +struct ngram_occurrence +{ + ngram_document_id doc_id; + int pos; + + [[nodiscard]] constexpr bool operator==(ngram_occurrence const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_occurrence const&) const noexcept = default; +}; + } // iris template @@ -36,6 +46,17 @@ struct std::formatter } }; +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_occurrence const& occ, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}:{}", occ.doc_id, occ.pos); + } +}; + namespace iris { namespace detail { @@ -44,15 +65,6 @@ inline constexpr std::size_t N_GRAM_MAX_OPTIMIZED_N = 2; } // detail -struct ngram_occurrence -{ - ngram_document_id doc_id; - int pos; - - [[nodiscard]] constexpr bool operator==(ngram_occurrence const&) const noexcept = default; - [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_occurrence const&) const noexcept = default; -}; - template struct ngram { @@ -170,7 +182,7 @@ class ngram_database template [[nodiscard]] - std::vector const* get_occurrences(ngram ng) const noexcept + std::vector const* find_occurrences(ngram ng) const noexcept { auto const& idx = get_index(); auto const it = idx.occs.find(ng); diff --git a/test/ngram.cpp b/test/ngram.cpp index 0c51e41..88a237a 100644 --- a/test/ngram.cpp +++ b/test/ngram.cpp @@ -14,7 +14,7 @@ namespace iris { inline std::ostream& operator<<(std::ostream& os, iris::ngram_occurrence const& occ) { - return os << std::format("{}:{}", occ.doc_id, occ.pos); + return os << std::format("{}", occ); } } // iris @@ -31,12 +31,12 @@ constexpr auto make_occurrences(std::initializer_list occs) } #define IRIS_CHECK_NO_OCCURRENCE(ng_str) do { \ - CHECK(!ngram_db.get_occurrences(iris::to_ngram(U ## ng_str))); \ + CHECK(!ngram_db.find_occurrences(iris::to_ngram(U ## ng_str))); \ } while (false); #define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ std::vector const* occs = nullptr; \ - CHECK((occs = ngram_db.get_occurrences(iris::to_ngram(U ## ng_str)))); \ + CHECK((occs = ngram_db.find_occurrences(iris::to_ngram(U ## ng_str)))); \ if (occs) { \ CHECK(*occs == make_occurrences({__VA_ARGS__})); \ } \ From ada5846f9fa265aec4dca643a635d89f1b765ffb Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Wed, 12 Aug 2026 04:35:46 +0900 Subject: [PATCH 05/10] Implement search functionality --- include/iris/format.hpp | 2 + include/iris/interval.hpp | 129 +++++++++++ include/iris/ngram.hpp | 475 ++++++++++++++++++++++++++++++++++---- test/CMakeLists.txt | 1 + test/interval.cpp | 64 +++++ test/ngram.cpp | 46 +++- 6 files changed, 661 insertions(+), 56 deletions(-) create mode 100644 include/iris/interval.hpp create mode 100644 test/interval.cpp diff --git a/include/iris/format.hpp b/include/iris/format.hpp index 5539380..4c63876 100644 --- a/include/iris/format.hpp +++ b/include/iris/format.hpp @@ -3,6 +3,8 @@ // SPDX-License-Identifier: MIT +// SPDX-License-Identifier: MIT + #include #include diff --git a/include/iris/interval.hpp b/include/iris/interval.hpp new file mode 100644 index 0000000..1131ad7 --- /dev/null +++ b/include/iris/interval.hpp @@ -0,0 +1,129 @@ +#ifndef IRIS_INTERVAL_HPP +#define IRIS_INTERVAL_HPP + +// SPDX-License-Identifier: MIT + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace iris { + +template +struct interval +{ + using value_type = T; + T left, right; + + [[nodiscard]] + constexpr bool operator==(interval const&) const noexcept = default; + + [[nodiscard]] + constexpr std::strong_ordering operator<=>(interval const&) const noexcept = default; +}; + +template +[[nodiscard]] constexpr T& get(interval& iv) noexcept +{ + static_assert(I == 0 || I == 1); + if constexpr (I == 0) { return iv.left; } else { return iv.right; } +} +template +[[nodiscard]] constexpr T const& get(interval const& iv) noexcept +{ + static_assert(I == 0 || I == 1); + if constexpr (I == 0) { return iv.left; } else { return iv.right; } +} +template +[[nodiscard]] constexpr T&& get(interval&& iv) noexcept +{ + static_assert(I == 0 || I == 1); + if constexpr (I == 0) { return std::move(iv).left; } else { return std::move(iv).right; } +} +template +[[nodiscard]] constexpr T const&& get(interval const&& iv) noexcept +{ + static_assert(I == 0 || I == 1); + if constexpr (I == 0) { return std::move(iv).left; } else { return std::move(iv).right; } +} + +} // iris + +template +struct std::tuple_size> + : std::integral_constant +{}; + +template +struct std::tuple_element> +{ + using type = T; +}; + +template +struct std::formatter, CharT> +{ + [[nodiscard]] constexpr std::basic_format_parse_context::const_iterator + parse(std::basic_format_parse_context& ctx) + { + auto const first = ctx.begin(); + if (first == ctx.end()) return first; + if (*first == iris::format_traits::brace_close) return first; + + // Bound the search to this replacement field + auto const close_it = std::find( + first, ctx.end(), + iris::format_traits::brace_close + ); + if (close_it == ctx.end()) { + throw std::format_error("unterminated format specifier"); + } + + auto const comma_it = std::find( + first, close_it, + iris::format_traits::comma + ); + if (comma_it == close_it) { + throw std::format_error("expected ',' in format specifier"); + } + + { + std::basic_format_parse_context left_ctx{ + std::basic_string_view{first, comma_it}, 1 + }; + if (left_fmt_.parse(left_ctx) != left_ctx.end()) { + throw std::format_error("trailing characters in left format specifier"); + } + } + { + std::basic_format_parse_context right_ctx{ + std::basic_string_view{std::next(comma_it), close_it}, 1 + }; + if (right_fmt_.parse(right_ctx) != right_ctx.end()) { + throw std::format_error("trailing characters in right format specifier"); + } + } + return close_it; + } + + template + Ctx::iterator format(iris::interval const& iv, Ctx& ctx) const + { + ctx.advance_to(std::format_to(ctx.out(), "{}", iris::format_traits::square_brace_open)); + left_fmt_.format(iv.left, ctx); + ctx.advance_to(std::format_to(ctx.out(), "{}", iris::format_traits::comma)); + right_fmt_.format(iv.right, ctx); + return std::format_to(ctx.out(), "{}", iris::format_traits::paren_close); + } + +private: + std::formatter left_fmt_, right_fmt_; +}; + +#endif diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index 61fee8f..513652b 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -1,8 +1,15 @@ #ifndef IRIS_NGRAM_HPP #define IRIS_NGRAM_HPP +// SPDX-License-Identifier: MIT + +#include +#include +#include +#include #include +#include #include #include #include @@ -12,9 +19,9 @@ #include #include #include -#include #include #include +#include #include @@ -73,23 +80,34 @@ struct ngram // TODO: optimize for N=1 std::array chars; - template - [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + template + [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(*it++)) { - assert(chars[Len - 1] == static_cast(0)); - if constexpr (N == 1) { - return ngram{chars[0]}; + ngram ng; + ng.chars[0] = *it; + return ng; } else if constexpr (N == 2) { - return ngram{chars[0], chars[1]}; + ngram ng; + ng.chars[0] = *it++; + ng.chars[1] = *it; + return ng; } else { static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); ngram ng; - std::ranges::copy_n(chars, Len - 1, ng.chars.begin()); + std::ranges::copy_n(it, N, ng.chars.begin()); return ng; } } + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + assert(chars[Len - 1] == static_cast(0)); + return ngram::from_copy_n(std::ranges::begin(chars)); + } + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; }; @@ -104,12 +122,14 @@ inline namespace ngram_literals { [[nodiscard]] constexpr auto operator ""_2gram(char32_t const* str, std::size_t len) noexcept { assert(len == 2); + (void)len; return ngram<2, char32_t>{str[0], str[1]}; } [[nodiscard]] constexpr auto operator ""_1gram(char32_t const* str, std::size_t len) noexcept { assert(len == 1); + (void)len; return ngram<1, char32_t>{str[0]}; } @@ -124,82 +144,439 @@ template namespace detail { -template +enum struct [[nodiscard]] search_continuation : bool +{ + abort = false, + proceed = true, +}; + +struct ngram_posting +{ + ngram_document_id doc_id; + unsigned pos_offset = 0; + unsigned pos_count = 0; +}; + +struct ngram_posting_list +{ + std::vector postings; + std::vector positions; + + void append(ngram_document_id const doc_id, int pos) + { + if (postings.empty() || postings.back().doc_id != doc_id) { + if (!postings.empty() && postings.back().doc_id > doc_id) { + throw std::invalid_argument{"documents must be indexed in non-decreasing order of document ID"}; + } + postings.emplace_back( + doc_id, + static_cast(positions.size()), + 0 + ); + } + ++postings.back().pos_count; + positions.emplace_back(pos); + } + + void to_occurrence_list(std::vector& occs) const + { + occs.clear(); + for (auto const& posting : postings) { + for (std::size_t i = posting.pos_offset; i < posting.pos_offset + posting.pos_count; ++i) { + occs.emplace_back(posting.doc_id, positions[i]); + } + } + } + + template + void for_each_documents(F&& f) const + { + static_assert(std::invocable>); + + constexpr bool f_returns_continuation = std::same_as< + std::invoke_result_t>, + search_continuation + >; + + for (auto const& posting : postings) { + std::span const posting_span{ + positions.begin() + posting.pos_offset, + static_cast(posting.pos_count) + }; + + if constexpr (f_returns_continuation) { + search_continuation const cont = f(posting.doc_id, posting_span); + if (cont == search_continuation::abort) break; + } else { + f(posting.doc_id, posting_span); + } + } + } +}; + +template struct ngram_index { - void append(ngram ng, ngram_document_id doc_id, int pos) + void append(ngram const ng, ngram_document_id const doc_id, int const pos) + { + gram_entries[ng].append(doc_id, pos); + } + + [[nodiscard]] + bool empty() const noexcept + { + return gram_entries.empty(); + } + + void find_occurrences(ngram const ng, std::vector& occs) const { - occs[ng].emplace_back(doc_id, pos); + occs.clear(); + auto const it = gram_entries.find(ng); + if (it == gram_entries.end()) return; + + it->second.to_occurrence_list(occs); } - std::flat_map, std::vector> occs; + template + void search(ngram const ng, F&& f) const + { + auto const it = gram_entries.find(ng); + if (it == gram_entries.end()) return; + it->second.for_each_documents(f); + } + + std::flat_map, PostingListT> gram_entries; }; -template + +template struct ngram_index_storage { - ngram_index<1, CharT> uni_idx; - ngram_index<2, CharT> bi_idx; + ngram_index<1, CharT, PostingListT> uni_idx; + ngram_index<2, CharT, PostingListT> bi_idx; + + template + [[nodiscard]] auto& get_index(this auto& self) noexcept + { + if constexpr (N == 1) { + return self.uni_idx; + } else if constexpr (N == 2) { + return self.bi_idx; + } else { + static_assert(N_GRAM_MAX_OPTIMIZED_N == 2); + } + } + + void append_index(ngram_document_id const doc_id, std::basic_string_view const input) + { + this->template append_index<1>(doc_id, this->template get_index<1>(), input); + this->template append_index<2>(doc_id, this->template get_index<2>(), input); + } + + [[nodiscard]] + bool empty() const noexcept + { + return uni_idx.empty() && bi_idx.empty(); + } + + template + void search(ngram const ng, F&& f) const + { + this->template get_index().search(ng, f); + } + +private: + template + void append_index( + ngram_document_id const doc_id, + ngram_index& idx, + std::basic_string_view const input + ) + { + if (input.size() < N) return; + + if constexpr (N == 1) { + for (std::size_t i = 0; i < input.size(); ++i) { + idx.append(ngram<1, CharT>{input[i]}, doc_id, int(i)); + } + + } else { + auto ng = ngram::from_copy_n(input.begin()); + idx.append(ng, doc_id, 0); + + for (std::size_t i = N; i < input.size(); ++i) { + std::shift_left(ng.chars.begin(), ng.chars.end(), 1); + ng.chars[N - 1] = input[i]; + idx.append(ng, doc_id, int(i - N + 1)); + } + } + } }; } // detail + template -class ngram_database +struct ngram_search_query { + explicit ngram_search_query(std::basic_string_view input_sv) + { + std::basic_string input{input_sv}; + iris::compact_spaces(input); + if (input.empty()) return; + + words_ = input + | std::views::split(detail::string_algo_traits::space) + | std::views::transform([](auto const& r) { + return std::basic_string{std::from_range, r}; + }) + | std::ranges::to(); + + std::ranges::sort(words_); + { + auto const [first, last] = std::ranges::unique(words_); + words_.erase(first, last); + } + } + + // ------------------------------------------ + + [[nodiscard]] + auto const& words() const noexcept + { + return words_; + } + + [[nodiscard]] bool empty() const noexcept + { + return words_.empty(); + } + + [[nodiscard]] bool operator==(ngram_search_query const& other) const noexcept + { + return words_ == other.words_; + } + +private: + std::vector> words_; +}; + +} // iris + +template +struct std::formatter, CharT> + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_search_query const& query, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { + return iris::unicode::transcode(ustr); + })); + } +}; + + +namespace iris { + +class [[nodiscard]] ngram_search_result +{ + struct word_matches_t + { + int word_id = 0; + std::vector> matches; + }; + + using doc_matches_map = std::flat_map>; + + struct word_matches_handle + { + doc_matches_map::iterator map_it; + std::vector>* word_matches = nullptr; + + [[nodiscard]] + std::vector>* operator->() const noexcept + { + return word_matches; + } + + [[nodiscard]] explicit operator bool() const noexcept + { + return word_matches; + } + }; + public: [[nodiscard]] - ngram_document_id add_document(std::basic_string_view const doc_text) + bool has_document(ngram_document_id const doc_id) const noexcept { - ngram_document_id const doc_id{max_doc_id_}; - max_doc_id_ = ngram_document_id{std::to_underlying(max_doc_id_) + 1u}; + return doc_matches_.contains(doc_id); + } + + [[nodiscard]] + auto const& doc_matches() const noexcept { return doc_matches_; } - for (std::size_t i = 0; i < doc_text.size(); ++i) { - store_.uni_idx.append(ngram<1, CharT>{doc_text[i]}, doc_id, int(i)); + // Returns whether search must continue + template + [[nodiscard]] + bool init_word_matches(ngram_document_id const doc_id, int const word_id, std::span const positions) + { + auto doc_matches_it = doc_matches_.find(doc_id); + if (doc_matches_it == doc_matches_.end()) { + if (word_id != 0) return false; + doc_matches_it = doc_matches_.try_emplace(doc_id).first; } - auto const do_ngram = [&](detail::ngram_index& idx) { - if (doc_text.size() < N) return; + auto& word_matches = doc_matches_it->second.emplace_back(word_id); + word_matches.matches.assign_range(positions | std::views::transform([](int const pos) -> interval { + return {pos, pos + static_cast(N)}; + })); + return true; + } - ngram ng; - std::size_t i = 0; - for (; i < N; ++i) { - ng.chars[i] = doc_text[i]; - } - idx.append(ng, doc_id, 0); + [[nodiscard]] + word_matches_handle get_word_matches(ngram_document_id const doc_id, int const word_id) + { + auto const doc_matches_it = doc_matches_.find(doc_id); + if (doc_matches_it == doc_matches_.end()) return {}; - for (; i < doc_text.size(); ++i) { - std::shift_left(ng.chars.begin(), ng.chars.end(), 1); - ng.chars[N - 1] = doc_text[i]; - idx.append(ng, doc_id, int(i - N + 1)); - } - }; - do_ngram(get_index<2>()); - static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); + auto const it = std::ranges::find(doc_matches_it->second, word_id, &word_matches_t::word_id); + if (it == doc_matches_it->second.end()) return {}; + return {doc_matches_it, &it->matches}; + } + void erase_word_matches(word_matches_handle const& handle) + { + doc_matches_.erase(handle.map_it); + } + + void clear() noexcept + { + doc_matches_.clear(); + } + + [[nodiscard]] + bool empty() const noexcept + { + return doc_matches_.empty(); + } + + [[nodiscard]] + explicit operator bool() const noexcept + { + return !this->empty(); + } + +private: + doc_matches_map doc_matches_; +}; + +template +class ngram_database +{ +public: + [[nodiscard]] + ngram_document_id add_document(std::basic_string_view const doc_text) + { + ngram_document_id const doc_id{max_doc_id_}; + max_doc_id_ = ngram_document_id{std::to_underlying(max_doc_id_) + 1u}; + + store_.append_index(doc_id, doc_text); return doc_id; } template + void find_occurrences(ngram ng, std::vector& occs) const noexcept + { + occs.clear(); + auto const& idx = store_.template get_index(); + idx.find_occurrences(ng, occs); + } + [[nodiscard]] - std::vector const* find_occurrences(ngram ng) const noexcept + ngram_search_result search(ngram_search_query const& query) const { - auto const& idx = get_index(); - auto const it = idx.occs.find(ng); - if (it == idx.occs.end()) return nullptr; - return &it->second; + if (query.empty()) return {}; + if (store_.empty()) return {}; + + ngram_search_result search_res; + + int word_id = 0; + auto it = query.words().begin(); + assert(!it->empty()); + this->search_word(search_res, word_id++, *it++); + if (search_res.empty()) return search_res; + + for (; it != query.words().end(); ++it) { + assert(!it->empty()); + this->search_word(search_res, word_id++, *it); + if (search_res.empty()) break; + } + return search_res; } private: - template - [[nodiscard]] auto& get_index(this auto& self) noexcept + template + void search_word(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const { - if constexpr (N == 1) { - return self.store_.uni_idx; - } else if constexpr (N == 2) { - return self.store_.bi_idx; + if (word.empty()) return; + + if (word.size() == 1) { + this->search_word_impl(search_res, word_id, word); } else { - static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); + this->search_word_impl(search_res, word_id, word); + } + } + + template + void search_word_impl(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const + { + auto ng = ngram::from_copy_n(word.begin()); + + std::size_t available_doc_count = 0; + store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + if (search_res.init_word_matches(doc_id, word_id, positions)) { + ++available_doc_count; + } + }); + + if constexpr (!IsFirstWord) { + if (available_doc_count == 0) { + search_res.clear(); + return; + } + } + + for (std::size_t i = N; i < word.size(); i += N) { + std::ranges::copy_n(word.begin() + i, N, ng.chars.begin()); + + store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + auto word_matches = search_res.get_word_matches(doc_id, word_id); + if (!word_matches) return detail::search_continuation::proceed; + assert(!word_matches->empty()); + + for (auto it = word_matches->begin(); it != word_matches->end();) { + auto& prev_pos = *it; + + // TODO: make this binary search + if (std::ranges::any_of(positions, [prev_pos](int const pos) { + return pos == prev_pos.right; + })) { + // Matched; the current word's current n-gram is contiguous to the previous n-gram + prev_pos.right += N; + ++it; + continue; + } + + it = word_matches->erase(it); + } + + if (word_matches->empty()) { + search_res.erase_word_matches(word_matches); + if (search_res.empty()) return detail::search_continuation::abort; + } + return detail::search_continuation::proceed; + }); } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8b7f62e..2fa897b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -183,6 +183,7 @@ if(PROJECT_IS_TOP_LEVEL) indirect colorize_format preprocess + interval string_algo ngram ) diff --git a/test/interval.cpp b/test/interval.cpp new file mode 100644 index 0000000..2f9cef7 --- /dev/null +++ b/test/interval.cpp @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: MIT + +#include "iris_test.hpp" + +#include + +#include +#include +#include +#include + +using namespace std::string_view_literals; + +// NOLINTBEGIN(readability-container-size-empty) + +TEST_CASE("interval: type traits") +{ + STATIC_CHECK(std::is_trivial_v>); +} + +TEST_CASE("interval: tuple") +{ + { + iris::interval const iv{1, 2}; + auto const [left, right] = iv; // structured bindings + CHECK(left == 1); + CHECK(right == 2); + } + + { + iris::interval iv{1, 2}; + auto&& left = iris::get<0>(iv); + STATIC_CHECK(std::same_as); + CHECK(left == 1); + } + { + iris::interval const iv{1, 2}; + auto&& left = iris::get<0>(iv); + STATIC_CHECK(std::same_as); + CHECK(left == 1); + } + { + iris::interval iv{1, 2}; + auto&& left = iris::get<0>(std::move(iv)); + STATIC_CHECK(std::same_as); + CHECK(left == 1); + } + { + iris::interval const iv{1, 2}; + auto&& left = iris::get<0>(std::move(iv)); + STATIC_CHECK(std::same_as); + CHECK(left == 1); + } +} + +TEST_CASE("interval: format") +{ + CHECK(std::format("{}", iris::interval{}) == "[0,0)"sv); + CHECK(std::format("{}", iris::interval{1, 2}) == "[1,2)"sv); + CHECK(std::format("{:2d,}", iris::interval{1, 2}) == "[ 1,2)"sv); + CHECK(std::format("{:2d,3d}", iris::interval{1, 2}) == "[ 1, 2)"sv); +} + +// NOLINTEND(readability-container-size-empty) diff --git a/test/ngram.cpp b/test/ngram.cpp index 88a237a..e96313e 100644 --- a/test/ngram.cpp +++ b/test/ngram.cpp @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: MIT + #include "iris_test.hpp" #include @@ -12,11 +14,17 @@ namespace iris { -inline std::ostream& operator<<(std::ostream& os, iris::ngram_occurrence const& occ) +inline std::ostream& operator<<(std::ostream& os, ngram_occurrence const& occ) { return os << std::format("{}", occ); } +template +inline std::ostream& operator<<(std::ostream& os, interval const& iv) +{ + return os << std::format("{}", iv); +} + } // iris // -------------------------------------------------- @@ -31,15 +39,15 @@ constexpr auto make_occurrences(std::initializer_list occs) } #define IRIS_CHECK_NO_OCCURRENCE(ng_str) do { \ - CHECK(!ngram_db.find_occurrences(iris::to_ngram(U ## ng_str))); \ + std::vector occs; \ + ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ + CHECK(occs.empty()); \ } while (false); #define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ - std::vector const* occs = nullptr; \ - CHECK((occs = ngram_db.find_occurrences(iris::to_ngram(U ## ng_str)))); \ - if (occs) { \ - CHECK(*occs == make_occurrences({__VA_ARGS__})); \ - } \ + std::vector occs; \ + ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ + CHECK(occs == make_occurrences({__VA_ARGS__})); \ } while (false); TEST_CASE("ngram (minimal input)") @@ -162,3 +170,27 @@ TEST_CASE("ngram (realistic input)") IRIS_CHECK_OCCURRENCE("う。", {2_doc_id, 13}); } } + +TEST_CASE("ngram search") +{ + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は良い天気です。"); + //(void)ngram_db.add_document(U"今日は大雨です。"); + //(void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); + + iris::ngram_search_query<> query{U"良い天気"}; + + auto const search_res = ngram_db.search(query); + + auto const& doc_matches = search_res.doc_matches(); + + REQUIRE(doc_matches.contains(0_doc_id)); + auto const& word_map = doc_matches.at(0_doc_id); + + REQUIRE(word_map.size() == 1); + CHECK(word_map[0].word_id == 0); + REQUIRE(word_map[0].matches.size() == 1); + CHECK(word_map[0].matches[0] == iris::interval{3, 7}); + } +} From e33c337426b8e5fa25d0e98b35b41a120395cad9 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Wed, 12 Aug 2026 20:51:24 +0900 Subject: [PATCH 06/10] Update formatter to not use vendor specific internal func --- include/iris/interval.hpp | 4 ++-- test/interval.cpp | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/iris/interval.hpp b/include/iris/interval.hpp index 1131ad7..507b0d2 100644 --- a/include/iris/interval.hpp +++ b/include/iris/interval.hpp @@ -95,7 +95,7 @@ struct std::formatter, CharT> { std::basic_format_parse_context left_ctx{ - std::basic_string_view{first, comma_it}, 1 + std::basic_string_view{first, comma_it} }; if (left_fmt_.parse(left_ctx) != left_ctx.end()) { throw std::format_error("trailing characters in left format specifier"); @@ -103,7 +103,7 @@ struct std::formatter, CharT> } { std::basic_format_parse_context right_ctx{ - std::basic_string_view{std::next(comma_it), close_it}, 1 + std::basic_string_view{std::next(comma_it), close_it} }; if (right_fmt_.parse(right_ctx) != right_ctx.end()) { throw std::format_error("trailing characters in right format specifier"); diff --git a/test/interval.cpp b/test/interval.cpp index 2f9cef7..b52f12a 100644 --- a/test/interval.cpp +++ b/test/interval.cpp @@ -15,7 +15,8 @@ using namespace std::string_view_literals; TEST_CASE("interval: type traits") { - STATIC_CHECK(std::is_trivial_v>); + STATIC_CHECK(std::is_trivially_default_constructible_v>); + STATIC_CHECK(std::is_trivially_copyable_v>); } TEST_CASE("interval: tuple") From 06c66c67042037cf6496ee16eb7c9112bfe17d15 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Thu, 13 Aug 2026 04:55:09 +0900 Subject: [PATCH 07/10] Complete implementation for basic search functionality --- include/iris/ngram.hpp | 435 ++++++++++++++++++++-------- test/ngram.cpp | 624 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 894 insertions(+), 165 deletions(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index 513652b..ec6c646 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -4,6 +4,7 @@ // SPDX-License-Identifier: MIT #include +#include #include #include #include @@ -38,34 +39,6 @@ struct ngram_occurrence [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_occurrence const&) const noexcept = default; }; -} // iris - -template -struct std::formatter - : std::formatter, CharT> -{ - using base_type = std::formatter, CharT>; - - template - Ctx::iterator format(iris::ngram_document_id doc_id, Ctx& ctx) const - { - return base_type::format(std::to_underlying(doc_id), ctx); - } -}; - -template -struct std::formatter - : iris::no_spec_formatter -{ - template - Ctx::iterator format(iris::ngram_occurrence const& occ, Ctx& ctx) const - { - return std::format_to(ctx.out(), "{}:{}", occ.doc_id, occ.pos); - } -}; - -namespace iris { - namespace detail { inline constexpr std::size_t N_GRAM_MAX_OPTIMIZED_N = 2; @@ -248,68 +221,137 @@ struct ngram_index std::flat_map, PostingListT> gram_entries; }; +template +struct ngram_pos_t +{ + ngram ng; + int pos; + + [[nodiscard]] constexpr bool operator==(ngram_pos_t const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_pos_t const&) const noexcept = default; +}; template struct ngram_index_storage { - ngram_index<1, CharT, PostingListT> uni_idx; - ngram_index<2, CharT, PostingListT> bi_idx; - - template - [[nodiscard]] auto& get_index(this auto& self) noexcept - { - if constexpr (N == 1) { - return self.uni_idx; - } else if constexpr (N == 2) { - return self.bi_idx; - } else { - static_assert(N_GRAM_MAX_OPTIMIZED_N == 2); - } - } - void append_index(ngram_document_id const doc_id, std::basic_string_view const input) { - this->template append_index<1>(doc_id, this->template get_index<1>(), input); - this->template append_index<2>(doc_id, this->template get_index<2>(), input); + this->template append_index<1>(doc_id, input); + this->template append_index<2>(doc_id, input); } [[nodiscard]] bool empty() const noexcept { - return uni_idx.empty() && bi_idx.empty(); + return + this->template get_data<1>().idx.empty() && + this->template get_data<2>().idx.empty(); } template void search(ngram const ng, F&& f) const { - this->template get_index().search(ng, f); + this->template get_data().idx.search(ng, f); + } + + template + [[nodiscard]] auto& get_index(this auto& self) noexcept IRIS_LIFETIMEBOUND + { + return self.template get_data().idx; } private: + template + struct ngram_index_storage_data + { + ngram_index idx; + + // Caches + std::vector, default_init_allocator>> + batch_grams; + + std::vector, PostingListT>> + batch_pending; + }; + + ngram_index_storage_data<1> uni_data_; + ngram_index_storage_data<2> bi_data_; + + template + [[nodiscard]] auto& get_data(this auto& self) noexcept IRIS_LIFETIMEBOUND + { + if constexpr (N == 1) { + return self.uni_data_; + } else if constexpr (N == 2) { + return self.bi_data_; + } else { + static_assert(N_GRAM_MAX_OPTIMIZED_N == 2); + } + } + template void append_index( ngram_document_id const doc_id, - ngram_index& idx, std::basic_string_view const input ) { if (input.size() < N) return; + ngram_index_storage_data& data = this->template get_data(); + + // Naive per-gram insertion into flat_map is expensive: each *new* key + // shifts the underlying vectors, so building an index of vocabulary + // size V costs O(V^2) overall. Instead, per document: + // + // 1. Collect grams+positions --- O(G) G = grams in this doc + // 2. Sort them ----------------- O(G log G) + // 3. Existing keys ------------- O(D log V) D = distinct grams (D <= G) + // 4. New keys ------------------ O(V + P) P = brand-new keys (P <= D) + // + // Once the vocabulary saturates (P ~ 0, typical after a few documents), + // step 4 is a no-op and each document costs only O(G log G + D log V). + // + // Note: initially implemented by @saki7, then the complexity math is + // double-checked by Claude. + + data.batch_grams.clear(); + data.batch_grams.resize(input.size() - N + 1); if constexpr (N == 1) { for (std::size_t i = 0; i < input.size(); ++i) { - idx.append(ngram<1, CharT>{input[i]}, doc_id, int(i)); + data.batch_grams[i].ng.chars[0] = input[i]; + data.batch_grams[i].pos = static_cast(i); } } else { - auto ng = ngram::from_copy_n(input.begin()); - idx.append(ng, doc_id, 0); + for (std::size_t i = 0; i + N <= input.size(); ++i) { + std::ranges::copy_n(input.begin() + i, N, data.batch_grams[i].ng.chars.begin()); + data.batch_grams[i].pos = static_cast(i); + } + } + std::ranges::sort(data.batch_grams); + + data.batch_pending.clear(); - for (std::size_t i = N; i < input.size(); ++i) { - std::shift_left(ng.chars.begin(), ng.chars.end(), 1); - ng.chars[N - 1] = input[i]; - idx.append(ng, doc_id, int(i - N + 1)); + for (auto const& chunk : data.batch_grams | std::views::chunk_by( + [](auto const& a, auto const& b) { return a.ng == b.ng; } + )) { + auto const& key = chunk.front().ng; + if (auto const it = data.idx.gram_entries.find(key); it != data.idx.gram_entries.end()) { + for (auto const& gp : chunk) { + it->second.append(doc_id, gp.pos); + } + } else { + auto& pl = data.batch_pending.emplace_back(key, PostingListT{}).second; + for (auto const& gp : chunk) { + pl.append(doc_id, gp.pos); + } } } + data.idx.gram_entries.insert( + std::sorted_unique, + std::make_move_iterator(data.batch_pending.begin()), + std::make_move_iterator(data.batch_pending.end()) + ); } }; @@ -361,48 +403,52 @@ struct ngram_search_query std::vector> words_; }; -} // iris +template +ngram_search_query(CharT const(&)[N]) -> ngram_search_query; -template -struct std::formatter, CharT> - : iris::no_spec_formatter + +struct [[nodiscard]] ngram_search_word_match { - template - Ctx::iterator format(iris::ngram_search_query const& query, Ctx& ctx) const + ngram_search_word_match() = default; + + explicit ngram_search_word_match(int word_id) + : word_id(word_id) + {} + + ngram_search_word_match(int word_id, std::initializer_list> spans) + : word_id(word_id) + , spans(spans) + {} + + int word_id = 0; + unsigned successful_ngrams = 1; // due to the class layout, this must be placed here + std::vector> spans; + + [[nodiscard]] + bool operator==(ngram_search_word_match const& other) const noexcept { - return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { - return iris::unicode::transcode(ustr); - })); + return word_id == other.word_id && spans == other.spans; } }; - -namespace iris { - class [[nodiscard]] ngram_search_result { - struct word_matches_t - { - int word_id = 0; - std::vector> matches; - }; - - using doc_matches_map = std::flat_map>; + using doc_matches_map = std::flat_map>; struct word_matches_handle { - doc_matches_map::iterator map_it; - std::vector>* word_matches = nullptr; + doc_matches_map::iterator doc_it; + ngram_search_word_match* word_match = nullptr; [[nodiscard]] - std::vector>* operator->() const noexcept + ngram_search_word_match* operator->() const noexcept { - return word_matches; + return word_match; } [[nodiscard]] explicit operator bool() const noexcept { - return word_matches; + return word_match; } }; @@ -417,18 +463,25 @@ class [[nodiscard]] ngram_search_result auto const& doc_matches() const noexcept { return doc_matches_; } // Returns whether search must continue - template + template [[nodiscard]] bool init_word_matches(ngram_document_id const doc_id, int const word_id, std::span const positions) { - auto doc_matches_it = doc_matches_.find(doc_id); - if (doc_matches_it == doc_matches_.end()) { - if (word_id != 0) return false; - doc_matches_it = doc_matches_.try_emplace(doc_id).first; + assert(!positions.empty()); + + doc_matches_map::iterator doc_matches_it; + if constexpr (IsFirstWord) { + assert(word_id == 0); + assert(doc_matches_.empty() || doc_matches_.rbegin()->first < doc_id); + doc_matches_it = doc_matches_.try_emplace(doc_matches_.end(), doc_id); // hint: append + } else { + doc_matches_it = doc_matches_.find(doc_id); + if (doc_matches_it == doc_matches_.end()) return false; // no new docs after word 0 } - auto& word_matches = doc_matches_it->second.emplace_back(word_id); - word_matches.matches.assign_range(positions | std::views::transform([](int const pos) -> interval { + assert(!std::ranges::contains(doc_matches_it->second, word_id, &ngram_search_word_match::word_id)); + auto& word_match = doc_matches_it->second.emplace_back(word_id); + word_match.spans.assign_range(positions | std::views::transform([](int const pos) -> interval { return {pos, pos + static_cast(N)}; })); return true; @@ -440,14 +493,52 @@ class [[nodiscard]] ngram_search_result auto const doc_matches_it = doc_matches_.find(doc_id); if (doc_matches_it == doc_matches_.end()) return {}; - auto const it = std::ranges::find(doc_matches_it->second, word_id, &word_matches_t::word_id); - if (it == doc_matches_it->second.end()) return {}; - return {doc_matches_it, &it->matches}; + // We don't need to do *full* `std::find` here; the word match is + // always inserted sequentially so if it exists, it is always placed + // at the *back* of the vector. + if ( + doc_matches_it->second.empty() || + doc_matches_it->second.back().word_id != word_id + ) { + assert( + doc_matches_it->second.empty() || + // Make sure the matching element does not exist at the position except for *back* + !std::ranges::contains(doc_matches_it->second, word_id, &ngram_search_word_match::word_id) + ); + return {}; + } + assert(!doc_matches_it->second.back().spans.empty()); + return {doc_matches_it, &doc_matches_it->second.back()}; + } + + void erase_document(word_matches_handle const& handle) + { + doc_matches_.erase(handle.doc_it); } - void erase_word_matches(word_matches_handle const& handle) + void remove_stale_document_matches(int const word_id, unsigned const expected_ngrams) { - doc_matches_.erase(handle.map_it); + auto [keys, values] = std::move(doc_matches_).extract(); + + std::size_t out = 0; + for (std::size_t in = 0; in < keys.size(); ++in) { + auto& word_matches = values[in]; + bool has_word = false; + std::erase_if(word_matches, [&](ngram_search_word_match const& wm) { + if (wm.word_id != word_id) return false; + has_word = true; + return wm.successful_ngrams != expected_ngrams; + }); + if (!has_word || word_matches.empty()) continue; + if (out != in) { + keys[out] = keys[in]; + values[out] = std::move(values[in]); + } + ++out; + } + keys.resize(out); + values.resize(out); + doc_matches_.replace(std::move(keys), std::move(values)); } void clear() noexcept @@ -519,7 +610,7 @@ class ngram_database template void search_word(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const { - if (word.empty()) return; + assert(!word.empty()); if (word.size() == 1) { this->search_word_impl(search_res, word_id, word); @@ -531,60 +622,168 @@ class ngram_database template void search_word_impl(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const { + assert(word.size() >= N); auto ng = ngram::from_copy_n(word.begin()); - std::size_t available_doc_count = 0; - store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { - if (search_res.init_word_matches(doc_id, word_id, positions)) { - ++available_doc_count; - } - }); + if constexpr (IsFirstWord) { + store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + (void)search_res.init_word_matches(doc_id, word_id, positions); + }); + if (search_res.empty()) return; - if constexpr (!IsFirstWord) { + } else { + std::size_t available_doc_count = 0; + store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + if (search_res.init_word_matches(doc_id, word_id, positions)) { + ++available_doc_count; + } + }); if (available_doc_count == 0) { search_res.clear(); return; } } - for (std::size_t i = N; i < word.size(); i += N) { - std::ranges::copy_n(word.begin() + i, N, ng.chars.begin()); - - store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { - auto word_matches = search_res.get_word_matches(doc_id, word_id); - if (!word_matches) return detail::search_continuation::proceed; - assert(!word_matches->empty()); + unsigned current_ngram = 1; + auto const do_search = [&](int remaining_chars) { + return [&, remaining_chars, overlapping_chars = int(N) - remaining_chars](ngram_document_id const doc_id, std::span const positions) { + // Find the existing match set from the previous iteration. + // If none exists, any subsequent characters of the document will not match. + // + // For example, when the document is "今日は晴れです" and current `ng` is "は晴", + // - When previous `ng` was "昨日", `search_res` contians no matches => omit further sequence + // - When previous `ng` was "今日", `search_res` contains matches => proceed with "は晴" + auto word_match = search_res.get_word_matches(doc_id, word_id); + if (!word_match) return detail::search_continuation::proceed; + + // Prevent *resurrecting* the false-positive match on "match -> unmatch -> match" pattern. + // For example, when the document is "abef" and the query is "abXXef", + // - ngram{"ab"} -> match (successful_ngrams = 1) + // - ngram{"XX"} -> no match (successful_ngrams is untouched) + // - ngram{"ef"} -> successful_ngrams does not match current_ngram! + if (word_match->successful_ngrams != current_ngram) { + search_res.erase_document(word_match); + if (search_res.empty()) return detail::search_continuation::abort; + return detail::search_continuation::proceed; + } - for (auto it = word_matches->begin(); it != word_matches->end();) { + // Find contiguous match; document has [previous ng, current ng] + for (auto it = word_match->spans.begin(); it != word_match->spans.end();) { auto& prev_pos = *it; - // TODO: make this binary search - if (std::ranges::any_of(positions, [prev_pos](int const pos) { - return pos == prev_pos.right; - })) { + if (std::ranges::binary_search(positions, prev_pos.right - overlapping_chars)) { // Matched; the current word's current n-gram is contiguous to the previous n-gram - prev_pos.right += N; + prev_pos.right += remaining_chars; ++it; continue; } - - it = word_matches->erase(it); + // Erase exiting match that indicates the below structure + // [previous ng, ...some unrelated chars..., current ng] + it = word_match->spans.erase(it); } - if (word_matches->empty()) { - search_res.erase_word_matches(word_matches); + // Even if *all* existing matches fit + // [previous ng, ...some unrelated chars..., current ng], + // we can always remove the entire document from the candidate pool. + if (word_match->spans.empty()) { + search_res.erase_document(word_match); if (search_res.empty()) return detail::search_continuation::abort; + return detail::search_continuation::proceed; } + + ++word_match->successful_ngrams; return detail::search_continuation::proceed; - }); + }; + }; + + std::size_t i = N; + for (; i + N <= word.size(); i += N) { + std::ranges::copy_n(word.begin() + i, N, ng.chars.begin()); + store_.search(ng, do_search(N)); + if (search_res.empty()) return; + ++current_ngram; + } + + // When the remaining character count is remainder of `word.size() % N`, + // search by the *slided* remaining characters. + // + // For example, when the document is "今日は晴れです": + // + // When doing 3-gram search with "今日は雨": + // 1. Search by "今日は" in the normal loop + // + // 2. Then, + // i == 3 + // remaining_chars == word.size() - i == 1 + // overlapping_chars == N - remaining_chars == 2 + // next_search_pos = i - overlapping_chars == 1 + // + // 3. Try to match "日は雨" in the last loop + if (int const remaining_chars = static_cast(word.size() - i); remaining_chars > 0) { + assert(remaining_chars < N); + std::shift_left(ng.chars.begin(), ng.chars.end(), remaining_chars); + std::ranges::copy_n(word.begin() + i, remaining_chars, ng.chars.begin() + (N - remaining_chars)); + store_.search(ng, do_search(remaining_chars)); + if (search_res.empty()) return; + ++current_ngram; } + + search_res.remove_stale_document_matches(word_id, current_ngram); } ngram_document_id max_doc_id_{0_doc_id}; detail::ngram_index_storage store_; }; - } // iris + +template +struct std::formatter + : std::formatter, CharT> +{ + using base_type = std::formatter, CharT>; + + template + Ctx::iterator format(iris::ngram_document_id doc_id, Ctx& ctx) const + { + return base_type::format(std::to_underlying(doc_id), ctx); + } +}; + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_occurrence const& occ, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}:{}", occ.doc_id, occ.pos); + } +}; + +template +struct std::formatter, CharT> + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_search_query const& query, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { + return iris::unicode::transcode(ustr); + })); + } +}; + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_search_word_match const& word_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{{word: #{}, spans: {}}}", word_match.word_id, word_match.spans); + } +}; + #endif diff --git a/test/ngram.cpp b/test/ngram.cpp index e96313e..7e81475 100644 --- a/test/ngram.cpp +++ b/test/ngram.cpp @@ -12,25 +12,9 @@ # include #endif -namespace iris { - -inline std::ostream& operator<<(std::ostream& os, ngram_occurrence const& occ) -{ - return os << std::format("{}", occ); -} - -template -inline std::ostream& operator<<(std::ostream& os, interval const& iv) -{ - return os << std::format("{}", iv); -} - -} // iris - -// -------------------------------------------------- - using namespace iris::ngram_literals; using iris::ngram_occurrence; +using iris::interval; [[nodiscard]] constexpr auto make_occurrences(std::initializer_list occs) @@ -42,13 +26,13 @@ constexpr auto make_occurrences(std::initializer_list occs) std::vector occs; \ ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ CHECK(occs.empty()); \ - } while (false); + } while (false) #define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ std::vector occs; \ ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ CHECK(occs == make_occurrences({__VA_ARGS__})); \ - } while (false); + } while (false) TEST_CASE("ngram (minimal input)") { @@ -59,30 +43,62 @@ TEST_CASE("ngram (minimal input)") { iris::ngram_database<> ngram_db; (void)ngram_db.add_document(U""); - IRIS_CHECK_NO_OCCURRENCE("今"); + IRIS_CHECK_NO_OCCURRENCE("a"); } { iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"今"); - IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); - IRIS_CHECK_NO_OCCURRENCE("無"); - IRIS_CHECK_NO_OCCURRENCE("今日"); + (void)ngram_db.add_document(U"a"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_NO_OCCURRENCE("XX"); } { iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"今"); - IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); - IRIS_CHECK_NO_OCCURRENCE("無"); - IRIS_CHECK_NO_OCCURRENCE("今日"); + (void)ngram_db.add_document(U"ab"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("XX"); } { iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"今日"); - IRIS_CHECK_OCCURRENCE("今", {0_doc_id, 0}); - IRIS_CHECK_OCCURRENCE("日", {0_doc_id, 1}); - IRIS_CHECK_NO_OCCURRENCE("無"); - IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); - IRIS_CHECK_NO_OCCURRENCE("今無"); + (void)ngram_db.add_document(U"abc"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); + IRIS_CHECK_NO_OCCURRENCE("XX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("d", {0_doc_id, 3}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("cd", {0_doc_id, 2}); + IRIS_CHECK_NO_OCCURRENCE("XX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcde"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("d", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("e", {0_doc_id, 4}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("cd", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("de", {0_doc_id, 3}); + IRIS_CHECK_NO_OCCURRENCE("XX"); } } @@ -171,26 +187,540 @@ TEST_CASE("ngram (realistic input)") } } -TEST_CASE("ngram search") +struct DocumentMatch +{ + iris::ngram_document_id doc_id; + std::vector word_matches; + + DocumentMatch(iris::ngram_document_id doc_id, std::initializer_list word_matches) + : doc_id(doc_id) + , word_matches(word_matches) + {} + + DocumentMatch(iris::ngram_document_id doc_id, std::vector word_matches) + : doc_id(doc_id) + , word_matches(std::move(word_matches)) + {} + + [[nodiscard]] + bool operator==(DocumentMatch const&) const noexcept = default; +}; + +template +struct std::formatter + : iris::no_spec_formatter { + template + Ctx::iterator format(DocumentMatch const& doc_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "(doc: #{}, word_matches: {})", doc_match.doc_id, doc_match.word_matches); + } +}; + +#define IRIS_CHECK_SEARCH(query_input, ...) do { \ + iris::ngram_search_query const query{U ## query_input}; \ + auto const search_res = ngram_db.search(query); \ + auto const& doc_matches = search_res.doc_matches(); \ + \ + std::vector const expected_doc_matches{ \ + std::initializer_list{__VA_ARGS__} \ + }; \ + \ + auto const actual_doc_matches = doc_matches | std::views::transform([](auto const& kv) { \ + return DocumentMatch{kv.first, kv.second}; \ + }) | std::ranges::to(); \ + CHECK(actual_doc_matches == expected_doc_matches); \ + } while (false) + +TEST_CASE("ngram search (document chars = 0)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + { iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"今日は良い天気です。"); - //(void)ngram_db.add_document(U"今日は大雨です。"); - //(void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} - iris::ngram_search_query<> query{U"良い天気"}; +// 1-gram document +TEST_CASE("ngram search (document chars = 1)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif - auto const search_res = ngram_db.search(query); + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"a"); + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} - auto const& doc_matches = search_res.doc_matches(); +// 2-gram document +TEST_CASE("ngram search (document chars = 2)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aa"); + IRIS_CHECK_SEARCH(""); - REQUIRE(doc_matches.contains(0_doc_id)); - auto const& word_map = doc_matches.at(0_doc_id); + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, aaa/baa)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif - REQUIRE(word_map.size() == 1); - CHECK(word_map[0].word_id == 0); - REQUIRE(word_map[0].matches.size() == 1); - CHECK(word_map[0].matches[0] == iris::interval{3, 7}); + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aaa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}, interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aaa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aaaX"); + IRIS_CHECK_SEARCH("Xaaa"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"baa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "baa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("baaX"); + IRIS_CHECK_SEARCH("Xbaa"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, aba/aab)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aba"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aba", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abaX"); + IRIS_CHECK_SEARCH("Xaba"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aab", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aabX"); + IRIS_CHECK_SEARCH("Xaab"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, abc)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abc"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "c", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("bcX"); + IRIS_CHECK_SEARCH("Xbc"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abcX"); + IRIS_CHECK_SEARCH("Xabc"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXbc"); + IRIS_CHECK_SEARCH("bcXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +TEST_CASE("ngram search (document chars = 4)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + // TODO + + // 2x 2-gram document + //{ + // iris::ngram_database<> ngram_db; + // (void)ngram_db.add_document(U"aaaa"); + // IRIS_CHECK_SEARCH(""); + + // IRIS_CHECK_SEARCH( + // "a", + // {0_doc_id, { + // {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}, interval{3, 4}}}, + // }}, + // ); + // IRIS_CHECK_SEARCH("X"); + + // IRIS_CHECK_SEARCH( + // "aa", + // {0_doc_id, { + // {0, {interval{0, 2}, interval{2, 4}}}, + // }}, + // ); + // IRIS_CHECK_SEARCH("XX"); + //} + +} + +TEST_CASE("ngram search (minimal input, dependency on previous match)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abefef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abxcd"); // trap document + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "abcd", + {1_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "ab cd", + {1_doc_id, { + {0, {interval{0, 2}}}, + {1, {interval{2, 4}}}, + }}, + ); } } From 9539362703cd58f1f57ff44d4ee064a16c920952 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:28:12 +0900 Subject: [PATCH 08/10] Optimize implementation and add tests --- include/iris/ngram.hpp | 364 ++++++++++++++++------- test/CMakeLists.txt | 3 + test/ngram.cpp | 563 +----------------------------------- test/ngram_search_2.cpp | 104 +++++++ test/ngram_search_3.cpp | 287 ++++++++++++++++++ test/ngram_search_4_dep.cpp | 407 ++++++++++++++++++++++++++ test/ngram_test.hpp | 67 +++++ 7 files changed, 1144 insertions(+), 651 deletions(-) create mode 100644 test/ngram_search_2.cpp create mode 100644 test/ngram_search_3.cpp create mode 100644 test/ngram_search_4_dep.cpp create mode 100644 test/ngram_test.hpp diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index ec6c646..1cf1e5e 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -25,6 +26,7 @@ #include #include +#include namespace iris { @@ -41,42 +43,131 @@ struct ngram_occurrence namespace detail { -inline constexpr std::size_t N_GRAM_MAX_OPTIMIZED_N = 2; +template struct ngram_value; +template<> struct ngram_value<1> { using type = std::uint8_t; }; +template<> struct ngram_value<2> { using type = std::uint16_t; }; +template<> struct ngram_value<4> { using type = std::uint32_t; }; +template<> struct ngram_value<8> { using type = std::uint64_t; }; + +template +using ngram_value_t = ngram_value::type; } // detail template struct ngram { - static_assert(N >= 1); + static_assert(N >= 3); - // TODO: optimize for N=1 - std::array chars; + std::array data; + template + constexpr void copy_n(It it) + noexcept(noexcept(*it++)) + { + std::ranges::copy_n(it, N, data.begin()); + } template [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + ngram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + constexpr void shift_copy(It it, int const remaining_chars) + noexcept( + noexcept(std::shift_left(data.begin(), data.end(), remaining_chars)) && + noexcept(std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars))) + ) + { + assert(remaining_chars < N); + std::shift_left(data.begin(), data.end(), remaining_chars); + std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars)); + } + + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == N + 1); + assert(chars[Len - 1] == static_cast(0)); + return ngram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; +}; + +template +struct ngram<1, CharT> +{ + CharT data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it)) + { + data = *it; + } + template + [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + ngram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == 1 + 1); + assert(chars[Len - 1] == static_cast(0)); + return ngram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; +}; + +template +struct ngram<2, CharT> +{ + using value_type = detail::ngram_value_t; + value_type data; + + template + constexpr void copy_n(It it) noexcept(noexcept(*it++)) { - if constexpr (N == 1) { - ngram ng; - ng.chars[0] = *it; - return ng; - } else if constexpr (N == 2) { - ngram ng; - ng.chars[0] = *it++; - ng.chars[1] = *it; - return ng; - } else { - static_assert(detail::N_GRAM_MAX_OPTIMIZED_N == 2); - ngram ng; - std::ranges::copy_n(it, N, ng.chars.begin()); - return ng; - } + using uchar = std::make_unsigned_t; + data = value_type(static_cast(*it++)) << (sizeof(CharT) * 8); + data |= value_type(static_cast(*it)); + } + template + [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + ngram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + constexpr void shift_copy(It it, int const remaining_chars) + noexcept(noexcept(*it)) + { + assert(remaining_chars == 1); + (void)remaining_chars; + data = (data << (sizeof(CharT) * 8)) | value_type(static_cast>(*it)); } template [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept { + static_assert(Len == 2 + 1); assert(chars[Len - 1] == static_cast(0)); return ngram::from_copy_n(std::ranges::begin(chars)); } @@ -92,20 +183,6 @@ inline namespace ngram_literals { return ngram_document_id{static_cast>(id)}; } -[[nodiscard]] constexpr auto operator ""_2gram(char32_t const* str, std::size_t len) noexcept -{ - assert(len == 2); - (void)len; - return ngram<2, char32_t>{str[0], str[1]}; -} - -[[nodiscard]] constexpr auto operator ""_1gram(char32_t const* str, std::size_t len) noexcept -{ - assert(len == 1); - (void)len; - return ngram<1, char32_t>{str[0]}; -} - } // ngram_literals template @@ -188,37 +265,101 @@ struct ngram_posting_list }; template -struct ngram_index +class ngram_index { - void append(ngram const ng, ngram_document_id const doc_id, int const pos) - { - gram_entries[ng].append(doc_id, pos); - } + using entry_map = std::flat_map, std::unique_ptr>; + static constexpr std::size_t side_merge_threshold = 2048; +public: [[nodiscard]] - bool empty() const noexcept + auto find_list(this auto&& self, ngram const ng) { - return gram_entries.empty(); + if (auto const it = self.gram_entries_.find(ng); it != self.gram_entries_.end()) { + return it->second.get(); + } + if (auto const it = self.side_entries_.find(ng); it != self.side_entries_.end()) { + return it->second.get(); + } + return static_cast(nullptr); } void find_occurrences(ngram const ng, std::vector& occs) const { occs.clear(); - auto const it = gram_entries.find(ng); - if (it == gram_entries.end()) return; - - it->second.to_occurrence_list(occs); + auto const* list = this->find_list(ng); + if (!list) return; + list->to_occurrence_list(occs); } template void search(ngram const ng, F&& f) const { - auto const it = gram_entries.find(ng); - if (it == gram_entries.end()) return; - it->second.for_each_documents(f); + auto const* list = this->find_list(ng); + if (!list) return; + list->for_each_documents(f); + } + + [[nodiscard]] + bool empty() const noexcept + { + return gram_entries_.empty() && side_entries_.empty(); + } + + void merge_new_entries(std::vector, std::unique_ptr>>& pending) + { + if (pending.empty()) return; // vocabulary saturated + + for (auto& [key, pl] : pending) { + [[maybe_unused]] + auto const it = side_entries_.try_emplace( + side_entries_.end(), // hint + key, std::move(pl) + ); + assert(it->second != nullptr && pl == nullptr); + } + if (side_entries_.size() >= side_merge_threshold) { + this->flush_side(); + } + } + +private: + void flush_side() + { + if (side_entries_.empty()) return; + + auto [skeys, svalues] = std::move(side_entries_).extract(); + auto [keys, values] = std::move(gram_entries_).extract(); + + std::size_t const old_size = keys.size(); + std::size_t const add = skeys.size(); + keys.resize(old_size + add); + values.resize(old_size + add); + + // Backward merge + std::size_t out = old_size + add; + std::size_t i = old_size; + std::size_t j = add; + while (j > 0) { + if (i > 0 && skeys[j - 1] < keys[i - 1]) { + --out; + --i; + keys[out] = keys[i]; + values[out] = std::move(values[i]); + } else { + assert(i == 0 || keys[i - 1] < skeys[j - 1]); + --out; + --j; + keys[out] = skeys[j]; + values[out] = std::move(svalues[j]); + } + } + assert(out == i); + + gram_entries_.replace(std::move(keys), std::move(values)); } - std::flat_map, PostingListT> gram_entries; + // Double-buffered to reduce insertion cost + entry_map gram_entries_, side_entries_; }; template @@ -255,7 +396,7 @@ struct ngram_index_storage } template - [[nodiscard]] auto& get_index(this auto& self) noexcept IRIS_LIFETIMEBOUND + [[nodiscard]] auto& get_index(this auto& self IRIS_LIFETIMEBOUND) noexcept { return self.template get_data().idx; } @@ -270,7 +411,7 @@ struct ngram_index_storage std::vector, default_init_allocator>> batch_grams; - std::vector, PostingListT>> + std::vector, std::unique_ptr>> batch_pending; }; @@ -278,14 +419,14 @@ struct ngram_index_storage ngram_index_storage_data<2> bi_data_; template - [[nodiscard]] auto& get_data(this auto& self) noexcept IRIS_LIFETIMEBOUND + [[nodiscard]] auto& get_data(this auto& self IRIS_LIFETIMEBOUND) noexcept { if constexpr (N == 1) { return self.uni_data_; } else if constexpr (N == 2) { return self.bi_data_; } else { - static_assert(N_GRAM_MAX_OPTIMIZED_N == 2); + static_assert(false); } } @@ -318,13 +459,13 @@ struct ngram_index_storage if constexpr (N == 1) { for (std::size_t i = 0; i < input.size(); ++i) { - data.batch_grams[i].ng.chars[0] = input[i]; + data.batch_grams[i].ng.data = input[i]; data.batch_grams[i].pos = static_cast(i); } } else { for (std::size_t i = 0; i + N <= input.size(); ++i) { - std::ranges::copy_n(input.begin() + i, N, data.batch_grams[i].ng.chars.begin()); + data.batch_grams[i].ng.copy_n(input.begin() + i); data.batch_grams[i].pos = static_cast(i); } } @@ -336,22 +477,20 @@ struct ngram_index_storage [](auto const& a, auto const& b) { return a.ng == b.ng; } )) { auto const& key = chunk.front().ng; - if (auto const it = data.idx.gram_entries.find(key); it != data.idx.gram_entries.end()) { + if (PostingListT* const pl = data.idx.find_list(key)) { for (auto const& gp : chunk) { - it->second.append(doc_id, gp.pos); + pl->append(doc_id, gp.pos); } + } else { - auto& pl = data.batch_pending.emplace_back(key, PostingListT{}).second; + auto& new_pl = data.batch_pending.emplace_back(key, std::make_unique()).second; for (auto const& gp : chunk) { - pl.append(doc_id, gp.pos); + new_pl->append(doc_id, gp.pos); } } } - data.idx.gram_entries.insert( - std::sorted_unique, - std::make_move_iterator(data.batch_pending.begin()), - std::make_move_iterator(data.batch_pending.end()) - ); + + data.idx.merge_new_entries(data.batch_pending); } }; @@ -456,7 +595,10 @@ class [[nodiscard]] ngram_search_result [[nodiscard]] bool has_document(ngram_document_id const doc_id) const noexcept { - return doc_matches_.contains(doc_id); + auto const it = doc_matches_.find(doc_id); + // An entry with no word matches is a tombstone (soft-erased document + // awaiting the next sweep), not a match. + return it != doc_matches_.end() && !it->second.empty(); } [[nodiscard]] @@ -474,9 +616,12 @@ class [[nodiscard]] ngram_search_result assert(word_id == 0); assert(doc_matches_.empty() || doc_matches_.rbegin()->first < doc_id); doc_matches_it = doc_matches_.try_emplace(doc_matches_.end(), doc_id); // hint: append + ++live_doc_count_; + } else { doc_matches_it = doc_matches_.find(doc_id); if (doc_matches_it == doc_matches_.end()) return false; // no new docs after word 0 + if (doc_matches_it->second.empty()) return false; // tombstoned (soft-erased) document; skip } assert(!std::ranges::contains(doc_matches_it->second, word_id, &ngram_search_word_match::word_id)); @@ -513,7 +658,14 @@ class [[nodiscard]] ngram_search_result void erase_document(word_matches_handle const& handle) { - doc_matches_.erase(handle.doc_it); + // This is slow because + // k erases x O(n) shift each =~ O(n^2) per word + //doc_matches_.erase(handle.doc_it); + + assert(!handle.doc_it->second.empty()); // never double-tombstone + handle.doc_it->second.clear(); // make this tombstone + assert(live_doc_count_ >= 1); + --live_doc_count_; } void remove_stale_document_matches(int const word_id, unsigned const expected_ngrams) @@ -523,13 +675,15 @@ class [[nodiscard]] ngram_search_result std::size_t out = 0; for (std::size_t in = 0; in < keys.size(); ++in) { auto& word_matches = values[in]; - bool has_word = false; + bool is_word_survived = false; std::erase_if(word_matches, [&](ngram_search_word_match const& wm) { if (wm.word_id != word_id) return false; - has_word = true; - return wm.successful_ngrams != expected_ngrams; + if (wm.successful_ngrams != expected_ngrams) return true; + is_word_survived = true; + return false; }); - if (!has_word || word_matches.empty()) continue; + if (!is_word_survived || word_matches.empty()) continue; + if (out != in) { keys[out] = keys[in]; values[out] = std::move(values[in]); @@ -539,17 +693,19 @@ class [[nodiscard]] ngram_search_result keys.resize(out); values.resize(out); doc_matches_.replace(std::move(keys), std::move(values)); + live_doc_count_ = out; } - void clear() noexcept + void reset() noexcept { doc_matches_.clear(); + live_doc_count_ = 0; } [[nodiscard]] bool empty() const noexcept { - return doc_matches_.empty(); + return live_doc_count_ == 0; } [[nodiscard]] @@ -560,6 +716,7 @@ class [[nodiscard]] ngram_search_result private: doc_matches_map doc_matches_; + std::size_t live_doc_count_ = 0; }; template @@ -577,7 +734,7 @@ class ngram_database } template - void find_occurrences(ngram ng, std::vector& occs) const noexcept + void find_occurrences(ngram ng, std::vector& occs) const { occs.clear(); auto const& idx = store_.template get_index(); @@ -596,12 +753,18 @@ class ngram_database auto it = query.words().begin(); assert(!it->empty()); this->search_word(search_res, word_id++, *it++); - if (search_res.empty()) return search_res; + if (search_res.empty()) { + search_res.reset(); // remove tombstones + return search_res; + } for (; it != query.words().end(); ++it) { assert(!it->empty()); this->search_word(search_res, word_id++, *it); - if (search_res.empty()) break; + if (search_res.empty()) { + search_res.reset(); // remove tombstones + break; + } } return search_res; } @@ -639,7 +802,7 @@ class ngram_database } }); if (available_doc_count == 0) { - search_res.clear(); + search_res.reset(); return; } } @@ -698,36 +861,41 @@ class ngram_database std::size_t i = N; for (; i + N <= word.size(); i += N) { - std::ranges::copy_n(word.begin() + i, N, ng.chars.begin()); + ng.copy_n(word.begin() + i); store_.search(ng, do_search(N)); if (search_res.empty()) return; ++current_ngram; } - // When the remaining character count is remainder of `word.size() % N`, - // search by the *slided* remaining characters. - // - // For example, when the document is "今日は晴れです": - // - // When doing 3-gram search with "今日は雨": - // 1. Search by "今日は" in the normal loop - // - // 2. Then, - // i == 3 - // remaining_chars == word.size() - i == 1 - // overlapping_chars == N - remaining_chars == 2 - // next_search_pos = i - overlapping_chars == 1 - // - // 3. Try to match "日は雨" in the last loop - if (int const remaining_chars = static_cast(word.size() - i); remaining_chars > 0) { - assert(remaining_chars < N); - std::shift_left(ng.chars.begin(), ng.chars.end(), remaining_chars); - std::ranges::copy_n(word.begin() + i, remaining_chars, ng.chars.begin() + (N - remaining_chars)); - store_.search(ng, do_search(remaining_chars)); - if (search_res.empty()) return; - ++current_ngram; + if constexpr (N >= 2) { + // When the remaining character count is remainder of `word.size() % N`, + // search by the *slided* remaining characters. + // + // For example, when the document is "今日は晴れです": + // + // When doing 3-gram search with "今日は雨": + // 1. Search by "今日は" in the normal loop + // + // 2. Then, + // i == 3 + // remaining_chars == word.size() - i == 1 + // overlapping_chars == N - remaining_chars == 2 + // next_search_pos = i - overlapping_chars == 1 + // + // 3. Try to match "日は雨" in the last loop + if (int const remaining_chars = static_cast(word.size() - i); remaining_chars > 0) { + assert(remaining_chars < N); + ng.shift_copy(word.begin() + i, remaining_chars); + store_.search(ng, do_search(remaining_chars)); + if (search_res.empty()) return; + ++current_ngram; + } } + if constexpr (IsFirstWord) { + // A first word of exactly one n-gram runs no continuation searches + if (current_ngram == 1) return; + } search_res.remove_stale_document_matches(word_id, current_ngram); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2fa897b..93eb1bf 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -186,6 +186,9 @@ if(PROJECT_IS_TOP_LEVEL) interval string_algo ngram + ngram_search_2 + ngram_search_3 + ngram_search_4_dep ) foreach(test_name IN LISTS IRIS_TEST_IRIS_TESTS) diff --git a/test/ngram.cpp b/test/ngram.cpp index 7e81475..7a1d39c 100644 --- a/test/ngram.cpp +++ b/test/ngram.cpp @@ -1,20 +1,6 @@ // SPDX-License-Identifier: MIT -#include "iris_test.hpp" - -#include - -#include -#include -#include - -#ifdef _MSC_VER -# include -#endif - -using namespace iris::ngram_literals; -using iris::ngram_occurrence; -using iris::interval; +#include "ngram_test.hpp" [[nodiscard]] constexpr auto make_occurrences(std::initializer_list occs) @@ -34,6 +20,15 @@ constexpr auto make_occurrences(std::initializer_list occs) CHECK(occs == make_occurrences({__VA_ARGS__})); \ } while (false) +TEST_CASE("ngram (type traits)") +{ + STATIC_CHECK(std::same_as::data), char>); + STATIC_CHECK(std::same_as::data), std::uint16_t>); + + STATIC_CHECK(std::same_as::data), char32_t>); + STATIC_CHECK(std::same_as::data), std::uint64_t>); +} + TEST_CASE("ngram (minimal input)") { #ifdef _MSC_VER @@ -186,541 +181,3 @@ TEST_CASE("ngram (realistic input)") IRIS_CHECK_OCCURRENCE("う。", {2_doc_id, 13}); } } - -struct DocumentMatch -{ - iris::ngram_document_id doc_id; - std::vector word_matches; - - DocumentMatch(iris::ngram_document_id doc_id, std::initializer_list word_matches) - : doc_id(doc_id) - , word_matches(word_matches) - {} - - DocumentMatch(iris::ngram_document_id doc_id, std::vector word_matches) - : doc_id(doc_id) - , word_matches(std::move(word_matches)) - {} - - [[nodiscard]] - bool operator==(DocumentMatch const&) const noexcept = default; -}; - -template -struct std::formatter - : iris::no_spec_formatter -{ - template - Ctx::iterator format(DocumentMatch const& doc_match, Ctx& ctx) const - { - return std::format_to(ctx.out(), "(doc: #{}, word_matches: {})", doc_match.doc_id, doc_match.word_matches); - } -}; - -#define IRIS_CHECK_SEARCH(query_input, ...) do { \ - iris::ngram_search_query const query{U ## query_input}; \ - auto const search_res = ngram_db.search(query); \ - auto const& doc_matches = search_res.doc_matches(); \ - \ - std::vector const expected_doc_matches{ \ - std::initializer_list{__VA_ARGS__} \ - }; \ - \ - auto const actual_doc_matches = doc_matches | std::views::transform([](auto const& kv) { \ - return DocumentMatch{kv.first, kv.second}; \ - }) | std::ranges::to(); \ - CHECK(actual_doc_matches == expected_doc_matches); \ - } while (false) - -TEST_CASE("ngram search (document chars = 0)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - IRIS_CHECK_SEARCH(""); - IRIS_CHECK_SEARCH("X"); - IRIS_CHECK_SEARCH("XX"); - } -} - -// 1-gram document -TEST_CASE("ngram search (document chars = 1)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"a"); - IRIS_CHECK_SEARCH(""); - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - IRIS_CHECK_SEARCH("XX"); - } -} - -// 2-gram document -TEST_CASE("ngram search (document chars = 2)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"aa"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}, interval{1, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "aa", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH("aaX"); - IRIS_CHECK_SEARCH("Xaa"); - IRIS_CHECK_SEARCH("XXX"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "b", - {0_doc_id, { - {0, {interval{1, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "ab", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH("abX"); - IRIS_CHECK_SEARCH("Xab"); - IRIS_CHECK_SEARCH("XXX"); - } -} - -// 2-gram + 1-gram document -TEST_CASE("ngram search (document chars = 3, aaa/baa)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"aaa"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "aa", - {0_doc_id, { - {0, {interval{0, 2}, interval{1, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH( - "aaa", - {0_doc_id, { - {0, {interval{0, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("aaX"); - IRIS_CHECK_SEARCH("Xaa"); - IRIS_CHECK_SEARCH("XXX"); - - IRIS_CHECK_SEARCH("aaaX"); - IRIS_CHECK_SEARCH("Xaaa"); - IRIS_CHECK_SEARCH("XXaa"); - IRIS_CHECK_SEARCH("aaXX"); - IRIS_CHECK_SEARCH("XXXX"); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"baa"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{1, 2}, interval{2, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "b", - {0_doc_id, { - {0, {interval{0, 1}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "aa", - {0_doc_id, { - {0, {interval{1, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "ba", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH( - "baa", - {0_doc_id, { - {0, {interval{0, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("aaX"); - IRIS_CHECK_SEARCH("Xaa"); - IRIS_CHECK_SEARCH("baX"); - IRIS_CHECK_SEARCH("Xba"); - IRIS_CHECK_SEARCH("XXX"); - - IRIS_CHECK_SEARCH("baaX"); - IRIS_CHECK_SEARCH("Xbaa"); - IRIS_CHECK_SEARCH("baXX"); - IRIS_CHECK_SEARCH("XXba"); - IRIS_CHECK_SEARCH("aaXX"); - IRIS_CHECK_SEARCH("XXaa"); - IRIS_CHECK_SEARCH("XXXX"); - } -} - -// 2-gram + 1-gram document -TEST_CASE("ngram search (document chars = 3, aba/aab)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"aba"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}, interval{2, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "b", - {0_doc_id, { - {0, {interval{1, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "ab", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "ba", - {0_doc_id, { - {0, {interval{1, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH( - "aba", - {0_doc_id, { - {0, {interval{0, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("abX"); - IRIS_CHECK_SEARCH("Xab"); - IRIS_CHECK_SEARCH("baX"); - IRIS_CHECK_SEARCH("Xba"); - IRIS_CHECK_SEARCH("XXX"); - - IRIS_CHECK_SEARCH("abaX"); - IRIS_CHECK_SEARCH("Xaba"); - IRIS_CHECK_SEARCH("XXab"); - IRIS_CHECK_SEARCH("abXX"); - IRIS_CHECK_SEARCH("XXba"); - IRIS_CHECK_SEARCH("baXX"); - IRIS_CHECK_SEARCH("XXXX"); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"aab"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}, interval{1, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "b", - {0_doc_id, { - {0, {interval{2, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "aa", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "ab", - {0_doc_id, { - {0, {interval{1, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH( - "aab", - {0_doc_id, { - {0, {interval{0, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("aaX"); - IRIS_CHECK_SEARCH("Xaa"); - IRIS_CHECK_SEARCH("abX"); - IRIS_CHECK_SEARCH("Xab"); - IRIS_CHECK_SEARCH("XXX"); - - IRIS_CHECK_SEARCH("aabX"); - IRIS_CHECK_SEARCH("Xaab"); - IRIS_CHECK_SEARCH("XXaa"); - IRIS_CHECK_SEARCH("aaXX"); - IRIS_CHECK_SEARCH("XXab"); - IRIS_CHECK_SEARCH("abXX"); - IRIS_CHECK_SEARCH("XXXX"); - } -} - -// 2-gram + 1-gram document -TEST_CASE("ngram search (document chars = 3, abc)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abc"); - IRIS_CHECK_SEARCH(""); - - IRIS_CHECK_SEARCH( - "a", - {0_doc_id, { - {0, {interval{0, 1}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "b", - {0_doc_id, { - {0, {interval{1, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "c", - {0_doc_id, { - {0, {interval{2, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("X"); - - IRIS_CHECK_SEARCH( - "ab", - {0_doc_id, { - {0, {interval{0, 2}}}, - }}, - ); - IRIS_CHECK_SEARCH( - "bc", - {0_doc_id, { - {0, {interval{1, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("XX"); - - IRIS_CHECK_SEARCH( - "abc", - {0_doc_id, { - {0, {interval{0, 3}}}, - }}, - ); - IRIS_CHECK_SEARCH("abX"); - IRIS_CHECK_SEARCH("Xab"); - IRIS_CHECK_SEARCH("bcX"); - IRIS_CHECK_SEARCH("Xbc"); - IRIS_CHECK_SEARCH("XXX"); - - IRIS_CHECK_SEARCH("abcX"); - IRIS_CHECK_SEARCH("Xabc"); - IRIS_CHECK_SEARCH("XXab"); - IRIS_CHECK_SEARCH("abXX"); - IRIS_CHECK_SEARCH("XXbc"); - IRIS_CHECK_SEARCH("bcXX"); - IRIS_CHECK_SEARCH("XXXX"); - } -} - -TEST_CASE("ngram search (document chars = 4)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - // TODO - - // 2x 2-gram document - //{ - // iris::ngram_database<> ngram_db; - // (void)ngram_db.add_document(U"aaaa"); - // IRIS_CHECK_SEARCH(""); - - // IRIS_CHECK_SEARCH( - // "a", - // {0_doc_id, { - // {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}, interval{3, 4}}}, - // }}, - // ); - // IRIS_CHECK_SEARCH("X"); - - // IRIS_CHECK_SEARCH( - // "aa", - // {0_doc_id, { - // {0, {interval{0, 2}, interval{2, 4}}}, - // }}, - // ); - // IRIS_CHECK_SEARCH("XX"); - //} - -} - -TEST_CASE("ngram search (minimal input, dependency on previous match)") -{ -#ifdef _MSC_VER - SetConsoleOutputCP(CP_UTF8); -#endif - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abef"); - IRIS_CHECK_SEARCH("abXXef"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abef"); - IRIS_CHECK_SEARCH("abXXefef"); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab..ef"); - IRIS_CHECK_SEARCH("abef"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab..ef"); - IRIS_CHECK_SEARCH("abefef"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab..ef"); - IRIS_CHECK_SEARCH("abXXef"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab..ef"); - IRIS_CHECK_SEARCH("abXXefef"); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abef"); - IRIS_CHECK_SEARCH("abXef"); - } - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abef"); - IRIS_CHECK_SEARCH("abXefef"); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"abxcd"); // trap document - (void)ngram_db.add_document(U"abcd"); - IRIS_CHECK_SEARCH( - "abcd", - {1_doc_id, { - {0, {interval{0, 4}}}, - }}, - ); - } - - { - iris::ngram_database<> ngram_db; - (void)ngram_db.add_document(U"ab"); - (void)ngram_db.add_document(U"abcd"); - IRIS_CHECK_SEARCH( - "ab cd", - {1_doc_id, { - {0, {interval{0, 2}}}, - {1, {interval{2, 4}}}, - }}, - ); - } -} diff --git a/test/ngram_search_2.cpp b/test/ngram_search_2.cpp new file mode 100644 index 0000000..2f85570 --- /dev/null +++ b/test/ngram_search_2.cpp @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +TEST_CASE("ngram search (document chars = 0)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} + +// 1-gram document +TEST_CASE("ngram search (document chars = 1)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"a"); + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} + +// 2-gram document +TEST_CASE("ngram search (document chars = 2)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + } +} diff --git a/test/ngram_search_3.cpp b/test/ngram_search_3.cpp new file mode 100644 index 0000000..dc9040d --- /dev/null +++ b/test/ngram_search_3.cpp @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, aaa/baa)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aaa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}, interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aaa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aaaX"); + IRIS_CHECK_SEARCH("Xaaa"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"baa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "baa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("baaX"); + IRIS_CHECK_SEARCH("Xbaa"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, aba/aab)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aba"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aba", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abaX"); + IRIS_CHECK_SEARCH("Xaba"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aab", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aabX"); + IRIS_CHECK_SEARCH("Xaab"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, abc)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abc"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "c", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("bcX"); + IRIS_CHECK_SEARCH("Xbc"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abcX"); + IRIS_CHECK_SEARCH("Xabc"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXbc"); + IRIS_CHECK_SEARCH("bcXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} diff --git a/test/ngram_search_4_dep.cpp b/test/ngram_search_4_dep.cpp new file mode 100644 index 0000000..8045632 --- /dev/null +++ b/test/ngram_search_4_dep.cpp @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, aaaa)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aaaa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}, interval{1, 3}, interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aaa", + {0_doc_id, { + {0, {interval{0, 3}, interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "aaaa", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaaX"); + IRIS_CHECK_SEARCH("Xaaa"); + IRIS_CHECK_SEARCH("XXXX"); + + IRIS_CHECK_SEARCH("aaaaX"); + IRIS_CHECK_SEARCH("Xaaaa"); + IRIS_CHECK_SEARCH("XXaaa"); + IRIS_CHECK_SEARCH("aaaXX"); + IRIS_CHECK_SEARCH("XXXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, abab)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}, interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aba", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bab", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abab", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("ababX"); + IRIS_CHECK_SEARCH("Xabab"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, abca)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abca"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "c", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ca", + {0_doc_id, { + {0, {interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bca", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xbc"); + IRIS_CHECK_SEARCH("caX"); + IRIS_CHECK_SEARCH("Xca"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abca", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abcaX"); + IRIS_CHECK_SEARCH("Xabca"); + IRIS_CHECK_SEARCH("caab"); // "ca" and "ab" both exist but not contiguous as "caab" + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, abcd)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "d", + {0_doc_id, { + {0, {interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "cd", + {0_doc_id, { + {0, {interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bcd", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abcX"); + IRIS_CHECK_SEARCH("Xbcd"); + IRIS_CHECK_SEARCH("abcdX"); + IRIS_CHECK_SEARCH("Xabcd"); + IRIS_CHECK_SEARCH("abXcd"); // both halves exist; broken by X in the middle... but see note below! + IRIS_CHECK_SEARCH("acbd"); // all chars exist; order scrambled + IRIS_CHECK_SEARCH("XXXX"); + } +} + +TEST_CASE("ngram search (dependency on previous match)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH("abXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abefef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abxcd"); // trap document + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "abcd", + {1_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + (void)ngram_db.add_document(U"abxcd"); // trap document + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "ab cd", + {1_doc_id, { + {0, {interval{0, 2}}}, + {1, {interval{2, 4}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abXcd"); + (void)ngram_db.add_document(U"abcdX"); + IRIS_CHECK_SEARCH("abcdc"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abXXabcd"); + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{4, 8}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab cdXf"); + (void)ngram_db.add_document(U"ab cdef"); + IRIS_CHECK_SEARCH( + "ab cdef", + {1_doc_id, { + {0, {interval{0, 2}}}, + {1, {interval{3, 7}}}, + }}, + ); + } +} diff --git a/test/ngram_test.hpp b/test/ngram_test.hpp new file mode 100644 index 0000000..ff49a87 --- /dev/null +++ b/test/ngram_test.hpp @@ -0,0 +1,67 @@ +#ifndef IRIS_ZZ_TEST_NGRAM_TEST_HPP +#define IRIS_ZZ_TEST_NGRAM_TEST_HPP + +// SPDX-License-Identifier: MIT + +#include "iris_test.hpp" + +#include + +#include +#include +#include + +#ifdef _MSC_VER +# include +#endif + +using namespace iris::ngram_literals; +using iris::ngram_occurrence; +using iris::interval; + +struct DocumentMatch +{ + iris::ngram_document_id doc_id; + std::vector word_matches; + + DocumentMatch(iris::ngram_document_id doc_id, std::initializer_list word_matches) + : doc_id(doc_id) + , word_matches(word_matches) + {} + + DocumentMatch(iris::ngram_document_id doc_id, std::vector word_matches) + : doc_id(doc_id) + , word_matches(std::move(word_matches)) + {} + + [[nodiscard]] + bool operator==(DocumentMatch const&) const noexcept = default; +}; + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(DocumentMatch const& doc_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "(doc: #{}, word_matches: {})", doc_match.doc_id, doc_match.word_matches); + } +}; + +#define IRIS_CHECK_SEARCH(query_input, ...) do { \ + iris::ngram_search_query const query{U ## query_input}; \ + auto const search_res = ngram_db.search(query); \ + auto const& doc_matches = search_res.doc_matches(); \ + \ + std::vector const expected_doc_matches{ \ + std::initializer_list{__VA_ARGS__} \ + }; \ + \ + auto const actual_doc_matches = doc_matches | std::views::transform([](auto const& kv) { \ + return DocumentMatch{kv.first, kv.second}; \ + }) | std::ranges::to(); \ + CHECK(actual_doc_matches == expected_doc_matches); \ + } while (false) + +#endif From 93b4750f8f5e8a292e8bfb3e7e222dc96d4cfe39 Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Thu, 13 Aug 2026 07:36:45 +0900 Subject: [PATCH 09/10] Fix sign comparison --- include/iris/ngram.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index 1cf1e5e..b3319e2 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -83,7 +83,7 @@ struct ngram noexcept(std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars))) ) { - assert(remaining_chars < N); + assert(remaining_chars < int(N)); std::shift_left(data.begin(), data.end(), remaining_chars); std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars)); } @@ -884,7 +884,7 @@ class ngram_database // // 3. Try to match "日は雨" in the last loop if (int const remaining_chars = static_cast(word.size() - i); remaining_chars > 0) { - assert(remaining_chars < N); + assert(remaining_chars < int(N)); ng.shift_copy(word.begin() + i, remaining_chars); store_.search(ng, do_search(remaining_chars)); if (search_res.empty()) return; From 0ac9b285e1ea0322b251bf7dab1e9a4f5a66dcce Mon Sep 17 00:00:00 2001 From: Nana Sakisaka <1901813+saki7@users.noreply.github.com> Date: Fri, 14 Aug 2026 03:50:56 +0900 Subject: [PATCH 10/10] Current --- include/iris/format.hpp | 2 - include/iris/interval.hpp | 98 ++++++-- include/iris/interval_set.hpp | 257 +++++++++++++++++++++ include/iris/iterator.hpp | 66 ++++++ include/iris/ngram.hpp | 8 +- test/CMakeLists.txt | 1 + test/interval.cpp | 224 +++++++++++++++--- test/interval_set.cpp | 423 ++++++++++++++++++++++++++++++++++ 8 files changed, 1032 insertions(+), 47 deletions(-) create mode 100644 include/iris/interval_set.hpp create mode 100644 include/iris/iterator.hpp create mode 100644 test/interval_set.cpp diff --git a/include/iris/format.hpp b/include/iris/format.hpp index 4c63876..5539380 100644 --- a/include/iris/format.hpp +++ b/include/iris/format.hpp @@ -3,8 +3,6 @@ // SPDX-License-Identifier: MIT -// SPDX-License-Identifier: MIT - #include #include diff --git a/include/iris/interval.hpp b/include/iris/interval.hpp index 507b0d2..2f07d54 100644 --- a/include/iris/interval.hpp +++ b/include/iris/interval.hpp @@ -1,5 +1,5 @@ -#ifndef IRIS_INTERVAL_HPP -#define IRIS_INTERVAL_HPP +#ifndef IRIS_ZZ_INTERVAL_HPP +#define IRIS_ZZ_INTERVAL_HPP // SPDX-License-Identifier: MIT @@ -19,38 +19,106 @@ template struct interval { using value_type = T; - T left, right; + T lower, upper; - [[nodiscard]] - constexpr bool operator==(interval const&) const noexcept = default; + constexpr interval() noexcept + : lower{} + , upper{} + {} - [[nodiscard]] - constexpr std::strong_ordering operator<=>(interval const&) const noexcept = default; + constexpr interval(T lower, T upper) noexcept + : lower(lower) + , upper(upper) + {} + + [[nodiscard]] constexpr bool empty() const noexcept + { + // Malformed interval is treated as "empty" + return lower >= upper; + } + + [[nodiscard]] constexpr value_type length() const noexcept + { + return static_cast(upper - lower); + } + + template + [[nodiscard]] constexpr bool intersects(interval other) const noexcept + { + return (lower < other.upper && other.lower < upper) && !empty() && !other.empty(); + } + + // !intersects + template + [[nodiscard]] constexpr bool disjoint(interval other) const noexcept + { + return (upper <= other.lower || other.upper <= lower) || empty() || other.empty(); + } + + template + [[nodiscard]] constexpr bool touches(interval other) const noexcept + { + return (upper == other.lower || other.upper == lower) && !empty() && !other.empty(); + } + + // intersects || touches + template + [[nodiscard]] constexpr bool connected(interval other) const noexcept + { + return (lower <= other.upper && other.lower <= upper) && !empty() && !other.empty(); + } + + template + [[nodiscard]] constexpr bool covers(interval other) const noexcept + { + return (lower <= other.lower && other.upper <= upper) || other.empty(); + } + + template + [[nodiscard]] constexpr bool equals(interval other) const noexcept + { + return (lower == other.lower && upper == other.upper) || (empty() && other.empty()); + } + + // ------------------------------------------- + + [[nodiscard]] constexpr bool contains(value_type p) const noexcept + { + return lower <= p && p < upper; + } + + // ------------------------------------------- + + // Note: This does not reflect mathematical definition like `equals(...)`; this always checks exact data representation + [[nodiscard]] constexpr bool operator==(interval const&) const noexcept = default; + + // Note: This does not reflect mathematical definition like `equals(...)`; this always checks exact data representation + [[nodiscard]] constexpr std::strong_ordering operator<=>(interval const&) const noexcept = default; }; template [[nodiscard]] constexpr T& get(interval& iv) noexcept { static_assert(I == 0 || I == 1); - if constexpr (I == 0) { return iv.left; } else { return iv.right; } + if constexpr (I == 0) { return iv.lower; } else { return iv.upper; } } template [[nodiscard]] constexpr T const& get(interval const& iv) noexcept { static_assert(I == 0 || I == 1); - if constexpr (I == 0) { return iv.left; } else { return iv.right; } + if constexpr (I == 0) { return iv.lower; } else { return iv.upper; } } template [[nodiscard]] constexpr T&& get(interval&& iv) noexcept { static_assert(I == 0 || I == 1); - if constexpr (I == 0) { return std::move(iv).left; } else { return std::move(iv).right; } + if constexpr (I == 0) { return std::move(iv).lower; } else { return std::move(iv).upper; } } template [[nodiscard]] constexpr T const&& get(interval const&& iv) noexcept { static_assert(I == 0 || I == 1); - if constexpr (I == 0) { return std::move(iv).left; } else { return std::move(iv).right; } + if constexpr (I == 0) { return std::move(iv).lower; } else { return std::move(iv).upper; } } } // iris @@ -98,7 +166,7 @@ struct std::formatter, CharT> std::basic_string_view{first, comma_it} }; if (left_fmt_.parse(left_ctx) != left_ctx.end()) { - throw std::format_error("trailing characters in left format specifier"); + throw std::format_error("trailing characters in lower format specifier"); } } { @@ -106,7 +174,7 @@ struct std::formatter, CharT> std::basic_string_view{std::next(comma_it), close_it} }; if (right_fmt_.parse(right_ctx) != right_ctx.end()) { - throw std::format_error("trailing characters in right format specifier"); + throw std::format_error("trailing characters in upper format specifier"); } } return close_it; @@ -116,9 +184,9 @@ struct std::formatter, CharT> Ctx::iterator format(iris::interval const& iv, Ctx& ctx) const { ctx.advance_to(std::format_to(ctx.out(), "{}", iris::format_traits::square_brace_open)); - left_fmt_.format(iv.left, ctx); + left_fmt_.format(iv.lower, ctx); ctx.advance_to(std::format_to(ctx.out(), "{}", iris::format_traits::comma)); - right_fmt_.format(iv.right, ctx); + right_fmt_.format(iv.upper, ctx); return std::format_to(ctx.out(), "{}", iris::format_traits::paren_close); } diff --git a/include/iris/interval_set.hpp b/include/iris/interval_set.hpp new file mode 100644 index 0000000..dad688c --- /dev/null +++ b/include/iris/interval_set.hpp @@ -0,0 +1,257 @@ +#ifndef IRIS_ZZ_IntervalT_SET_HPP +#define IRIS_ZZ_IntervalT_SET_HPP + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace iris { + +template< + class IntervalT, + class MapT = std::map +> +class interval_set +{ +public: + using interval_type = IntervalT; + using map_type = MapT; + using offset_type = IntervalT::value_type; + + class const_iterator : public iterator_base + { + using typename iterator_base::iterator_base_type; + static_assert(std::bidirectional_iterator); + iterator_base_type it_; + + public: + using value_type = IntervalT; + using pointer = IntervalT const*; + using reference = IntervalT; + + constexpr const_iterator() noexcept = default; + + constexpr explicit const_iterator(iterator_base_type it) noexcept + : it_(std::move(it)) + {} + + [[nodiscard]] constexpr IntervalT operator*() const noexcept + { + return {it_->first, it_->second}; + } + + constexpr const_iterator& operator++() noexcept + { + ++it_; + return *this; + } + + [[nodiscard]] constexpr const_iterator operator++(int) noexcept + { + auto temp{*this}; + ++it_; + return temp; + } + + constexpr const_iterator& operator--() noexcept + { + --it_; + return *this; + } + + [[nodiscard]] constexpr const_iterator operator--(int) noexcept + { + auto temp{*this}; + --it_; + return temp; + } + + [[nodiscard]] constexpr bool operator==(const_iterator const&) const noexcept = default; + [[nodiscard]] constexpr auto operator<=>(const_iterator const&) const noexcept = default; + }; + + using iterator = const_iterator; + + constexpr interval_set() = default; + + constexpr explicit interval_set(std::initializer_list il) + { + auto it = il.begin(); + if (it == il.end()) return; + if (!it->empty()) { + map_.emplace(it->lower, it->upper); + } + for (++it; it != il.end(); ++it) { + this->insert(*it); + } + } + + template Se> + requires std::convertible_to, IntervalT> + constexpr interval_set(It it, Se se) + { + if (it == se) return; + if (IntervalT const iv = *it; !iv.empty()) { + map_.emplace(iv.lower, iv.upper); + } + for (++it; it != se; ++it) { + this->insert(*it); + } + } + + template + requires std::convertible_to, IntervalT> + constexpr interval_set(std::from_range_t, R&& r) + : interval_set(std::ranges::begin(r), std::ranges::end(r)) + {} + + [[nodiscard]] constexpr const_iterator begin() const noexcept + { + return const_iterator{map_.begin()}; + } + + [[nodiscard]] constexpr const_iterator end() const noexcept + { + return const_iterator{map_.end()}; + } + + [[nodiscard]] constexpr bool empty() const noexcept { return map_.empty(); } + [[nodiscard]] constexpr std::size_t size() const noexcept { return map_.size(); } + + constexpr void clear() noexcept + { + map_.clear(); + } + + // Total number of covered offsets (sum of lengths) + constexpr offset_type coverage() const noexcept + { + offset_type total = 0; + for (auto const& [lower, upper] : map_) { + total += upper - lower; + } + return total; + } + + // O(1) + [[nodiscard]] constexpr IntervalT extent() const noexcept + { + if (map_.empty()) return {}; + return {map_.begin()->first, std::prev(map_.end())->second}; + } + + // Insert [iv.lower, iv.upper), merging with any interval it overlaps or touches + constexpr void insert(IntervalT iv) + { + if (iv.empty()) return; + + auto it = map_.lower_bound(iv.lower); + if (it != map_.begin()) { + auto prev = std::prev(it); + if (prev->second >= iv.lower) { + it = prev; + } + } + + // Absorb every interval that overlaps or touches [iv.lower, iv.upper) + while (it != map_.end() && it->first <= iv.upper) { + if (it->first < iv.lower) iv.lower = it->first; + if (it->second > iv.upper) iv.upper = it->second; + it = map_.erase(it); + } + + map_.emplace(iv.lower, iv.upper); + } + + constexpr void insert(offset_type lower, offset_type upper) + { + this->insert(IntervalT{lower, upper}); + } + + // -------------------------------------- + + [[nodiscard]] constexpr bool intersects(IntervalT const iv) const + { + if (iv.empty()) return false; + auto const it = map_.upper_bound(iv.lower); + if (it != map_.begin() && std::prev(it)->second > iv.lower) return true; + return it != map_.end() && it->first < iv.upper; + } + + [[nodiscard]] constexpr bool covers(IntervalT const iv) const + { + if (iv.empty()) return true; + auto const it = map_.upper_bound(iv.lower); + if (it == map_.begin()) return false; + auto const& [lower, upper] = *std::prev(it); + return lower <= iv.lower && iv.upper <= upper; + } + + [[nodiscard]] constexpr bool contains(offset_type p) const + { + auto const it = map_.upper_bound(p); + if (it == map_.begin()) return false; + return std::prev(it)->second > p; + } + + [[nodiscard]] constexpr bool operator==(interval_set const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(interval_set const&) const noexcept = default; + + constexpr void swap(interval_set& other) noexcept + { + using std::swap; + swap(map_, other.map_); + } + +private: + MapT map_; +}; + +template +constexpr void swap(interval_set& a, interval_set& b) noexcept +{ + a.swap(b); +} + +} // iris + +template +struct std::formatter, CharT> +{ + [[nodiscard]] constexpr std::basic_format_parse_context::const_iterator + parse(std::basic_format_parse_context& ctx) + { + return iv_fmt_.parse(ctx); + } + + template + Ctx::iterator format(iris::interval_set const& ivs, Ctx& ctx) const + { + ctx.advance_to(std::format_to(ctx.out(), "{{")); + bool is_first = true; + for (auto const& iv : ivs) { + if (is_first) { + ctx.advance_to(iv_fmt_.format(iv, ctx)); + is_first = false; + } else { + ctx.advance_to(std::format_to(ctx.out(), " ")); + ctx.advance_to(iv_fmt_.format(iv, ctx)); + } + } + return std::format_to(ctx.out(), "}}"); + } + +private: + std::formatter iv_fmt_; +}; + +#endif diff --git a/include/iris/iterator.hpp b/include/iris/iterator.hpp new file mode 100644 index 0000000..9bc42ed --- /dev/null +++ b/include/iris/iterator.hpp @@ -0,0 +1,66 @@ +#ifndef IRIS_ZZ_ITERATOR_HPP +#define IRIS_ZZ_ITERATOR_HPP + +#include +#include + +namespace iris { + +template +struct iterator_tags_base; + +template + requires requires { + typename std::iterator_traits::iterator_category; + typename std::iterator_traits::iterator_concept; + } +struct iterator_tags_base +{ + using iterator_base_type = It; + using iterator_category = std::iterator_traits::iterator_category; + using iterator_concept = std::iterator_traits::iterator_concept; + + [[nodiscard]] constexpr bool operator==(iterator_tags_base const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(iterator_tags_base const&) const noexcept = default; +}; + +template + requires + requires { typename std::iterator_traits::iterator_category; } && + (!requires { typename std::iterator_traits::iterator_concept; }) +struct iterator_tags_base +{ + using iterator_base_type = It; + using iterator_category = std::iterator_traits::iterator_category; + + [[nodiscard]] constexpr bool operator==(iterator_tags_base const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(iterator_tags_base const&) const noexcept = default; +}; + +template + requires + (!requires { typename std::iterator_traits::iterator_category; }) && + requires { typename std::iterator_traits::iterator_concept; } +struct iterator_tags_base +{ + using iterator_base_type = It; + using iterator_concept = std::iterator_traits::iterator_concept; + + [[nodiscard]] constexpr bool operator==(iterator_tags_base const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(iterator_tags_base const&) const noexcept = default; +}; + +// ---------------------------------------------- + +template +struct iterator_base : iterator_tags_base +{ + using difference_type = std::iterator_traits::difference_type; + + [[nodiscard]] constexpr bool operator==(iterator_base const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(iterator_base const&) const noexcept = default; +}; + +} // iris + +#endif diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp index b3319e2..60ded71 100644 --- a/include/iris/ngram.hpp +++ b/include/iris/ngram.hpp @@ -1,5 +1,5 @@ -#ifndef IRIS_NGRAM_HPP -#define IRIS_NGRAM_HPP +#ifndef IRIS_ZZ_NGRAM_HPP +#define IRIS_ZZ_NGRAM_HPP // SPDX-License-Identifier: MIT @@ -834,9 +834,9 @@ class ngram_database for (auto it = word_match->spans.begin(); it != word_match->spans.end();) { auto& prev_pos = *it; - if (std::ranges::binary_search(positions, prev_pos.right - overlapping_chars)) { + if (std::ranges::binary_search(positions, prev_pos.upper - overlapping_chars)) { // Matched; the current word's current n-gram is contiguous to the previous n-gram - prev_pos.right += remaining_chars; + prev_pos.upper += remaining_chars; ++it; continue; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 93eb1bf..2fe6f92 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -184,6 +184,7 @@ if(PROJECT_IS_TOP_LEVEL) colorize_format preprocess interval + interval_set string_algo ngram ngram_search_2 diff --git a/test/interval.cpp b/test/interval.cpp index b52f12a..de757c8 100644 --- a/test/interval.cpp +++ b/test/interval.cpp @@ -11,55 +11,227 @@ using namespace std::string_view_literals; +using iris::interval; + // NOLINTBEGIN(readability-container-size-empty) TEST_CASE("interval: type traits") { - STATIC_CHECK(std::is_trivially_default_constructible_v>); - STATIC_CHECK(std::is_trivially_copyable_v>); + STATIC_CHECK(std::is_trivially_copyable_v>); + + STATIC_CHECK(std::is_nothrow_default_constructible_v>); + STATIC_CHECK(std::is_nothrow_constructible_v, int, int>); + STATIC_CHECK(std::is_nothrow_copy_constructible_v>); + STATIC_CHECK(std::is_nothrow_move_constructible_v>); + STATIC_CHECK(std::is_nothrow_copy_assignable_v>); + STATIC_CHECK(std::is_nothrow_move_assignable_v>); + STATIC_CHECK(std::is_nothrow_destructible_v>); + STATIC_CHECK(std::is_nothrow_swappable_v>); +} + +TEST_CASE("interval: members") +{ + // Check in constexpr context to detect uninitialized value and other UBs + + // [x, x) + { + constexpr interval iv{}; + STATIC_CHECK(iv.lower == 0); + STATIC_CHECK(iv.upper == 0); + STATIC_CHECK(iv.length() == 0); + STATIC_CHECK(iv.empty()); + STATIC_CHECK(iv == iv); + STATIC_CHECK((iv <=> iv) == std::strong_ordering::equal); + } + + // [-, -) + { + constexpr interval iv{-5, -2}; + STATIC_CHECK(iv.lower == -5); + STATIC_CHECK(iv.upper == -2); + STATIC_CHECK(iv.length() == 3); + STATIC_CHECK(!iv.empty()); + STATIC_CHECK(iv == iv); + STATIC_CHECK((iv <=> iv) == std::strong_ordering::equal); + } + + // [-, +) + { + constexpr interval iv{-5, 2}; + STATIC_CHECK(iv.lower == -5); + STATIC_CHECK(iv.upper == 2); + STATIC_CHECK(iv.length() == 7); + STATIC_CHECK(!iv.empty()); + STATIC_CHECK(iv == iv); + STATIC_CHECK((iv <=> iv) == std::strong_ordering::equal); + } + + // [+, +) + { + constexpr interval iv{2, 5}; + STATIC_CHECK(iv.lower == 2); + STATIC_CHECK(iv.upper == 5); + STATIC_CHECK(iv.length() == 3); + STATIC_CHECK(!iv.empty()); + STATIC_CHECK(iv == iv); + STATIC_CHECK((iv <=> iv) == std::strong_ordering::equal); + } +} + +TEST_CASE("interval: relationship") +{ + STATIC_CHECK(interval{2, 5}.equals(interval{2, 5})); + STATIC_CHECK(interval{2, 5} == interval{2, 5}); + STATIC_CHECK((interval{2, 5} <=> interval{2, 5}) == std::strong_ordering::equal); + + // b = empty interval + STATIC_CHECK(interval{0, 0}.equals(interval{0, 0})); + STATIC_CHECK(interval{0, 0}.equals(interval{1, 1})); + STATIC_CHECK(interval{0, 0} == interval{0, 0}); + STATIC_CHECK(interval{0, 0} != interval{1, 1}); + STATIC_CHECK((interval{0, 0} <=> interval{0, 0}) == std::strong_ordering::equal); + STATIC_CHECK((interval{0, 0} <=> interval{1, 1}) == std::strong_ordering::less); + + // ---------------------------------------------------- + + STATIC_CHECK(!interval{2, 5}.intersects({0, 1})); STATIC_CHECK( interval{2, 5}.disjoint({0, 1})); + STATIC_CHECK(!interval{2, 5}.intersects({0, 2})); STATIC_CHECK( interval{2, 5}.disjoint({0, 2})); + STATIC_CHECK( interval{2, 5}.intersects({0, 3})); STATIC_CHECK(!interval{2, 5}.disjoint({0, 3})); + STATIC_CHECK( interval{2, 5}.intersects({4, 7})); STATIC_CHECK(!interval{2, 5}.disjoint({4, 7})); + STATIC_CHECK(!interval{2, 5}.intersects({5, 7})); STATIC_CHECK( interval{2, 5}.disjoint({5, 7})); + STATIC_CHECK(!interval{2, 5}.intersects({6, 7})); STATIC_CHECK( interval{2, 5}.disjoint({6, 7})); + + // b = empty interval + STATIC_CHECK(!interval{2, 5}.intersects({0, 0})); STATIC_CHECK( interval{2, 5}.disjoint({0, 0})); + STATIC_CHECK(!interval{2, 5}.intersects({1, 1})); STATIC_CHECK( interval{2, 5}.disjoint({1, 1})); + STATIC_CHECK(!interval{2, 5}.intersects({2, 2})); STATIC_CHECK( interval{2, 5}.disjoint({2, 2})); + STATIC_CHECK(!interval{2, 5}.intersects({3, 3})); STATIC_CHECK( interval{2, 5}.disjoint({3, 3})); + STATIC_CHECK(!interval{2, 5}.intersects({4, 4})); STATIC_CHECK( interval{2, 5}.disjoint({4, 4})); + STATIC_CHECK(!interval{2, 5}.intersects({5, 5})); STATIC_CHECK( interval{2, 5}.disjoint({5, 5})); + STATIC_CHECK(!interval{2, 5}.intersects({6, 6})); STATIC_CHECK( interval{2, 5}.disjoint({6, 6})); + + // ---------------------------------------------------- + + STATIC_CHECK(!interval{2, 5}.touches({0, 1})); + STATIC_CHECK( interval{2, 5}.touches({0, 2})); + STATIC_CHECK(!interval{2, 5}.touches({0, 3})); + STATIC_CHECK(!interval{2, 5}.touches({4, 7})); + STATIC_CHECK( interval{2, 5}.touches({5, 7})); + STATIC_CHECK(!interval{2, 5}.touches({6, 7})); + + // b = empty interval + STATIC_CHECK(!interval{2, 5}.touches({0, 0})); + STATIC_CHECK(!interval{2, 5}.touches({1, 1})); + STATIC_CHECK(!interval{2, 5}.touches({2, 2})); + STATIC_CHECK(!interval{2, 5}.touches({3, 3})); + STATIC_CHECK(!interval{2, 5}.touches({4, 4})); + STATIC_CHECK(!interval{2, 5}.touches({5, 5})); + STATIC_CHECK(!interval{2, 5}.touches({6, 6})); + + // ---------------------------------------------------- + + STATIC_CHECK(!interval{2, 5}.connected({0, 1})); + STATIC_CHECK( interval{2, 5}.connected({0, 2})); + STATIC_CHECK( interval{2, 5}.connected({0, 3})); + STATIC_CHECK( interval{2, 5}.connected({4, 7})); + STATIC_CHECK( interval{2, 5}.connected({5, 7})); + STATIC_CHECK(!interval{2, 5}.connected({6, 7})); + + // b = empty interval + STATIC_CHECK(!interval{2, 5}.connected({0, 0})); + STATIC_CHECK(!interval{2, 5}.connected({1, 1})); + STATIC_CHECK(!interval{2, 5}.connected({2, 2})); + STATIC_CHECK(!interval{2, 5}.connected({3, 3})); + STATIC_CHECK(!interval{2, 5}.connected({4, 4})); + STATIC_CHECK(!interval{2, 5}.connected({5, 5})); + STATIC_CHECK(!interval{2, 5}.connected({6, 6})); + + // ---------------------------------------------------- + + STATIC_CHECK(!interval{2, 5}.covers({0, 1})); + STATIC_CHECK(!interval{2, 5}.covers({0, 2})); + STATIC_CHECK(!interval{2, 5}.covers({0, 3})); + + STATIC_CHECK(!interval{2, 5}.covers({1, 2})); + STATIC_CHECK(!interval{2, 5}.covers({1, 3})); + + STATIC_CHECK( interval{2, 5}.covers({2, 3})); + STATIC_CHECK( interval{2, 5}.covers({2, 4})); + STATIC_CHECK( interval{2, 5}.covers({2, 5})); + STATIC_CHECK(!interval{2, 5}.covers({2, 6})); + + STATIC_CHECK( interval{2, 5}.covers({3, 4})); + STATIC_CHECK( interval{2, 5}.covers({3, 5})); + STATIC_CHECK(!interval{2, 5}.covers({3, 6})); + + STATIC_CHECK( interval{2, 5}.covers({4, 5})); + STATIC_CHECK(!interval{2, 5}.covers({4, 6})); + + STATIC_CHECK(!interval{2, 5}.covers({5, 6})); + + STATIC_CHECK(!interval{2, 5}.covers({6, 7})); + + // b = empty interval + STATIC_CHECK( interval{2, 5}.covers({0, 0})); + STATIC_CHECK( interval{2, 5}.covers({1, 1})); + STATIC_CHECK( interval{2, 5}.covers({2, 2})); + STATIC_CHECK( interval{2, 5}.covers({3, 3})); + STATIC_CHECK( interval{2, 5}.covers({4, 4})); + STATIC_CHECK( interval{2, 5}.covers({5, 5})); + STATIC_CHECK( interval{2, 5}.covers({6, 6})); + + // ---------------------------------------------------- + + STATIC_CHECK(!interval{2, 5}.contains(0)); + STATIC_CHECK(!interval{2, 5}.contains(1)); + STATIC_CHECK( interval{2, 5}.contains(2)); + STATIC_CHECK( interval{2, 5}.contains(3)); + STATIC_CHECK( interval{2, 5}.contains(4)); + STATIC_CHECK(!interval{2, 5}.contains(5)); + STATIC_CHECK(!interval{2, 5}.contains(6)); } TEST_CASE("interval: tuple") { { - iris::interval const iv{1, 2}; - auto const [left, right] = iv; // structured bindings - CHECK(left == 1); - CHECK(right == 2); + interval const iv{1, 2}; + auto const [lower, upper] = iv; // structured bindings + CHECK(lower == 1); + CHECK(upper == 2); } { - iris::interval iv{1, 2}; - auto&& left = iris::get<0>(iv); - STATIC_CHECK(std::same_as); - CHECK(left == 1); + interval iv{1, 2}; + auto&& lower = iris::get<0>(iv); + STATIC_CHECK(std::same_as); + CHECK(lower == 1); } { - iris::interval const iv{1, 2}; - auto&& left = iris::get<0>(iv); - STATIC_CHECK(std::same_as); - CHECK(left == 1); + interval const iv{1, 2}; + auto&& lower = iris::get<0>(iv); + STATIC_CHECK(std::same_as); + CHECK(lower == 1); } { - iris::interval iv{1, 2}; - auto&& left = iris::get<0>(std::move(iv)); - STATIC_CHECK(std::same_as); - CHECK(left == 1); + interval iv{1, 2}; + auto&& lower = iris::get<0>(std::move(iv)); + STATIC_CHECK(std::same_as); + CHECK(lower == 1); } { - iris::interval const iv{1, 2}; - auto&& left = iris::get<0>(std::move(iv)); - STATIC_CHECK(std::same_as); - CHECK(left == 1); + interval const iv{1, 2}; + auto&& lower = iris::get<0>(std::move(iv)); + STATIC_CHECK(std::same_as); + CHECK(lower == 1); } } TEST_CASE("interval: format") { - CHECK(std::format("{}", iris::interval{}) == "[0,0)"sv); - CHECK(std::format("{}", iris::interval{1, 2}) == "[1,2)"sv); - CHECK(std::format("{:2d,}", iris::interval{1, 2}) == "[ 1,2)"sv); - CHECK(std::format("{:2d,3d}", iris::interval{1, 2}) == "[ 1, 2)"sv); + CHECK(std::format("{}", interval{}) == "[0,0)"sv); + CHECK(std::format("{}", interval{1, 2}) == "[1,2)"sv); + CHECK(std::format("{:2d,}", interval{1, 2}) == "[ 1,2)"sv); + CHECK(std::format("{:2d,3d}", interval{1, 2}) == "[ 1, 2)"sv); } // NOLINTEND(readability-container-size-empty) diff --git a/test/interval_set.cpp b/test/interval_set.cpp new file mode 100644 index 0000000..c55b0b7 --- /dev/null +++ b/test/interval_set.cpp @@ -0,0 +1,423 @@ +#include "iris_test.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace std::string_view_literals; +using iris::interval; +using IVS = iris::interval_set>; + +// NOLINTBEGIN(readability-container-size-empty) + +TEST_CASE("interval_set: type_traits") +{ + STATIC_CHECK(std::bidirectional_iterator); + STATIC_CHECK(std::bidirectional_iterator); + + STATIC_CHECK(std::ranges::bidirectional_range); + STATIC_CHECK(std::ranges::sized_range); + + STATIC_CHECK(std::is_default_constructible_v); + STATIC_CHECK(std::is_nothrow_destructible_v); + STATIC_CHECK(std::is_copy_constructible_v); + STATIC_CHECK(std::is_move_constructible_v); + STATIC_CHECK(std::is_nothrow_move_constructible_v == std::is_nothrow_move_constructible_v); + STATIC_CHECK(std::is_nothrow_move_assignable_v == std::is_nothrow_move_assignable_v); + STATIC_CHECK(std::is_copy_assignable_v); + STATIC_CHECK(std::is_nothrow_swappable_v); + + STATIC_CHECK(!std::is_constructible_v>); + STATIC_CHECK(std::is_constructible_v>>); +} + +TEST_CASE("interval_set: construction") +{ + { + STATIC_CHECK(std::is_constructible_v>::iterator, std::vector>::iterator>); + std::vector> v; + [[maybe_unused]] IVS ivs{v.begin(), v.end()}; + } + + { + STATIC_CHECK(std::is_constructible_v>>); + [[maybe_unused]] IVS ivs{std::from_range, std::vector>{}}; + } + + { + std::vector ivec; + auto view = ivec | std::views::transform([](int const lower) -> interval { + return {lower, lower + 1}; + }); + using View = decltype(view); + STATIC_CHECK(std::is_constructible_v); + STATIC_CHECK(std::is_constructible_v); + STATIC_CHECK(std::is_constructible_v); + + IVS const ivs{std::from_range, view}; + CHECK(ivs.empty()); + } + { + std::vector ivec{0}; + auto view = ivec | std::views::transform([](int const lower) -> interval { + return {lower, lower + 1}; + }); + IVS const ivs{std::from_range, view}; + CHECK(ivs.size() == 1); + CHECK(ivs == IVS{{0, 1}}); + } + { + std::vector ivec{0}; + auto view = ivec | std::views::transform([](int const lower) -> interval { + return {lower, lower}; + }); + IVS const ivs{std::from_range, view}; + CHECK(ivs.empty()); + } + + { + std::vector ivec{0, 4, 8}; + auto view = ivec | std::views::transform([](int const lower) -> interval { + return {lower, lower + 1}; + }); + + IVS const ivs{std::from_range, view}; + CHECK(ivs.size() == 3); + CHECK(ivs == IVS{{0, 1}, {4, 5}, {8, 9}}); + } + { + std::vector ivec{0, 1, 2}; + auto view = ivec | std::views::transform([](int const lower) -> interval { + return {lower, lower + 1}; + }); + IVS const ivs{std::from_range, view}; + CHECK(ivs.size() == 1); + CHECK(ivs == IVS{{0, 3}}); + } +} + +TEST_CASE("interval_set: identity") +{ + { + IVS ivs; + CHECK(ivs.size() == 0); + CHECK(ivs.empty()); + CHECK(ivs.begin() == ivs.end()); + CHECK(ivs.extent() == interval{}); + CHECK(ivs == ivs); + CHECK((ivs <=> ivs) == std::strong_ordering::equal); + } + + { + IVS ivs{{0, 0}}; + CHECK(ivs.size() == 0); + CHECK(ivs.empty()); + CHECK(ivs.begin() == ivs.end()); + CHECK(ivs.extent() == interval{}); + CHECK(ivs == ivs); + CHECK((ivs <=> ivs) == std::strong_ordering::equal); + } + { + IVS ivs{{1, 1}}; + CHECK(ivs.size() == 0); + CHECK(ivs.empty()); + CHECK(ivs.begin() == ivs.end()); + CHECK(ivs.extent() == interval{}); // not {1, 1} + CHECK(ivs == ivs); + CHECK((ivs <=> ivs) == std::strong_ordering::equal); + } + + { + IVS ivs{{0, 1}}; + CHECK(ivs.size() == 1); + CHECK(!ivs.empty()); + CHECK(ivs.begin() != ivs.end()); + CHECK(std::distance(ivs.begin(), ivs.end()) == 1); + CHECK(ivs == ivs); + CHECK((ivs <=> ivs) == std::strong_ordering::equal); + } +} + +TEST_CASE("interval_set: extent") +{ + CHECK(IVS{}.extent() == interval{0, 0}); + CHECK(IVS{{0, 0}}.extent() == interval{0, 0}); + CHECK(IVS{{1, 1}}.extent() == interval{0, 0}); + + CHECK(IVS{{2, 5}}.extent() == interval{2, 5}); + + CHECK(IVS{{2, 5}, {0, 1}}.extent() == interval{0, 5}); + CHECK(IVS{{2, 5}, {0, 2}}.extent() == interval{0, 5}); + CHECK(IVS{{2, 5}, {0, 3}}.extent() == interval{0, 5}); + CHECK(IVS{{2, 5}, {0, 4}}.extent() == interval{0, 5}); + CHECK(IVS{{2, 5}, {0, 5}}.extent() == interval{0, 5}); + CHECK(IVS{{2, 5}, {0, 6}}.extent() == interval{0, 6}); + CHECK(IVS{{2, 5}, {0, 7}}.extent() == interval{0, 7}); + + CHECK(IVS{{2, 5}, {1, 2}}.extent() == interval{1, 5}); + CHECK(IVS{{2, 5}, {1, 3}}.extent() == interval{1, 5}); + CHECK(IVS{{2, 5}, {1, 4}}.extent() == interval{1, 5}); + CHECK(IVS{{2, 5}, {1, 5}}.extent() == interval{1, 5}); + CHECK(IVS{{2, 5}, {1, 6}}.extent() == interval{1, 6}); + CHECK(IVS{{2, 5}, {1, 7}}.extent() == interval{1, 7}); + + CHECK(IVS{{2, 5}, {2, 3}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {2, 4}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {2, 5}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {2, 6}}.extent() == interval{2, 6}); + CHECK(IVS{{2, 5}, {2, 7}}.extent() == interval{2, 7}); + + CHECK(IVS{{2, 5}, {3, 4}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {3, 5}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {3, 6}}.extent() == interval{2, 6}); + CHECK(IVS{{2, 5}, {3, 7}}.extent() == interval{2, 7}); + + CHECK(IVS{{2, 5}, {4, 5}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {4, 6}}.extent() == interval{2, 6}); + CHECK(IVS{{2, 5}, {4, 7}}.extent() == interval{2, 7}); + + CHECK(IVS{{2, 5}, {5, 6}}.extent() == interval{2, 6}); + CHECK(IVS{{2, 5}, {5, 7}}.extent() == interval{2, 7}); + + CHECK(IVS{{2, 5}, {6, 7}}.extent() == interval{2, 7}); + + CHECK(IVS{{-5, -3}, {4, 10}}.extent() == interval{-5, 10}); + CHECK(IVS{{-5, -3}, {-1, 2}, {4, 10}}.extent() == interval{-5, 10}); +} + +#define IRIS_CHECK_REL(rel, a0, a1, b0, b1) \ + CHECK(interval a0, a1 .rel(b0, b1) == IVS{a0, a1}.rel(b0, b1)) + +#define IRIS_CHECK_REL_P(rel, a0, a1, p) \ + CHECK(interval a0, a1 .rel(p) == IVS{a0, a1}.rel(p)) + +TEST_CASE("interval_set: relationship") +{ + IRIS_CHECK_REL(intersects, {2, 5}, {0, 1}); + IRIS_CHECK_REL(intersects, {2, 5}, {0, 2}); + IRIS_CHECK_REL(intersects, {2, 5}, {0, 3}); + IRIS_CHECK_REL(intersects, {2, 5}, {4, 7}); + IRIS_CHECK_REL(intersects, {2, 5}, {5, 7}); + IRIS_CHECK_REL(intersects, {2, 5}, {6, 7}); + + // b = empty interval + IRIS_CHECK_REL(intersects, {2, 5}, {0, 0}); + IRIS_CHECK_REL(intersects, {2, 5}, {1, 1}); + IRIS_CHECK_REL(intersects, {2, 5}, {2, 2}); + IRIS_CHECK_REL(intersects, {2, 5}, {3, 3}); + IRIS_CHECK_REL(intersects, {2, 5}, {4, 4}); + IRIS_CHECK_REL(intersects, {2, 5}, {5, 5}); + IRIS_CHECK_REL(intersects, {2, 5}, {6, 6}); + + // ---------------------------------------------------- + + IRIS_CHECK_REL(covers, {2, 5}, {0, 1}); + IRIS_CHECK_REL(covers, {2, 5}, {0, 2}); + IRIS_CHECK_REL(covers, {2, 5}, {0, 3}); + + IRIS_CHECK_REL(covers, {2, 5}, {1, 2}); + IRIS_CHECK_REL(covers, {2, 5}, {1, 3}); + + IRIS_CHECK_REL(covers, {2, 5}, {2, 3}); + IRIS_CHECK_REL(covers, {2, 5}, {2, 4}); + IRIS_CHECK_REL(covers, {2, 5}, {2, 5}); + IRIS_CHECK_REL(covers, {2, 5}, {2, 6}); + + IRIS_CHECK_REL(covers, {2, 5}, {3, 4}); + IRIS_CHECK_REL(covers, {2, 5}, {3, 5}); + IRIS_CHECK_REL(covers, {2, 5}, {3, 6}); + + IRIS_CHECK_REL(covers, {2, 5}, {4, 5}); + IRIS_CHECK_REL(covers, {2, 5}, {4, 6}); + + IRIS_CHECK_REL(covers, {2, 5}, {5, 6}); + + IRIS_CHECK_REL(covers, {2, 5}, {6, 7}); + + // b = empty interval + IRIS_CHECK_REL(covers, {2, 5}, {0, 0}); + IRIS_CHECK_REL(covers, {2, 5}, {1, 1}); + IRIS_CHECK_REL(covers, {2, 5}, {2, 2}); + IRIS_CHECK_REL(covers, {2, 5}, {3, 3}); + IRIS_CHECK_REL(covers, {2, 5}, {4, 4}); + IRIS_CHECK_REL(covers, {2, 5}, {5, 5}); + IRIS_CHECK_REL(covers, {2, 5}, {6, 6}); + + // ---------------------------------------------------- + + IRIS_CHECK_REL_P(contains, {2, 5}, 0); + IRIS_CHECK_REL_P(contains, {2, 5}, 1); + IRIS_CHECK_REL_P(contains, {2, 5}, 2); + IRIS_CHECK_REL_P(contains, {2, 5}, 3); + IRIS_CHECK_REL_P(contains, {2, 5}, 4); + IRIS_CHECK_REL_P(contains, {2, 5}, 5); + IRIS_CHECK_REL_P(contains, {2, 5}, 6); +} + +TEST_CASE("interval_set: insertion") +{ + // Insertion of empty interval is no-op + { + IVS ivs{{2, 5}}; ivs.insert({0, 0}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 1}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({2, 2}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({3, 3}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({4, 4}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({5, 5}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({6, 6}); + CHECK(ivs == IVS{{2, 5}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({0, 1}); + CHECK(ivs == IVS{{0, 1}, {2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 2}); + CHECK(ivs == IVS{{0, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 3}); + CHECK(ivs == IVS{{0, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 4}); + CHECK(ivs == IVS{{0, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 5}); + CHECK(ivs == IVS{{0, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 6}); + CHECK(ivs == IVS{{0, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 7}); + CHECK(ivs == IVS{{0, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({1, 2}); + CHECK(ivs == IVS{{1, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 3}); + CHECK(ivs == IVS{{1, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 4}); + CHECK(ivs == IVS{{1, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 5}); + CHECK(ivs == IVS{{1, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 6}); + CHECK(ivs == IVS{{1, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 7}); + CHECK(ivs == IVS{{1, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({2, 3}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({2, 4}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({2, 5}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({2, 6}); + CHECK(ivs == IVS{{2, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({2, 7}); + CHECK(ivs == IVS{{2, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({3, 4}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({3, 5}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({3, 6}); + CHECK(ivs == IVS{{2, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({3, 7}); + CHECK(ivs == IVS{{2, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({4, 5}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({4, 6}); + CHECK(ivs == IVS{{2, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({4, 7}); + CHECK(ivs == IVS{{2, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({5, 6}); + CHECK(ivs == IVS{{2, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({5, 7}); + CHECK(ivs == IVS{{2, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({6, 7}); + CHECK(ivs == IVS{{2, 5}, {6, 7}}); + } +} + +TEST_CASE("interval_set: format") +{ + CHECK(std::format("{}", IVS{}) == "{}"sv); + CHECK(std::format("{}", IVS{{0, 1}}) == "{[0,1)}"sv); + CHECK(std::format("{}", IVS{{0, 1}, {2, 3}}) == "{[0,1) [2,3)}"sv); + CHECK(std::format("{:2d,3d}", IVS{{0, 1}, {2, 3}}) == "{[ 0, 1) [ 2, 3)}"sv); +} + +// NOLINTEND(readability-container-size-empty)