diff --git a/include/iris/interval.hpp b/include/iris/interval.hpp new file mode 100644 index 0000000..2f07d54 --- /dev/null +++ b/include/iris/interval.hpp @@ -0,0 +1,197 @@ +#ifndef IRIS_ZZ_INTERVAL_HPP +#define IRIS_ZZ_INTERVAL_HPP + +// SPDX-License-Identifier: MIT + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace iris { + +template +struct interval +{ + using value_type = T; + T lower, upper; + + constexpr interval() noexcept + : lower{} + , upper{} + {} + + constexpr interval(T lower, T upper) noexcept + : lower(lower) + , upper(upper) + {} + + [[nodiscard]] constexpr bool empty() const noexcept + { + // Malformed interval is treated as "empty" + return lower >= upper; + } + + [[nodiscard]] constexpr value_type length() const noexcept + { + return static_cast(upper - lower); + } + + template + [[nodiscard]] constexpr bool intersects(interval other) const noexcept + { + return (lower < other.upper && other.lower < upper) && !empty() && !other.empty(); + } + + // !intersects + template + [[nodiscard]] constexpr bool disjoint(interval other) const noexcept + { + return (upper <= other.lower || other.upper <= lower) || empty() || other.empty(); + } + + template + [[nodiscard]] constexpr bool touches(interval other) const noexcept + { + return (upper == other.lower || other.upper == lower) && !empty() && !other.empty(); + } + + // intersects || touches + template + [[nodiscard]] constexpr bool connected(interval other) const noexcept + { + return (lower <= other.upper && other.lower <= upper) && !empty() && !other.empty(); + } + + template + [[nodiscard]] constexpr bool covers(interval other) const noexcept + { + return (lower <= other.lower && other.upper <= upper) || other.empty(); + } + + template + [[nodiscard]] constexpr bool equals(interval other) const noexcept + { + return (lower == other.lower && upper == other.upper) || (empty() && other.empty()); + } + + // ------------------------------------------- + + [[nodiscard]] constexpr bool contains(value_type p) const noexcept + { + return lower <= p && p < upper; + } + + // ------------------------------------------- + + // Note: This does not reflect mathematical definition like `equals(...)`; this always checks exact data representation + [[nodiscard]] constexpr bool operator==(interval const&) const noexcept = default; + + // Note: This does not reflect mathematical definition like `equals(...)`; this always checks exact data representation + [[nodiscard]] constexpr std::strong_ordering operator<=>(interval const&) const noexcept = default; +}; + +template +[[nodiscard]] constexpr T& get(interval& iv) noexcept +{ + static_assert(I == 0 || I == 1); + if constexpr (I == 0) { return iv.lower; } else { return iv.upper; } +} +template +[[nodiscard]] constexpr T const& get(interval const& iv) noexcept +{ + static_assert(I == 0 || I == 1); + if constexpr (I == 0) { return iv.lower; } else { return iv.upper; } +} +template +[[nodiscard]] constexpr T&& get(interval&& iv) noexcept +{ + static_assert(I == 0 || I == 1); + if constexpr (I == 0) { return std::move(iv).lower; } else { return std::move(iv).upper; } +} +template +[[nodiscard]] constexpr T const&& get(interval const&& iv) noexcept +{ + static_assert(I == 0 || I == 1); + if constexpr (I == 0) { return std::move(iv).lower; } else { return std::move(iv).upper; } +} + +} // iris + +template +struct std::tuple_size> + : std::integral_constant +{}; + +template +struct std::tuple_element> +{ + using type = T; +}; + +template +struct std::formatter, CharT> +{ + [[nodiscard]] constexpr std::basic_format_parse_context::const_iterator + parse(std::basic_format_parse_context& ctx) + { + auto const first = ctx.begin(); + if (first == ctx.end()) return first; + if (*first == iris::format_traits::brace_close) return first; + + // Bound the search to this replacement field + auto const close_it = std::find( + first, ctx.end(), + iris::format_traits::brace_close + ); + if (close_it == ctx.end()) { + throw std::format_error("unterminated format specifier"); + } + + auto const comma_it = std::find( + first, close_it, + iris::format_traits::comma + ); + if (comma_it == close_it) { + throw std::format_error("expected ',' in format specifier"); + } + + { + std::basic_format_parse_context left_ctx{ + std::basic_string_view{first, comma_it} + }; + if (left_fmt_.parse(left_ctx) != left_ctx.end()) { + throw std::format_error("trailing characters in lower format specifier"); + } + } + { + std::basic_format_parse_context right_ctx{ + std::basic_string_view{std::next(comma_it), close_it} + }; + if (right_fmt_.parse(right_ctx) != right_ctx.end()) { + throw std::format_error("trailing characters in upper format specifier"); + } + } + return close_it; + } + + template + Ctx::iterator format(iris::interval const& iv, Ctx& ctx) const + { + ctx.advance_to(std::format_to(ctx.out(), "{}", iris::format_traits::square_brace_open)); + left_fmt_.format(iv.lower, ctx); + ctx.advance_to(std::format_to(ctx.out(), "{}", iris::format_traits::comma)); + right_fmt_.format(iv.upper, ctx); + return std::format_to(ctx.out(), "{}", iris::format_traits::paren_close); + } + +private: + std::formatter left_fmt_, right_fmt_; +}; + +#endif diff --git a/include/iris/interval_set.hpp b/include/iris/interval_set.hpp new file mode 100644 index 0000000..dad688c --- /dev/null +++ b/include/iris/interval_set.hpp @@ -0,0 +1,257 @@ +#ifndef IRIS_ZZ_IntervalT_SET_HPP +#define IRIS_ZZ_IntervalT_SET_HPP + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace iris { + +template< + class IntervalT, + class MapT = std::map +> +class interval_set +{ +public: + using interval_type = IntervalT; + using map_type = MapT; + using offset_type = IntervalT::value_type; + + class const_iterator : public iterator_base + { + using typename iterator_base::iterator_base_type; + static_assert(std::bidirectional_iterator); + iterator_base_type it_; + + public: + using value_type = IntervalT; + using pointer = IntervalT const*; + using reference = IntervalT; + + constexpr const_iterator() noexcept = default; + + constexpr explicit const_iterator(iterator_base_type it) noexcept + : it_(std::move(it)) + {} + + [[nodiscard]] constexpr IntervalT operator*() const noexcept + { + return {it_->first, it_->second}; + } + + constexpr const_iterator& operator++() noexcept + { + ++it_; + return *this; + } + + [[nodiscard]] constexpr const_iterator operator++(int) noexcept + { + auto temp{*this}; + ++it_; + return temp; + } + + constexpr const_iterator& operator--() noexcept + { + --it_; + return *this; + } + + [[nodiscard]] constexpr const_iterator operator--(int) noexcept + { + auto temp{*this}; + --it_; + return temp; + } + + [[nodiscard]] constexpr bool operator==(const_iterator const&) const noexcept = default; + [[nodiscard]] constexpr auto operator<=>(const_iterator const&) const noexcept = default; + }; + + using iterator = const_iterator; + + constexpr interval_set() = default; + + constexpr explicit interval_set(std::initializer_list il) + { + auto it = il.begin(); + if (it == il.end()) return; + if (!it->empty()) { + map_.emplace(it->lower, it->upper); + } + for (++it; it != il.end(); ++it) { + this->insert(*it); + } + } + + template Se> + requires std::convertible_to, IntervalT> + constexpr interval_set(It it, Se se) + { + if (it == se) return; + if (IntervalT const iv = *it; !iv.empty()) { + map_.emplace(iv.lower, iv.upper); + } + for (++it; it != se; ++it) { + this->insert(*it); + } + } + + template + requires std::convertible_to, IntervalT> + constexpr interval_set(std::from_range_t, R&& r) + : interval_set(std::ranges::begin(r), std::ranges::end(r)) + {} + + [[nodiscard]] constexpr const_iterator begin() const noexcept + { + return const_iterator{map_.begin()}; + } + + [[nodiscard]] constexpr const_iterator end() const noexcept + { + return const_iterator{map_.end()}; + } + + [[nodiscard]] constexpr bool empty() const noexcept { return map_.empty(); } + [[nodiscard]] constexpr std::size_t size() const noexcept { return map_.size(); } + + constexpr void clear() noexcept + { + map_.clear(); + } + + // Total number of covered offsets (sum of lengths) + constexpr offset_type coverage() const noexcept + { + offset_type total = 0; + for (auto const& [lower, upper] : map_) { + total += upper - lower; + } + return total; + } + + // O(1) + [[nodiscard]] constexpr IntervalT extent() const noexcept + { + if (map_.empty()) return {}; + return {map_.begin()->first, std::prev(map_.end())->second}; + } + + // Insert [iv.lower, iv.upper), merging with any interval it overlaps or touches + constexpr void insert(IntervalT iv) + { + if (iv.empty()) return; + + auto it = map_.lower_bound(iv.lower); + if (it != map_.begin()) { + auto prev = std::prev(it); + if (prev->second >= iv.lower) { + it = prev; + } + } + + // Absorb every interval that overlaps or touches [iv.lower, iv.upper) + while (it != map_.end() && it->first <= iv.upper) { + if (it->first < iv.lower) iv.lower = it->first; + if (it->second > iv.upper) iv.upper = it->second; + it = map_.erase(it); + } + + map_.emplace(iv.lower, iv.upper); + } + + constexpr void insert(offset_type lower, offset_type upper) + { + this->insert(IntervalT{lower, upper}); + } + + // -------------------------------------- + + [[nodiscard]] constexpr bool intersects(IntervalT const iv) const + { + if (iv.empty()) return false; + auto const it = map_.upper_bound(iv.lower); + if (it != map_.begin() && std::prev(it)->second > iv.lower) return true; + return it != map_.end() && it->first < iv.upper; + } + + [[nodiscard]] constexpr bool covers(IntervalT const iv) const + { + if (iv.empty()) return true; + auto const it = map_.upper_bound(iv.lower); + if (it == map_.begin()) return false; + auto const& [lower, upper] = *std::prev(it); + return lower <= iv.lower && iv.upper <= upper; + } + + [[nodiscard]] constexpr bool contains(offset_type p) const + { + auto const it = map_.upper_bound(p); + if (it == map_.begin()) return false; + return std::prev(it)->second > p; + } + + [[nodiscard]] constexpr bool operator==(interval_set const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(interval_set const&) const noexcept = default; + + constexpr void swap(interval_set& other) noexcept + { + using std::swap; + swap(map_, other.map_); + } + +private: + MapT map_; +}; + +template +constexpr void swap(interval_set& a, interval_set& b) noexcept +{ + a.swap(b); +} + +} // iris + +template +struct std::formatter, CharT> +{ + [[nodiscard]] constexpr std::basic_format_parse_context::const_iterator + parse(std::basic_format_parse_context& ctx) + { + return iv_fmt_.parse(ctx); + } + + template + Ctx::iterator format(iris::interval_set const& ivs, Ctx& ctx) const + { + ctx.advance_to(std::format_to(ctx.out(), "{{")); + bool is_first = true; + for (auto const& iv : ivs) { + if (is_first) { + ctx.advance_to(iv_fmt_.format(iv, ctx)); + is_first = false; + } else { + ctx.advance_to(std::format_to(ctx.out(), " ")); + ctx.advance_to(iv_fmt_.format(iv, ctx)); + } + } + return std::format_to(ctx.out(), "}}"); + } + +private: + std::formatter iv_fmt_; +}; + +#endif diff --git a/include/iris/iterator.hpp b/include/iris/iterator.hpp new file mode 100644 index 0000000..9bc42ed --- /dev/null +++ b/include/iris/iterator.hpp @@ -0,0 +1,66 @@ +#ifndef IRIS_ZZ_ITERATOR_HPP +#define IRIS_ZZ_ITERATOR_HPP + +#include +#include + +namespace iris { + +template +struct iterator_tags_base; + +template + requires requires { + typename std::iterator_traits::iterator_category; + typename std::iterator_traits::iterator_concept; + } +struct iterator_tags_base +{ + using iterator_base_type = It; + using iterator_category = std::iterator_traits::iterator_category; + using iterator_concept = std::iterator_traits::iterator_concept; + + [[nodiscard]] constexpr bool operator==(iterator_tags_base const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(iterator_tags_base const&) const noexcept = default; +}; + +template + requires + requires { typename std::iterator_traits::iterator_category; } && + (!requires { typename std::iterator_traits::iterator_concept; }) +struct iterator_tags_base +{ + using iterator_base_type = It; + using iterator_category = std::iterator_traits::iterator_category; + + [[nodiscard]] constexpr bool operator==(iterator_tags_base const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(iterator_tags_base const&) const noexcept = default; +}; + +template + requires + (!requires { typename std::iterator_traits::iterator_category; }) && + requires { typename std::iterator_traits::iterator_concept; } +struct iterator_tags_base +{ + using iterator_base_type = It; + using iterator_concept = std::iterator_traits::iterator_concept; + + [[nodiscard]] constexpr bool operator==(iterator_tags_base const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(iterator_tags_base const&) const noexcept = default; +}; + +// ---------------------------------------------- + +template +struct iterator_base : iterator_tags_base +{ + using difference_type = std::iterator_traits::difference_type; + + [[nodiscard]] constexpr bool operator==(iterator_base const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(iterator_base const&) const noexcept = default; +}; + +} // iris + +#endif diff --git a/include/iris/ngram.hpp b/include/iris/ngram.hpp new file mode 100644 index 0000000..60ded71 --- /dev/null +++ b/include/iris/ngram.hpp @@ -0,0 +1,957 @@ +#ifndef IRIS_ZZ_NGRAM_HPP +#define IRIS_ZZ_NGRAM_HPP + +// SPDX-License-Identifier: MIT + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace iris { + +enum struct ngram_document_id : unsigned {}; + +struct ngram_occurrence +{ + ngram_document_id doc_id; + int pos; + + [[nodiscard]] constexpr bool operator==(ngram_occurrence const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_occurrence const&) const noexcept = default; +}; + +namespace detail { + +template struct ngram_value; +template<> struct ngram_value<1> { using type = std::uint8_t; }; +template<> struct ngram_value<2> { using type = std::uint16_t; }; +template<> struct ngram_value<4> { using type = std::uint32_t; }; +template<> struct ngram_value<8> { using type = std::uint64_t; }; + +template +using ngram_value_t = ngram_value::type; + +} // detail + +template +struct ngram +{ + static_assert(N >= 3); + + std::array data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it++)) + { + std::ranges::copy_n(it, N, data.begin()); + } + template + [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + ngram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + constexpr void shift_copy(It it, int const remaining_chars) + noexcept( + noexcept(std::shift_left(data.begin(), data.end(), remaining_chars)) && + noexcept(std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars))) + ) + { + assert(remaining_chars < int(N)); + std::shift_left(data.begin(), data.end(), remaining_chars); + std::ranges::copy_n(it, remaining_chars, data.begin() + (N - remaining_chars)); + } + + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == N + 1); + assert(chars[Len - 1] == static_cast(0)); + return ngram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; +}; + +template +struct ngram<1, CharT> +{ + CharT data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it)) + { + data = *it; + } + template + [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + ngram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == 1 + 1); + assert(chars[Len - 1] == static_cast(0)); + return ngram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; +}; + +template +struct ngram<2, CharT> +{ + using value_type = detail::ngram_value_t; + value_type data; + + template + constexpr void copy_n(It it) + noexcept(noexcept(*it++)) + { + using uchar = std::make_unsigned_t; + data = value_type(static_cast(*it++)) << (sizeof(CharT) * 8); + data |= value_type(static_cast(*it)); + } + template + [[nodiscard]] static constexpr ngram from_copy_n(It it) + noexcept(noexcept(std::declval().copy_n(std::move(it)))) + { + ngram ng; + ng.copy_n(std::move(it)); + return ng; + } + + template + constexpr void shift_copy(It it, int const remaining_chars) + noexcept(noexcept(*it)) + { + assert(remaining_chars == 1); + (void)remaining_chars; + data = (data << (sizeof(CharT) * 8)) | value_type(static_cast>(*it)); + } + + template + [[nodiscard]] static constexpr ngram from_c_array(CharT const (&chars)[Len]) noexcept + { + static_assert(Len == 2 + 1); + assert(chars[Len - 1] == static_cast(0)); + return ngram::from_copy_n(std::ranges::begin(chars)); + } + + [[nodiscard]] constexpr bool operator==(ngram const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram const&) const noexcept = default; +}; + +inline namespace ngram_literals { + +[[nodiscard]] constexpr ngram_document_id operator ""_doc_id(unsigned long long id) noexcept +{ + return ngram_document_id{static_cast>(id)}; +} + +} // ngram_literals + +template +[[nodiscard]] ngram to_ngram(CharT const (&chars)[N]) noexcept +{ + return ngram::from_c_array(chars); +} + + +namespace detail { + +enum struct [[nodiscard]] search_continuation : bool +{ + abort = false, + proceed = true, +}; + +struct ngram_posting +{ + ngram_document_id doc_id; + unsigned pos_offset = 0; + unsigned pos_count = 0; +}; + +struct ngram_posting_list +{ + std::vector postings; + std::vector positions; + + void append(ngram_document_id const doc_id, int pos) + { + if (postings.empty() || postings.back().doc_id != doc_id) { + if (!postings.empty() && postings.back().doc_id > doc_id) { + throw std::invalid_argument{"documents must be indexed in non-decreasing order of document ID"}; + } + postings.emplace_back( + doc_id, + static_cast(positions.size()), + 0 + ); + } + ++postings.back().pos_count; + positions.emplace_back(pos); + } + + void to_occurrence_list(std::vector& occs) const + { + occs.clear(); + for (auto const& posting : postings) { + for (std::size_t i = posting.pos_offset; i < posting.pos_offset + posting.pos_count; ++i) { + occs.emplace_back(posting.doc_id, positions[i]); + } + } + } + + template + void for_each_documents(F&& f) const + { + static_assert(std::invocable>); + + constexpr bool f_returns_continuation = std::same_as< + std::invoke_result_t>, + search_continuation + >; + + for (auto const& posting : postings) { + std::span const posting_span{ + positions.begin() + posting.pos_offset, + static_cast(posting.pos_count) + }; + + if constexpr (f_returns_continuation) { + search_continuation const cont = f(posting.doc_id, posting_span); + if (cont == search_continuation::abort) break; + } else { + f(posting.doc_id, posting_span); + } + } + } +}; + +template +class ngram_index +{ + using entry_map = std::flat_map, std::unique_ptr>; + static constexpr std::size_t side_merge_threshold = 2048; + +public: + [[nodiscard]] + auto find_list(this auto&& self, ngram const ng) + { + if (auto const it = self.gram_entries_.find(ng); it != self.gram_entries_.end()) { + return it->second.get(); + } + if (auto const it = self.side_entries_.find(ng); it != self.side_entries_.end()) { + return it->second.get(); + } + return static_cast(nullptr); + } + + void find_occurrences(ngram const ng, std::vector& occs) const + { + occs.clear(); + auto const* list = this->find_list(ng); + if (!list) return; + list->to_occurrence_list(occs); + } + + template + void search(ngram const ng, F&& f) const + { + auto const* list = this->find_list(ng); + if (!list) return; + list->for_each_documents(f); + } + + [[nodiscard]] + bool empty() const noexcept + { + return gram_entries_.empty() && side_entries_.empty(); + } + + void merge_new_entries(std::vector, std::unique_ptr>>& pending) + { + if (pending.empty()) return; // vocabulary saturated + + for (auto& [key, pl] : pending) { + [[maybe_unused]] + auto const it = side_entries_.try_emplace( + side_entries_.end(), // hint + key, std::move(pl) + ); + assert(it->second != nullptr && pl == nullptr); + } + if (side_entries_.size() >= side_merge_threshold) { + this->flush_side(); + } + } + +private: + void flush_side() + { + if (side_entries_.empty()) return; + + auto [skeys, svalues] = std::move(side_entries_).extract(); + auto [keys, values] = std::move(gram_entries_).extract(); + + std::size_t const old_size = keys.size(); + std::size_t const add = skeys.size(); + keys.resize(old_size + add); + values.resize(old_size + add); + + // Backward merge + std::size_t out = old_size + add; + std::size_t i = old_size; + std::size_t j = add; + while (j > 0) { + if (i > 0 && skeys[j - 1] < keys[i - 1]) { + --out; + --i; + keys[out] = keys[i]; + values[out] = std::move(values[i]); + } else { + assert(i == 0 || keys[i - 1] < skeys[j - 1]); + --out; + --j; + keys[out] = skeys[j]; + values[out] = std::move(svalues[j]); + } + } + assert(out == i); + + gram_entries_.replace(std::move(keys), std::move(values)); + } + + // Double-buffered to reduce insertion cost + entry_map gram_entries_, side_entries_; +}; + +template +struct ngram_pos_t +{ + ngram ng; + int pos; + + [[nodiscard]] constexpr bool operator==(ngram_pos_t const&) const noexcept = default; + [[nodiscard]] constexpr std::strong_ordering operator<=>(ngram_pos_t const&) const noexcept = default; +}; + +template +struct ngram_index_storage +{ + void append_index(ngram_document_id const doc_id, std::basic_string_view const input) + { + this->template append_index<1>(doc_id, input); + this->template append_index<2>(doc_id, input); + } + + [[nodiscard]] + bool empty() const noexcept + { + return + this->template get_data<1>().idx.empty() && + this->template get_data<2>().idx.empty(); + } + + template + void search(ngram const ng, F&& f) const + { + this->template get_data().idx.search(ng, f); + } + + template + [[nodiscard]] auto& get_index(this auto& self IRIS_LIFETIMEBOUND) noexcept + { + return self.template get_data().idx; + } + +private: + template + struct ngram_index_storage_data + { + ngram_index idx; + + // Caches + std::vector, default_init_allocator>> + batch_grams; + + std::vector, std::unique_ptr>> + batch_pending; + }; + + ngram_index_storage_data<1> uni_data_; + ngram_index_storage_data<2> bi_data_; + + template + [[nodiscard]] auto& get_data(this auto& self IRIS_LIFETIMEBOUND) noexcept + { + if constexpr (N == 1) { + return self.uni_data_; + } else if constexpr (N == 2) { + return self.bi_data_; + } else { + static_assert(false); + } + } + + template + void append_index( + ngram_document_id const doc_id, + std::basic_string_view const input + ) + { + if (input.size() < N) return; + ngram_index_storage_data& data = this->template get_data(); + + // Naive per-gram insertion into flat_map is expensive: each *new* key + // shifts the underlying vectors, so building an index of vocabulary + // size V costs O(V^2) overall. Instead, per document: + // + // 1. Collect grams+positions --- O(G) G = grams in this doc + // 2. Sort them ----------------- O(G log G) + // 3. Existing keys ------------- O(D log V) D = distinct grams (D <= G) + // 4. New keys ------------------ O(V + P) P = brand-new keys (P <= D) + // + // Once the vocabulary saturates (P ~ 0, typical after a few documents), + // step 4 is a no-op and each document costs only O(G log G + D log V). + // + // Note: initially implemented by @saki7, then the complexity math is + // double-checked by Claude. + + data.batch_grams.clear(); + data.batch_grams.resize(input.size() - N + 1); + + if constexpr (N == 1) { + for (std::size_t i = 0; i < input.size(); ++i) { + data.batch_grams[i].ng.data = input[i]; + data.batch_grams[i].pos = static_cast(i); + } + + } else { + for (std::size_t i = 0; i + N <= input.size(); ++i) { + data.batch_grams[i].ng.copy_n(input.begin() + i); + data.batch_grams[i].pos = static_cast(i); + } + } + std::ranges::sort(data.batch_grams); + + data.batch_pending.clear(); + + for (auto const& chunk : data.batch_grams | std::views::chunk_by( + [](auto const& a, auto const& b) { return a.ng == b.ng; } + )) { + auto const& key = chunk.front().ng; + if (PostingListT* const pl = data.idx.find_list(key)) { + for (auto const& gp : chunk) { + pl->append(doc_id, gp.pos); + } + + } else { + auto& new_pl = data.batch_pending.emplace_back(key, std::make_unique()).second; + for (auto const& gp : chunk) { + new_pl->append(doc_id, gp.pos); + } + } + } + + data.idx.merge_new_entries(data.batch_pending); + } +}; + +} // detail + + +template +struct ngram_search_query +{ + explicit ngram_search_query(std::basic_string_view input_sv) + { + std::basic_string input{input_sv}; + iris::compact_spaces(input); + if (input.empty()) return; + + words_ = input + | std::views::split(detail::string_algo_traits::space) + | std::views::transform([](auto const& r) { + return std::basic_string{std::from_range, r}; + }) + | std::ranges::to(); + + std::ranges::sort(words_); + { + auto const [first, last] = std::ranges::unique(words_); + words_.erase(first, last); + } + } + + // ------------------------------------------ + + [[nodiscard]] + auto const& words() const noexcept + { + return words_; + } + + [[nodiscard]] bool empty() const noexcept + { + return words_.empty(); + } + + [[nodiscard]] bool operator==(ngram_search_query const& other) const noexcept + { + return words_ == other.words_; + } + +private: + std::vector> words_; +}; + +template +ngram_search_query(CharT const(&)[N]) -> ngram_search_query; + + +struct [[nodiscard]] ngram_search_word_match +{ + ngram_search_word_match() = default; + + explicit ngram_search_word_match(int word_id) + : word_id(word_id) + {} + + ngram_search_word_match(int word_id, std::initializer_list> spans) + : word_id(word_id) + , spans(spans) + {} + + int word_id = 0; + unsigned successful_ngrams = 1; // due to the class layout, this must be placed here + std::vector> spans; + + [[nodiscard]] + bool operator==(ngram_search_word_match const& other) const noexcept + { + return word_id == other.word_id && spans == other.spans; + } +}; + +class [[nodiscard]] ngram_search_result +{ + using doc_matches_map = std::flat_map>; + + struct word_matches_handle + { + doc_matches_map::iterator doc_it; + ngram_search_word_match* word_match = nullptr; + + [[nodiscard]] + ngram_search_word_match* operator->() const noexcept + { + return word_match; + } + + [[nodiscard]] explicit operator bool() const noexcept + { + return word_match; + } + }; + +public: + [[nodiscard]] + bool has_document(ngram_document_id const doc_id) const noexcept + { + auto const it = doc_matches_.find(doc_id); + // An entry with no word matches is a tombstone (soft-erased document + // awaiting the next sweep), not a match. + return it != doc_matches_.end() && !it->second.empty(); + } + + [[nodiscard]] + auto const& doc_matches() const noexcept { return doc_matches_; } + + // Returns whether search must continue + template + [[nodiscard]] + bool init_word_matches(ngram_document_id const doc_id, int const word_id, std::span const positions) + { + assert(!positions.empty()); + + doc_matches_map::iterator doc_matches_it; + if constexpr (IsFirstWord) { + assert(word_id == 0); + assert(doc_matches_.empty() || doc_matches_.rbegin()->first < doc_id); + doc_matches_it = doc_matches_.try_emplace(doc_matches_.end(), doc_id); // hint: append + ++live_doc_count_; + + } else { + doc_matches_it = doc_matches_.find(doc_id); + if (doc_matches_it == doc_matches_.end()) return false; // no new docs after word 0 + if (doc_matches_it->second.empty()) return false; // tombstoned (soft-erased) document; skip + } + + assert(!std::ranges::contains(doc_matches_it->second, word_id, &ngram_search_word_match::word_id)); + auto& word_match = doc_matches_it->second.emplace_back(word_id); + word_match.spans.assign_range(positions | std::views::transform([](int const pos) -> interval { + return {pos, pos + static_cast(N)}; + })); + return true; + } + + [[nodiscard]] + word_matches_handle get_word_matches(ngram_document_id const doc_id, int const word_id) + { + auto const doc_matches_it = doc_matches_.find(doc_id); + if (doc_matches_it == doc_matches_.end()) return {}; + + // We don't need to do *full* `std::find` here; the word match is + // always inserted sequentially so if it exists, it is always placed + // at the *back* of the vector. + if ( + doc_matches_it->second.empty() || + doc_matches_it->second.back().word_id != word_id + ) { + assert( + doc_matches_it->second.empty() || + // Make sure the matching element does not exist at the position except for *back* + !std::ranges::contains(doc_matches_it->second, word_id, &ngram_search_word_match::word_id) + ); + return {}; + } + assert(!doc_matches_it->second.back().spans.empty()); + return {doc_matches_it, &doc_matches_it->second.back()}; + } + + void erase_document(word_matches_handle const& handle) + { + // This is slow because + // k erases x O(n) shift each =~ O(n^2) per word + //doc_matches_.erase(handle.doc_it); + + assert(!handle.doc_it->second.empty()); // never double-tombstone + handle.doc_it->second.clear(); // make this tombstone + assert(live_doc_count_ >= 1); + --live_doc_count_; + } + + void remove_stale_document_matches(int const word_id, unsigned const expected_ngrams) + { + auto [keys, values] = std::move(doc_matches_).extract(); + + std::size_t out = 0; + for (std::size_t in = 0; in < keys.size(); ++in) { + auto& word_matches = values[in]; + bool is_word_survived = false; + std::erase_if(word_matches, [&](ngram_search_word_match const& wm) { + if (wm.word_id != word_id) return false; + if (wm.successful_ngrams != expected_ngrams) return true; + is_word_survived = true; + return false; + }); + if (!is_word_survived || word_matches.empty()) continue; + + if (out != in) { + keys[out] = keys[in]; + values[out] = std::move(values[in]); + } + ++out; + } + keys.resize(out); + values.resize(out); + doc_matches_.replace(std::move(keys), std::move(values)); + live_doc_count_ = out; + } + + void reset() noexcept + { + doc_matches_.clear(); + live_doc_count_ = 0; + } + + [[nodiscard]] + bool empty() const noexcept + { + return live_doc_count_ == 0; + } + + [[nodiscard]] + explicit operator bool() const noexcept + { + return !this->empty(); + } + +private: + doc_matches_map doc_matches_; + std::size_t live_doc_count_ = 0; +}; + +template +class ngram_database +{ +public: + [[nodiscard]] + ngram_document_id add_document(std::basic_string_view const doc_text) + { + ngram_document_id const doc_id{max_doc_id_}; + max_doc_id_ = ngram_document_id{std::to_underlying(max_doc_id_) + 1u}; + + store_.append_index(doc_id, doc_text); + return doc_id; + } + + template + void find_occurrences(ngram ng, std::vector& occs) const + { + occs.clear(); + auto const& idx = store_.template get_index(); + idx.find_occurrences(ng, occs); + } + + [[nodiscard]] + ngram_search_result search(ngram_search_query const& query) const + { + if (query.empty()) return {}; + if (store_.empty()) return {}; + + ngram_search_result search_res; + + int word_id = 0; + auto it = query.words().begin(); + assert(!it->empty()); + this->search_word(search_res, word_id++, *it++); + if (search_res.empty()) { + search_res.reset(); // remove tombstones + return search_res; + } + + for (; it != query.words().end(); ++it) { + assert(!it->empty()); + this->search_word(search_res, word_id++, *it); + if (search_res.empty()) { + search_res.reset(); // remove tombstones + break; + } + } + return search_res; + } + +private: + template + void search_word(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const + { + assert(!word.empty()); + + if (word.size() == 1) { + this->search_word_impl(search_res, word_id, word); + } else { + this->search_word_impl(search_res, word_id, word); + } + } + + template + void search_word_impl(ngram_search_result& search_res, int const word_id, std::basic_string_view const word) const + { + assert(word.size() >= N); + auto ng = ngram::from_copy_n(word.begin()); + + if constexpr (IsFirstWord) { + store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + (void)search_res.init_word_matches(doc_id, word_id, positions); + }); + if (search_res.empty()) return; + + } else { + std::size_t available_doc_count = 0; + store_.search(ng, [&](ngram_document_id const doc_id, std::span const positions) { + if (search_res.init_word_matches(doc_id, word_id, positions)) { + ++available_doc_count; + } + }); + if (available_doc_count == 0) { + search_res.reset(); + return; + } + } + + unsigned current_ngram = 1; + auto const do_search = [&](int remaining_chars) { + return [&, remaining_chars, overlapping_chars = int(N) - remaining_chars](ngram_document_id const doc_id, std::span const positions) { + // Find the existing match set from the previous iteration. + // If none exists, any subsequent characters of the document will not match. + // + // For example, when the document is "今日は晴れです" and current `ng` is "は晴", + // - When previous `ng` was "昨日", `search_res` contians no matches => omit further sequence + // - When previous `ng` was "今日", `search_res` contains matches => proceed with "は晴" + auto word_match = search_res.get_word_matches(doc_id, word_id); + if (!word_match) return detail::search_continuation::proceed; + + // Prevent *resurrecting* the false-positive match on "match -> unmatch -> match" pattern. + // For example, when the document is "abef" and the query is "abXXef", + // - ngram{"ab"} -> match (successful_ngrams = 1) + // - ngram{"XX"} -> no match (successful_ngrams is untouched) + // - ngram{"ef"} -> successful_ngrams does not match current_ngram! + if (word_match->successful_ngrams != current_ngram) { + search_res.erase_document(word_match); + if (search_res.empty()) return detail::search_continuation::abort; + return detail::search_continuation::proceed; + } + + // Find contiguous match; document has [previous ng, current ng] + for (auto it = word_match->spans.begin(); it != word_match->spans.end();) { + auto& prev_pos = *it; + + if (std::ranges::binary_search(positions, prev_pos.upper - overlapping_chars)) { + // Matched; the current word's current n-gram is contiguous to the previous n-gram + prev_pos.upper += remaining_chars; + ++it; + continue; + } + // Erase exiting match that indicates the below structure + // [previous ng, ...some unrelated chars..., current ng] + it = word_match->spans.erase(it); + } + + // Even if *all* existing matches fit + // [previous ng, ...some unrelated chars..., current ng], + // we can always remove the entire document from the candidate pool. + if (word_match->spans.empty()) { + search_res.erase_document(word_match); + if (search_res.empty()) return detail::search_continuation::abort; + return detail::search_continuation::proceed; + } + + ++word_match->successful_ngrams; + return detail::search_continuation::proceed; + }; + }; + + std::size_t i = N; + for (; i + N <= word.size(); i += N) { + ng.copy_n(word.begin() + i); + store_.search(ng, do_search(N)); + if (search_res.empty()) return; + ++current_ngram; + } + + if constexpr (N >= 2) { + // When the remaining character count is remainder of `word.size() % N`, + // search by the *slided* remaining characters. + // + // For example, when the document is "今日は晴れです": + // + // When doing 3-gram search with "今日は雨": + // 1. Search by "今日は" in the normal loop + // + // 2. Then, + // i == 3 + // remaining_chars == word.size() - i == 1 + // overlapping_chars == N - remaining_chars == 2 + // next_search_pos = i - overlapping_chars == 1 + // + // 3. Try to match "日は雨" in the last loop + if (int const remaining_chars = static_cast(word.size() - i); remaining_chars > 0) { + assert(remaining_chars < int(N)); + ng.shift_copy(word.begin() + i, remaining_chars); + store_.search(ng, do_search(remaining_chars)); + if (search_res.empty()) return; + ++current_ngram; + } + } + + if constexpr (IsFirstWord) { + // A first word of exactly one n-gram runs no continuation searches + if (current_ngram == 1) return; + } + search_res.remove_stale_document_matches(word_id, current_ngram); + } + + ngram_document_id max_doc_id_{0_doc_id}; + detail::ngram_index_storage store_; +}; + +} // iris + + +template +struct std::formatter + : std::formatter, CharT> +{ + using base_type = std::formatter, CharT>; + + template + Ctx::iterator format(iris::ngram_document_id doc_id, Ctx& ctx) const + { + return base_type::format(std::to_underlying(doc_id), ctx); + } +}; + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_occurrence const& occ, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}:{}", occ.doc_id, occ.pos); + } +}; + +template +struct std::formatter, CharT> + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_search_query const& query, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{}", query.words() | std::views::transform([](std::u32string_view ustr) { + return iris::unicode::transcode(ustr); + })); + } +}; + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(iris::ngram_search_word_match const& word_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "{{word: #{}, spans: {}}}", word_match.word_id, word_match.spans); + } +}; + +#endif diff --git a/iris.natvis b/iris.natvis index b928d79..76f86a6 100644 --- a/iris.natvis +++ b/iris.natvis @@ -252,4 +252,19 @@ (int)index_ + + + + + {chars._Elems,na1} + + + {chars._Elems,na2} + + + {chars._Elems,na3} + + + {chars._Elems,na4} + diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 66a345c..2fe6f92 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -183,7 +183,13 @@ if(PROJECT_IS_TOP_LEVEL) indirect colorize_format preprocess + interval + interval_set string_algo + ngram + ngram_search_2 + ngram_search_3 + ngram_search_4_dep ) foreach(test_name IN LISTS IRIS_TEST_IRIS_TESTS) diff --git a/test/interval.cpp b/test/interval.cpp new file mode 100644 index 0000000..de757c8 --- /dev/null +++ b/test/interval.cpp @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: MIT + +#include "iris_test.hpp" + +#include + +#include +#include +#include +#include + +using namespace std::string_view_literals; + +using iris::interval; + +// NOLINTBEGIN(readability-container-size-empty) + +TEST_CASE("interval: type traits") +{ + STATIC_CHECK(std::is_trivially_copyable_v>); + + STATIC_CHECK(std::is_nothrow_default_constructible_v>); + STATIC_CHECK(std::is_nothrow_constructible_v, int, int>); + STATIC_CHECK(std::is_nothrow_copy_constructible_v>); + STATIC_CHECK(std::is_nothrow_move_constructible_v>); + STATIC_CHECK(std::is_nothrow_copy_assignable_v>); + STATIC_CHECK(std::is_nothrow_move_assignable_v>); + STATIC_CHECK(std::is_nothrow_destructible_v>); + STATIC_CHECK(std::is_nothrow_swappable_v>); +} + +TEST_CASE("interval: members") +{ + // Check in constexpr context to detect uninitialized value and other UBs + + // [x, x) + { + constexpr interval iv{}; + STATIC_CHECK(iv.lower == 0); + STATIC_CHECK(iv.upper == 0); + STATIC_CHECK(iv.length() == 0); + STATIC_CHECK(iv.empty()); + STATIC_CHECK(iv == iv); + STATIC_CHECK((iv <=> iv) == std::strong_ordering::equal); + } + + // [-, -) + { + constexpr interval iv{-5, -2}; + STATIC_CHECK(iv.lower == -5); + STATIC_CHECK(iv.upper == -2); + STATIC_CHECK(iv.length() == 3); + STATIC_CHECK(!iv.empty()); + STATIC_CHECK(iv == iv); + STATIC_CHECK((iv <=> iv) == std::strong_ordering::equal); + } + + // [-, +) + { + constexpr interval iv{-5, 2}; + STATIC_CHECK(iv.lower == -5); + STATIC_CHECK(iv.upper == 2); + STATIC_CHECK(iv.length() == 7); + STATIC_CHECK(!iv.empty()); + STATIC_CHECK(iv == iv); + STATIC_CHECK((iv <=> iv) == std::strong_ordering::equal); + } + + // [+, +) + { + constexpr interval iv{2, 5}; + STATIC_CHECK(iv.lower == 2); + STATIC_CHECK(iv.upper == 5); + STATIC_CHECK(iv.length() == 3); + STATIC_CHECK(!iv.empty()); + STATIC_CHECK(iv == iv); + STATIC_CHECK((iv <=> iv) == std::strong_ordering::equal); + } +} + +TEST_CASE("interval: relationship") +{ + STATIC_CHECK(interval{2, 5}.equals(interval{2, 5})); + STATIC_CHECK(interval{2, 5} == interval{2, 5}); + STATIC_CHECK((interval{2, 5} <=> interval{2, 5}) == std::strong_ordering::equal); + + // b = empty interval + STATIC_CHECK(interval{0, 0}.equals(interval{0, 0})); + STATIC_CHECK(interval{0, 0}.equals(interval{1, 1})); + STATIC_CHECK(interval{0, 0} == interval{0, 0}); + STATIC_CHECK(interval{0, 0} != interval{1, 1}); + STATIC_CHECK((interval{0, 0} <=> interval{0, 0}) == std::strong_ordering::equal); + STATIC_CHECK((interval{0, 0} <=> interval{1, 1}) == std::strong_ordering::less); + + // ---------------------------------------------------- + + STATIC_CHECK(!interval{2, 5}.intersects({0, 1})); STATIC_CHECK( interval{2, 5}.disjoint({0, 1})); + STATIC_CHECK(!interval{2, 5}.intersects({0, 2})); STATIC_CHECK( interval{2, 5}.disjoint({0, 2})); + STATIC_CHECK( interval{2, 5}.intersects({0, 3})); STATIC_CHECK(!interval{2, 5}.disjoint({0, 3})); + STATIC_CHECK( interval{2, 5}.intersects({4, 7})); STATIC_CHECK(!interval{2, 5}.disjoint({4, 7})); + STATIC_CHECK(!interval{2, 5}.intersects({5, 7})); STATIC_CHECK( interval{2, 5}.disjoint({5, 7})); + STATIC_CHECK(!interval{2, 5}.intersects({6, 7})); STATIC_CHECK( interval{2, 5}.disjoint({6, 7})); + + // b = empty interval + STATIC_CHECK(!interval{2, 5}.intersects({0, 0})); STATIC_CHECK( interval{2, 5}.disjoint({0, 0})); + STATIC_CHECK(!interval{2, 5}.intersects({1, 1})); STATIC_CHECK( interval{2, 5}.disjoint({1, 1})); + STATIC_CHECK(!interval{2, 5}.intersects({2, 2})); STATIC_CHECK( interval{2, 5}.disjoint({2, 2})); + STATIC_CHECK(!interval{2, 5}.intersects({3, 3})); STATIC_CHECK( interval{2, 5}.disjoint({3, 3})); + STATIC_CHECK(!interval{2, 5}.intersects({4, 4})); STATIC_CHECK( interval{2, 5}.disjoint({4, 4})); + STATIC_CHECK(!interval{2, 5}.intersects({5, 5})); STATIC_CHECK( interval{2, 5}.disjoint({5, 5})); + STATIC_CHECK(!interval{2, 5}.intersects({6, 6})); STATIC_CHECK( interval{2, 5}.disjoint({6, 6})); + + // ---------------------------------------------------- + + STATIC_CHECK(!interval{2, 5}.touches({0, 1})); + STATIC_CHECK( interval{2, 5}.touches({0, 2})); + STATIC_CHECK(!interval{2, 5}.touches({0, 3})); + STATIC_CHECK(!interval{2, 5}.touches({4, 7})); + STATIC_CHECK( interval{2, 5}.touches({5, 7})); + STATIC_CHECK(!interval{2, 5}.touches({6, 7})); + + // b = empty interval + STATIC_CHECK(!interval{2, 5}.touches({0, 0})); + STATIC_CHECK(!interval{2, 5}.touches({1, 1})); + STATIC_CHECK(!interval{2, 5}.touches({2, 2})); + STATIC_CHECK(!interval{2, 5}.touches({3, 3})); + STATIC_CHECK(!interval{2, 5}.touches({4, 4})); + STATIC_CHECK(!interval{2, 5}.touches({5, 5})); + STATIC_CHECK(!interval{2, 5}.touches({6, 6})); + + // ---------------------------------------------------- + + STATIC_CHECK(!interval{2, 5}.connected({0, 1})); + STATIC_CHECK( interval{2, 5}.connected({0, 2})); + STATIC_CHECK( interval{2, 5}.connected({0, 3})); + STATIC_CHECK( interval{2, 5}.connected({4, 7})); + STATIC_CHECK( interval{2, 5}.connected({5, 7})); + STATIC_CHECK(!interval{2, 5}.connected({6, 7})); + + // b = empty interval + STATIC_CHECK(!interval{2, 5}.connected({0, 0})); + STATIC_CHECK(!interval{2, 5}.connected({1, 1})); + STATIC_CHECK(!interval{2, 5}.connected({2, 2})); + STATIC_CHECK(!interval{2, 5}.connected({3, 3})); + STATIC_CHECK(!interval{2, 5}.connected({4, 4})); + STATIC_CHECK(!interval{2, 5}.connected({5, 5})); + STATIC_CHECK(!interval{2, 5}.connected({6, 6})); + + // ---------------------------------------------------- + + STATIC_CHECK(!interval{2, 5}.covers({0, 1})); + STATIC_CHECK(!interval{2, 5}.covers({0, 2})); + STATIC_CHECK(!interval{2, 5}.covers({0, 3})); + + STATIC_CHECK(!interval{2, 5}.covers({1, 2})); + STATIC_CHECK(!interval{2, 5}.covers({1, 3})); + + STATIC_CHECK( interval{2, 5}.covers({2, 3})); + STATIC_CHECK( interval{2, 5}.covers({2, 4})); + STATIC_CHECK( interval{2, 5}.covers({2, 5})); + STATIC_CHECK(!interval{2, 5}.covers({2, 6})); + + STATIC_CHECK( interval{2, 5}.covers({3, 4})); + STATIC_CHECK( interval{2, 5}.covers({3, 5})); + STATIC_CHECK(!interval{2, 5}.covers({3, 6})); + + STATIC_CHECK( interval{2, 5}.covers({4, 5})); + STATIC_CHECK(!interval{2, 5}.covers({4, 6})); + + STATIC_CHECK(!interval{2, 5}.covers({5, 6})); + + STATIC_CHECK(!interval{2, 5}.covers({6, 7})); + + // b = empty interval + STATIC_CHECK( interval{2, 5}.covers({0, 0})); + STATIC_CHECK( interval{2, 5}.covers({1, 1})); + STATIC_CHECK( interval{2, 5}.covers({2, 2})); + STATIC_CHECK( interval{2, 5}.covers({3, 3})); + STATIC_CHECK( interval{2, 5}.covers({4, 4})); + STATIC_CHECK( interval{2, 5}.covers({5, 5})); + STATIC_CHECK( interval{2, 5}.covers({6, 6})); + + // ---------------------------------------------------- + + STATIC_CHECK(!interval{2, 5}.contains(0)); + STATIC_CHECK(!interval{2, 5}.contains(1)); + STATIC_CHECK( interval{2, 5}.contains(2)); + STATIC_CHECK( interval{2, 5}.contains(3)); + STATIC_CHECK( interval{2, 5}.contains(4)); + STATIC_CHECK(!interval{2, 5}.contains(5)); + STATIC_CHECK(!interval{2, 5}.contains(6)); +} + +TEST_CASE("interval: tuple") +{ + { + interval const iv{1, 2}; + auto const [lower, upper] = iv; // structured bindings + CHECK(lower == 1); + CHECK(upper == 2); + } + + { + interval iv{1, 2}; + auto&& lower = iris::get<0>(iv); + STATIC_CHECK(std::same_as); + CHECK(lower == 1); + } + { + interval const iv{1, 2}; + auto&& lower = iris::get<0>(iv); + STATIC_CHECK(std::same_as); + CHECK(lower == 1); + } + { + interval iv{1, 2}; + auto&& lower = iris::get<0>(std::move(iv)); + STATIC_CHECK(std::same_as); + CHECK(lower == 1); + } + { + interval const iv{1, 2}; + auto&& lower = iris::get<0>(std::move(iv)); + STATIC_CHECK(std::same_as); + CHECK(lower == 1); + } +} + +TEST_CASE("interval: format") +{ + CHECK(std::format("{}", interval{}) == "[0,0)"sv); + CHECK(std::format("{}", interval{1, 2}) == "[1,2)"sv); + CHECK(std::format("{:2d,}", interval{1, 2}) == "[ 1,2)"sv); + CHECK(std::format("{:2d,3d}", interval{1, 2}) == "[ 1, 2)"sv); +} + +// NOLINTEND(readability-container-size-empty) diff --git a/test/interval_set.cpp b/test/interval_set.cpp new file mode 100644 index 0000000..c55b0b7 --- /dev/null +++ b/test/interval_set.cpp @@ -0,0 +1,423 @@ +#include "iris_test.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace std::string_view_literals; +using iris::interval; +using IVS = iris::interval_set>; + +// NOLINTBEGIN(readability-container-size-empty) + +TEST_CASE("interval_set: type_traits") +{ + STATIC_CHECK(std::bidirectional_iterator); + STATIC_CHECK(std::bidirectional_iterator); + + STATIC_CHECK(std::ranges::bidirectional_range); + STATIC_CHECK(std::ranges::sized_range); + + STATIC_CHECK(std::is_default_constructible_v); + STATIC_CHECK(std::is_nothrow_destructible_v); + STATIC_CHECK(std::is_copy_constructible_v); + STATIC_CHECK(std::is_move_constructible_v); + STATIC_CHECK(std::is_nothrow_move_constructible_v == std::is_nothrow_move_constructible_v); + STATIC_CHECK(std::is_nothrow_move_assignable_v == std::is_nothrow_move_assignable_v); + STATIC_CHECK(std::is_copy_assignable_v); + STATIC_CHECK(std::is_nothrow_swappable_v); + + STATIC_CHECK(!std::is_constructible_v>); + STATIC_CHECK(std::is_constructible_v>>); +} + +TEST_CASE("interval_set: construction") +{ + { + STATIC_CHECK(std::is_constructible_v>::iterator, std::vector>::iterator>); + std::vector> v; + [[maybe_unused]] IVS ivs{v.begin(), v.end()}; + } + + { + STATIC_CHECK(std::is_constructible_v>>); + [[maybe_unused]] IVS ivs{std::from_range, std::vector>{}}; + } + + { + std::vector ivec; + auto view = ivec | std::views::transform([](int const lower) -> interval { + return {lower, lower + 1}; + }); + using View = decltype(view); + STATIC_CHECK(std::is_constructible_v); + STATIC_CHECK(std::is_constructible_v); + STATIC_CHECK(std::is_constructible_v); + + IVS const ivs{std::from_range, view}; + CHECK(ivs.empty()); + } + { + std::vector ivec{0}; + auto view = ivec | std::views::transform([](int const lower) -> interval { + return {lower, lower + 1}; + }); + IVS const ivs{std::from_range, view}; + CHECK(ivs.size() == 1); + CHECK(ivs == IVS{{0, 1}}); + } + { + std::vector ivec{0}; + auto view = ivec | std::views::transform([](int const lower) -> interval { + return {lower, lower}; + }); + IVS const ivs{std::from_range, view}; + CHECK(ivs.empty()); + } + + { + std::vector ivec{0, 4, 8}; + auto view = ivec | std::views::transform([](int const lower) -> interval { + return {lower, lower + 1}; + }); + + IVS const ivs{std::from_range, view}; + CHECK(ivs.size() == 3); + CHECK(ivs == IVS{{0, 1}, {4, 5}, {8, 9}}); + } + { + std::vector ivec{0, 1, 2}; + auto view = ivec | std::views::transform([](int const lower) -> interval { + return {lower, lower + 1}; + }); + IVS const ivs{std::from_range, view}; + CHECK(ivs.size() == 1); + CHECK(ivs == IVS{{0, 3}}); + } +} + +TEST_CASE("interval_set: identity") +{ + { + IVS ivs; + CHECK(ivs.size() == 0); + CHECK(ivs.empty()); + CHECK(ivs.begin() == ivs.end()); + CHECK(ivs.extent() == interval{}); + CHECK(ivs == ivs); + CHECK((ivs <=> ivs) == std::strong_ordering::equal); + } + + { + IVS ivs{{0, 0}}; + CHECK(ivs.size() == 0); + CHECK(ivs.empty()); + CHECK(ivs.begin() == ivs.end()); + CHECK(ivs.extent() == interval{}); + CHECK(ivs == ivs); + CHECK((ivs <=> ivs) == std::strong_ordering::equal); + } + { + IVS ivs{{1, 1}}; + CHECK(ivs.size() == 0); + CHECK(ivs.empty()); + CHECK(ivs.begin() == ivs.end()); + CHECK(ivs.extent() == interval{}); // not {1, 1} + CHECK(ivs == ivs); + CHECK((ivs <=> ivs) == std::strong_ordering::equal); + } + + { + IVS ivs{{0, 1}}; + CHECK(ivs.size() == 1); + CHECK(!ivs.empty()); + CHECK(ivs.begin() != ivs.end()); + CHECK(std::distance(ivs.begin(), ivs.end()) == 1); + CHECK(ivs == ivs); + CHECK((ivs <=> ivs) == std::strong_ordering::equal); + } +} + +TEST_CASE("interval_set: extent") +{ + CHECK(IVS{}.extent() == interval{0, 0}); + CHECK(IVS{{0, 0}}.extent() == interval{0, 0}); + CHECK(IVS{{1, 1}}.extent() == interval{0, 0}); + + CHECK(IVS{{2, 5}}.extent() == interval{2, 5}); + + CHECK(IVS{{2, 5}, {0, 1}}.extent() == interval{0, 5}); + CHECK(IVS{{2, 5}, {0, 2}}.extent() == interval{0, 5}); + CHECK(IVS{{2, 5}, {0, 3}}.extent() == interval{0, 5}); + CHECK(IVS{{2, 5}, {0, 4}}.extent() == interval{0, 5}); + CHECK(IVS{{2, 5}, {0, 5}}.extent() == interval{0, 5}); + CHECK(IVS{{2, 5}, {0, 6}}.extent() == interval{0, 6}); + CHECK(IVS{{2, 5}, {0, 7}}.extent() == interval{0, 7}); + + CHECK(IVS{{2, 5}, {1, 2}}.extent() == interval{1, 5}); + CHECK(IVS{{2, 5}, {1, 3}}.extent() == interval{1, 5}); + CHECK(IVS{{2, 5}, {1, 4}}.extent() == interval{1, 5}); + CHECK(IVS{{2, 5}, {1, 5}}.extent() == interval{1, 5}); + CHECK(IVS{{2, 5}, {1, 6}}.extent() == interval{1, 6}); + CHECK(IVS{{2, 5}, {1, 7}}.extent() == interval{1, 7}); + + CHECK(IVS{{2, 5}, {2, 3}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {2, 4}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {2, 5}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {2, 6}}.extent() == interval{2, 6}); + CHECK(IVS{{2, 5}, {2, 7}}.extent() == interval{2, 7}); + + CHECK(IVS{{2, 5}, {3, 4}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {3, 5}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {3, 6}}.extent() == interval{2, 6}); + CHECK(IVS{{2, 5}, {3, 7}}.extent() == interval{2, 7}); + + CHECK(IVS{{2, 5}, {4, 5}}.extent() == interval{2, 5}); + CHECK(IVS{{2, 5}, {4, 6}}.extent() == interval{2, 6}); + CHECK(IVS{{2, 5}, {4, 7}}.extent() == interval{2, 7}); + + CHECK(IVS{{2, 5}, {5, 6}}.extent() == interval{2, 6}); + CHECK(IVS{{2, 5}, {5, 7}}.extent() == interval{2, 7}); + + CHECK(IVS{{2, 5}, {6, 7}}.extent() == interval{2, 7}); + + CHECK(IVS{{-5, -3}, {4, 10}}.extent() == interval{-5, 10}); + CHECK(IVS{{-5, -3}, {-1, 2}, {4, 10}}.extent() == interval{-5, 10}); +} + +#define IRIS_CHECK_REL(rel, a0, a1, b0, b1) \ + CHECK(interval a0, a1 .rel(b0, b1) == IVS{a0, a1}.rel(b0, b1)) + +#define IRIS_CHECK_REL_P(rel, a0, a1, p) \ + CHECK(interval a0, a1 .rel(p) == IVS{a0, a1}.rel(p)) + +TEST_CASE("interval_set: relationship") +{ + IRIS_CHECK_REL(intersects, {2, 5}, {0, 1}); + IRIS_CHECK_REL(intersects, {2, 5}, {0, 2}); + IRIS_CHECK_REL(intersects, {2, 5}, {0, 3}); + IRIS_CHECK_REL(intersects, {2, 5}, {4, 7}); + IRIS_CHECK_REL(intersects, {2, 5}, {5, 7}); + IRIS_CHECK_REL(intersects, {2, 5}, {6, 7}); + + // b = empty interval + IRIS_CHECK_REL(intersects, {2, 5}, {0, 0}); + IRIS_CHECK_REL(intersects, {2, 5}, {1, 1}); + IRIS_CHECK_REL(intersects, {2, 5}, {2, 2}); + IRIS_CHECK_REL(intersects, {2, 5}, {3, 3}); + IRIS_CHECK_REL(intersects, {2, 5}, {4, 4}); + IRIS_CHECK_REL(intersects, {2, 5}, {5, 5}); + IRIS_CHECK_REL(intersects, {2, 5}, {6, 6}); + + // ---------------------------------------------------- + + IRIS_CHECK_REL(covers, {2, 5}, {0, 1}); + IRIS_CHECK_REL(covers, {2, 5}, {0, 2}); + IRIS_CHECK_REL(covers, {2, 5}, {0, 3}); + + IRIS_CHECK_REL(covers, {2, 5}, {1, 2}); + IRIS_CHECK_REL(covers, {2, 5}, {1, 3}); + + IRIS_CHECK_REL(covers, {2, 5}, {2, 3}); + IRIS_CHECK_REL(covers, {2, 5}, {2, 4}); + IRIS_CHECK_REL(covers, {2, 5}, {2, 5}); + IRIS_CHECK_REL(covers, {2, 5}, {2, 6}); + + IRIS_CHECK_REL(covers, {2, 5}, {3, 4}); + IRIS_CHECK_REL(covers, {2, 5}, {3, 5}); + IRIS_CHECK_REL(covers, {2, 5}, {3, 6}); + + IRIS_CHECK_REL(covers, {2, 5}, {4, 5}); + IRIS_CHECK_REL(covers, {2, 5}, {4, 6}); + + IRIS_CHECK_REL(covers, {2, 5}, {5, 6}); + + IRIS_CHECK_REL(covers, {2, 5}, {6, 7}); + + // b = empty interval + IRIS_CHECK_REL(covers, {2, 5}, {0, 0}); + IRIS_CHECK_REL(covers, {2, 5}, {1, 1}); + IRIS_CHECK_REL(covers, {2, 5}, {2, 2}); + IRIS_CHECK_REL(covers, {2, 5}, {3, 3}); + IRIS_CHECK_REL(covers, {2, 5}, {4, 4}); + IRIS_CHECK_REL(covers, {2, 5}, {5, 5}); + IRIS_CHECK_REL(covers, {2, 5}, {6, 6}); + + // ---------------------------------------------------- + + IRIS_CHECK_REL_P(contains, {2, 5}, 0); + IRIS_CHECK_REL_P(contains, {2, 5}, 1); + IRIS_CHECK_REL_P(contains, {2, 5}, 2); + IRIS_CHECK_REL_P(contains, {2, 5}, 3); + IRIS_CHECK_REL_P(contains, {2, 5}, 4); + IRIS_CHECK_REL_P(contains, {2, 5}, 5); + IRIS_CHECK_REL_P(contains, {2, 5}, 6); +} + +TEST_CASE("interval_set: insertion") +{ + // Insertion of empty interval is no-op + { + IVS ivs{{2, 5}}; ivs.insert({0, 0}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 1}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({2, 2}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({3, 3}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({4, 4}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({5, 5}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({6, 6}); + CHECK(ivs == IVS{{2, 5}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({0, 1}); + CHECK(ivs == IVS{{0, 1}, {2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 2}); + CHECK(ivs == IVS{{0, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 3}); + CHECK(ivs == IVS{{0, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 4}); + CHECK(ivs == IVS{{0, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 5}); + CHECK(ivs == IVS{{0, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 6}); + CHECK(ivs == IVS{{0, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({0, 7}); + CHECK(ivs == IVS{{0, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({1, 2}); + CHECK(ivs == IVS{{1, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 3}); + CHECK(ivs == IVS{{1, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 4}); + CHECK(ivs == IVS{{1, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 5}); + CHECK(ivs == IVS{{1, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 6}); + CHECK(ivs == IVS{{1, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({1, 7}); + CHECK(ivs == IVS{{1, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({2, 3}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({2, 4}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({2, 5}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({2, 6}); + CHECK(ivs == IVS{{2, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({2, 7}); + CHECK(ivs == IVS{{2, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({3, 4}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({3, 5}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({3, 6}); + CHECK(ivs == IVS{{2, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({3, 7}); + CHECK(ivs == IVS{{2, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({4, 5}); + CHECK(ivs == IVS{{2, 5}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({4, 6}); + CHECK(ivs == IVS{{2, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({4, 7}); + CHECK(ivs == IVS{{2, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({5, 6}); + CHECK(ivs == IVS{{2, 6}}); + } + { + IVS ivs{{2, 5}}; ivs.insert({5, 7}); + CHECK(ivs == IVS{{2, 7}}); + } + // -------------------------------------------------------------- + { + IVS ivs{{2, 5}}; ivs.insert({6, 7}); + CHECK(ivs == IVS{{2, 5}, {6, 7}}); + } +} + +TEST_CASE("interval_set: format") +{ + CHECK(std::format("{}", IVS{}) == "{}"sv); + CHECK(std::format("{}", IVS{{0, 1}}) == "{[0,1)}"sv); + CHECK(std::format("{}", IVS{{0, 1}, {2, 3}}) == "{[0,1) [2,3)}"sv); + CHECK(std::format("{:2d,3d}", IVS{{0, 1}, {2, 3}}) == "{[ 0, 1) [ 2, 3)}"sv); +} + +// NOLINTEND(readability-container-size-empty) diff --git a/test/ngram.cpp b/test/ngram.cpp new file mode 100644 index 0000000..7a1d39c --- /dev/null +++ b/test/ngram.cpp @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +[[nodiscard]] +constexpr auto make_occurrences(std::initializer_list occs) +{ + return std::vector{occs}; +} + +#define IRIS_CHECK_NO_OCCURRENCE(ng_str) do { \ + std::vector occs; \ + ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ + CHECK(occs.empty()); \ + } while (false) + +#define IRIS_CHECK_OCCURRENCE(ng_str, ...) do { \ + std::vector occs; \ + ngram_db.find_occurrences(iris::to_ngram(U ## ng_str), occs); \ + CHECK(occs == make_occurrences({__VA_ARGS__})); \ + } while (false) + +TEST_CASE("ngram (type traits)") +{ + STATIC_CHECK(std::same_as::data), char>); + STATIC_CHECK(std::same_as::data), std::uint16_t>); + + STATIC_CHECK(std::same_as::data), char32_t>); + STATIC_CHECK(std::same_as::data), std::uint64_t>); +} + +TEST_CASE("ngram (minimal input)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U""); + IRIS_CHECK_NO_OCCURRENCE("a"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"a"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_NO_OCCURRENCE("XX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_NO_OCCURRENCE("XX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abc"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); + IRIS_CHECK_NO_OCCURRENCE("XX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("d", {0_doc_id, 3}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("cd", {0_doc_id, 2}); + IRIS_CHECK_NO_OCCURRENCE("XX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcde"); + IRIS_CHECK_OCCURRENCE("a", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("b", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("c", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("d", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("e", {0_doc_id, 4}); + IRIS_CHECK_NO_OCCURRENCE("X"); + IRIS_CHECK_OCCURRENCE("ab", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("bc", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("cd", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("de", {0_doc_id, 3}); + IRIS_CHECK_NO_OCCURRENCE("XX"); + } +} + +TEST_CASE("ngram (realistic input)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + // https://gihyo.jp/dev/serial/01/make-findspot/0005 + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は良い天気です。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("は良", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("良い", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("い天", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("天気", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("気で", {0_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 7}); + IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 8}); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は大雨です。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("は大", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("大雨", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("雨で", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 6}); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日の", {0_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("の東", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("東海", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("海地", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("地方", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("方は", {0_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("は大", {0_doc_id, 7}); + IRIS_CHECK_OCCURRENCE("大雨", {0_doc_id, 8}); + IRIS_CHECK_OCCURRENCE("雨で", {0_doc_id, 9}); + IRIS_CHECK_OCCURRENCE("でし", {0_doc_id, 10}); + IRIS_CHECK_OCCURRENCE("しょ", {0_doc_id, 11}); + IRIS_CHECK_OCCURRENCE("ょう", {0_doc_id, 12}); + IRIS_CHECK_OCCURRENCE("う。", {0_doc_id, 13}); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"今日は良い天気です。"); + (void)ngram_db.add_document(U"今日は大雨です。"); + (void)ngram_db.add_document(U"今日の東海地方は大雨でしょう。"); + + IRIS_CHECK_OCCURRENCE("今日", {0_doc_id, 0}, {1_doc_id, 0}, {2_doc_id, 0}); + IRIS_CHECK_OCCURRENCE("日は", {0_doc_id, 1}, {1_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("は良", {0_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("良い", {0_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("い天", {0_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("天気", {0_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("気で", {0_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("です", {0_doc_id, 7}, {1_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("す。", {0_doc_id, 8}, {1_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("は大", {1_doc_id, 2}, {2_doc_id, 7}); + IRIS_CHECK_OCCURRENCE("大雨", {1_doc_id, 3}, {2_doc_id, 8}); + IRIS_CHECK_OCCURRENCE("雨で", {1_doc_id, 4}, {2_doc_id, 9}); + IRIS_CHECK_OCCURRENCE("日の", {2_doc_id, 1}); + IRIS_CHECK_OCCURRENCE("の東", {2_doc_id, 2}); + IRIS_CHECK_OCCURRENCE("東海", {2_doc_id, 3}); + IRIS_CHECK_OCCURRENCE("海地", {2_doc_id, 4}); + IRIS_CHECK_OCCURRENCE("地方", {2_doc_id, 5}); + IRIS_CHECK_OCCURRENCE("方は", {2_doc_id, 6}); + IRIS_CHECK_OCCURRENCE("でし", {2_doc_id, 10}); + IRIS_CHECK_OCCURRENCE("しょ", {2_doc_id, 11}); + IRIS_CHECK_OCCURRENCE("ょう", {2_doc_id, 12}); + IRIS_CHECK_OCCURRENCE("う。", {2_doc_id, 13}); + } +} diff --git a/test/ngram_search_2.cpp b/test/ngram_search_2.cpp new file mode 100644 index 0000000..2f85570 --- /dev/null +++ b/test/ngram_search_2.cpp @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +TEST_CASE("ngram search (document chars = 0)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} + +// 1-gram document +TEST_CASE("ngram search (document chars = 1)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"a"); + IRIS_CHECK_SEARCH(""); + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + IRIS_CHECK_SEARCH("XX"); + } +} + +// 2-gram document +TEST_CASE("ngram search (document chars = 2)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + } +} diff --git a/test/ngram_search_3.cpp b/test/ngram_search_3.cpp new file mode 100644 index 0000000..dc9040d --- /dev/null +++ b/test/ngram_search_3.cpp @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, aaa/baa)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aaa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}, interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aaa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aaaX"); + IRIS_CHECK_SEARCH("Xaaa"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"baa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{1, 2}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "baa", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("baaX"); + IRIS_CHECK_SEARCH("Xbaa"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, aba/aab)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aba"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aba", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("baX"); + IRIS_CHECK_SEARCH("Xba"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abaX"); + IRIS_CHECK_SEARCH("Xaba"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXba"); + IRIS_CHECK_SEARCH("baXX"); + IRIS_CHECK_SEARCH("XXXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aab", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaX"); + IRIS_CHECK_SEARCH("Xaa"); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("aabX"); + IRIS_CHECK_SEARCH("Xaab"); + IRIS_CHECK_SEARCH("XXaa"); + IRIS_CHECK_SEARCH("aaXX"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2-gram + 1-gram document +TEST_CASE("ngram search (document chars = 3, abc)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abc"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "c", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("bcX"); + IRIS_CHECK_SEARCH("Xbc"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH("abcX"); + IRIS_CHECK_SEARCH("Xabc"); + IRIS_CHECK_SEARCH("XXab"); + IRIS_CHECK_SEARCH("abXX"); + IRIS_CHECK_SEARCH("XXbc"); + IRIS_CHECK_SEARCH("bcXX"); + IRIS_CHECK_SEARCH("XXXX"); + } +} diff --git a/test/ngram_search_4_dep.cpp b/test/ngram_search_4_dep.cpp new file mode 100644 index 0000000..8045632 --- /dev/null +++ b/test/ngram_search_4_dep.cpp @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: MIT + +#include "ngram_test.hpp" + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, aaaa)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"aaaa"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{1, 2}, interval{2, 3}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "aa", + {0_doc_id, { + {0, {interval{0, 2}, interval{1, 3}, interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aaa", + {0_doc_id, { + {0, {interval{0, 3}, interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "aaaa", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("aaaX"); + IRIS_CHECK_SEARCH("Xaaa"); + IRIS_CHECK_SEARCH("XXXX"); + + IRIS_CHECK_SEARCH("aaaaX"); + IRIS_CHECK_SEARCH("Xaaaa"); + IRIS_CHECK_SEARCH("XXaaa"); + IRIS_CHECK_SEARCH("aaaXX"); + IRIS_CHECK_SEARCH("XXXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, abab)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abab"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}, interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ba", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "aba", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bab", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xab"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abab", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("ababX"); + IRIS_CHECK_SEARCH("Xabab"); + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, abca)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abca"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}, interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "b", + {0_doc_id, { + {0, {interval{1, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "c", + {0_doc_id, { + {0, {interval{2, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "ca", + {0_doc_id, { + {0, {interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bca", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abX"); + IRIS_CHECK_SEARCH("Xbc"); + IRIS_CHECK_SEARCH("caX"); + IRIS_CHECK_SEARCH("Xca"); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abca", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abcaX"); + IRIS_CHECK_SEARCH("Xabca"); + IRIS_CHECK_SEARCH("caab"); // "ca" and "ab" both exist but not contiguous as "caab" + IRIS_CHECK_SEARCH("XXXX"); + } +} + +// 2x 2-gram document +TEST_CASE("ngram search (document chars = 4, abcd)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH(""); + + IRIS_CHECK_SEARCH( + "a", + {0_doc_id, { + {0, {interval{0, 1}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "d", + {0_doc_id, { + {0, {interval{3, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("X"); + + IRIS_CHECK_SEARCH( + "ab", + {0_doc_id, { + {0, {interval{0, 2}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bc", + {0_doc_id, { + {0, {interval{1, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "cd", + {0_doc_id, { + {0, {interval{2, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XX"); + + IRIS_CHECK_SEARCH( + "abc", + {0_doc_id, { + {0, {interval{0, 3}}}, + }}, + ); + IRIS_CHECK_SEARCH( + "bcd", + {0_doc_id, { + {0, {interval{1, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("XXX"); + + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + IRIS_CHECK_SEARCH("abcX"); + IRIS_CHECK_SEARCH("Xbcd"); + IRIS_CHECK_SEARCH("abcdX"); + IRIS_CHECK_SEARCH("Xabcd"); + IRIS_CHECK_SEARCH("abXcd"); // both halves exist; broken by X in the middle... but see note below! + IRIS_CHECK_SEARCH("acbd"); // all chars exist; order scrambled + IRIS_CHECK_SEARCH("XXXX"); + } +} + +TEST_CASE("ngram search (dependency on previous match)") +{ +#ifdef _MSC_VER + SetConsoleOutputCP(CP_UTF8); +#endif + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH("abXX"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abefef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab..ef"); + IRIS_CHECK_SEARCH("abXXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXef"); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abef"); + IRIS_CHECK_SEARCH("abXefef"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abxcd"); // trap document + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "abcd", + {1_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + } + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abcd"); + (void)ngram_db.add_document(U"abxcd"); // trap document + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{0, 4}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab"); + (void)ngram_db.add_document(U"abcd"); + IRIS_CHECK_SEARCH( + "ab cd", + {1_doc_id, { + {0, {interval{0, 2}}}, + {1, {interval{2, 4}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abXcd"); + (void)ngram_db.add_document(U"abcdX"); + IRIS_CHECK_SEARCH("abcdc"); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"abXXabcd"); + IRIS_CHECK_SEARCH( + "abcd", + {0_doc_id, { + {0, {interval{4, 8}}}, + }}, + ); + } + + { + iris::ngram_database<> ngram_db; + (void)ngram_db.add_document(U"ab cdXf"); + (void)ngram_db.add_document(U"ab cdef"); + IRIS_CHECK_SEARCH( + "ab cdef", + {1_doc_id, { + {0, {interval{0, 2}}}, + {1, {interval{3, 7}}}, + }}, + ); + } +} diff --git a/test/ngram_test.hpp b/test/ngram_test.hpp new file mode 100644 index 0000000..ff49a87 --- /dev/null +++ b/test/ngram_test.hpp @@ -0,0 +1,67 @@ +#ifndef IRIS_ZZ_TEST_NGRAM_TEST_HPP +#define IRIS_ZZ_TEST_NGRAM_TEST_HPP + +// SPDX-License-Identifier: MIT + +#include "iris_test.hpp" + +#include + +#include +#include +#include + +#ifdef _MSC_VER +# include +#endif + +using namespace iris::ngram_literals; +using iris::ngram_occurrence; +using iris::interval; + +struct DocumentMatch +{ + iris::ngram_document_id doc_id; + std::vector word_matches; + + DocumentMatch(iris::ngram_document_id doc_id, std::initializer_list word_matches) + : doc_id(doc_id) + , word_matches(word_matches) + {} + + DocumentMatch(iris::ngram_document_id doc_id, std::vector word_matches) + : doc_id(doc_id) + , word_matches(std::move(word_matches)) + {} + + [[nodiscard]] + bool operator==(DocumentMatch const&) const noexcept = default; +}; + +template +struct std::formatter + : iris::no_spec_formatter +{ + template + Ctx::iterator format(DocumentMatch const& doc_match, Ctx& ctx) const + { + return std::format_to(ctx.out(), "(doc: #{}, word_matches: {})", doc_match.doc_id, doc_match.word_matches); + } +}; + +#define IRIS_CHECK_SEARCH(query_input, ...) do { \ + iris::ngram_search_query const query{U ## query_input}; \ + auto const search_res = ngram_db.search(query); \ + auto const& doc_matches = search_res.doc_matches(); \ + \ + std::vector const expected_doc_matches{ \ + std::initializer_list{__VA_ARGS__} \ + }; \ + \ + auto const actual_doc_matches = doc_matches | std::views::transform([](auto const& kv) { \ + return DocumentMatch{kv.first, kv.second}; \ + }) | std::ranges::to(); \ + CHECK(actual_doc_matches == expected_doc_matches); \ + } while (false) + +#endif