From fb02b08a4975abd4ec8bc6946e014cc92ae658bf Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 11:40:57 +0000 Subject: [PATCH 01/10] Add tile-backed proof engine and hash sources Add HashSourceT (abstract subtree-root resolver), TileHashSourceT (resolves from full tiles, with an LRU tile cache), ProofEngineT (inclusion/consistency proofs and their verifiers, built on mth_range), MemoryHashSourceT (resolves from a resident in-memory tree), and CombinedHashSourceT (memory first, falling back to tiles). Add the sole essential core change: TreeT::subtree_root(), a read-only, non-hashing accessor that lets proofs be served from the resident tree, plus the include it needs. No other merklecpp.h changes. Add tiles_proofs tests, cross-checking tile-derived inclusion and consistency proofs against merkle::TreeT as the oracle across a range of sizes, including tile-boundary crossings. Move the memory-only subtree_root proof coverage and the ProofEngineProbe hostile-arithmetic edge cases here from tree coverage, since they exercise the proof engine and core accessor rather than TiledTree lifecycle. Add tiles_level2 for end-to-end coverage of the level-2 tile path. Introduce the LONG_TESTS CMake option, gate tiles_level2 behind it, and enable it in CI (and CodeQL) so long-running tile coverage runs on pull requests. Document the compatibility statement, the optional core accessor, and the HashSource/ProofEngineT API and algorithms in the design doc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 11 +- .github/workflows/codeql-analysis.yml | 2 +- CMakeLists.txt | 1 + doc/design/tlog-tiles.md | 124 +++++- merklecpp.h | 77 ++++ merklecpp_tiles.h | 521 ++++++++++++++++++++++++++ test/CMakeLists.txt | 15 +- test/tiles_level2.cpp | 138 +++++++ test/tiles_proofs.cpp | 383 +++++++++++++++++++ 9 files changed, 1257 insertions(+), 15 deletions(-) create mode 100644 test/tiles_level2.cpp create mode 100644 test/tiles_proofs.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b301f9a..492e06c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,10 +53,17 @@ jobs: shell: bash working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} run: | + long_tests=OFF + if [ "$RUNNER_OS" == "Linux" ] && + [ "${{ matrix.compiler }}" == "g++" ] && + [ "${{ matrix.build_type }}" == "Release" ] && + [ "${{ matrix.openssl }}" == "OFF" ]; then + long_tests=ON + fi if [ "$RUNNER_OS" == "Linux" ]; then - cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=$long_tests -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON else - cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} + cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=$long_tests -DOPENSSL=${{ matrix.openssl }} fi - name: Build diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d052011..1b61053 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: - name: Configure merklecpp working-directory: ${{github.workspace}}/build - run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON $GITHUB_WORKSPACE + run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON -DLONG_TESTS=ON $GITHUB_WORKSPACE - name: Build merklecpp working-directory: ${{github.workspace}}/build diff --git a/CMakeLists.txt b/CMakeLists.txt index b6f5ad2..85f9952 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ option(TESTS "enable testing" OFF) option(OPENSSL "enable OpenSSL" OFF) option(TRACE "enable debug traces" OFF) option(CLANG_TIDY "enable clang-tidy checks during build" OFF) +option(LONG_TESTS "enable long-running tests" OFF) if(CLANG_TIDY) find_program(CLANG_TIDY_PROGRAM clang-tidy) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 5219ef2..7525624 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -194,9 +194,9 @@ hashes, not bytes. | `ProofEngineT` | Roots, inclusion/consistency proofs, and verification | | `TiledTreeT` | `append`, `flush`, proof APIs, and compaction | -The planned `TileHashSource` owns the proof-read LRU cache; `TileStoreT` does -not cache. A combined source may require one read-only, non-hashing core -`subtree_root` accessor. +`TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. +`MemoryHashSourceT` uses the logically read-only, non-hashing +`TreeT::subtree_root` accessor. ### 5.1 Types and aliases @@ -297,20 +297,126 @@ stores the supplied leaf hash unchanged. `EntryBundleWriterT` mirrors `TileWriterT`: it writes only complete 256-entry bundles, confirms existing bundles before reusing them, and leaves the incomplete tail with the application. +### 5.5 `TreeT::subtree_root` + +Proofs over the resident frontier use one logically read-only core accessor: + +```cpp +bool subtree_root(uint8_t level, size_t index, Hash& out); +``` + +It returns the existing root of the complete subtree spanning +`[index << level, (index + 1) << level)`. The method rejects overflow, flushed +or out-of-range leaves, and non-perfect frontier nodes. It may realize a dirty +node hash exactly as `root()` and `path()` do, but does not change tree shape or +hashing semantics. + +### 5.6 Hash sources + +```cpp +struct HashSourceT { + // MTH(D[index< inclusion_proof(uint64_t index, uint64_t size) const; + + // RFC 6962 consistency proof that size `m` is a prefix of size `n` (m<=n). + std::vector consistency_proof(uint64_t m, uint64_t n) const; + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) const; + + // Verifier (consistency is new to merklecpp; inclusion reuses PathT::verify). + static bool verify_consistency(uint64_t m, uint64_t n, + const Hash& old_root, const Hash& new_root, + const std::vector& proof); +}; +``` + +Inclusion (top-down; element order/`direction` chosen to match `Tree::path`): + +``` +elements = [] # leaf→root order via push_front +lo = 0, hi = size, idx = index +while hi - lo > 1: + k = largest_pow2_lt(hi - lo) # split at lo+k + if idx - lo < k: # target in left ⇒ sibling on the RIGHT + sib = mth_range(lo+k, hi); dir = PATH_RIGHT; hi = lo + k + else: # target in right ⇒ sibling on the LEFT + sib = mth_range(lo, lo+k); dir = PATH_LEFT; lo = lo + k + elements.push_front({sib, dir}) +leaf = src.leaf(index) +return Path(leaf, index, elements, max_index = size - 1) +``` + +Consistency (RFC 6962 `SUBPROOF`): + +``` +consistency_proof(m, n): # 0 < m <= n + if m == n: return [] + subproof(m, lo=0, hi=n, complete=true) + +subproof(m, lo, hi, complete): + if m == hi - lo: + if not complete: proof.push_back(mth_range(lo, hi)) + return + k = largest_pow2_lt(hi - lo) + if m <= k: + subproof(m, lo, lo+k, complete) + proof.push_back(mth_range(lo+k, hi)) + else: + subproof(m-k, lo+k, hi, false) + proof.push_back(mth_range(lo, lo+k)) +``` + +Because every emitted hash is an `mth_range` computed with `HASH_FUNCTION`, the +consistency proof reconciles `Tree::past_root(m-1)` with +`Tree::past_root(n-1)` — i.e. it is consistent with the existing library. + ## 6. Delivery plan -Phases 0-2 now deliver the storage primitives plus incremental tile and -entry-bundle writers. Later PRs deliver the remaining independently testable -phases; phase 3 needs no further core changes, while phase 4 may add one -non-hashing accessor. +Phases 0-3 now deliver the storage primitives, incremental tile and entry-bundle +writers, hash sources, proof engine, and the only required core accessor. Later +PRs deliver the lifecycle wrapper, user documentation, and performance coverage; +no further core changes are planned. | Phase | Scope | Key tests | |---|---|---| | 0. Scaffolding | Headers, PAL, namespace, geometry, aliases, and CMake test wiring | Public-header and build integration | | 1. Coordinates/store | `TileRef`, index/path encoding, `TileStoreT`, durable atomic I/O, entry-bundle primitives | Encoding vectors; algorithm roots; 256-hash SHA-256/384 tiles; round trips; file/symlink collisions | | 2. Writers | Incremental `TileWriterT::write_up_to` from `leaf_at`, roll-ups, and `EntryBundleWriterT`; full resources only | Sizes 256 and 70,000 produce the exact tile set; repeated writes preserve immutability | -| 3. Proof engine | `TileHashSource`, `mth_range`, roots, inclusion/consistency proofs, and verification | Tile roots equal tree roots; inclusion equals `path()` / `past_path()` and verifies; consistency reconciles `past_root()` values | -| 4. Combined tree | Optional `subtree_root`; memory/combined sources; `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | +| 3. Proof engine | `TreeT::subtree_root`; tile, memory, and combined hash sources; roots; inclusion/consistency proofs and verification | Tile roots equal tree roots; inclusion equals `path()` / `past_path()` and verifies; consistency reconciles `past_root()` values | +| 4. Combined tree | `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | | 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | Deliverables are `merklecpp_tiles.h`, `merklecpp_pal.h`, `test/tiles_*.cpp`, diff --git a/merklecpp.h b/merklecpp.h index 5709da7..db061b8 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1336,6 +1337,82 @@ namespace merkle leaf_node(index)->hash, index, std::move(path), as_of); } + /// @brief Extracts the root hash of a complete subtree resident in memory + /// @param level The height of the subtree (it spans 2**level leaves) + /// @param index The index of the subtree at that height + /// @param out Set to the subtree root hash on success + /// @return Whether the subtree is a complete (balanced) subtree fully + /// resident in memory + /// @note This is read-only and does not change the hashing of the tree: it + /// returns an existing node hash (computing it on demand exactly as root() + /// and path() do). It returns false if any leaf of the subtree has been + /// flushed, if the subtree extends past the last leaf, or if the node at + /// that position is not a full subtree. The subtree spans leaf indices + /// [index << level, (index + 1) << level). + bool subtree_root(uint8_t level, size_t index, Hash& out) + { + const size_t leaves = num_leaves(); + if (leaves == 0 || level >= std::numeric_limits::digits) + { + return false; + } + if (index > (std::numeric_limits::max() >> level)) + { + return false; + } + + const size_t lo = index << level; + const size_t count = (size_t)1 << level; + + if (lo < min_index() || count > leaves || lo > leaves - count) + { + return false; + } + + if (level == 0) + { + out = leaf(lo); + return true; + } + + compute_root(); + + const uint8_t target_height = level + 1; + if (!_root || _root->height < target_height) + { + return false; + } + + Node* cur = _root; + size_t it = lo << (sizeof(lo) * 8 - _root->height + 1); + for (uint8_t height = _root->height; height > target_height;) + { + const bool go_right = ((it >> (8 * sizeof(it) - 1)) & 0x01) != 0U; + if (cur->height == height) + { + Node* next = go_right ? cur->right : cur->left; + if (!next) + { + return false; // conflated/flushed: not resident + } + cur = next; + } + it <<= 1; + height--; + } + + if (cur->height != target_height || !cur->is_full()) + { + return false; + } + if (cur->dirty) + { + hash(cur); + } + out = cur->hash; + return true; + } + /// @brief Serialises the tree /// @param bytes The vector of bytes to serialise to void serialise(std::vector& bytes) diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 5169d14..7ebabf7 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -935,6 +935,512 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } }; + /// @brief Abstract source of Merkle subtree roots for proof generation. + /// @note Implementations resolve the root of a complete (balanced) subtree + /// from tiles, from an in-memory tree, or from a combination of the two. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + struct HashSourceT + { + /// @brief The type of hashes resolved. + using Hash = HashT; + + virtual ~HashSourceT() = default; + + /// @brief Resolves MTH(D[index << level : (index + 1) << level]). + /// @param level The subtree height (the subtree spans 2**level leaves) + /// @param index The subtree index at that height + /// @param out Set to the subtree root on success + /// @return Whether the complete, balanced subtree could be resolved + virtual bool subtree_root( + uint8_t level, uint64_t index, Hash& out) const = 0; + + /// @brief Resolves the level-0 leaf hash at @p index. + virtual bool leaf(uint64_t index, Hash& out) const + { + return subtree_root(0, index, out); + } + }; + + /// @brief Resolves subtree roots from tlog-tiles tile files. + /// @note @p available_size is rounded down to a whole number of full tiles: + /// only complete, durably-written full tiles are read. A complete subtree + /// within that full-tile prefix is resolvable; anything reaching into the + /// incomplete frontier yields false so that a proof builder can fall back + /// to another source (e.g. an in-memory tree). + /// @warning No internal synchronization is provided. Even const operations + /// update the internal LRU cache, so callers must serialize all access to a + /// shared source. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Store = TileStoreT; + + /// @brief Constructs a source over @p store for trees up to + /// @p available_size leaves. @p available_size is rounded down to a whole + /// number of full tiles, since only full tiles are durable. + TileHashSourceT(const Store& store, uint64_t available_size) : + store(store), available_size((available_size / TILE_WIDTH) * TILE_WIDTH) + { + tile_cache.reserve(TILE_CACHE_SIZE); + } + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + // The subtree covers leaves [index << level, (index + 1) << level). It + // is resolvable only when it lies entirely within the full-tile-covered + // prefix; the incomplete frontier is served from another source. + if (level >= 64 || index >= (available_size >> level)) + { + return false; + } + resolve(level, index, out); + return true; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Store& store; + uint64_t available_size; // full-tile prefix length (a multiple of WIDTH) + + /// @brief Combines the @p span entries at @p off of @p tile into a root. + static Hash roll_up( + const std::vector& tile, uint64_t off, uint64_t span) + { + if (span == 1) + { + return tile.at(off); + } + return perfect_root(std::vector( + tile.begin() + (std::ptrdiff_t)off, + tile.begin() + (std::ptrdiff_t)(off + span))); + } + + /// @brief Resolves a complete subtree known to lie within the full-tile + /// prefix, reading the highest-level full tile that holds it (and rolling + /// up); descends to lower full tiles when a higher-level full tile has not + /// completed. Terminates because full level-0 tiles always cover the + /// prefix. + void resolve(uint8_t level, uint64_t index, Hash& out) const + { + if (level <= TILE_HEIGHT) + { + // Spans 2**level <= TILE_WIDTH leaves: held by one level-0 tile. + const uint64_t span = (uint64_t)1 << level; + const uint64_t start = index << level; + const std::vector tile = + read_tile(TileRef{0, start / TILE_WIDTH}); + out = roll_up(tile, start % TILE_WIDTH, span); + return; + } + + const uint8_t L = level / TILE_HEIGHT; + const uint8_t r = level % TILE_HEIGHT; + const uint64_t first = index << r; // first level-L entry + const uint64_t n = first / TILE_WIDTH; // level-L tile index + const unsigned full_shift = 8U * ((unsigned)L + 1U); + const uint64_t full_tiles = + full_shift >= 64 ? 0 : (available_size >> full_shift); + + if (n < full_tiles) + { + // One full level-L tile holds all 2**r entries of this subtree. + const std::vector tile = read_tile(TileRef{L, n}); + out = roll_up(tile, first % TILE_WIDTH, (uint64_t)1 << r); + return; + } + + // No full level-L tile here: split into two level-(level-1) subtrees. + Hash lo; + Hash hi; + resolve((uint8_t)(level - 1), index * 2, lo); + resolve((uint8_t)(level - 1), index * 2 + 1, hi); + HASH_FUNCTION(lo, hi, out); + } + + struct TileCacheEntry + { + TileRef ref; + std::vector hashes; + }; + + static constexpr size_t TILE_CACHE_SIZE = 64; + mutable std::vector tile_cache; + + std::vector read_tile(const TileRef& ref) const + { + for (auto it = tile_cache.begin(); it != tile_cache.end(); it++) + { + if (it->ref.level == ref.level && it->ref.index == ref.index) + { + TileCacheEntry entry = std::move(*it); + tile_cache.erase(it); + std::vector hashes = entry.hashes; + tile_cache.push_back(std::move(entry)); + return hashes; + } + } + + if (tile_cache.size() >= TILE_CACHE_SIZE) + { + tile_cache.erase(tile_cache.begin()); + } + tile_cache.push_back(TileCacheEntry{ref, store.read_tile(ref)}); + return tile_cache.back().hashes; + } + }; + + /// @brief Builds and verifies inclusion and consistency proofs. + /// @note Proofs are assembled from a HashSource using the tree's + /// HASH_FUNCTION, so an inclusion proof is byte-identical to the one + /// produced by merkle::TreeT::path()/past_path() and verifies with + /// PathT::verify(). + /// @warning Thread safety is inherited from the supplied HashSource. Callers + /// must serialize operations when the source is shared. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class ProofEngineT + { + public: + using Hash = HashT; + using Path = PathT; + using Source = HashSourceT; + + explicit ProofEngineT(const Source& source) : source(source) {} + + /// @brief The Merkle root of a tree of @p size leaves. + Hash root(uint64_t size) const + { + if (size == 0) + { + throw std::runtime_error("empty tree has no root"); + } + Hash out; + if (!mth_range(0, size, out)) + { + throw std::runtime_error("unresolved subtree while computing root"); + } + return out; + } + + /// @brief Inclusion proof for leaf @p index in a tree of @p size leaves. + /// @note Equivalent to TreeT::path(index) when size == num_leaves(), and + /// to TreeT::past_path(index, size - 1) otherwise. + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size) const + { + if (index >= size) + { + throw std::runtime_error("leaf index out of bounds"); + } + + std::list elements; // leaf -> root order + uint64_t lo = 0; + uint64_t hi = size; + while (hi - lo > 1) + { + const uint64_t k = largest_pow2_lt(hi - lo); + typename Path::Element e; + if (index - lo < k) + { + if (!mth_range(lo + k, hi, e.hash)) + { + throw std::runtime_error("unresolved subtree in inclusion proof"); + } + e.direction = Path::PATH_RIGHT; + hi = lo + k; + } + else + { + if (!mth_range(lo, lo + k, e.hash)) + { + throw std::runtime_error("unresolved subtree in inclusion proof"); + } + e.direction = Path::PATH_LEFT; + lo = lo + k; + } + elements.push_front(std::move(e)); + } + + Hash leaf; + if (!source.leaf(index, leaf)) + { + throw std::runtime_error("unresolved leaf in inclusion proof"); + } + return std::make_shared( + leaf, index, std::move(elements), size - 1); + } + + /// @brief Consistency proof that a tree of @p m leaves is a prefix of a + /// tree of @p n leaves (RFC 6962). + std::vector consistency_proof(uint64_t m, uint64_t n) const + { + if (m == 0 || m > n) + { + throw std::runtime_error("invalid consistency proof sizes"); + } + std::vector proof; + if (m == n) + { + return proof; + } + subproof(m, 0, n, true, proof); + return proof; + } + + /// @brief Consistency proof between the trees whose last leaves are at + /// indices @p first_index and @p second_index (first_index <= + /// second_index). + /// @note Equivalent to consistency_proof(first_index + 1, + /// second_index + 1): it proves the tree of the first first_index + 1 + /// leaves is a prefix of the tree of the first second_index + 1 leaves. + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) const + { + return consistency_proof(first_index + 1, second_index + 1); + } + + /// @brief Verifies an RFC 6962 consistency proof reconciling the roots of + /// trees of @p m and @p n leaves. + static bool verify_consistency( + uint64_t m, + uint64_t n, + const Hash& first_hash, + const Hash& second_hash, + std::vector proof) + { + if (m > n) + { + return false; + } + if (m == n) + { + return proof.empty() && first_hash == second_hash; + } + if (m == 0) + { + return proof.empty(); + } + + if (is_pow2(m)) + { + proof.insert(proof.begin(), first_hash); + } + if (proof.empty()) + { + return false; + } + + uint64_t fn = m - 1; + uint64_t sn = n - 1; + while ((fn & 1) != 0) + { + fn >>= 1; + sn >>= 1; + } + + Hash fr = proof[0]; + Hash sr = proof[0]; + for (size_t i = 1; i < proof.size(); i++) + { + if (sn == 0) + { + return false; + } + const Hash& c = proof[i]; + if ((fn & 1) != 0 || fn == sn) + { + HASH_FUNCTION(c, fr, fr); + HASH_FUNCTION(c, sr, sr); + if ((fn & 1) == 0) + { + while ((fn & 1) == 0 && fn != 0) + { + fn >>= 1; + sn >>= 1; + } + } + } + else + { + HASH_FUNCTION(sr, c, sr); + } + fn >>= 1; + sn >>= 1; + } + + return fr == first_hash && sr == second_hash && sn == 0; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& source; + + static bool is_pow2(uint64_t n) + { + return n != 0 && (n & (n - 1)) == 0; + } + + static uint64_t largest_pow2_lt(uint64_t n) + { + uint64_t k = 1; + while (k <= (n - 1) / 2) + { + k <<= 1; + } + return k; + } + + static uint8_t log2_exact(uint64_t n) + { + uint8_t r = 0; + while (n > 1) + { + n >>= 1; + r++; + } + return r; + } + + /// @brief MTH(D[a:b]) via the source; falls back to splitting when a + /// perfect subtree cannot be resolved directly. + bool mth_range(uint64_t a, uint64_t b, Hash& out) const + { + const uint64_t w = b - a; + if (w == 0) + { + return false; + } + if (w == 1) + { + return source.leaf(a, out); + } + if (is_pow2(w) && (a % w == 0)) + { + if (source.subtree_root(log2_exact(w), a / w, out)) + { + return true; + } + } + const uint64_t k = largest_pow2_lt(w); + Hash left; + Hash right; + if (!mth_range(a, a + k, left) || !mth_range(a + k, b, right)) + { + return false; + } + HASH_FUNCTION(left, right, out); + return true; + } + + void subproof( + uint64_t m, + uint64_t lo, + uint64_t hi, + bool complete, + std::vector& proof) const + { + if (m == hi - lo) + { + if (!complete) + { + Hash h; + if (!mth_range(lo, hi, h)) + { + throw std::runtime_error( + "unresolved subtree in consistency proof"); + } + proof.push_back(h); + } + return; + } + const uint64_t k = largest_pow2_lt(hi - lo); + Hash h; + if (m <= k) + { + subproof(m, lo, lo + k, complete, proof); + if (!mth_range(lo + k, hi, h)) + { + throw std::runtime_error("unresolved subtree in consistency proof"); + } + } + else + { + subproof(m - k, lo + k, hi, false, proof); + if (!mth_range(lo, lo + k, h)) + { + throw std::runtime_error("unresolved subtree in consistency proof"); + } + } + proof.push_back(h); + } + }; + + /// @brief Resolves subtree roots from an in-memory merkle::TreeT. + /// @note Resolves only complete subtrees that are fully resident (not + /// flushed), returning false otherwise so that a builder can fall back to + /// another source. Performs no hashing changes (see TreeT::subtree_root). + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class MemoryHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Tree = TreeT; + + explicit MemoryHashSourceT(Tree& tree) : tree(tree) {} + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + return tree.subtree_root(level, (size_t)index, out); + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Tree& tree; + }; + + /// @brief Resolves subtree roots from a primary source, falling back to a + /// secondary source. + /// @note Used to combine an in-memory tree (primary: no I/O, serves the + /// resident frontier) with tile files (secondary: serve the flushed past). + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class CombinedHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Source = HashSourceT; + + CombinedHashSourceT(const Source& primary, const Source& secondary) : + primary(primary), secondary(secondary) + {} + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + return primary.subtree_root(level, index, out) || + secondary.subtree_root(level, index, out); + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& primary; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& secondary; + }; + /// @brief Writes tlog-tiles entry bundles (raw log entries) for a growing /// log. /// @note Entry bundles are level-0 only and application-owned: merklecpp @@ -1039,6 +1545,21 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) using TileWriter = TileWriterT; + /// @brief Default abstract hash source (SHA256, default hash function). + using HashSource = HashSourceT<32, sha256_compress>; + + /// @brief Default tile-backed hash source (SHA256, default hash function). + using TileHashSource = TileHashSourceT<32, sha256_compress>; + + /// @brief Default proof engine (SHA256, default hash function). + using ProofEngine = ProofEngineT<32, sha256_compress>; + + /// @brief Default in-memory hash source (SHA256, default hash function). + using MemoryHashSource = MemoryHashSourceT<32, sha256_compress>; + + /// @brief Default combined hash source (SHA256, default hash function). + using CombinedHashSource = CombinedHashSourceT<32, sha256_compress>; + /// @brief Default entry-bundle writer (SHA256, default hash function). using EntryBundleWriter = EntryBundleWriterT< merkle::Tree::Hash::size_bytes, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9e61e03..724c37c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -37,12 +37,21 @@ add_merklecpp_test(serialisation serialisation.cpp) add_merklecpp_test(partial_serialisation partial_serialisation.cpp) add_merklecpp_test(serialise_to_file serialise_to_file.cpp) add_merklecpp_test(coverage coverage.cpp) +add_merklecpp_test(tiles_store tiles_store.cpp) +add_merklecpp_test(tiles_writer tiles_writer.cpp) +add_merklecpp_test(tiles_proofs tiles_proofs.cpp) +add_merklecpp_test(tiles_entries tiles_entries.cpp) + +if(LONG_TESTS) + add_merklecpp_test(tiles_level2 tiles_level2.cpp) + set_tests_properties( + ${MERKLECPP_TEST_PREFIX}tiles_level2 + PROPERTIES TIMEOUT 900 + ) +endif() if(OPENSSL) add_merklecpp_test(compare_hash_functions compare_hash_functions.cpp) endif() -add_merklecpp_test(tiles_store tiles_store.cpp) -add_merklecpp_test(tiles_writer tiles_writer.cpp) -add_merklecpp_test(tiles_entries tiles_entries.cpp) add_merklecpp_test(unit_tests unit_tests.cpp) diff --git a/test/tiles_level2.cpp b/test/tiles_level2.cpp new file mode 100644 index 0000000..e1ea541 --- /dev/null +++ b/test/tiles_level2.cpp @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// End-to-end coverage of the level-2 tile paths. A full level-2 tile requires +// 256^3 == 16,777,216 leaves, so this is a deliberately large test: it writes +// the tiles for that many leaves (~65k tile files) from a deterministic leaf +// source -- no in-memory tree is built -- and cross-checks the writer's +// level-by-level roll-up against TileHashSourceT::resolve (which reads the +// level-2 tile) and against the underlying leaves. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TILE_WIDTH; +using merkle::tiles::TileHashSource; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Roll up a perfect (power-of-two) set of hashes with the default combiner. +static Hash rollup(const std::vector& hashes) +{ + return merkle::tiles::perfect_root<32, merkle::sha256_compress>(hashes); +} + +int main() +{ + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_tiles_level2_" + + std::to_string((unsigned long long)std::time(nullptr))); + + try + { + // 256^3 leaves == exactly one full level-2 tile. + const uint64_t n = + (uint64_t)TILE_WIDTH * (uint64_t)TILE_WIDTH * (uint64_t)TILE_WIDTH; + + TileStore store(dir); + TileWriter writer(store); + + // Deterministic leaf hash derived from the index (low 8 bytes); avoids + // materialising a 16.7M-entry vector. + Hash leaf; + const auto leaf_at = [&](uint64_t i) -> const Hash& { + leaf.zero(); + for (int b = 0; b < 8; b++) + { + leaf.bytes[b] = (uint8_t)(i >> (8 * b)); + } + return leaf; + }; + + const auto stats = writer.write_up_to(n, leaf_at); + + // 65536 full L0 tiles + 256 full L1 tiles + 1 full L2 tile. + expect( + stats.full_written == (uint64_t)TILE_WIDTH * TILE_WIDTH + TILE_WIDTH + 1, + "level2: tile counts"); + expect(store.has_full_tile(2, 0), "level2: L2 tile present"); + expect(!store.has_full_tile(2, 1), "level2: no second L2 tile"); + expect(!store.has_full_tile(3, 0), "level2: no L3 tile"); + + const auto l2 = store.read_tile(TileRef{2, 0}); + expect(l2.size() == TILE_WIDTH, "level2: L2 tile width"); + + const TileHashSource src(store, n); + + // Each level-2 entry j is the root of level-1 tile j, which rolls up + // level-0 tiles, which are the leaves verbatim. Cross-check the writer's + // roll-up, resolve's level-2 read, and the leaf chain on a sample of + // indices. + for (const uint64_t j : + {(uint64_t)0, (uint64_t)1, (uint64_t)200, (uint64_t)255}) + { + const auto l1j = store.read_tile(TileRef{1, j}); + expect(l2[j] == rollup(l1j), "level2: L2[j] == rollup(L1 tile j)"); + + // resolve reads the level-2 tile for the 2^16-leaf subtree at index j. + Hash r16; + expect( + src.subtree_root(16, j, r16), "level2: subtree_root(16,j) resolves"); + expect(r16 == l2[j], "level2: resolve(16,j) == L2[j]"); + + // Anchor to leaves: L1 tile j entry 0 == root of L0 tile (j*256), whose + // first entry is leaf (j * 65536). + const auto l0 = store.read_tile(TileRef{0, j * TILE_WIDTH}); + expect(l1j[0] == rollup(l0), "level2: L1[j][0] == rollup(L0 tile)"); + expect( + l0[0] == leaf_at(j * (uint64_t)TILE_WIDTH * TILE_WIDTH), + "level2: L0 entry == leaf"); + } + + // Intra-tile roll-up: the 2^17-leaf subtree at 0 hashes L2[0] and L2[1]. + Hash r17; + expect(src.subtree_root(17, 0, r17), "level2: subtree_root(17,0) resolves"); + expect( + r17 == rollup({l2[0], l2[1]}), "level2: resolve(17,0) == H(L2[0],L2[1])"); + + std::cout << "tiles_level2: OK" << '\n'; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp new file mode 100644 index 0000000..1697dbd --- /dev/null +++ b/test/tiles_proofs.cpp @@ -0,0 +1,383 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::CombinedHashSource; +using merkle::tiles::MemoryHashSource; +using merkle::tiles::ProofEngine; +using merkle::tiles::TileHashSource; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +class ProofEngineProbe : public ProofEngine +{ +public: + using ProofEngine::largest_pow2_lt; +}; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Validates the TreeT::subtree_root accessor via memory-only proofs: they must +// match the library exactly. +static void check_memory_source(uint64_t n, const std::vector& hashes) +{ + const std::string at = " @n=" + std::to_string(n); + + merkle::Tree tree; + for (uint64_t i = 0; i < n; i++) + { + tree.insert(hashes[i]); + } + const Hash root = tree.root(); + + const MemoryHashSource source(tree); + const ProofEngine engine(source); + + expect(engine.root(n) == root, "mem root" + at); + + std::vector indices; + if (n <= 16) + { + for (uint64_t i = 0; i < n; i++) + { + indices.push_back(i); + } + } + else + { + for (const uint64_t i : {(uint64_t)0, (uint64_t)1, n / 2, n - 1}) + { + indices.push_back(i); + } + } + + for (const uint64_t i : indices) + { + const auto p = engine.inclusion_proof(i, n); + expect( + *p == *tree.path(i), "mem inclusion==path i=" + std::to_string(i) + at); + expect(p->verify(root), "mem inclusion verify i=" + std::to_string(i) + at); + } + + std::vector> pairs; + if (n <= 16) + { + for (uint64_t m = 1; m < n; m++) + { + for (uint64_t k = m + 1; k <= n; k++) + { + pairs.emplace_back(m, k); + } + } + } + else + { + pairs = {{1, n}, {n / 2, n}, {n - 1, n}}; + } + + for (const auto& pr : pairs) + { + const uint64_t m = pr.first; + const uint64_t k = pr.second; + const Hash rm = engine.root(m); + const Hash rk = engine.root(k); + expect(rm == *tree.past_root(m - 1), "mem past_root m" + at); + + const auto pp = engine.inclusion_proof(m / 2, m); + expect( + *pp == *tree.past_path(m / 2, m - 1), "mem inclusion(m)==past_path" + at); + + const auto cp = engine.consistency_proof(m, k); + expect( + ProofEngine::verify_consistency(m, k, rm, rk, cp), + "mem consistency" + at); + } +} + +// Exercises tile-derived proofs for a tree of `n` leaves against the existing +// library (which acts as the oracle: proofs must be byte-identical). Full tiles +// serve the covered prefix and an in-memory tree serves the un-tiled frontier, +// exactly as TiledTree combines them. +static void check_size( + const fs::path& dir, uint64_t n, const std::vector& hashes) +{ + const std::string at = " @n=" + std::to_string(n); + + TileStore store(dir); + TileWriter writer(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + writer.write_up_to(n, leaf_at); + + // Oracle: a full, never-flushed tree with the same leaves. + merkle::Tree tree; + for (uint64_t i = 0; i < n; i++) + { + tree.insert(hashes[i]); + } + const Hash root = tree.root(); + + // Production-shaped source: full tiles serve the covered prefix, an in-memory + // tree serves the un-tiled frontier. Drop the tiled past from the frontier + // tree so proofs over it are genuinely served from the tiles. merklecpp keeps + // at least one resident leaf, so never flush the whole tree. + const uint64_t covered = (n / 256) * 256; // 256 == TILE_WIDTH + merkle::Tree frontier; + for (uint64_t i = 0; i < n; i++) + { + frontier.insert(hashes[i]); + } + uint64_t drop_to = covered; + if (n > 0 && drop_to >= n) + { + drop_to = n - 1; + } + if (drop_to > 0) + { + frontier.flush_to((size_t)drop_to); + } + const MemoryHashSource mem(frontier); + const TileHashSource tiles(store, covered); + const CombinedHashSource source(mem, tiles); + const ProofEngine engine(source); + + // Root recomputed from tiles equals the library root. + expect(engine.root(n) == root, "root" + at); + + // Indices to probe: all of them for small trees, else a spread. + std::vector indices; + if (n <= 16) + { + for (uint64_t i = 0; i < n; i++) + { + indices.push_back(i); + } + } + else + { + for (const uint64_t i : + {(uint64_t)0, (uint64_t)1, n / 3, n / 2, n - 2, n - 1}) + { + indices.push_back(i); + } + } + + // Inclusion proofs are identical to TreeT::path and verify. + for (const uint64_t i : indices) + { + if (i >= n) + { + continue; + } + const auto p = engine.inclusion_proof(i, n); + expect(*p == *tree.path(i), "inclusion==path i=" + std::to_string(i) + at); + expect(p->verify(root), "inclusion verify i=" + std::to_string(i) + at); + } + + // Consistency pairs: exhaustive for small trees, else a fixed spread. + std::vector> pairs; + if (n <= 16) + { + for (uint64_t m = 1; m < n; m++) + { + for (uint64_t k = m + 1; k <= n; k++) + { + pairs.emplace_back(m, k); + } + } + } + else + { + pairs = {{1, n}, {n / 2, n}, {n - 1, n}, {1, 2}}; + // Tile-boundary crossings. + if (n > 256) + { + pairs.emplace_back(256, n); + pairs.emplace_back(257, n); + } + } + + for (const auto& pr : pairs) + { + const uint64_t m = pr.first; + const uint64_t k = pr.second; + if (m == 0 || m >= k || k > n) + { + continue; + } + + const Hash rm = engine.root(m); + const Hash rk = engine.root(k); + expect(rm == *tree.past_root(m - 1), "past_root m" + at); + expect(rk == *tree.past_root(k - 1), "past_root k" + at); + + // Past inclusion proof matches TreeT::past_path. + const uint64_t i = m / 2; + const auto pp = engine.inclusion_proof(i, m); + expect( + *pp == *tree.past_path(i, m - 1), + "inclusion(m)==past_path i=" + std::to_string(i) + at); + expect(pp->verify(rm), "inclusion(m) verify" + at); + + // Consistency proof reconciles the two roots. + const auto cp = engine.consistency_proof(m, k); + expect( + ProofEngine::verify_consistency(m, k, rm, rk, cp), + "consistency " + std::to_string(m) + "->" + std::to_string(k) + at); + + // The index-based variant is consistency_proof(i+1, j+1). + expect( + engine.consistency_proof_from_indices(m - 1, k - 1) == cp, + "consistency index variant" + at); + + // Tampering with a proof element or a root is rejected. + auto bad = cp; + bad[0].bytes[0] ^= 0xFFU; + expect( + !ProofEngine::verify_consistency(m, k, rm, rk, bad), + "consistency tamper rejected" + at); + + Hash wrong = rk; + wrong.bytes[0] ^= 0xFFU; + expect( + !ProofEngine::verify_consistency(m, k, rm, wrong, cp), + "consistency wrong root rejected" + at); + } + + std::error_code ec; + fs::remove_all(dir, ec); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_proofs_" + std::to_string((unsigned long long)seed) + + "_" + std::to_string(std::rand())); + + try + { + const auto hashes = make_hashes(300000); + + // ---- Memory-only proofs (exercises TreeT::subtree_root). + for (const uint64_t n : + {(uint64_t)1, + (uint64_t)2, + (uint64_t)3, + (uint64_t)5, + (uint64_t)8, + (uint64_t)13, + (uint64_t)16, + (uint64_t)256, + (uint64_t)257, + (uint64_t)1000}) + { + check_memory_source(n, hashes); + } + std::cout << "memory source: OK" << '\n'; + + // ---- Hostile arithmetic inputs are rejected without UB or overflow + // loops. + { + merkle::Tree tree; + tree.insert(hashes[0]); + Hash out; + expect(!tree.subtree_root(64, 0, out), "subtree_root rejects level 64"); + expect(!tree.subtree_root(100, 0, out), "subtree_root rejects level 100"); + expect( + !tree.subtree_root(1, std::numeric_limits::max(), out), + "subtree_root rejects overflowing index"); + + expect(ProofEngineProbe::largest_pow2_lt(2) == 1, "pow2_lt 2"); + expect( + ProofEngineProbe::largest_pow2_lt((uint64_t)1 << 63) == + ((uint64_t)1 << 62), + "pow2_lt 2^63"); + expect( + ProofEngineProbe::largest_pow2_lt(((uint64_t)1 << 63) + 1) == + ((uint64_t)1 << 63), + "pow2_lt 2^63+1"); + expect( + ProofEngineProbe::largest_pow2_lt( + std::numeric_limits::max()) == ((uint64_t)1 << 63), + "pow2_lt uint64 max"); + std::cout << "hostile arithmetic inputs: OK" << '\n'; + } + + for (const uint64_t n : + {(uint64_t)1, + (uint64_t)2, + (uint64_t)3, + (uint64_t)4, + (uint64_t)5, + (uint64_t)7, + (uint64_t)8, + (uint64_t)13, + (uint64_t)16, + (uint64_t)255, + (uint64_t)256, + (uint64_t)257, + (uint64_t)1000}) + { + check_size(base / ("n" + std::to_string(n)), n, hashes); + } + std::cout << "small/medium sizes: OK" << '\n'; + + // Large trees. 65536 == 256 full L0 tiles == one full L1 tile (exact L1 + // boundary); 65537 is one past it; 70000 exercises a full L1 tile plus an + // in-memory frontier; 300000 forces proofs over height->=16 subtrees, so + // TileHashSource::resolve descends through level-2 logic (full_shift = 24) + // before reaching the level-1 tiles -- the only coverage of the L>=2 path. + for (const uint64_t n : + {(uint64_t)65536, (uint64_t)65537, (uint64_t)70000, (uint64_t)300000}) + { + check_size(base / ("big" + std::to_string(n)), n, hashes); + std::cout << "size " << n << ": OK" << '\n'; + } + + std::cout << "tiles_proofs: OK" << '\n'; + + std::error_code ec; + fs::remove_all(base, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} From 77c129a9387e33090f57ed533fb2fd847df794bf Mon Sep 17 00:00:00 2001 From: achamayou Date: Sat, 25 Jul 2026 22:12:58 +0100 Subject: [PATCH 02/10] Fix proof aliases and index bounds Use the current default tree hash function for public proof aliases, reject index conversions that cannot be represented safely, and align the design and level-2 coverage notes with the rebased implementation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c504572c-322f-4950-8682-edf4a7fd2c5b --- doc/design/tlog-tiles.md | 22 +++++++++---------- merklecpp_tiles.h | 46 ++++++++++++++++++++++++++++++++-------- test/tiles_level2.cpp | 4 +++- test/tiles_proofs.cpp | 18 ++++++++++++++-- 4 files changed, 67 insertions(+), 23 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 7525624..d0f76e4 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -174,10 +174,10 @@ hashes, not bytes. │ owns Tree ▲ inclusion / consistency proofs │ tiles ▼ │ - ProofEngine ──▶ HashSource ◀──────────┘ - ├─ MemoryHashSource - ├─ TileHashSource - └─ CombinedHashSource + ProofEngineT ──▶ HashSourceT ◀─────────┘ + ├─ MemoryHashSourceT + ├─ TileHashSourceT + └─ CombinedHashSourceT ``` `merklecpp_tiles.h` contains the public `merkle::tiles` API and includes @@ -192,7 +192,7 @@ hashes, not bytes. | `TileWriterT` | Persist newly completed full tiles | | Hash sources | Resolve subtree roots from memory, tiles, or both | | `ProofEngineT` | Roots, inclusion/consistency proofs, and verification | -| `TiledTreeT` | `append`, `flush`, proof APIs, and compaction | +| `TiledTreeT` (planned) | `append`, `flush`, proof APIs, and compaction | `TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. `MemoryHashSourceT` uses the logically read-only, non-hashing @@ -215,9 +215,8 @@ class TileStoreT; using TileStore = TileStoreT; -using TiledTree = - TiledTreeT; -// Equivalent SHA-384 and SHA-512 aliases. +// Hash-source and proof-engine aliases follow the same pattern. +// The TiledTree alias arrives with the phase-4 lifecycle wrapper. } ``` @@ -336,13 +335,14 @@ struct HashSourceT { ### 5.7 `ProofEngineT` -All three proof building blocks reduce to `mth_range` over a `HashSource`. +All three proof building blocks reduce to `mth_range` over a `HashSourceT`. Returned `PathT` objects are byte-identical to `Tree::path` / `Tree::past_path`. ```cpp class ProofEngineT { public: - explicit ProofEngineT(const HashSource& src); + using Source = HashSourceT; + explicit ProofEngineT(const Source& source); Hash root(uint64_t size) const; // = mth_range(0, size) @@ -420,4 +420,4 @@ no further core changes are planned. | 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | Deliverables are `merklecpp_tiles.h`, `merklecpp_pal.h`, `test/tiles_*.cpp`, -CMake wiring, the optional core accessor, and README/design updates. +CMake wiring, the core accessor, and README/design updates. diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 7ebabf7..19e8c5c 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -1098,11 +1099,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) }; /// @brief Builds and verifies inclusion and consistency proofs. - /// @note Proofs are assembled from a HashSource using the tree's + /// @note Proofs are assembled from a HashSourceT using the tree's /// HASH_FUNCTION, so an inclusion proof is byte-identical to the one /// produced by merkle::TreeT::path()/past_path() and verifies with /// PathT::verify(). - /// @warning Thread safety is inherited from the supplied HashSource. Callers + /// @warning Thread safety is inherited from the supplied HashSourceT. Callers /// must serialize operations when the source is shared. template < size_t HASH_SIZE, @@ -1141,6 +1142,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { throw std::runtime_error("leaf index out of bounds"); } + if ( + index > std::numeric_limits::max() || + size - 1 > std::numeric_limits::max()) + { + throw std::runtime_error("inclusion proof exceeds PathT index range"); + } std::list elements; // leaf -> root order uint64_t lo = 0; @@ -1176,7 +1183,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) throw std::runtime_error("unresolved leaf in inclusion proof"); } return std::make_shared( - leaf, index, std::move(elements), size - 1); + leaf, + static_cast(index), + std::move(elements), + static_cast(size - 1)); } /// @brief Consistency proof that a tree of @p m leaves is a prefix of a @@ -1205,6 +1215,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::vector consistency_proof_from_indices( uint64_t first_index, uint64_t second_index) const { + if ( + first_index == std::numeric_limits::max() || + second_index == std::numeric_limits::max()) + { + throw std::runtime_error("consistency proof index out of bounds"); + } return consistency_proof(first_index + 1, second_index + 1); } @@ -1402,7 +1418,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override { - return tree.subtree_root(level, (size_t)index, out); + if (index > std::numeric_limits::max()) + { + return false; + } + return tree.subtree_root(level, static_cast(index), out); } protected: @@ -1546,19 +1566,27 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) TileWriterT; /// @brief Default abstract hash source (SHA256, default hash function). - using HashSource = HashSourceT<32, sha256_compress>; + using HashSource = + HashSourceT; /// @brief Default tile-backed hash source (SHA256, default hash function). - using TileHashSource = TileHashSourceT<32, sha256_compress>; + using TileHashSource = TileHashSourceT< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>; /// @brief Default proof engine (SHA256, default hash function). - using ProofEngine = ProofEngineT<32, sha256_compress>; + using ProofEngine = + ProofEngineT; /// @brief Default in-memory hash source (SHA256, default hash function). - using MemoryHashSource = MemoryHashSourceT<32, sha256_compress>; + using MemoryHashSource = MemoryHashSourceT< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>; /// @brief Default combined hash source (SHA256, default hash function). - using CombinedHashSource = CombinedHashSourceT<32, sha256_compress>; + using CombinedHashSource = CombinedHashSourceT< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>; /// @brief Default entry-bundle writer (SHA256, default hash function). using EntryBundleWriter = EntryBundleWriterT< diff --git a/test/tiles_level2.cpp b/test/tiles_level2.cpp index e1ea541..d1a97c0 100644 --- a/test/tiles_level2.cpp +++ b/test/tiles_level2.cpp @@ -38,7 +38,9 @@ static void expect(bool cond, const std::string& what) // Roll up a perfect (power-of-two) set of hashes with the default combiner. static Hash rollup(const std::vector& hashes) { - return merkle::tiles::perfect_root<32, merkle::sha256_compress>(hashes); + return merkle::tiles::perfect_root< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>(hashes); } int main() diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp index 1697dbd..cbebb92 100644 --- a/test/tiles_proofs.cpp +++ b/test/tiles_proofs.cpp @@ -305,6 +305,8 @@ int main() { merkle::Tree tree; tree.insert(hashes[0]); + const MemoryHashSource source(tree); + const ProofEngine engine(source); Hash out; expect(!tree.subtree_root(64, 0, out), "subtree_root rejects level 64"); expect(!tree.subtree_root(100, 0, out), "subtree_root rejects level 100"); @@ -325,6 +327,17 @@ int main() ProofEngineProbe::largest_pow2_lt( std::numeric_limits::max()) == ((uint64_t)1 << 63), "pow2_lt uint64 max"); + bool rejected = false; + try + { + engine.consistency_proof_from_indices( + 0, std::numeric_limits::max()); + } + catch (const std::runtime_error&) + { + rejected = true; + } + expect(rejected, "consistency index rejects overflow"); std::cout << "hostile arithmetic inputs: OK" << '\n'; } @@ -349,9 +362,10 @@ int main() // Large trees. 65536 == 256 full L0 tiles == one full L1 tile (exact L1 // boundary); 65537 is one past it; 70000 exercises a full L1 tile plus an - // in-memory frontier; 300000 forces proofs over height->=16 subtrees, so + // in-memory frontier; 300000 forces proofs over height >= 16 subtrees, so // TileHashSource::resolve descends through level-2 logic (full_shift = 24) - // before reaching the level-1 tiles -- the only coverage of the L>=2 path. + // before falling back to level-1 tiles. tiles_level2 separately covers a + // completed level-2 tile. for (const uint64_t n : {(uint64_t)65536, (uint64_t)65537, (uint64_t)70000, (uint64_t)300000}) { From 96a6a31e00ed88bf653d675d73576507047f98ca Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 3 Aug 2026 13:15:57 +0000 Subject: [PATCH 03/10] Parallelize CI builds and release tests --- .github/processor_count.cmake | 9 +++++++++ .github/workflows/ci.yml | 13 +++++++------ test/CMakeLists.txt | 6 +++++- 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 .github/processor_count.cmake diff --git a/.github/processor_count.cmake b/.github/processor_count.cmake new file mode 100644 index 0000000..58cfd2d --- /dev/null +++ b/.github/processor_count.cmake @@ -0,0 +1,9 @@ +include(ProcessorCount) + +ProcessorCount(PARALLELISM) +if(PARALLELISM EQUAL 0) + set(PARALLELISM 1) +endif() + +message(STATUS "Using ${PARALLELISM} parallel jobs") +file(APPEND "$ENV{GITHUB_ENV}" "PARALLELISM=${PARALLELISM}\n") \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 492e06c..89d1f08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,10 @@ jobs: - name: Create Build Environment run: cmake -E make_directory ${{github.workspace}}/build/${{ matrix.build_type }} + - name: Detect Runner Parallelism + shell: bash + run: cmake -P "$GITHUB_WORKSPACE/.github/processor_count.cmake" + - name: Configure CMake # Use a bash shell so we can use the same syntax for environment variable # access regardless of the host operating system @@ -54,10 +58,7 @@ jobs: working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} run: | long_tests=OFF - if [ "$RUNNER_OS" == "Linux" ] && - [ "${{ matrix.compiler }}" == "g++" ] && - [ "${{ matrix.build_type }}" == "Release" ] && - [ "${{ matrix.openssl }}" == "OFF" ]; then + if [ "${{ matrix.build_type }}" == "Release" ]; then long_tests=ON fi if [ "$RUNNER_OS" == "Linux" ]; then @@ -69,7 +70,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash - run: cmake --build . --config ${{ matrix.build_type }} + run: cmake --build . --config ${{ matrix.build_type }} --parallel "$PARALLELISM" - name: Clang-Tidy Header if: matrix.os == 'ubuntu-latest' @@ -82,6 +83,6 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash - run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 + run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 --parallel "$PARALLELISM" env: ASAN_OPTIONS: use_sigaltstack=false # To avoid SetAlternateSignalStack with clang-11 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 724c37c..e555eee 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -44,9 +44,13 @@ add_merklecpp_test(tiles_entries tiles_entries.cpp) if(LONG_TESTS) add_merklecpp_test(tiles_level2 tiles_level2.cpp) + set(TILES_LEVEL2_TIMEOUT 900) + if(WIN32) + set(TILES_LEVEL2_TIMEOUT 3600) + endif() set_tests_properties( ${MERKLECPP_TEST_PREFIX}tiles_level2 - PROPERTIES TIMEOUT 900 + PROPERTIES TIMEOUT ${TILES_LEVEL2_TIMEOUT} ) endif() From 9d669bff0a3d920e6cc7c15aa1d86379af26c688 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 3 Aug 2026 13:32:50 +0000 Subject: [PATCH 04/10] Address proof generation review feedback --- merklecpp_tiles.h | 32 +++++++++++++++++--------------- test/tiles_proofs.cpp | 3 ++- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 19e8c5c..7c9828a 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -1036,7 +1036,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) // Spans 2**level <= TILE_WIDTH leaves: held by one level-0 tile. const uint64_t span = (uint64_t)1 << level; const uint64_t start = index << level; - const std::vector tile = + const std::vector& tile = read_tile(TileRef{0, start / TILE_WIDTH}); out = roll_up(tile, start % TILE_WIDTH, span); return; @@ -1053,7 +1053,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (n < full_tiles) { // One full level-L tile holds all 2**r entries of this subtree. - const std::vector tile = read_tile(TileRef{L, n}); + const std::vector& tile = read_tile(TileRef{L, n}); out = roll_up(tile, first % TILE_WIDTH, (uint64_t)1 << r); return; } @@ -1075,7 +1075,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) static constexpr size_t TILE_CACHE_SIZE = 64; mutable std::vector tile_cache; - std::vector read_tile(const TileRef& ref) const + const std::vector& read_tile(const TileRef& ref) const { for (auto it = tile_cache.begin(); it != tile_cache.end(); it++) { @@ -1083,9 +1083,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { TileCacheEntry entry = std::move(*it); tile_cache.erase(it); - std::vector hashes = entry.hashes; tile_cache.push_back(std::move(entry)); - return hashes; + return tile_cache.back().hashes; } } @@ -1231,7 +1230,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) uint64_t n, const Hash& first_hash, const Hash& second_hash, - std::vector proof) + const std::vector& proof) { if (m > n) { @@ -1246,13 +1245,18 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) return proof.empty(); } - if (is_pow2(m)) + size_t proof_index = 0; + Hash fr = first_hash; + Hash sr = first_hash; + if (!is_pow2(m)) { - proof.insert(proof.begin(), first_hash); - } - if (proof.empty()) - { - return false; + if (proof.empty()) + { + return false; + } + fr = proof[0]; + sr = proof[0]; + proof_index = 1; } uint64_t fn = m - 1; @@ -1263,9 +1267,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) sn >>= 1; } - Hash fr = proof[0]; - Hash sr = proof[0]; - for (size_t i = 1; i < proof.size(); i++) + for (size_t i = proof_index; i < proof.size(); i++) { if (sn == 0) { diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp index cbebb92..a1a339c 100644 --- a/test/tiles_proofs.cpp +++ b/test/tiles_proofs.cpp @@ -21,6 +21,7 @@ using merkle::Hash; using merkle::tiles::CombinedHashSource; using merkle::tiles::MemoryHashSource; using merkle::tiles::ProofEngine; +using merkle::tiles::TILE_WIDTH; using merkle::tiles::TileHashSource; using merkle::tiles::TileStore; using merkle::tiles::TileWriter; @@ -142,7 +143,7 @@ static void check_size( // tree serves the un-tiled frontier. Drop the tiled past from the frontier // tree so proofs over it are genuinely served from the tiles. merklecpp keeps // at least one resident leaf, so never flush the whole tree. - const uint64_t covered = (n / 256) * 256; // 256 == TILE_WIDTH + const uint64_t covered = (n / TILE_WIDTH) * TILE_WIDTH; merkle::Tree frontier; for (uint64_t i = 0; i < n; i++) { From 791ddb0d8f19556e151c8b84a40636685e1ebf74 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 3 Aug 2026 14:16:49 +0000 Subject: [PATCH 05/10] Use standard CMake parallelism variables --- .github/processor_count.cmake | 11 +++++++++-- .github/workflows/ci.yml | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/processor_count.cmake b/.github/processor_count.cmake index 58cfd2d..3993f24 100644 --- a/.github/processor_count.cmake +++ b/.github/processor_count.cmake @@ -5,5 +5,12 @@ if(PARALLELISM EQUAL 0) set(PARALLELISM 1) endif() -message(STATUS "Using ${PARALLELISM} parallel jobs") -file(APPEND "$ENV{GITHUB_ENV}" "PARALLELISM=${PARALLELISM}\n") \ No newline at end of file +message( + STATUS + "Setting CMAKE_BUILD_PARALLEL_LEVEL and CTEST_PARALLEL_LEVEL to ${PARALLELISM}" +) +file( + APPEND "$ENV{GITHUB_ENV}" + "CMAKE_BUILD_PARALLEL_LEVEL=${PARALLELISM}\n" + "CTEST_PARALLEL_LEVEL=${PARALLELISM}\n" +) \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89d1f08..270792e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash - run: cmake --build . --config ${{ matrix.build_type }} --parallel "$PARALLELISM" + run: cmake --build . --config ${{ matrix.build_type }} - name: Clang-Tidy Header if: matrix.os == 'ubuntu-latest' @@ -83,6 +83,6 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash - run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 --parallel "$PARALLELISM" + run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 env: ASAN_OPTIONS: use_sigaltstack=false # To avoid SetAlternateSignalStack with clang-11 From ebfa8d8f2d1b6f8360a346210de8f179e590a469 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 3 Aug 2026 14:30:20 +0000 Subject: [PATCH 06/10] Adopt standard CMake test configuration --- .github/workflows/ci.yml | 18 +++++++++++++----- .github/workflows/codeql-analysis.yml | 2 +- CMakeLists.txt | 6 ++---- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 270792e..0027c96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,12 +61,22 @@ jobs: if [ "${{ matrix.build_type }}" == "Release" ]; then long_tests=ON fi + + linux_options=() if [ "$RUNNER_OS" == "Linux" ]; then - cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=$long_tests -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - else - cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=$long_tests -DOPENSSL=${{ matrix.openssl }} + linux_options=( + "-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}" + -DCLANG_TIDY=ON + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + ) fi + cmake "$GITHUB_WORKSPACE" \ + "-DCMAKE_BUILD_TYPE=${{ matrix.build_type }}" \ + "-DLONG_TESTS=$long_tests" \ + "-DOPENSSL=${{ matrix.openssl }}" \ + "${linux_options[@]}" + - name: Build working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash @@ -84,5 +94,3 @@ jobs: working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 - env: - ASAN_OPTIONS: use_sigaltstack=false # To avoid SetAlternateSignalStack with clang-11 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1b61053..88c9f6e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: - name: Configure merklecpp working-directory: ${{github.workspace}}/build - run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON -DLONG_TESTS=ON $GITHUB_WORKSPACE + run: cmake -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON $GITHUB_WORKSPACE - name: Build merklecpp working-directory: ${{github.workspace}}/build diff --git a/CMakeLists.txt b/CMakeLists.txt index 85f9952..6252f35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ cmake_minimum_required(VERSION 3.14) project(merklecpp LANGUAGES CXX) include(GNUInstallDirs) +include(CTest) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -13,7 +14,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(MERKLECPP_DIR ${CMAKE_CURRENT_SOURCE_DIR}) option(PROFILE "enable profiling" OFF) -option(TESTS "enable testing" OFF) option(OPENSSL "enable OpenSSL" OFF) option(TRACE "enable debug traces" OFF) option(CLANG_TIDY "enable clang-tidy checks during build" OFF) @@ -46,9 +46,7 @@ if(OPENSSL) target_link_libraries(merklecpp INTERFACE OpenSSL::Crypto) endif() -if(TESTS) - enable_testing() - +if(BUILD_TESTING) function(add_unit_test NAME SRC) add_executable(${NAME} ${SRC}) target_link_libraries(${NAME} PRIVATE $) From 0d1666885865ddf474b6e0ed41bb820eb2113280 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 4 Aug 2026 08:54:30 +0000 Subject: [PATCH 07/10] Address large-tree and proof review feedback --- merklecpp.h | 9 +++++++- merklecpp_tiles.h | 23 +++++++++++++++----- test/tiles_level2.cpp | 8 +++---- test/tiles_proofs.cpp | 49 +++++++++++++++++++++++++++++-------------- 4 files changed, 63 insertions(+), 26 deletions(-) diff --git a/merklecpp.h b/merklecpp.h index db061b8..4e7a8b8 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -704,7 +704,14 @@ namespace merkle /// 2**height-1. [[nodiscard]] bool is_full() const { - size_t max_size = (1 << height) - 1; + constexpr size_t size_digits = std::numeric_limits::digits; + if (height > size_digits) + { + return false; + } + const size_t max_size = height == size_digits ? + std::numeric_limits::max() : + (size_t{1} << height) - 1; assert(size <= max_size); return size == max_size; } diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 7c9828a..cbca67d 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -1015,13 +1015,26 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) static Hash roll_up( const std::vector& tile, uint64_t off, uint64_t span) { - if (span == 1) + if ( + span == 0 || span > TILE_WIDTH || (span & (span - 1)) != 0 || + off > tile.size() || span > tile.size() - off) + { + throw std::runtime_error("invalid tile roll-up range"); + } + + std::array level; + for (size_t i = 0; i < span; i++) { - return tile.at(off); + level[i] = tile[off + i]; + } + for (size_t width = span; width > 1; width /= 2) + { + for (size_t i = 0; i < width; i += 2) + { + HASH_FUNCTION(level[i], level[i + 1], level[i / 2]); + } } - return perfect_root(std::vector( - tile.begin() + (std::ptrdiff_t)off, - tile.begin() + (std::ptrdiff_t)(off + span))); + return level[0]; } /// @brief Resolves a complete subtree known to lie within the full-tile diff --git a/test/tiles_level2.cpp b/test/tiles_level2.cpp index d1a97c0..2936ec6 100644 --- a/test/tiles_level2.cpp +++ b/test/tiles_level2.cpp @@ -8,14 +8,15 @@ // level-by-level roll-up against TileHashSourceT::resolve (which reads the // level-2 tile) and against the underlying leaves. +#include "tiles_test_util.h" #include "util.h" #include -#include #include #include #include #include +#include #include #include @@ -45,9 +46,8 @@ static Hash rollup(const std::vector& hashes) int main() { - const fs::path dir = fs::temp_directory_path() / - ("merklecpp_tiles_level2_" + - std::to_string((unsigned long long)std::time(nullptr))); + const TemporaryDirectory temporary_directory("merklecpp_tiles_level2"); + const fs::path& dir = temporary_directory.path(); try { diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp index a1a339c..2153093 100644 --- a/test/tiles_proofs.cpp +++ b/test/tiles_proofs.cpp @@ -1,17 +1,17 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#include "tiles_test_util.h" #include "util.h" #include -#include -#include #include #include #include #include #include #include +#include #include #include #include @@ -32,6 +32,18 @@ class ProofEngineProbe : public ProofEngine using ProofEngine::largest_pow2_lt; }; +class TreeProbe : public merkle::Tree +{ +public: + static bool node_is_full(uint8_t height, size_t size) + { + Node node{}; + node.height = height; + node.size = size; + return node.is_full(); + } +}; + static void expect(bool cond, const std::string& what) { if (!cond) @@ -272,13 +284,8 @@ static void check_size( int main() { - const auto seed = std::time(nullptr); - std::srand((unsigned)seed); - std::cout << "seed=" << seed << '\n'; - - const fs::path base = fs::temp_directory_path() / - ("merklecpp_tiles_proofs_" + std::to_string((unsigned long long)seed) + - "_" + std::to_string(std::rand())); + const TemporaryDirectory temporary_directory("merklecpp_tiles_proofs"); + const fs::path& base = temporary_directory.path(); try { @@ -315,6 +322,23 @@ int main() !tree.subtree_root(1, std::numeric_limits::max(), out), "subtree_root rejects overflowing index"); + const auto signed_shift_boundary = + static_cast(std::numeric_limits::digits); + if ( + signed_shift_boundary < std::numeric_limits::digits) + { + const size_t full_size = + (size_t{1} << signed_shift_boundary) - 1; + expect( + TreeProbe::node_is_full(signed_shift_boundary, full_size), + "is_full handles signed-shift boundary"); + } + expect( + TreeProbe::node_is_full( + static_cast(std::numeric_limits::digits), + std::numeric_limits::max()), + "is_full handles maximum size_t height"); + expect(ProofEngineProbe::largest_pow2_lt(2) == 1, "pow2_lt 2"); expect( ProofEngineProbe::largest_pow2_lt((uint64_t)1 << 63) == @@ -375,22 +399,15 @@ int main() } std::cout << "tiles_proofs: OK" << '\n'; - - std::error_code ec; - fs::remove_all(base, ec); } catch (std::exception& ex) { std::cout << "Error: " << ex.what() << '\n'; - std::error_code ec; - fs::remove_all(base, ec); return 1; } catch (...) { std::cout << "Error" << '\n'; - std::error_code ec; - fs::remove_all(base, ec); return 1; } From 18355a2a6728fea777f9574f42f7290b2186adb0 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 4 Aug 2026 10:48:36 +0000 Subject: [PATCH 08/10] Tighten proof roll-up and documentation --- doc/design/tlog-tiles.md | 6 +++--- merklecpp_tiles.h | 4 ++++ test/tiles_proofs.cpp | 35 ++++++++++++++++++++--------------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index d0f76e4..7d717f8 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -195,7 +195,7 @@ hashes, not bytes. | `TiledTreeT` (planned) | `append`, `flush`, proof APIs, and compaction | `TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. -`MemoryHashSourceT` uses the logically read-only, non-hashing +`MemoryHashSourceT` uses the logically read-only `TreeT::subtree_root` accessor. ### 5.1 Types and aliases @@ -419,5 +419,5 @@ no further core changes are planned. | 4. Combined tree | `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | | 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | -Deliverables are `merklecpp_tiles.h`, `merklecpp_pal.h`, `test/tiles_*.cpp`, -CMake wiring, the core accessor, and README/design updates. +Delivered through phase 3 are `merklecpp_tiles.h`, `merklecpp_pal.h`, +`test/tiles_*.cpp`, CMake wiring, the core accessor, and design updates. diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index cbca67d..e6aabe3 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -1021,6 +1021,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { throw std::runtime_error("invalid tile roll-up range"); } + if (span == 1) + { + return tile[off]; + } std::array level; for (size_t i = 0; i < span; i++) diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp index 2153093..ab985aa 100644 --- a/test/tiles_proofs.cpp +++ b/test/tiles_proofs.cpp @@ -224,10 +224,10 @@ static void check_size( { pairs = {{1, n}, {n / 2, n}, {n - 1, n}, {1, 2}}; // Tile-boundary crossings. - if (n > 256) + if (n > TILE_WIDTH) { - pairs.emplace_back(256, n); - pairs.emplace_back(257, n); + pairs.emplace_back(TILE_WIDTH, n); + pairs.emplace_back((uint64_t)TILE_WIDTH + 1, n); } } @@ -286,6 +286,8 @@ int main() { const TemporaryDirectory temporary_directory("merklecpp_tiles_proofs"); const fs::path& base = temporary_directory.path(); + const uint64_t tile_width = TILE_WIDTH; + const uint64_t level1_width = tile_width * tile_width; try { @@ -300,8 +302,8 @@ int main() (uint64_t)8, (uint64_t)13, (uint64_t)16, - (uint64_t)256, - (uint64_t)257, + tile_width, + tile_width + 1, (uint64_t)1000}) { check_memory_source(n, hashes); @@ -376,23 +378,26 @@ int main() (uint64_t)8, (uint64_t)13, (uint64_t)16, - (uint64_t)255, - (uint64_t)256, - (uint64_t)257, + tile_width - 1, + tile_width, + tile_width + 1, (uint64_t)1000}) { check_size(base / ("n" + std::to_string(n)), n, hashes); } std::cout << "small/medium sizes: OK" << '\n'; - // Large trees. 65536 == 256 full L0 tiles == one full L1 tile (exact L1 - // boundary); 65537 is one past it; 70000 exercises a full L1 tile plus an - // in-memory frontier; 300000 forces proofs over height >= 16 subtrees, so - // TileHashSource::resolve descends through level-2 logic (full_shift = 24) - // before falling back to level-1 tiles. tiles_level2 separately covers a - // completed level-2 tile. + // Large trees. TILE_WIDTH * TILE_WIDTH is one full L1 tile (the exact L1 + // boundary); the next size is one past it. 70000 exercises a full L1 tile + // plus an in-memory frontier; 300000 forces proofs over height >= 16 + // subtrees, so TileHashSource::resolve descends through level-2 logic + // (full_shift = 24) before falling back to level-1 tiles. tiles_level2 + // separately covers a completed level-2 tile. for (const uint64_t n : - {(uint64_t)65536, (uint64_t)65537, (uint64_t)70000, (uint64_t)300000}) + {level1_width, + level1_width + 1, + (uint64_t)70000, + (uint64_t)300000}) { check_size(base / ("big" + std::to_string(n)), n, hashes); std::cout << "size " << n << ": OK" << '\n'; From 6a9e58aff74e4beae2a09b5a88e0932cc41ca216 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:40:26 +0000 Subject: [PATCH 09/10] Clarify subtree root result and effects Co-authored-by: eddyashton <6000239+eddyashton@users.noreply.github.com> --- doc/design/tlog-tiles.md | 12 ++++++------ merklecpp.h | 36 ++++++++++++++++-------------------- merklecpp_tiles.h | 11 +++++++++-- test/tiles_proofs.cpp | 10 ++++++---- 4 files changed, 37 insertions(+), 32 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 7d717f8..23bd0f1 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -195,8 +195,7 @@ hashes, not bytes. | `TiledTreeT` (planned) | `append`, `flush`, proof APIs, and compaction | `TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. -`MemoryHashSourceT` uses the logically read-only -`TreeT::subtree_root` accessor. +`MemoryHashSourceT` uses the `TreeT::subtree_root` accessor. ### 5.1 Types and aliases @@ -298,16 +297,17 @@ bundles before reusing them, and leaves the incomplete tail with the application ### 5.5 `TreeT::subtree_root` -Proofs over the resident frontier use one logically read-only core accessor: +Proofs over the resident frontier use one core accessor: ```cpp -bool subtree_root(uint8_t level, size_t index, Hash& out); +std::optional subtree_root(uint8_t level, size_t index); ``` It returns the existing root of the complete subtree spanning `[index << level, (index + 1) << level)`. The method rejects overflow, flushed -or out-of-range leaves, and non-perfect frontier nodes. It may realize a dirty -node hash exactly as `root()` and `path()` do, but does not change tree shape or +or out-of-range leaves, and non-perfect frontier nodes by returning +`std::nullopt`. Like `root()` and `path()`, it may materialize pending +insertions and cache computed hashes without changing the leaf sequence or hashing semantics. ### 5.6 Hash sources diff --git a/merklecpp.h b/merklecpp.h index 4e7a8b8..ce00544 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -1347,25 +1348,22 @@ namespace merkle /// @brief Extracts the root hash of a complete subtree resident in memory /// @param level The height of the subtree (it spans 2**level leaves) /// @param index The index of the subtree at that height - /// @param out Set to the subtree root hash on success - /// @return Whether the subtree is a complete (balanced) subtree fully - /// resident in memory - /// @note This is read-only and does not change the hashing of the tree: it - /// returns an existing node hash (computing it on demand exactly as root() - /// and path() do). It returns false if any leaf of the subtree has been - /// flushed, if the subtree extends past the last leaf, or if the node at - /// that position is not a full subtree. The subtree spans leaf indices - /// [index << level, (index + 1) << level). - bool subtree_root(uint8_t level, size_t index, Hash& out) + /// @return The subtree root hash if the subtree is complete (balanced) and + /// fully resident in memory; otherwise, std::nullopt + /// @note This may materialize pending insertions and cache computed hashes, + /// exactly as root() and path() do. It does not change the leaf sequence or + /// hashing semantics. The subtree spans leaf indices [index << level, + /// (index + 1) << level). + std::optional subtree_root(uint8_t level, size_t index) { const size_t leaves = num_leaves(); if (leaves == 0 || level >= std::numeric_limits::digits) { - return false; + return std::nullopt; } if (index > (std::numeric_limits::max() >> level)) { - return false; + return std::nullopt; } const size_t lo = index << level; @@ -1373,13 +1371,12 @@ namespace merkle if (lo < min_index() || count > leaves || lo > leaves - count) { - return false; + return std::nullopt; } if (level == 0) { - out = leaf(lo); - return true; + return leaf(lo); } compute_root(); @@ -1387,7 +1384,7 @@ namespace merkle const uint8_t target_height = level + 1; if (!_root || _root->height < target_height) { - return false; + return std::nullopt; } Node* cur = _root; @@ -1400,7 +1397,7 @@ namespace merkle Node* next = go_right ? cur->right : cur->left; if (!next) { - return false; // conflated/flushed: not resident + return std::nullopt; // conflated/flushed: not resident } cur = next; } @@ -1410,14 +1407,13 @@ namespace merkle if (cur->height != target_height || !cur->is_full()) { - return false; + return std::nullopt; } if (cur->dirty) { hash(cur); } - out = cur->hash; - return true; + return cur->hash; } /// @brief Serialises the tree diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index e6aabe3..26b83b4 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -1422,7 +1422,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Resolves subtree roots from an in-memory merkle::TreeT. /// @note Resolves only complete subtrees that are fully resident (not /// flushed), returning false otherwise so that a builder can fall back to - /// another source. Performs no hashing changes (see TreeT::subtree_root). + /// another source. Resolution may materialize the tree's pending insertions + /// and cache hashes (see TreeT::subtree_root). template < size_t HASH_SIZE, void HASH_FUNCTION( @@ -1441,7 +1442,13 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { return false; } - return tree.subtree_root(level, static_cast(index), out); + const auto root = tree.subtree_root(level, static_cast(index)); + if (!root) + { + return false; + } + out = *root; + return true; } protected: diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp index ab985aa..1f7d453 100644 --- a/test/tiles_proofs.cpp +++ b/test/tiles_proofs.cpp @@ -317,11 +317,13 @@ int main() tree.insert(hashes[0]); const MemoryHashSource source(tree); const ProofEngine engine(source); - Hash out; - expect(!tree.subtree_root(64, 0, out), "subtree_root rejects level 64"); - expect(!tree.subtree_root(100, 0, out), "subtree_root rejects level 100"); expect( - !tree.subtree_root(1, std::numeric_limits::max(), out), + tree.subtree_root(0, 0) == hashes[0], + "subtree_root returns a resident leaf hash"); + expect(!tree.subtree_root(64, 0), "subtree_root rejects level 64"); + expect(!tree.subtree_root(100, 0), "subtree_root rejects level 100"); + expect( + !tree.subtree_root(1, std::numeric_limits::max()), "subtree_root rejects overflowing index"); const auto signed_shift_boundary = From 741fb70fa89c1e08ade5ac7ace3d51687ea45853 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 4 Aug 2026 14:16:35 +0000 Subject: [PATCH 10/10] Consolidate proof subtree reduction --- doc/design/tlog-tiles.md | 6 ++-- merklecpp.h | 11 ++++--- merklecpp_tiles.h | 69 ++++++++++++++++++++-------------------- 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 23bd0f1..583ccf2 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -306,9 +306,9 @@ std::optional subtree_root(uint8_t level, size_t index); It returns the existing root of the complete subtree spanning `[index << level, (index + 1) << level)`. The method rejects overflow, flushed or out-of-range leaves, and non-perfect frontier nodes by returning -`std::nullopt`. Like `root()` and `path()`, it may materialize pending -insertions and cache computed hashes without changing the leaf sequence or -hashing semantics. +`std::nullopt`. Like `root()` and `path()`, it may materialize pending nodes and +compute dirty hashes, but does not change logical leaf contents or hashing +semantics. ### 5.6 Hash sources diff --git a/merklecpp.h b/merklecpp.h index ce00544..0306f2e 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -1350,10 +1350,13 @@ namespace merkle /// @param index The index of the subtree at that height /// @return The subtree root hash if the subtree is complete (balanced) and /// fully resident in memory; otherwise, std::nullopt - /// @note This may materialize pending insertions and cache computed hashes, - /// exactly as root() and path() do. It does not change the leaf sequence or - /// hashing semantics. The subtree spans leaf indices [index << level, - /// (index + 1) << level). + /// @note Like root() and path(), it may + /// materialize pending nodes and compute dirty hashes, but does not change + /// logical leaf contents or hashing semantics. It returns std::nullopt if + /// any leaf of the subtree has been flushed, if the subtree extends past + /// the last leaf, or if the node at that position is not a full subtree. + /// The subtree spans leaf indices + /// [index << level, (index + 1) << level). std::optional subtree_root(uint8_t level, size_t index) { const size_t leaves = num_leaves(); diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 26b83b4..6957da3 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -732,6 +732,32 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @endcond }; + namespace detail + { + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + HashT perfect_root_range( + const std::vector>& hashes, + size_t offset, + size_t count) + { + if (count == 1) + { + return hashes[offset]; + } + const size_t half = count / 2; + const auto left = + perfect_root_range(hashes, offset, half); + const auto right = perfect_root_range( + hashes, offset + half, half); + HashT out; + HASH_FUNCTION(left, right, out); + return out; + } + } + /// @brief Computes the Merkle Tree Hash of a perfect (balanced) subtree. /// @param leaves The subtree's leaves; the count MUST be a power of two. /// @return The subtree root, computed with the tree's HASH_FUNCTION. @@ -755,20 +781,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) "perfect_root requires a power-of-two number of leaves"); } - std::vector> level = leaves; - while (level.size() > 1) - { - std::vector> next; - next.reserve(level.size() / 2); - for (size_t i = 0; i + 1 < level.size(); i += 2) - { - HashT h; - HASH_FUNCTION(level[i], level[i + 1], h); - next.push_back(h); - } - level.swap(next); - } - return level.front(); + return detail::perfect_root_range( + leaves, 0, leaves.size()); } /// @brief Computes and persists tlog-tiles tiles for a growing tree. @@ -1021,24 +1035,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { throw std::runtime_error("invalid tile roll-up range"); } - if (span == 1) - { - return tile[off]; - } - - std::array level; - for (size_t i = 0; i < span; i++) - { - level[i] = tile[off + i]; - } - for (size_t width = span; width > 1; width /= 2) - { - for (size_t i = 0; i < width; i += 2) - { - HASH_FUNCTION(level[i], level[i + 1], level[i / 2]); - } - } - return level[0]; + return detail::perfect_root_range( + tile, static_cast(off), static_cast(span)); } /// @brief Resolves a complete subtree known to lie within the full-tile @@ -1063,7 +1061,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const uint8_t r = level % TILE_HEIGHT; const uint64_t first = index << r; // first level-L entry const uint64_t n = first / TILE_WIDTH; // level-L tile index - const unsigned full_shift = 8U * ((unsigned)L + 1U); + const unsigned full_shift = static_cast(TILE_HEIGHT) * + (static_cast(L) + 1U); const uint64_t full_tiles = full_shift >= 64 ? 0 : (available_size >> full_shift); @@ -1422,8 +1421,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Resolves subtree roots from an in-memory merkle::TreeT. /// @note Resolves only complete subtrees that are fully resident (not /// flushed), returning false otherwise so that a builder can fall back to - /// another source. Resolution may materialize the tree's pending insertions - /// and cache hashes (see TreeT::subtree_root). + /// another source. It may materialize pending nodes and compute dirty + /// hashes but does not change logical contents or hashing semantics. template < size_t HASH_SIZE, void HASH_FUNCTION(