From fb02b08a4975abd4ec8bc6946e014cc92ae658bf Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 11:40:57 +0000 Subject: [PATCH 01/12] Add tile-backed proof engine and hash sources Add HashSourceT (abstract subtree-root resolver), TileHashSourceT (resolves from full tiles, with an LRU tile cache), ProofEngineT (inclusion/consistency proofs and their verifiers, built on mth_range), MemoryHashSourceT (resolves from a resident in-memory tree), and CombinedHashSourceT (memory first, falling back to tiles). Add the sole essential core change: TreeT::subtree_root(), a read-only, non-hashing accessor that lets proofs be served from the resident tree, plus the include it needs. No other merklecpp.h changes. Add tiles_proofs tests, cross-checking tile-derived inclusion and consistency proofs against merkle::TreeT as the oracle across a range of sizes, including tile-boundary crossings. Move the memory-only subtree_root proof coverage and the ProofEngineProbe hostile-arithmetic edge cases here from tree coverage, since they exercise the proof engine and core accessor rather than TiledTree lifecycle. Add tiles_level2 for end-to-end coverage of the level-2 tile path. Introduce the LONG_TESTS CMake option, gate tiles_level2 behind it, and enable it in CI (and CodeQL) so long-running tile coverage runs on pull requests. Document the compatibility statement, the optional core accessor, and the HashSource/ProofEngineT API and algorithms in the design doc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 11 +- .github/workflows/codeql-analysis.yml | 2 +- CMakeLists.txt | 1 + doc/design/tlog-tiles.md | 124 +++++- merklecpp.h | 77 ++++ merklecpp_tiles.h | 521 ++++++++++++++++++++++++++ test/CMakeLists.txt | 15 +- test/tiles_level2.cpp | 138 +++++++ test/tiles_proofs.cpp | 383 +++++++++++++++++++ 9 files changed, 1257 insertions(+), 15 deletions(-) create mode 100644 test/tiles_level2.cpp create mode 100644 test/tiles_proofs.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b301f9a..492e06c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,10 +53,17 @@ jobs: shell: bash working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} run: | + long_tests=OFF + if [ "$RUNNER_OS" == "Linux" ] && + [ "${{ matrix.compiler }}" == "g++" ] && + [ "${{ matrix.build_type }}" == "Release" ] && + [ "${{ matrix.openssl }}" == "OFF" ]; then + long_tests=ON + fi if [ "$RUNNER_OS" == "Linux" ]; then - cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=$long_tests -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON else - cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} + cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=$long_tests -DOPENSSL=${{ matrix.openssl }} fi - name: Build diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d052011..1b61053 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: - name: Configure merklecpp working-directory: ${{github.workspace}}/build - run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON $GITHUB_WORKSPACE + run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON -DLONG_TESTS=ON $GITHUB_WORKSPACE - name: Build merklecpp working-directory: ${{github.workspace}}/build diff --git a/CMakeLists.txt b/CMakeLists.txt index b6f5ad2..85f9952 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ option(TESTS "enable testing" OFF) option(OPENSSL "enable OpenSSL" OFF) option(TRACE "enable debug traces" OFF) option(CLANG_TIDY "enable clang-tidy checks during build" OFF) +option(LONG_TESTS "enable long-running tests" OFF) if(CLANG_TIDY) find_program(CLANG_TIDY_PROGRAM clang-tidy) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 5219ef2..7525624 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -194,9 +194,9 @@ hashes, not bytes. | `ProofEngineT` | Roots, inclusion/consistency proofs, and verification | | `TiledTreeT` | `append`, `flush`, proof APIs, and compaction | -The planned `TileHashSource` owns the proof-read LRU cache; `TileStoreT` does -not cache. A combined source may require one read-only, non-hashing core -`subtree_root` accessor. +`TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. +`MemoryHashSourceT` uses the logically read-only, non-hashing +`TreeT::subtree_root` accessor. ### 5.1 Types and aliases @@ -297,20 +297,126 @@ stores the supplied leaf hash unchanged. `EntryBundleWriterT` mirrors `TileWriterT`: it writes only complete 256-entry bundles, confirms existing bundles before reusing them, and leaves the incomplete tail with the application. +### 5.5 `TreeT::subtree_root` + +Proofs over the resident frontier use one logically read-only core accessor: + +```cpp +bool subtree_root(uint8_t level, size_t index, Hash& out); +``` + +It returns the existing root of the complete subtree spanning +`[index << level, (index + 1) << level)`. The method rejects overflow, flushed +or out-of-range leaves, and non-perfect frontier nodes. It may realize a dirty +node hash exactly as `root()` and `path()` do, but does not change tree shape or +hashing semantics. + +### 5.6 Hash sources + +```cpp +struct HashSourceT { + // MTH(D[index< inclusion_proof(uint64_t index, uint64_t size) const; + + // RFC 6962 consistency proof that size `m` is a prefix of size `n` (m<=n). + std::vector consistency_proof(uint64_t m, uint64_t n) const; + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) const; + + // Verifier (consistency is new to merklecpp; inclusion reuses PathT::verify). + static bool verify_consistency(uint64_t m, uint64_t n, + const Hash& old_root, const Hash& new_root, + const std::vector& proof); +}; +``` + +Inclusion (top-down; element order/`direction` chosen to match `Tree::path`): + +``` +elements = [] # leaf→root order via push_front +lo = 0, hi = size, idx = index +while hi - lo > 1: + k = largest_pow2_lt(hi - lo) # split at lo+k + if idx - lo < k: # target in left ⇒ sibling on the RIGHT + sib = mth_range(lo+k, hi); dir = PATH_RIGHT; hi = lo + k + else: # target in right ⇒ sibling on the LEFT + sib = mth_range(lo, lo+k); dir = PATH_LEFT; lo = lo + k + elements.push_front({sib, dir}) +leaf = src.leaf(index) +return Path(leaf, index, elements, max_index = size - 1) +``` + +Consistency (RFC 6962 `SUBPROOF`): + +``` +consistency_proof(m, n): # 0 < m <= n + if m == n: return [] + subproof(m, lo=0, hi=n, complete=true) + +subproof(m, lo, hi, complete): + if m == hi - lo: + if not complete: proof.push_back(mth_range(lo, hi)) + return + k = largest_pow2_lt(hi - lo) + if m <= k: + subproof(m, lo, lo+k, complete) + proof.push_back(mth_range(lo+k, hi)) + else: + subproof(m-k, lo+k, hi, false) + proof.push_back(mth_range(lo, lo+k)) +``` + +Because every emitted hash is an `mth_range` computed with `HASH_FUNCTION`, the +consistency proof reconciles `Tree::past_root(m-1)` with +`Tree::past_root(n-1)` — i.e. it is consistent with the existing library. + ## 6. Delivery plan -Phases 0-2 now deliver the storage primitives plus incremental tile and -entry-bundle writers. Later PRs deliver the remaining independently testable -phases; phase 3 needs no further core changes, while phase 4 may add one -non-hashing accessor. +Phases 0-3 now deliver the storage primitives, incremental tile and entry-bundle +writers, hash sources, proof engine, and the only required core accessor. Later +PRs deliver the lifecycle wrapper, user documentation, and performance coverage; +no further core changes are planned. | Phase | Scope | Key tests | |---|---|---| | 0. Scaffolding | Headers, PAL, namespace, geometry, aliases, and CMake test wiring | Public-header and build integration | | 1. Coordinates/store | `TileRef`, index/path encoding, `TileStoreT`, durable atomic I/O, entry-bundle primitives | Encoding vectors; algorithm roots; 256-hash SHA-256/384 tiles; round trips; file/symlink collisions | | 2. Writers | Incremental `TileWriterT::write_up_to` from `leaf_at`, roll-ups, and `EntryBundleWriterT`; full resources only | Sizes 256 and 70,000 produce the exact tile set; repeated writes preserve immutability | -| 3. Proof engine | `TileHashSource`, `mth_range`, roots, inclusion/consistency proofs, and verification | Tile roots equal tree roots; inclusion equals `path()` / `past_path()` and verifies; consistency reconciles `past_root()` values | -| 4. Combined tree | Optional `subtree_root`; memory/combined sources; `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | +| 3. Proof engine | `TreeT::subtree_root`; tile, memory, and combined hash sources; roots; inclusion/consistency proofs and verification | Tile roots equal tree roots; inclusion equals `path()` / `past_path()` and verifies; consistency reconciles `past_root()` values | +| 4. Combined tree | `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | | 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | Deliverables are `merklecpp_tiles.h`, `merklecpp_pal.h`, `test/tiles_*.cpp`, diff --git a/merklecpp.h b/merklecpp.h index 5709da7..db061b8 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -1336,6 +1337,82 @@ namespace merkle leaf_node(index)->hash, index, std::move(path), as_of); } + /// @brief Extracts the root hash of a complete subtree resident in memory + /// @param level The height of the subtree (it spans 2**level leaves) + /// @param index The index of the subtree at that height + /// @param out Set to the subtree root hash on success + /// @return Whether the subtree is a complete (balanced) subtree fully + /// resident in memory + /// @note This is read-only and does not change the hashing of the tree: it + /// returns an existing node hash (computing it on demand exactly as root() + /// and path() do). It returns false if any leaf of the subtree has been + /// flushed, if the subtree extends past the last leaf, or if the node at + /// that position is not a full subtree. The subtree spans leaf indices + /// [index << level, (index + 1) << level). + bool subtree_root(uint8_t level, size_t index, Hash& out) + { + const size_t leaves = num_leaves(); + if (leaves == 0 || level >= std::numeric_limits::digits) + { + return false; + } + if (index > (std::numeric_limits::max() >> level)) + { + return false; + } + + const size_t lo = index << level; + const size_t count = (size_t)1 << level; + + if (lo < min_index() || count > leaves || lo > leaves - count) + { + return false; + } + + if (level == 0) + { + out = leaf(lo); + return true; + } + + compute_root(); + + const uint8_t target_height = level + 1; + if (!_root || _root->height < target_height) + { + return false; + } + + Node* cur = _root; + size_t it = lo << (sizeof(lo) * 8 - _root->height + 1); + for (uint8_t height = _root->height; height > target_height;) + { + const bool go_right = ((it >> (8 * sizeof(it) - 1)) & 0x01) != 0U; + if (cur->height == height) + { + Node* next = go_right ? cur->right : cur->left; + if (!next) + { + return false; // conflated/flushed: not resident + } + cur = next; + } + it <<= 1; + height--; + } + + if (cur->height != target_height || !cur->is_full()) + { + return false; + } + if (cur->dirty) + { + hash(cur); + } + out = cur->hash; + return true; + } + /// @brief Serialises the tree /// @param bytes The vector of bytes to serialise to void serialise(std::vector& bytes) diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 5169d14..7ebabf7 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -935,6 +935,512 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } }; + /// @brief Abstract source of Merkle subtree roots for proof generation. + /// @note Implementations resolve the root of a complete (balanced) subtree + /// from tiles, from an in-memory tree, or from a combination of the two. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + struct HashSourceT + { + /// @brief The type of hashes resolved. + using Hash = HashT; + + virtual ~HashSourceT() = default; + + /// @brief Resolves MTH(D[index << level : (index + 1) << level]). + /// @param level The subtree height (the subtree spans 2**level leaves) + /// @param index The subtree index at that height + /// @param out Set to the subtree root on success + /// @return Whether the complete, balanced subtree could be resolved + virtual bool subtree_root( + uint8_t level, uint64_t index, Hash& out) const = 0; + + /// @brief Resolves the level-0 leaf hash at @p index. + virtual bool leaf(uint64_t index, Hash& out) const + { + return subtree_root(0, index, out); + } + }; + + /// @brief Resolves subtree roots from tlog-tiles tile files. + /// @note @p available_size is rounded down to a whole number of full tiles: + /// only complete, durably-written full tiles are read. A complete subtree + /// within that full-tile prefix is resolvable; anything reaching into the + /// incomplete frontier yields false so that a proof builder can fall back + /// to another source (e.g. an in-memory tree). + /// @warning No internal synchronization is provided. Even const operations + /// update the internal LRU cache, so callers must serialize all access to a + /// shared source. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Store = TileStoreT; + + /// @brief Constructs a source over @p store for trees up to + /// @p available_size leaves. @p available_size is rounded down to a whole + /// number of full tiles, since only full tiles are durable. + TileHashSourceT(const Store& store, uint64_t available_size) : + store(store), available_size((available_size / TILE_WIDTH) * TILE_WIDTH) + { + tile_cache.reserve(TILE_CACHE_SIZE); + } + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + // The subtree covers leaves [index << level, (index + 1) << level). It + // is resolvable only when it lies entirely within the full-tile-covered + // prefix; the incomplete frontier is served from another source. + if (level >= 64 || index >= (available_size >> level)) + { + return false; + } + resolve(level, index, out); + return true; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Store& store; + uint64_t available_size; // full-tile prefix length (a multiple of WIDTH) + + /// @brief Combines the @p span entries at @p off of @p tile into a root. + static Hash roll_up( + const std::vector& tile, uint64_t off, uint64_t span) + { + if (span == 1) + { + return tile.at(off); + } + return perfect_root(std::vector( + tile.begin() + (std::ptrdiff_t)off, + tile.begin() + (std::ptrdiff_t)(off + span))); + } + + /// @brief Resolves a complete subtree known to lie within the full-tile + /// prefix, reading the highest-level full tile that holds it (and rolling + /// up); descends to lower full tiles when a higher-level full tile has not + /// completed. Terminates because full level-0 tiles always cover the + /// prefix. + void resolve(uint8_t level, uint64_t index, Hash& out) const + { + if (level <= TILE_HEIGHT) + { + // Spans 2**level <= TILE_WIDTH leaves: held by one level-0 tile. + const uint64_t span = (uint64_t)1 << level; + const uint64_t start = index << level; + const std::vector tile = + read_tile(TileRef{0, start / TILE_WIDTH}); + out = roll_up(tile, start % TILE_WIDTH, span); + return; + } + + const uint8_t L = level / TILE_HEIGHT; + const uint8_t r = level % TILE_HEIGHT; + const uint64_t first = index << r; // first level-L entry + const uint64_t n = first / TILE_WIDTH; // level-L tile index + const unsigned full_shift = 8U * ((unsigned)L + 1U); + const uint64_t full_tiles = + full_shift >= 64 ? 0 : (available_size >> full_shift); + + if (n < full_tiles) + { + // One full level-L tile holds all 2**r entries of this subtree. + const std::vector tile = read_tile(TileRef{L, n}); + out = roll_up(tile, first % TILE_WIDTH, (uint64_t)1 << r); + return; + } + + // No full level-L tile here: split into two level-(level-1) subtrees. + Hash lo; + Hash hi; + resolve((uint8_t)(level - 1), index * 2, lo); + resolve((uint8_t)(level - 1), index * 2 + 1, hi); + HASH_FUNCTION(lo, hi, out); + } + + struct TileCacheEntry + { + TileRef ref; + std::vector hashes; + }; + + static constexpr size_t TILE_CACHE_SIZE = 64; + mutable std::vector tile_cache; + + std::vector read_tile(const TileRef& ref) const + { + for (auto it = tile_cache.begin(); it != tile_cache.end(); it++) + { + if (it->ref.level == ref.level && it->ref.index == ref.index) + { + TileCacheEntry entry = std::move(*it); + tile_cache.erase(it); + std::vector hashes = entry.hashes; + tile_cache.push_back(std::move(entry)); + return hashes; + } + } + + if (tile_cache.size() >= TILE_CACHE_SIZE) + { + tile_cache.erase(tile_cache.begin()); + } + tile_cache.push_back(TileCacheEntry{ref, store.read_tile(ref)}); + return tile_cache.back().hashes; + } + }; + + /// @brief Builds and verifies inclusion and consistency proofs. + /// @note Proofs are assembled from a HashSource using the tree's + /// HASH_FUNCTION, so an inclusion proof is byte-identical to the one + /// produced by merkle::TreeT::path()/past_path() and verifies with + /// PathT::verify(). + /// @warning Thread safety is inherited from the supplied HashSource. Callers + /// must serialize operations when the source is shared. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class ProofEngineT + { + public: + using Hash = HashT; + using Path = PathT; + using Source = HashSourceT; + + explicit ProofEngineT(const Source& source) : source(source) {} + + /// @brief The Merkle root of a tree of @p size leaves. + Hash root(uint64_t size) const + { + if (size == 0) + { + throw std::runtime_error("empty tree has no root"); + } + Hash out; + if (!mth_range(0, size, out)) + { + throw std::runtime_error("unresolved subtree while computing root"); + } + return out; + } + + /// @brief Inclusion proof for leaf @p index in a tree of @p size leaves. + /// @note Equivalent to TreeT::path(index) when size == num_leaves(), and + /// to TreeT::past_path(index, size - 1) otherwise. + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size) const + { + if (index >= size) + { + throw std::runtime_error("leaf index out of bounds"); + } + + std::list elements; // leaf -> root order + uint64_t lo = 0; + uint64_t hi = size; + while (hi - lo > 1) + { + const uint64_t k = largest_pow2_lt(hi - lo); + typename Path::Element e; + if (index - lo < k) + { + if (!mth_range(lo + k, hi, e.hash)) + { + throw std::runtime_error("unresolved subtree in inclusion proof"); + } + e.direction = Path::PATH_RIGHT; + hi = lo + k; + } + else + { + if (!mth_range(lo, lo + k, e.hash)) + { + throw std::runtime_error("unresolved subtree in inclusion proof"); + } + e.direction = Path::PATH_LEFT; + lo = lo + k; + } + elements.push_front(std::move(e)); + } + + Hash leaf; + if (!source.leaf(index, leaf)) + { + throw std::runtime_error("unresolved leaf in inclusion proof"); + } + return std::make_shared( + leaf, index, std::move(elements), size - 1); + } + + /// @brief Consistency proof that a tree of @p m leaves is a prefix of a + /// tree of @p n leaves (RFC 6962). + std::vector consistency_proof(uint64_t m, uint64_t n) const + { + if (m == 0 || m > n) + { + throw std::runtime_error("invalid consistency proof sizes"); + } + std::vector proof; + if (m == n) + { + return proof; + } + subproof(m, 0, n, true, proof); + return proof; + } + + /// @brief Consistency proof between the trees whose last leaves are at + /// indices @p first_index and @p second_index (first_index <= + /// second_index). + /// @note Equivalent to consistency_proof(first_index + 1, + /// second_index + 1): it proves the tree of the first first_index + 1 + /// leaves is a prefix of the tree of the first second_index + 1 leaves. + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) const + { + return consistency_proof(first_index + 1, second_index + 1); + } + + /// @brief Verifies an RFC 6962 consistency proof reconciling the roots of + /// trees of @p m and @p n leaves. + static bool verify_consistency( + uint64_t m, + uint64_t n, + const Hash& first_hash, + const Hash& second_hash, + std::vector proof) + { + if (m > n) + { + return false; + } + if (m == n) + { + return proof.empty() && first_hash == second_hash; + } + if (m == 0) + { + return proof.empty(); + } + + if (is_pow2(m)) + { + proof.insert(proof.begin(), first_hash); + } + if (proof.empty()) + { + return false; + } + + uint64_t fn = m - 1; + uint64_t sn = n - 1; + while ((fn & 1) != 0) + { + fn >>= 1; + sn >>= 1; + } + + Hash fr = proof[0]; + Hash sr = proof[0]; + for (size_t i = 1; i < proof.size(); i++) + { + if (sn == 0) + { + return false; + } + const Hash& c = proof[i]; + if ((fn & 1) != 0 || fn == sn) + { + HASH_FUNCTION(c, fr, fr); + HASH_FUNCTION(c, sr, sr); + if ((fn & 1) == 0) + { + while ((fn & 1) == 0 && fn != 0) + { + fn >>= 1; + sn >>= 1; + } + } + } + else + { + HASH_FUNCTION(sr, c, sr); + } + fn >>= 1; + sn >>= 1; + } + + return fr == first_hash && sr == second_hash && sn == 0; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& source; + + static bool is_pow2(uint64_t n) + { + return n != 0 && (n & (n - 1)) == 0; + } + + static uint64_t largest_pow2_lt(uint64_t n) + { + uint64_t k = 1; + while (k <= (n - 1) / 2) + { + k <<= 1; + } + return k; + } + + static uint8_t log2_exact(uint64_t n) + { + uint8_t r = 0; + while (n > 1) + { + n >>= 1; + r++; + } + return r; + } + + /// @brief MTH(D[a:b]) via the source; falls back to splitting when a + /// perfect subtree cannot be resolved directly. + bool mth_range(uint64_t a, uint64_t b, Hash& out) const + { + const uint64_t w = b - a; + if (w == 0) + { + return false; + } + if (w == 1) + { + return source.leaf(a, out); + } + if (is_pow2(w) && (a % w == 0)) + { + if (source.subtree_root(log2_exact(w), a / w, out)) + { + return true; + } + } + const uint64_t k = largest_pow2_lt(w); + Hash left; + Hash right; + if (!mth_range(a, a + k, left) || !mth_range(a + k, b, right)) + { + return false; + } + HASH_FUNCTION(left, right, out); + return true; + } + + void subproof( + uint64_t m, + uint64_t lo, + uint64_t hi, + bool complete, + std::vector& proof) const + { + if (m == hi - lo) + { + if (!complete) + { + Hash h; + if (!mth_range(lo, hi, h)) + { + throw std::runtime_error( + "unresolved subtree in consistency proof"); + } + proof.push_back(h); + } + return; + } + const uint64_t k = largest_pow2_lt(hi - lo); + Hash h; + if (m <= k) + { + subproof(m, lo, lo + k, complete, proof); + if (!mth_range(lo + k, hi, h)) + { + throw std::runtime_error("unresolved subtree in consistency proof"); + } + } + else + { + subproof(m - k, lo + k, hi, false, proof); + if (!mth_range(lo, lo + k, h)) + { + throw std::runtime_error("unresolved subtree in consistency proof"); + } + } + proof.push_back(h); + } + }; + + /// @brief Resolves subtree roots from an in-memory merkle::TreeT. + /// @note Resolves only complete subtrees that are fully resident (not + /// flushed), returning false otherwise so that a builder can fall back to + /// another source. Performs no hashing changes (see TreeT::subtree_root). + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class MemoryHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Tree = TreeT; + + explicit MemoryHashSourceT(Tree& tree) : tree(tree) {} + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + return tree.subtree_root(level, (size_t)index, out); + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Tree& tree; + }; + + /// @brief Resolves subtree roots from a primary source, falling back to a + /// secondary source. + /// @note Used to combine an in-memory tree (primary: no I/O, serves the + /// resident frontier) with tile files (secondary: serve the flushed past). + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class CombinedHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Source = HashSourceT; + + CombinedHashSourceT(const Source& primary, const Source& secondary) : + primary(primary), secondary(secondary) + {} + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + return primary.subtree_root(level, index, out) || + secondary.subtree_root(level, index, out); + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& primary; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& secondary; + }; + /// @brief Writes tlog-tiles entry bundles (raw log entries) for a growing /// log. /// @note Entry bundles are level-0 only and application-owned: merklecpp @@ -1039,6 +1545,21 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) using TileWriter = TileWriterT; + /// @brief Default abstract hash source (SHA256, default hash function). + using HashSource = HashSourceT<32, sha256_compress>; + + /// @brief Default tile-backed hash source (SHA256, default hash function). + using TileHashSource = TileHashSourceT<32, sha256_compress>; + + /// @brief Default proof engine (SHA256, default hash function). + using ProofEngine = ProofEngineT<32, sha256_compress>; + + /// @brief Default in-memory hash source (SHA256, default hash function). + using MemoryHashSource = MemoryHashSourceT<32, sha256_compress>; + + /// @brief Default combined hash source (SHA256, default hash function). + using CombinedHashSource = CombinedHashSourceT<32, sha256_compress>; + /// @brief Default entry-bundle writer (SHA256, default hash function). using EntryBundleWriter = EntryBundleWriterT< merkle::Tree::Hash::size_bytes, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9e61e03..724c37c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -37,12 +37,21 @@ add_merklecpp_test(serialisation serialisation.cpp) add_merklecpp_test(partial_serialisation partial_serialisation.cpp) add_merklecpp_test(serialise_to_file serialise_to_file.cpp) add_merklecpp_test(coverage coverage.cpp) +add_merklecpp_test(tiles_store tiles_store.cpp) +add_merklecpp_test(tiles_writer tiles_writer.cpp) +add_merklecpp_test(tiles_proofs tiles_proofs.cpp) +add_merklecpp_test(tiles_entries tiles_entries.cpp) + +if(LONG_TESTS) + add_merklecpp_test(tiles_level2 tiles_level2.cpp) + set_tests_properties( + ${MERKLECPP_TEST_PREFIX}tiles_level2 + PROPERTIES TIMEOUT 900 + ) +endif() if(OPENSSL) add_merklecpp_test(compare_hash_functions compare_hash_functions.cpp) endif() -add_merklecpp_test(tiles_store tiles_store.cpp) -add_merklecpp_test(tiles_writer tiles_writer.cpp) -add_merklecpp_test(tiles_entries tiles_entries.cpp) add_merklecpp_test(unit_tests unit_tests.cpp) diff --git a/test/tiles_level2.cpp b/test/tiles_level2.cpp new file mode 100644 index 0000000..e1ea541 --- /dev/null +++ b/test/tiles_level2.cpp @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// End-to-end coverage of the level-2 tile paths. A full level-2 tile requires +// 256^3 == 16,777,216 leaves, so this is a deliberately large test: it writes +// the tiles for that many leaves (~65k tile files) from a deterministic leaf +// source -- no in-memory tree is built -- and cross-checks the writer's +// level-by-level roll-up against TileHashSourceT::resolve (which reads the +// level-2 tile) and against the underlying leaves. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TILE_WIDTH; +using merkle::tiles::TileHashSource; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Roll up a perfect (power-of-two) set of hashes with the default combiner. +static Hash rollup(const std::vector& hashes) +{ + return merkle::tiles::perfect_root<32, merkle::sha256_compress>(hashes); +} + +int main() +{ + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_tiles_level2_" + + std::to_string((unsigned long long)std::time(nullptr))); + + try + { + // 256^3 leaves == exactly one full level-2 tile. + const uint64_t n = + (uint64_t)TILE_WIDTH * (uint64_t)TILE_WIDTH * (uint64_t)TILE_WIDTH; + + TileStore store(dir); + TileWriter writer(store); + + // Deterministic leaf hash derived from the index (low 8 bytes); avoids + // materialising a 16.7M-entry vector. + Hash leaf; + const auto leaf_at = [&](uint64_t i) -> const Hash& { + leaf.zero(); + for (int b = 0; b < 8; b++) + { + leaf.bytes[b] = (uint8_t)(i >> (8 * b)); + } + return leaf; + }; + + const auto stats = writer.write_up_to(n, leaf_at); + + // 65536 full L0 tiles + 256 full L1 tiles + 1 full L2 tile. + expect( + stats.full_written == (uint64_t)TILE_WIDTH * TILE_WIDTH + TILE_WIDTH + 1, + "level2: tile counts"); + expect(store.has_full_tile(2, 0), "level2: L2 tile present"); + expect(!store.has_full_tile(2, 1), "level2: no second L2 tile"); + expect(!store.has_full_tile(3, 0), "level2: no L3 tile"); + + const auto l2 = store.read_tile(TileRef{2, 0}); + expect(l2.size() == TILE_WIDTH, "level2: L2 tile width"); + + const TileHashSource src(store, n); + + // Each level-2 entry j is the root of level-1 tile j, which rolls up + // level-0 tiles, which are the leaves verbatim. Cross-check the writer's + // roll-up, resolve's level-2 read, and the leaf chain on a sample of + // indices. + for (const uint64_t j : + {(uint64_t)0, (uint64_t)1, (uint64_t)200, (uint64_t)255}) + { + const auto l1j = store.read_tile(TileRef{1, j}); + expect(l2[j] == rollup(l1j), "level2: L2[j] == rollup(L1 tile j)"); + + // resolve reads the level-2 tile for the 2^16-leaf subtree at index j. + Hash r16; + expect( + src.subtree_root(16, j, r16), "level2: subtree_root(16,j) resolves"); + expect(r16 == l2[j], "level2: resolve(16,j) == L2[j]"); + + // Anchor to leaves: L1 tile j entry 0 == root of L0 tile (j*256), whose + // first entry is leaf (j * 65536). + const auto l0 = store.read_tile(TileRef{0, j * TILE_WIDTH}); + expect(l1j[0] == rollup(l0), "level2: L1[j][0] == rollup(L0 tile)"); + expect( + l0[0] == leaf_at(j * (uint64_t)TILE_WIDTH * TILE_WIDTH), + "level2: L0 entry == leaf"); + } + + // Intra-tile roll-up: the 2^17-leaf subtree at 0 hashes L2[0] and L2[1]. + Hash r17; + expect(src.subtree_root(17, 0, r17), "level2: subtree_root(17,0) resolves"); + expect( + r17 == rollup({l2[0], l2[1]}), "level2: resolve(17,0) == H(L2[0],L2[1])"); + + std::cout << "tiles_level2: OK" << '\n'; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp new file mode 100644 index 0000000..1697dbd --- /dev/null +++ b/test/tiles_proofs.cpp @@ -0,0 +1,383 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::CombinedHashSource; +using merkle::tiles::MemoryHashSource; +using merkle::tiles::ProofEngine; +using merkle::tiles::TileHashSource; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +class ProofEngineProbe : public ProofEngine +{ +public: + using ProofEngine::largest_pow2_lt; +}; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Validates the TreeT::subtree_root accessor via memory-only proofs: they must +// match the library exactly. +static void check_memory_source(uint64_t n, const std::vector& hashes) +{ + const std::string at = " @n=" + std::to_string(n); + + merkle::Tree tree; + for (uint64_t i = 0; i < n; i++) + { + tree.insert(hashes[i]); + } + const Hash root = tree.root(); + + const MemoryHashSource source(tree); + const ProofEngine engine(source); + + expect(engine.root(n) == root, "mem root" + at); + + std::vector indices; + if (n <= 16) + { + for (uint64_t i = 0; i < n; i++) + { + indices.push_back(i); + } + } + else + { + for (const uint64_t i : {(uint64_t)0, (uint64_t)1, n / 2, n - 1}) + { + indices.push_back(i); + } + } + + for (const uint64_t i : indices) + { + const auto p = engine.inclusion_proof(i, n); + expect( + *p == *tree.path(i), "mem inclusion==path i=" + std::to_string(i) + at); + expect(p->verify(root), "mem inclusion verify i=" + std::to_string(i) + at); + } + + std::vector> pairs; + if (n <= 16) + { + for (uint64_t m = 1; m < n; m++) + { + for (uint64_t k = m + 1; k <= n; k++) + { + pairs.emplace_back(m, k); + } + } + } + else + { + pairs = {{1, n}, {n / 2, n}, {n - 1, n}}; + } + + for (const auto& pr : pairs) + { + const uint64_t m = pr.first; + const uint64_t k = pr.second; + const Hash rm = engine.root(m); + const Hash rk = engine.root(k); + expect(rm == *tree.past_root(m - 1), "mem past_root m" + at); + + const auto pp = engine.inclusion_proof(m / 2, m); + expect( + *pp == *tree.past_path(m / 2, m - 1), "mem inclusion(m)==past_path" + at); + + const auto cp = engine.consistency_proof(m, k); + expect( + ProofEngine::verify_consistency(m, k, rm, rk, cp), + "mem consistency" + at); + } +} + +// Exercises tile-derived proofs for a tree of `n` leaves against the existing +// library (which acts as the oracle: proofs must be byte-identical). Full tiles +// serve the covered prefix and an in-memory tree serves the un-tiled frontier, +// exactly as TiledTree combines them. +static void check_size( + const fs::path& dir, uint64_t n, const std::vector& hashes) +{ + const std::string at = " @n=" + std::to_string(n); + + TileStore store(dir); + TileWriter writer(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + writer.write_up_to(n, leaf_at); + + // Oracle: a full, never-flushed tree with the same leaves. + merkle::Tree tree; + for (uint64_t i = 0; i < n; i++) + { + tree.insert(hashes[i]); + } + const Hash root = tree.root(); + + // Production-shaped source: full tiles serve the covered prefix, an in-memory + // tree serves the un-tiled frontier. Drop the tiled past from the frontier + // tree so proofs over it are genuinely served from the tiles. merklecpp keeps + // at least one resident leaf, so never flush the whole tree. + const uint64_t covered = (n / 256) * 256; // 256 == TILE_WIDTH + merkle::Tree frontier; + for (uint64_t i = 0; i < n; i++) + { + frontier.insert(hashes[i]); + } + uint64_t drop_to = covered; + if (n > 0 && drop_to >= n) + { + drop_to = n - 1; + } + if (drop_to > 0) + { + frontier.flush_to((size_t)drop_to); + } + const MemoryHashSource mem(frontier); + const TileHashSource tiles(store, covered); + const CombinedHashSource source(mem, tiles); + const ProofEngine engine(source); + + // Root recomputed from tiles equals the library root. + expect(engine.root(n) == root, "root" + at); + + // Indices to probe: all of them for small trees, else a spread. + std::vector indices; + if (n <= 16) + { + for (uint64_t i = 0; i < n; i++) + { + indices.push_back(i); + } + } + else + { + for (const uint64_t i : + {(uint64_t)0, (uint64_t)1, n / 3, n / 2, n - 2, n - 1}) + { + indices.push_back(i); + } + } + + // Inclusion proofs are identical to TreeT::path and verify. + for (const uint64_t i : indices) + { + if (i >= n) + { + continue; + } + const auto p = engine.inclusion_proof(i, n); + expect(*p == *tree.path(i), "inclusion==path i=" + std::to_string(i) + at); + expect(p->verify(root), "inclusion verify i=" + std::to_string(i) + at); + } + + // Consistency pairs: exhaustive for small trees, else a fixed spread. + std::vector> pairs; + if (n <= 16) + { + for (uint64_t m = 1; m < n; m++) + { + for (uint64_t k = m + 1; k <= n; k++) + { + pairs.emplace_back(m, k); + } + } + } + else + { + pairs = {{1, n}, {n / 2, n}, {n - 1, n}, {1, 2}}; + // Tile-boundary crossings. + if (n > 256) + { + pairs.emplace_back(256, n); + pairs.emplace_back(257, n); + } + } + + for (const auto& pr : pairs) + { + const uint64_t m = pr.first; + const uint64_t k = pr.second; + if (m == 0 || m >= k || k > n) + { + continue; + } + + const Hash rm = engine.root(m); + const Hash rk = engine.root(k); + expect(rm == *tree.past_root(m - 1), "past_root m" + at); + expect(rk == *tree.past_root(k - 1), "past_root k" + at); + + // Past inclusion proof matches TreeT::past_path. + const uint64_t i = m / 2; + const auto pp = engine.inclusion_proof(i, m); + expect( + *pp == *tree.past_path(i, m - 1), + "inclusion(m)==past_path i=" + std::to_string(i) + at); + expect(pp->verify(rm), "inclusion(m) verify" + at); + + // Consistency proof reconciles the two roots. + const auto cp = engine.consistency_proof(m, k); + expect( + ProofEngine::verify_consistency(m, k, rm, rk, cp), + "consistency " + std::to_string(m) + "->" + std::to_string(k) + at); + + // The index-based variant is consistency_proof(i+1, j+1). + expect( + engine.consistency_proof_from_indices(m - 1, k - 1) == cp, + "consistency index variant" + at); + + // Tampering with a proof element or a root is rejected. + auto bad = cp; + bad[0].bytes[0] ^= 0xFFU; + expect( + !ProofEngine::verify_consistency(m, k, rm, rk, bad), + "consistency tamper rejected" + at); + + Hash wrong = rk; + wrong.bytes[0] ^= 0xFFU; + expect( + !ProofEngine::verify_consistency(m, k, rm, wrong, cp), + "consistency wrong root rejected" + at); + } + + std::error_code ec; + fs::remove_all(dir, ec); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + + const fs::path base = fs::temp_directory_path() / + ("merklecpp_tiles_proofs_" + std::to_string((unsigned long long)seed) + + "_" + std::to_string(std::rand())); + + try + { + const auto hashes = make_hashes(300000); + + // ---- Memory-only proofs (exercises TreeT::subtree_root). + for (const uint64_t n : + {(uint64_t)1, + (uint64_t)2, + (uint64_t)3, + (uint64_t)5, + (uint64_t)8, + (uint64_t)13, + (uint64_t)16, + (uint64_t)256, + (uint64_t)257, + (uint64_t)1000}) + { + check_memory_source(n, hashes); + } + std::cout << "memory source: OK" << '\n'; + + // ---- Hostile arithmetic inputs are rejected without UB or overflow + // loops. + { + merkle::Tree tree; + tree.insert(hashes[0]); + Hash out; + expect(!tree.subtree_root(64, 0, out), "subtree_root rejects level 64"); + expect(!tree.subtree_root(100, 0, out), "subtree_root rejects level 100"); + expect( + !tree.subtree_root(1, std::numeric_limits::max(), out), + "subtree_root rejects overflowing index"); + + expect(ProofEngineProbe::largest_pow2_lt(2) == 1, "pow2_lt 2"); + expect( + ProofEngineProbe::largest_pow2_lt((uint64_t)1 << 63) == + ((uint64_t)1 << 62), + "pow2_lt 2^63"); + expect( + ProofEngineProbe::largest_pow2_lt(((uint64_t)1 << 63) + 1) == + ((uint64_t)1 << 63), + "pow2_lt 2^63+1"); + expect( + ProofEngineProbe::largest_pow2_lt( + std::numeric_limits::max()) == ((uint64_t)1 << 63), + "pow2_lt uint64 max"); + std::cout << "hostile arithmetic inputs: OK" << '\n'; + } + + for (const uint64_t n : + {(uint64_t)1, + (uint64_t)2, + (uint64_t)3, + (uint64_t)4, + (uint64_t)5, + (uint64_t)7, + (uint64_t)8, + (uint64_t)13, + (uint64_t)16, + (uint64_t)255, + (uint64_t)256, + (uint64_t)257, + (uint64_t)1000}) + { + check_size(base / ("n" + std::to_string(n)), n, hashes); + } + std::cout << "small/medium sizes: OK" << '\n'; + + // Large trees. 65536 == 256 full L0 tiles == one full L1 tile (exact L1 + // boundary); 65537 is one past it; 70000 exercises a full L1 tile plus an + // in-memory frontier; 300000 forces proofs over height->=16 subtrees, so + // TileHashSource::resolve descends through level-2 logic (full_shift = 24) + // before reaching the level-1 tiles -- the only coverage of the L>=2 path. + for (const uint64_t n : + {(uint64_t)65536, (uint64_t)65537, (uint64_t)70000, (uint64_t)300000}) + { + check_size(base / ("big" + std::to_string(n)), n, hashes); + std::cout << "size " << n << ": OK" << '\n'; + } + + std::cout << "tiles_proofs: OK" << '\n'; + + std::error_code ec; + fs::remove_all(base, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(base, ec); + return 1; + } + + return 0; +} From 77c129a9387e33090f57ed533fb2fd847df794bf Mon Sep 17 00:00:00 2001 From: achamayou Date: Sat, 25 Jul 2026 22:12:58 +0100 Subject: [PATCH 02/12] Fix proof aliases and index bounds Use the current default tree hash function for public proof aliases, reject index conversions that cannot be represented safely, and align the design and level-2 coverage notes with the rebased implementation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c504572c-322f-4950-8682-edf4a7fd2c5b --- doc/design/tlog-tiles.md | 22 +++++++++---------- merklecpp_tiles.h | 46 ++++++++++++++++++++++++++++++++-------- test/tiles_level2.cpp | 4 +++- test/tiles_proofs.cpp | 18 ++++++++++++++-- 4 files changed, 67 insertions(+), 23 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 7525624..d0f76e4 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -174,10 +174,10 @@ hashes, not bytes. │ owns Tree ▲ inclusion / consistency proofs │ tiles ▼ │ - ProofEngine ──▶ HashSource ◀──────────┘ - ├─ MemoryHashSource - ├─ TileHashSource - └─ CombinedHashSource + ProofEngineT ──▶ HashSourceT ◀─────────┘ + ├─ MemoryHashSourceT + ├─ TileHashSourceT + └─ CombinedHashSourceT ``` `merklecpp_tiles.h` contains the public `merkle::tiles` API and includes @@ -192,7 +192,7 @@ hashes, not bytes. | `TileWriterT` | Persist newly completed full tiles | | Hash sources | Resolve subtree roots from memory, tiles, or both | | `ProofEngineT` | Roots, inclusion/consistency proofs, and verification | -| `TiledTreeT` | `append`, `flush`, proof APIs, and compaction | +| `TiledTreeT` (planned) | `append`, `flush`, proof APIs, and compaction | `TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. `MemoryHashSourceT` uses the logically read-only, non-hashing @@ -215,9 +215,8 @@ class TileStoreT; using TileStore = TileStoreT; -using TiledTree = - TiledTreeT; -// Equivalent SHA-384 and SHA-512 aliases. +// Hash-source and proof-engine aliases follow the same pattern. +// The TiledTree alias arrives with the phase-4 lifecycle wrapper. } ``` @@ -336,13 +335,14 @@ struct HashSourceT { ### 5.7 `ProofEngineT` -All three proof building blocks reduce to `mth_range` over a `HashSource`. +All three proof building blocks reduce to `mth_range` over a `HashSourceT`. Returned `PathT` objects are byte-identical to `Tree::path` / `Tree::past_path`. ```cpp class ProofEngineT { public: - explicit ProofEngineT(const HashSource& src); + using Source = HashSourceT; + explicit ProofEngineT(const Source& source); Hash root(uint64_t size) const; // = mth_range(0, size) @@ -420,4 +420,4 @@ no further core changes are planned. | 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | Deliverables are `merklecpp_tiles.h`, `merklecpp_pal.h`, `test/tiles_*.cpp`, -CMake wiring, the optional core accessor, and README/design updates. +CMake wiring, the core accessor, and README/design updates. diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 7ebabf7..19e8c5c 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -1098,11 +1099,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) }; /// @brief Builds and verifies inclusion and consistency proofs. - /// @note Proofs are assembled from a HashSource using the tree's + /// @note Proofs are assembled from a HashSourceT using the tree's /// HASH_FUNCTION, so an inclusion proof is byte-identical to the one /// produced by merkle::TreeT::path()/past_path() and verifies with /// PathT::verify(). - /// @warning Thread safety is inherited from the supplied HashSource. Callers + /// @warning Thread safety is inherited from the supplied HashSourceT. Callers /// must serialize operations when the source is shared. template < size_t HASH_SIZE, @@ -1141,6 +1142,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { throw std::runtime_error("leaf index out of bounds"); } + if ( + index > std::numeric_limits::max() || + size - 1 > std::numeric_limits::max()) + { + throw std::runtime_error("inclusion proof exceeds PathT index range"); + } std::list elements; // leaf -> root order uint64_t lo = 0; @@ -1176,7 +1183,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) throw std::runtime_error("unresolved leaf in inclusion proof"); } return std::make_shared( - leaf, index, std::move(elements), size - 1); + leaf, + static_cast(index), + std::move(elements), + static_cast(size - 1)); } /// @brief Consistency proof that a tree of @p m leaves is a prefix of a @@ -1205,6 +1215,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::vector consistency_proof_from_indices( uint64_t first_index, uint64_t second_index) const { + if ( + first_index == std::numeric_limits::max() || + second_index == std::numeric_limits::max()) + { + throw std::runtime_error("consistency proof index out of bounds"); + } return consistency_proof(first_index + 1, second_index + 1); } @@ -1402,7 +1418,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override { - return tree.subtree_root(level, (size_t)index, out); + if (index > std::numeric_limits::max()) + { + return false; + } + return tree.subtree_root(level, static_cast(index), out); } protected: @@ -1546,19 +1566,27 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) TileWriterT; /// @brief Default abstract hash source (SHA256, default hash function). - using HashSource = HashSourceT<32, sha256_compress>; + using HashSource = + HashSourceT; /// @brief Default tile-backed hash source (SHA256, default hash function). - using TileHashSource = TileHashSourceT<32, sha256_compress>; + using TileHashSource = TileHashSourceT< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>; /// @brief Default proof engine (SHA256, default hash function). - using ProofEngine = ProofEngineT<32, sha256_compress>; + using ProofEngine = + ProofEngineT; /// @brief Default in-memory hash source (SHA256, default hash function). - using MemoryHashSource = MemoryHashSourceT<32, sha256_compress>; + using MemoryHashSource = MemoryHashSourceT< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>; /// @brief Default combined hash source (SHA256, default hash function). - using CombinedHashSource = CombinedHashSourceT<32, sha256_compress>; + using CombinedHashSource = CombinedHashSourceT< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>; /// @brief Default entry-bundle writer (SHA256, default hash function). using EntryBundleWriter = EntryBundleWriterT< diff --git a/test/tiles_level2.cpp b/test/tiles_level2.cpp index e1ea541..d1a97c0 100644 --- a/test/tiles_level2.cpp +++ b/test/tiles_level2.cpp @@ -38,7 +38,9 @@ static void expect(bool cond, const std::string& what) // Roll up a perfect (power-of-two) set of hashes with the default combiner. static Hash rollup(const std::vector& hashes) { - return merkle::tiles::perfect_root<32, merkle::sha256_compress>(hashes); + return merkle::tiles::perfect_root< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>(hashes); } int main() diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp index 1697dbd..cbebb92 100644 --- a/test/tiles_proofs.cpp +++ b/test/tiles_proofs.cpp @@ -305,6 +305,8 @@ int main() { merkle::Tree tree; tree.insert(hashes[0]); + const MemoryHashSource source(tree); + const ProofEngine engine(source); Hash out; expect(!tree.subtree_root(64, 0, out), "subtree_root rejects level 64"); expect(!tree.subtree_root(100, 0, out), "subtree_root rejects level 100"); @@ -325,6 +327,17 @@ int main() ProofEngineProbe::largest_pow2_lt( std::numeric_limits::max()) == ((uint64_t)1 << 63), "pow2_lt uint64 max"); + bool rejected = false; + try + { + engine.consistency_proof_from_indices( + 0, std::numeric_limits::max()); + } + catch (const std::runtime_error&) + { + rejected = true; + } + expect(rejected, "consistency index rejects overflow"); std::cout << "hostile arithmetic inputs: OK" << '\n'; } @@ -349,9 +362,10 @@ int main() // Large trees. 65536 == 256 full L0 tiles == one full L1 tile (exact L1 // boundary); 65537 is one past it; 70000 exercises a full L1 tile plus an - // in-memory frontier; 300000 forces proofs over height->=16 subtrees, so + // in-memory frontier; 300000 forces proofs over height >= 16 subtrees, so // TileHashSource::resolve descends through level-2 logic (full_shift = 24) - // before reaching the level-1 tiles -- the only coverage of the L>=2 path. + // before falling back to level-1 tiles. tiles_level2 separately covers a + // completed level-2 tile. for (const uint64_t n : {(uint64_t)65536, (uint64_t)65537, (uint64_t)70000, (uint64_t)300000}) { From 96a6a31e00ed88bf653d675d73576507047f98ca Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 3 Aug 2026 13:15:57 +0000 Subject: [PATCH 03/12] Parallelize CI builds and release tests --- .github/processor_count.cmake | 9 +++++++++ .github/workflows/ci.yml | 13 +++++++------ test/CMakeLists.txt | 6 +++++- 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 .github/processor_count.cmake diff --git a/.github/processor_count.cmake b/.github/processor_count.cmake new file mode 100644 index 0000000..58cfd2d --- /dev/null +++ b/.github/processor_count.cmake @@ -0,0 +1,9 @@ +include(ProcessorCount) + +ProcessorCount(PARALLELISM) +if(PARALLELISM EQUAL 0) + set(PARALLELISM 1) +endif() + +message(STATUS "Using ${PARALLELISM} parallel jobs") +file(APPEND "$ENV{GITHUB_ENV}" "PARALLELISM=${PARALLELISM}\n") \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 492e06c..89d1f08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,10 @@ jobs: - name: Create Build Environment run: cmake -E make_directory ${{github.workspace}}/build/${{ matrix.build_type }} + - name: Detect Runner Parallelism + shell: bash + run: cmake -P "$GITHUB_WORKSPACE/.github/processor_count.cmake" + - name: Configure CMake # Use a bash shell so we can use the same syntax for environment variable # access regardless of the host operating system @@ -54,10 +58,7 @@ jobs: working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} run: | long_tests=OFF - if [ "$RUNNER_OS" == "Linux" ] && - [ "${{ matrix.compiler }}" == "g++" ] && - [ "${{ matrix.build_type }}" == "Release" ] && - [ "${{ matrix.openssl }}" == "OFF" ]; then + if [ "${{ matrix.build_type }}" == "Release" ]; then long_tests=ON fi if [ "$RUNNER_OS" == "Linux" ]; then @@ -69,7 +70,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash - run: cmake --build . --config ${{ matrix.build_type }} + run: cmake --build . --config ${{ matrix.build_type }} --parallel "$PARALLELISM" - name: Clang-Tidy Header if: matrix.os == 'ubuntu-latest' @@ -82,6 +83,6 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash - run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 + run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 --parallel "$PARALLELISM" env: ASAN_OPTIONS: use_sigaltstack=false # To avoid SetAlternateSignalStack with clang-11 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 724c37c..e555eee 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -44,9 +44,13 @@ add_merklecpp_test(tiles_entries tiles_entries.cpp) if(LONG_TESTS) add_merklecpp_test(tiles_level2 tiles_level2.cpp) + set(TILES_LEVEL2_TIMEOUT 900) + if(WIN32) + set(TILES_LEVEL2_TIMEOUT 3600) + endif() set_tests_properties( ${MERKLECPP_TEST_PREFIX}tiles_level2 - PROPERTIES TIMEOUT 900 + PROPERTIES TIMEOUT ${TILES_LEVEL2_TIMEOUT} ) endif() From 9d669bff0a3d920e6cc7c15aa1d86379af26c688 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 3 Aug 2026 13:32:50 +0000 Subject: [PATCH 04/12] Address proof generation review feedback --- merklecpp_tiles.h | 32 +++++++++++++++++--------------- test/tiles_proofs.cpp | 3 ++- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 19e8c5c..7c9828a 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -1036,7 +1036,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) // Spans 2**level <= TILE_WIDTH leaves: held by one level-0 tile. const uint64_t span = (uint64_t)1 << level; const uint64_t start = index << level; - const std::vector tile = + const std::vector& tile = read_tile(TileRef{0, start / TILE_WIDTH}); out = roll_up(tile, start % TILE_WIDTH, span); return; @@ -1053,7 +1053,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) if (n < full_tiles) { // One full level-L tile holds all 2**r entries of this subtree. - const std::vector tile = read_tile(TileRef{L, n}); + const std::vector& tile = read_tile(TileRef{L, n}); out = roll_up(tile, first % TILE_WIDTH, (uint64_t)1 << r); return; } @@ -1075,7 +1075,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) static constexpr size_t TILE_CACHE_SIZE = 64; mutable std::vector tile_cache; - std::vector read_tile(const TileRef& ref) const + const std::vector& read_tile(const TileRef& ref) const { for (auto it = tile_cache.begin(); it != tile_cache.end(); it++) { @@ -1083,9 +1083,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { TileCacheEntry entry = std::move(*it); tile_cache.erase(it); - std::vector hashes = entry.hashes; tile_cache.push_back(std::move(entry)); - return hashes; + return tile_cache.back().hashes; } } @@ -1231,7 +1230,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) uint64_t n, const Hash& first_hash, const Hash& second_hash, - std::vector proof) + const std::vector& proof) { if (m > n) { @@ -1246,13 +1245,18 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) return proof.empty(); } - if (is_pow2(m)) + size_t proof_index = 0; + Hash fr = first_hash; + Hash sr = first_hash; + if (!is_pow2(m)) { - proof.insert(proof.begin(), first_hash); - } - if (proof.empty()) - { - return false; + if (proof.empty()) + { + return false; + } + fr = proof[0]; + sr = proof[0]; + proof_index = 1; } uint64_t fn = m - 1; @@ -1263,9 +1267,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) sn >>= 1; } - Hash fr = proof[0]; - Hash sr = proof[0]; - for (size_t i = 1; i < proof.size(); i++) + for (size_t i = proof_index; i < proof.size(); i++) { if (sn == 0) { diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp index cbebb92..a1a339c 100644 --- a/test/tiles_proofs.cpp +++ b/test/tiles_proofs.cpp @@ -21,6 +21,7 @@ using merkle::Hash; using merkle::tiles::CombinedHashSource; using merkle::tiles::MemoryHashSource; using merkle::tiles::ProofEngine; +using merkle::tiles::TILE_WIDTH; using merkle::tiles::TileHashSource; using merkle::tiles::TileStore; using merkle::tiles::TileWriter; @@ -142,7 +143,7 @@ static void check_size( // tree serves the un-tiled frontier. Drop the tiled past from the frontier // tree so proofs over it are genuinely served from the tiles. merklecpp keeps // at least one resident leaf, so never flush the whole tree. - const uint64_t covered = (n / 256) * 256; // 256 == TILE_WIDTH + const uint64_t covered = (n / TILE_WIDTH) * TILE_WIDTH; merkle::Tree frontier; for (uint64_t i = 0; i < n; i++) { From 791ddb0d8f19556e151c8b84a40636685e1ebf74 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 3 Aug 2026 14:16:49 +0000 Subject: [PATCH 05/12] Use standard CMake parallelism variables --- .github/processor_count.cmake | 11 +++++++++-- .github/workflows/ci.yml | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/processor_count.cmake b/.github/processor_count.cmake index 58cfd2d..3993f24 100644 --- a/.github/processor_count.cmake +++ b/.github/processor_count.cmake @@ -5,5 +5,12 @@ if(PARALLELISM EQUAL 0) set(PARALLELISM 1) endif() -message(STATUS "Using ${PARALLELISM} parallel jobs") -file(APPEND "$ENV{GITHUB_ENV}" "PARALLELISM=${PARALLELISM}\n") \ No newline at end of file +message( + STATUS + "Setting CMAKE_BUILD_PARALLEL_LEVEL and CTEST_PARALLEL_LEVEL to ${PARALLELISM}" +) +file( + APPEND "$ENV{GITHUB_ENV}" + "CMAKE_BUILD_PARALLEL_LEVEL=${PARALLELISM}\n" + "CTEST_PARALLEL_LEVEL=${PARALLELISM}\n" +) \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89d1f08..270792e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: - name: Build working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash - run: cmake --build . --config ${{ matrix.build_type }} --parallel "$PARALLELISM" + run: cmake --build . --config ${{ matrix.build_type }} - name: Clang-Tidy Header if: matrix.os == 'ubuntu-latest' @@ -83,6 +83,6 @@ jobs: - name: Test working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash - run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 --parallel "$PARALLELISM" + run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 env: ASAN_OPTIONS: use_sigaltstack=false # To avoid SetAlternateSignalStack with clang-11 From ebfa8d8f2d1b6f8360a346210de8f179e590a469 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 3 Aug 2026 14:30:20 +0000 Subject: [PATCH 06/12] Adopt standard CMake test configuration --- .github/workflows/ci.yml | 18 +++++++++++++----- .github/workflows/codeql-analysis.yml | 2 +- CMakeLists.txt | 6 ++---- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 270792e..0027c96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,12 +61,22 @@ jobs: if [ "${{ matrix.build_type }}" == "Release" ]; then long_tests=ON fi + + linux_options=() if [ "$RUNNER_OS" == "Linux" ]; then - cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=$long_tests -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - else - cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DLONG_TESTS=$long_tests -DOPENSSL=${{ matrix.openssl }} + linux_options=( + "-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}" + -DCLANG_TIDY=ON + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + ) fi + cmake "$GITHUB_WORKSPACE" \ + "-DCMAKE_BUILD_TYPE=${{ matrix.build_type }}" \ + "-DLONG_TESTS=$long_tests" \ + "-DOPENSSL=${{ matrix.openssl }}" \ + "${linux_options[@]}" + - name: Build working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash @@ -84,5 +94,3 @@ jobs: working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 - env: - ASAN_OPTIONS: use_sigaltstack=false # To avoid SetAlternateSignalStack with clang-11 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1b61053..88c9f6e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: - name: Configure merklecpp working-directory: ${{github.workspace}}/build - run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON -DLONG_TESTS=ON $GITHUB_WORKSPACE + run: cmake -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON $GITHUB_WORKSPACE - name: Build merklecpp working-directory: ${{github.workspace}}/build diff --git a/CMakeLists.txt b/CMakeLists.txt index 85f9952..6252f35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ cmake_minimum_required(VERSION 3.14) project(merklecpp LANGUAGES CXX) include(GNUInstallDirs) +include(CTest) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -13,7 +14,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(MERKLECPP_DIR ${CMAKE_CURRENT_SOURCE_DIR}) option(PROFILE "enable profiling" OFF) -option(TESTS "enable testing" OFF) option(OPENSSL "enable OpenSSL" OFF) option(TRACE "enable debug traces" OFF) option(CLANG_TIDY "enable clang-tidy checks during build" OFF) @@ -46,9 +46,7 @@ if(OPENSSL) target_link_libraries(merklecpp INTERFACE OpenSSL::Crypto) endif() -if(TESTS) - enable_testing() - +if(BUILD_TESTING) function(add_unit_test NAME SRC) add_executable(${NAME} ${SRC}) target_link_libraries(${NAME} PRIVATE $) From 0d1666885865ddf474b6e0ed41bb820eb2113280 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 4 Aug 2026 08:54:30 +0000 Subject: [PATCH 07/12] Address large-tree and proof review feedback --- merklecpp.h | 9 +++++++- merklecpp_tiles.h | 23 +++++++++++++++----- test/tiles_level2.cpp | 8 +++---- test/tiles_proofs.cpp | 49 +++++++++++++++++++++++++++++-------------- 4 files changed, 63 insertions(+), 26 deletions(-) diff --git a/merklecpp.h b/merklecpp.h index db061b8..4e7a8b8 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -704,7 +704,14 @@ namespace merkle /// 2**height-1. [[nodiscard]] bool is_full() const { - size_t max_size = (1 << height) - 1; + constexpr size_t size_digits = std::numeric_limits::digits; + if (height > size_digits) + { + return false; + } + const size_t max_size = height == size_digits ? + std::numeric_limits::max() : + (size_t{1} << height) - 1; assert(size <= max_size); return size == max_size; } diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 7c9828a..cbca67d 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -1015,13 +1015,26 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) static Hash roll_up( const std::vector& tile, uint64_t off, uint64_t span) { - if (span == 1) + if ( + span == 0 || span > TILE_WIDTH || (span & (span - 1)) != 0 || + off > tile.size() || span > tile.size() - off) + { + throw std::runtime_error("invalid tile roll-up range"); + } + + std::array level; + for (size_t i = 0; i < span; i++) { - return tile.at(off); + level[i] = tile[off + i]; + } + for (size_t width = span; width > 1; width /= 2) + { + for (size_t i = 0; i < width; i += 2) + { + HASH_FUNCTION(level[i], level[i + 1], level[i / 2]); + } } - return perfect_root(std::vector( - tile.begin() + (std::ptrdiff_t)off, - tile.begin() + (std::ptrdiff_t)(off + span))); + return level[0]; } /// @brief Resolves a complete subtree known to lie within the full-tile diff --git a/test/tiles_level2.cpp b/test/tiles_level2.cpp index d1a97c0..2936ec6 100644 --- a/test/tiles_level2.cpp +++ b/test/tiles_level2.cpp @@ -8,14 +8,15 @@ // level-by-level roll-up against TileHashSourceT::resolve (which reads the // level-2 tile) and against the underlying leaves. +#include "tiles_test_util.h" #include "util.h" #include -#include #include #include #include #include +#include #include #include @@ -45,9 +46,8 @@ static Hash rollup(const std::vector& hashes) int main() { - const fs::path dir = fs::temp_directory_path() / - ("merklecpp_tiles_level2_" + - std::to_string((unsigned long long)std::time(nullptr))); + const TemporaryDirectory temporary_directory("merklecpp_tiles_level2"); + const fs::path& dir = temporary_directory.path(); try { diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp index a1a339c..2153093 100644 --- a/test/tiles_proofs.cpp +++ b/test/tiles_proofs.cpp @@ -1,17 +1,17 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#include "tiles_test_util.h" #include "util.h" #include -#include -#include #include #include #include #include #include #include +#include #include #include #include @@ -32,6 +32,18 @@ class ProofEngineProbe : public ProofEngine using ProofEngine::largest_pow2_lt; }; +class TreeProbe : public merkle::Tree +{ +public: + static bool node_is_full(uint8_t height, size_t size) + { + Node node{}; + node.height = height; + node.size = size; + return node.is_full(); + } +}; + static void expect(bool cond, const std::string& what) { if (!cond) @@ -272,13 +284,8 @@ static void check_size( int main() { - const auto seed = std::time(nullptr); - std::srand((unsigned)seed); - std::cout << "seed=" << seed << '\n'; - - const fs::path base = fs::temp_directory_path() / - ("merklecpp_tiles_proofs_" + std::to_string((unsigned long long)seed) + - "_" + std::to_string(std::rand())); + const TemporaryDirectory temporary_directory("merklecpp_tiles_proofs"); + const fs::path& base = temporary_directory.path(); try { @@ -315,6 +322,23 @@ int main() !tree.subtree_root(1, std::numeric_limits::max(), out), "subtree_root rejects overflowing index"); + const auto signed_shift_boundary = + static_cast(std::numeric_limits::digits); + if ( + signed_shift_boundary < std::numeric_limits::digits) + { + const size_t full_size = + (size_t{1} << signed_shift_boundary) - 1; + expect( + TreeProbe::node_is_full(signed_shift_boundary, full_size), + "is_full handles signed-shift boundary"); + } + expect( + TreeProbe::node_is_full( + static_cast(std::numeric_limits::digits), + std::numeric_limits::max()), + "is_full handles maximum size_t height"); + expect(ProofEngineProbe::largest_pow2_lt(2) == 1, "pow2_lt 2"); expect( ProofEngineProbe::largest_pow2_lt((uint64_t)1 << 63) == @@ -375,22 +399,15 @@ int main() } std::cout << "tiles_proofs: OK" << '\n'; - - std::error_code ec; - fs::remove_all(base, ec); } catch (std::exception& ex) { std::cout << "Error: " << ex.what() << '\n'; - std::error_code ec; - fs::remove_all(base, ec); return 1; } catch (...) { std::cout << "Error" << '\n'; - std::error_code ec; - fs::remove_all(base, ec); return 1; } From 18355a2a6728fea777f9574f42f7290b2186adb0 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 4 Aug 2026 10:48:36 +0000 Subject: [PATCH 08/12] Tighten proof roll-up and documentation --- doc/design/tlog-tiles.md | 6 +++--- merklecpp_tiles.h | 4 ++++ test/tiles_proofs.cpp | 35 ++++++++++++++++++++--------------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index d0f76e4..7d717f8 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -195,7 +195,7 @@ hashes, not bytes. | `TiledTreeT` (planned) | `append`, `flush`, proof APIs, and compaction | `TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. -`MemoryHashSourceT` uses the logically read-only, non-hashing +`MemoryHashSourceT` uses the logically read-only `TreeT::subtree_root` accessor. ### 5.1 Types and aliases @@ -419,5 +419,5 @@ no further core changes are planned. | 4. Combined tree | `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | | 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | -Deliverables are `merklecpp_tiles.h`, `merklecpp_pal.h`, `test/tiles_*.cpp`, -CMake wiring, the core accessor, and README/design updates. +Delivered through phase 3 are `merklecpp_tiles.h`, `merklecpp_pal.h`, +`test/tiles_*.cpp`, CMake wiring, the core accessor, and design updates. diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index cbca67d..e6aabe3 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -1021,6 +1021,10 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { throw std::runtime_error("invalid tile roll-up range"); } + if (span == 1) + { + return tile[off]; + } std::array level; for (size_t i = 0; i < span; i++) diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp index 2153093..ab985aa 100644 --- a/test/tiles_proofs.cpp +++ b/test/tiles_proofs.cpp @@ -224,10 +224,10 @@ static void check_size( { pairs = {{1, n}, {n / 2, n}, {n - 1, n}, {1, 2}}; // Tile-boundary crossings. - if (n > 256) + if (n > TILE_WIDTH) { - pairs.emplace_back(256, n); - pairs.emplace_back(257, n); + pairs.emplace_back(TILE_WIDTH, n); + pairs.emplace_back((uint64_t)TILE_WIDTH + 1, n); } } @@ -286,6 +286,8 @@ int main() { const TemporaryDirectory temporary_directory("merklecpp_tiles_proofs"); const fs::path& base = temporary_directory.path(); + const uint64_t tile_width = TILE_WIDTH; + const uint64_t level1_width = tile_width * tile_width; try { @@ -300,8 +302,8 @@ int main() (uint64_t)8, (uint64_t)13, (uint64_t)16, - (uint64_t)256, - (uint64_t)257, + tile_width, + tile_width + 1, (uint64_t)1000}) { check_memory_source(n, hashes); @@ -376,23 +378,26 @@ int main() (uint64_t)8, (uint64_t)13, (uint64_t)16, - (uint64_t)255, - (uint64_t)256, - (uint64_t)257, + tile_width - 1, + tile_width, + tile_width + 1, (uint64_t)1000}) { check_size(base / ("n" + std::to_string(n)), n, hashes); } std::cout << "small/medium sizes: OK" << '\n'; - // Large trees. 65536 == 256 full L0 tiles == one full L1 tile (exact L1 - // boundary); 65537 is one past it; 70000 exercises a full L1 tile plus an - // in-memory frontier; 300000 forces proofs over height >= 16 subtrees, so - // TileHashSource::resolve descends through level-2 logic (full_shift = 24) - // before falling back to level-1 tiles. tiles_level2 separately covers a - // completed level-2 tile. + // Large trees. TILE_WIDTH * TILE_WIDTH is one full L1 tile (the exact L1 + // boundary); the next size is one past it. 70000 exercises a full L1 tile + // plus an in-memory frontier; 300000 forces proofs over height >= 16 + // subtrees, so TileHashSource::resolve descends through level-2 logic + // (full_shift = 24) before falling back to level-1 tiles. tiles_level2 + // separately covers a completed level-2 tile. for (const uint64_t n : - {(uint64_t)65536, (uint64_t)65537, (uint64_t)70000, (uint64_t)300000}) + {level1_width, + level1_width + 1, + (uint64_t)70000, + (uint64_t)300000}) { check_size(base / ("big" + std::to_string(n)), n, hashes); std::cout << "size " << n << ": OK" << '\n'; From 6a9e58aff74e4beae2a09b5a88e0932cc41ca216 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:40:26 +0000 Subject: [PATCH 09/12] Clarify subtree root result and effects Co-authored-by: eddyashton <6000239+eddyashton@users.noreply.github.com> --- doc/design/tlog-tiles.md | 12 ++++++------ merklecpp.h | 36 ++++++++++++++++-------------------- merklecpp_tiles.h | 11 +++++++++-- test/tiles_proofs.cpp | 10 ++++++---- 4 files changed, 37 insertions(+), 32 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 7d717f8..23bd0f1 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -195,8 +195,7 @@ hashes, not bytes. | `TiledTreeT` (planned) | `append`, `flush`, proof APIs, and compaction | `TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. -`MemoryHashSourceT` uses the logically read-only -`TreeT::subtree_root` accessor. +`MemoryHashSourceT` uses the `TreeT::subtree_root` accessor. ### 5.1 Types and aliases @@ -298,16 +297,17 @@ bundles before reusing them, and leaves the incomplete tail with the application ### 5.5 `TreeT::subtree_root` -Proofs over the resident frontier use one logically read-only core accessor: +Proofs over the resident frontier use one core accessor: ```cpp -bool subtree_root(uint8_t level, size_t index, Hash& out); +std::optional subtree_root(uint8_t level, size_t index); ``` It returns the existing root of the complete subtree spanning `[index << level, (index + 1) << level)`. The method rejects overflow, flushed -or out-of-range leaves, and non-perfect frontier nodes. It may realize a dirty -node hash exactly as `root()` and `path()` do, but does not change tree shape or +or out-of-range leaves, and non-perfect frontier nodes by returning +`std::nullopt`. Like `root()` and `path()`, it may materialize pending +insertions and cache computed hashes without changing the leaf sequence or hashing semantics. ### 5.6 Hash sources diff --git a/merklecpp.h b/merklecpp.h index 4e7a8b8..ce00544 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -1347,25 +1348,22 @@ namespace merkle /// @brief Extracts the root hash of a complete subtree resident in memory /// @param level The height of the subtree (it spans 2**level leaves) /// @param index The index of the subtree at that height - /// @param out Set to the subtree root hash on success - /// @return Whether the subtree is a complete (balanced) subtree fully - /// resident in memory - /// @note This is read-only and does not change the hashing of the tree: it - /// returns an existing node hash (computing it on demand exactly as root() - /// and path() do). It returns false if any leaf of the subtree has been - /// flushed, if the subtree extends past the last leaf, or if the node at - /// that position is not a full subtree. The subtree spans leaf indices - /// [index << level, (index + 1) << level). - bool subtree_root(uint8_t level, size_t index, Hash& out) + /// @return The subtree root hash if the subtree is complete (balanced) and + /// fully resident in memory; otherwise, std::nullopt + /// @note This may materialize pending insertions and cache computed hashes, + /// exactly as root() and path() do. It does not change the leaf sequence or + /// hashing semantics. The subtree spans leaf indices [index << level, + /// (index + 1) << level). + std::optional subtree_root(uint8_t level, size_t index) { const size_t leaves = num_leaves(); if (leaves == 0 || level >= std::numeric_limits::digits) { - return false; + return std::nullopt; } if (index > (std::numeric_limits::max() >> level)) { - return false; + return std::nullopt; } const size_t lo = index << level; @@ -1373,13 +1371,12 @@ namespace merkle if (lo < min_index() || count > leaves || lo > leaves - count) { - return false; + return std::nullopt; } if (level == 0) { - out = leaf(lo); - return true; + return leaf(lo); } compute_root(); @@ -1387,7 +1384,7 @@ namespace merkle const uint8_t target_height = level + 1; if (!_root || _root->height < target_height) { - return false; + return std::nullopt; } Node* cur = _root; @@ -1400,7 +1397,7 @@ namespace merkle Node* next = go_right ? cur->right : cur->left; if (!next) { - return false; // conflated/flushed: not resident + return std::nullopt; // conflated/flushed: not resident } cur = next; } @@ -1410,14 +1407,13 @@ namespace merkle if (cur->height != target_height || !cur->is_full()) { - return false; + return std::nullopt; } if (cur->dirty) { hash(cur); } - out = cur->hash; - return true; + return cur->hash; } /// @brief Serialises the tree diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index e6aabe3..26b83b4 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -1422,7 +1422,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Resolves subtree roots from an in-memory merkle::TreeT. /// @note Resolves only complete subtrees that are fully resident (not /// flushed), returning false otherwise so that a builder can fall back to - /// another source. Performs no hashing changes (see TreeT::subtree_root). + /// another source. Resolution may materialize the tree's pending insertions + /// and cache hashes (see TreeT::subtree_root). template < size_t HASH_SIZE, void HASH_FUNCTION( @@ -1441,7 +1442,13 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { return false; } - return tree.subtree_root(level, static_cast(index), out); + const auto root = tree.subtree_root(level, static_cast(index)); + if (!root) + { + return false; + } + out = *root; + return true; } protected: diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp index ab985aa..1f7d453 100644 --- a/test/tiles_proofs.cpp +++ b/test/tiles_proofs.cpp @@ -317,11 +317,13 @@ int main() tree.insert(hashes[0]); const MemoryHashSource source(tree); const ProofEngine engine(source); - Hash out; - expect(!tree.subtree_root(64, 0, out), "subtree_root rejects level 64"); - expect(!tree.subtree_root(100, 0, out), "subtree_root rejects level 100"); expect( - !tree.subtree_root(1, std::numeric_limits::max(), out), + tree.subtree_root(0, 0) == hashes[0], + "subtree_root returns a resident leaf hash"); + expect(!tree.subtree_root(64, 0), "subtree_root rejects level 64"); + expect(!tree.subtree_root(100, 0), "subtree_root rejects level 100"); + expect( + !tree.subtree_root(1, std::numeric_limits::max()), "subtree_root rejects overflowing index"); const auto signed_shift_boundary = From 741fb70fa89c1e08ade5ac7ace3d51687ea45853 Mon Sep 17 00:00:00 2001 From: achamayou Date: Tue, 4 Aug 2026 14:16:35 +0000 Subject: [PATCH 10/12] Consolidate proof subtree reduction --- doc/design/tlog-tiles.md | 6 ++-- merklecpp.h | 11 ++++--- merklecpp_tiles.h | 69 ++++++++++++++++++++-------------------- 3 files changed, 44 insertions(+), 42 deletions(-) diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 23bd0f1..583ccf2 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -306,9 +306,9 @@ std::optional subtree_root(uint8_t level, size_t index); It returns the existing root of the complete subtree spanning `[index << level, (index + 1) << level)`. The method rejects overflow, flushed or out-of-range leaves, and non-perfect frontier nodes by returning -`std::nullopt`. Like `root()` and `path()`, it may materialize pending -insertions and cache computed hashes without changing the leaf sequence or -hashing semantics. +`std::nullopt`. Like `root()` and `path()`, it may materialize pending nodes and +compute dirty hashes, but does not change logical leaf contents or hashing +semantics. ### 5.6 Hash sources diff --git a/merklecpp.h b/merklecpp.h index ce00544..0306f2e 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -1350,10 +1350,13 @@ namespace merkle /// @param index The index of the subtree at that height /// @return The subtree root hash if the subtree is complete (balanced) and /// fully resident in memory; otherwise, std::nullopt - /// @note This may materialize pending insertions and cache computed hashes, - /// exactly as root() and path() do. It does not change the leaf sequence or - /// hashing semantics. The subtree spans leaf indices [index << level, - /// (index + 1) << level). + /// @note Like root() and path(), it may + /// materialize pending nodes and compute dirty hashes, but does not change + /// logical leaf contents or hashing semantics. It returns std::nullopt if + /// any leaf of the subtree has been flushed, if the subtree extends past + /// the last leaf, or if the node at that position is not a full subtree. + /// The subtree spans leaf indices + /// [index << level, (index + 1) << level). std::optional subtree_root(uint8_t level, size_t index) { const size_t leaves = num_leaves(); diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 26b83b4..6957da3 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -732,6 +732,32 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @endcond }; + namespace detail + { + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + HashT perfect_root_range( + const std::vector>& hashes, + size_t offset, + size_t count) + { + if (count == 1) + { + return hashes[offset]; + } + const size_t half = count / 2; + const auto left = + perfect_root_range(hashes, offset, half); + const auto right = perfect_root_range( + hashes, offset + half, half); + HashT out; + HASH_FUNCTION(left, right, out); + return out; + } + } + /// @brief Computes the Merkle Tree Hash of a perfect (balanced) subtree. /// @param leaves The subtree's leaves; the count MUST be a power of two. /// @return The subtree root, computed with the tree's HASH_FUNCTION. @@ -755,20 +781,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) "perfect_root requires a power-of-two number of leaves"); } - std::vector> level = leaves; - while (level.size() > 1) - { - std::vector> next; - next.reserve(level.size() / 2); - for (size_t i = 0; i + 1 < level.size(); i += 2) - { - HashT h; - HASH_FUNCTION(level[i], level[i + 1], h); - next.push_back(h); - } - level.swap(next); - } - return level.front(); + return detail::perfect_root_range( + leaves, 0, leaves.size()); } /// @brief Computes and persists tlog-tiles tiles for a growing tree. @@ -1021,24 +1035,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) { throw std::runtime_error("invalid tile roll-up range"); } - if (span == 1) - { - return tile[off]; - } - - std::array level; - for (size_t i = 0; i < span; i++) - { - level[i] = tile[off + i]; - } - for (size_t width = span; width > 1; width /= 2) - { - for (size_t i = 0; i < width; i += 2) - { - HASH_FUNCTION(level[i], level[i + 1], level[i / 2]); - } - } - return level[0]; + return detail::perfect_root_range( + tile, static_cast(off), static_cast(span)); } /// @brief Resolves a complete subtree known to lie within the full-tile @@ -1063,7 +1061,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const uint8_t r = level % TILE_HEIGHT; const uint64_t first = index << r; // first level-L entry const uint64_t n = first / TILE_WIDTH; // level-L tile index - const unsigned full_shift = 8U * ((unsigned)L + 1U); + const unsigned full_shift = static_cast(TILE_HEIGHT) * + (static_cast(L) + 1U); const uint64_t full_tiles = full_shift >= 64 ? 0 : (available_size >> full_shift); @@ -1422,8 +1421,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Resolves subtree roots from an in-memory merkle::TreeT. /// @note Resolves only complete subtrees that are fully resident (not /// flushed), returning false otherwise so that a builder can fall back to - /// another source. Resolution may materialize the tree's pending insertions - /// and cache hashes (see TreeT::subtree_root). + /// another source. It may materialize pending nodes and compute dirty + /// hashes but does not change logical contents or hashing semantics. template < size_t HASH_SIZE, void HASH_FUNCTION( From d8e3b9e439aacf563c6be1bb84b69f8de5220b19 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 11:46:54 +0000 Subject: [PATCH 11/12] Add TiledTree lifecycle wrapper Add TiledTreeT: a fresh-only tiled tree (rejects an existing tile namespace rather than adopting it), append/flush/root, flushed/ immutable size tracking, interrupted-flush recovery (a failed flush seals the attempted full-tile boundary without advancing flushed size), optional compaction with configurable retention, a rollback boundary that only ever permits retracting the un-tiled frontier, noexcept move construction with no copy, mixed tile+memory proofs, and an explicit no-internal-synchronization / external-serialization contract for the store and tree it wraps. Add tiles_tree tests covering the empty tree, move construction, fresh-only rejection of an existing tile namespace, flush/compaction (including exact-multiple and retention-margin cases), and rollback (pre-flush, post-flush, exact-boundary, compacted, and interrupted- flush recovery). The memory-only subtree_root/ProofEngineProbe cases already moved to tiles_proofs are not duplicated here. Add tiles_hashes, exercising the tiled tree, writer and proof engine under SHA384/SHA512, and wire the OpenSSL 384/512 aliases for every tiled-storage component. Document the TiledTreeT API, the flush/compaction invariants and progressive-production algorithm, pruning, and the consolidated lifecycle/safety risks and edge cases in the design doc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/design/tlog-tiles.md | 175 ++++++++- merklecpp_tiles.h | 421 ++++++++++++++++++-- test/CMakeLists.txt | 5 + test/tiles_hashes.cpp | 93 +++++ test/tiles_tree.cpp | 805 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 1460 insertions(+), 39 deletions(-) create mode 100644 test/tiles_hashes.cpp create mode 100644 test/tiles_tree.cpp diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 583ccf2..b2aa05b 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -31,7 +31,7 @@ serving the static resources. |---|---| | Tile | `/-256w/tile//`, `application/octet-stream`; `L` is decimal `0..63` without leading zeros, and `N` uses the grouped encoding in [section 4](#4-storage-layout-and-publication). | | Full tile | 256 hashes. Level 0 stores leaf hashes; each level-`L` entry for `L >= 1` is the Merkle Tree Hash of one complete level-`L-1` tile. | -| Entry bundle | `/-256w/tile/entries/`; 256 raw entries encoded as big-endian `uint16` length-prefixed values. See [section 5.3](#53-entry-bundles-optional). | +| Entry bundle | `/-256w/tile/entries/`; 256 raw entries encoded as big-endian `uint16` length-prefixed values. See [section 5.4](#54-entry-bundles-optional). | | Pruning | Tiles or bundles ending at or before a log's minimum index may be denied, matching `flush_to()` / `min_index()`. | A tile spans eight tree levels. Tile `n` at level `l` contains, for @@ -192,7 +192,7 @@ hashes, not bytes. | `TileWriterT` | Persist newly completed full tiles | | Hash sources | Resolve subtree roots from memory, tiles, or both | | `ProofEngineT` | Roots, inclusion/consistency proofs, and verification | -| `TiledTreeT` (planned) | `append`, `flush`, proof APIs, and compaction | +| `TiledTreeT` | `append`, `flush`, proof APIs, and compaction | `TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. `MemoryHashSourceT` uses the `TreeT::subtree_root` accessor. @@ -214,8 +214,10 @@ class TileStoreT; using TileStore = TileStoreT; -// Hash-source and proof-engine aliases follow the same pattern. -// The TiledTree alias arrives with the phase-4 lifecycle wrapper. +using TiledTree = + TiledTreeT; +// Writer, hash-source, proof-engine, and entry-bundle aliases follow the same +// template pattern. } ``` @@ -403,12 +405,129 @@ Because every emitted hash is an `mth_range` computed with `HASH_FUNCTION`, the consistency proof reconciles `Tree::past_root(m-1)` with `Tree::past_root(n-1)` — i.e. it is consistent with the existing library. -## 6. Delivery plan +### 5.8 `TiledTreeT` - convenience wrapper -Phases 0-3 now deliver the storage primitives, incremental tile and entry-bundle -writers, hash sources, proof engine, and the only required core accessor. Later -PRs deliver the lifecycle wrapper, user documentation, and performance coverage; -no further core changes are planned. +```cpp +class TiledTreeT { +public: + struct Config { + std::filesystem::path prefix; + uint64_t retention_margin = 0; // keep at least this many resident + bool compact_on_flush = false; // opt in to dropping tiled leaves + }; + explicit TiledTreeT(Config); + TiledTreeT(Config, const std::string& hash_algorithm_short_name); + + void append(const Hash& leaf_hash); // tree.insert + uint64_t size() const; // tree.num_leaves + Hash root(); // tree.root + uint64_t flushed_size() const; // successful tile boundary + uint64_t immutable_size() const; // rollback boundary + + // Write newly-complete full tiles. Compaction (dropping already-tiled + // leaves from memory) happens only if compact_on_flush. + Stats flush(); + + // Drop from memory the leaves already covered by a full tile (opt-in); the + // un-tiled frontier is always retained, and proofs for dropped leaves remain + // available from the tiles. + uint64_t compact(); + + // Roll back only beyond immutable_size(). + void retract_to(size_t index); + + // Proofs over tiles ∪ resident tree (works for flushed indices). + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size); + std::vector consistency_proof(uint64_t m, uint64_t n); + std::vector consistency_proof_from_indices(uint64_t i, uint64_t j); + + Tree& tree_ref(); // mutable escape hatch + Store& store_ref(); // mutable escape hatch +}; +``` + +`TiledTreeT` performs no internal locking. The caller must serialize every +operation on a shared instance, including proof calls. + +Callers with their own storage can construct a `TileWriterT` and call +`write_up_to`, then build a `ProofEngineT` on a `CombinedHashSource`, so the +wrapper is optional sugar. + +## 6. Progressive production and compaction + +The pairing of tile writing with `flush_to` gives two central correctness +invariants: + +> **Compaction invariant.** Retain the final leaf of the last fully successful +> flush: `flushed_size == 0 || min_index() < flushed_size`. +> +> **Immutability invariant.** Never roll back below a full-tile boundary that a +> flush may have published: `size >= immutable_size`. + +`TiledTreeT` is fresh-only: its configured directory may exist, but the +algorithm-qualified `tile` subdirectory must be absent. Construction atomically +creates that directory to claim exclusive ownership. Tile files do not carry +the size, root, hash identity, or ownership information needed to reopen or +share a tree safely. The lower-level `TileWriterT` supports resume for +applications that persist and validate the matching tree state themselves. + +Per flush: + +1. Append new leaf hashes and compute the root as needed. +2. Compute `covered = floor(size / 256) * 256` and advance `immutable_size` to + `covered` before any write can publish a full tile. +3. Call `write_up_to(size, leaf_at)` to persist newly complete full tiles at all + levels. +4. After every level succeeds, set `flushed_size = covered`. +5. Optionally, `compact()` computes an aligned retention target capped below + nonzero `covered`, then calls `flush_to(target)`. This reclaims memory only + when `compact_on_flush` is set or `compact()` is called explicitly. It keeps + at least `retention_margin` recent leaves; alignment can retain up to 255 + additional tiled leaves, and a zero margin keeps the final tiled leaf. The + entire un-tiled frontier always remains resident. + +If tile writing fails, the final two steps do not run. `immutable_size` stays +advanced to prevent stale-tile rollback, while `flushed_size` stays at the last +complete all-level write so proofs and compaction do not trust an incomplete +flush. + +Given these invariants, every leaf and every perfect subtree is resolvable: + +- A leaf below `covered` is in a full level-0 tile; a leaf at or above + `min_index` is resident. Since `min_index <= covered`, every leaf is in tiles, + memory, or both. The frontier `[covered, size)` is always resident because + compaction never flushes past `covered`. +- `mth_range` resolves a perfect subtree directly when it lies wholly in tiles + (`end <= covered`) or wholly in memory (`start >= min_index`). Otherwise it + splits and recurses until each piece is resolvable. A subtree within + `covered` whose level has no completed full tile descends to the highest + available full tile. + +Inclusion and consistency proofs therefore remain available after compaction, +from full tiles for the tiled prefix, memory for the resident frontier, or the +combination. A flush costs `O(new full tiles)`; higher-level tiles are roll-ups +of 256 child hashes. Proof generation performs `O(log(size))` range operations, +with repeated tile reads served from the per-source cache. + +## 7. Pruning and minimum index + +tlog-tiles pruning maps directly onto merklecpp: + +- The log's minimum index is `tree.min_index()` (equal to `num_flushed`). +- A serving layer can deny tiles or bundles whose end index is at or below the + minimum index. On-disk tiles may instead be retained so historical proofs + remain producible. +- The unpruned default is `min_index() == 0` (no `flush_to`). + +`flush_to` provides the mechanism; the application owns the retention policy, +as in the tlog-tiles ecosystem. + +## 8. Delivery plan + +Phases 0-4 now deliver the storage primitives, incremental tile and entry-bundle +writers, hash sources, proof engine, the only required core accessor, and the +lifecycle wrapper. Later PRs deliver user documentation and performance +coverage; no further core changes are planned. | Phase | Scope | Key tests | |---|---|---| @@ -419,5 +538,39 @@ no further core changes are planned. | 4. Combined tree | `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | | 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | -Delivered through phase 3 are `merklecpp_tiles.h`, `merklecpp_pal.h`, -`test/tiles_*.cpp`, CMake wiring, the core accessor, and design updates. +Delivered through phase 4 are `merklecpp_tiles.h`, `merklecpp_pal.h`, +`test/tiles_*.cpp`, CMake wiring, the core accessor, the lifecycle wrapper, and +design updates. + +## 9. Risks and edge cases + +- **External interop (by design, no).** With the default combiner the tiles are + not byte-compatible with RFC 6962 tooling. See + [section 2.3](#23-algorithms-and-namespaces); opting into a compatible + `HASH_FUNCTION` is the consumer's choice and out of scope. +- **Filesystem dependency.** Tile I/O needs ``/``; isolated + in the companion header so the core stays dependency-free. +- **Immutable full tiles.** A tile is emitted only after all of its entries are + final, and every emitted tile is write-once. A stand-alone tile reader cannot + serve the frontier; that is the in-memory tree's job (or the application must + keep it elsewhere). +- **`flush_to` alignment.** Compaction normally flushes to a 256-multiple + derived from retention. When that target equals `flushed_size`, it stops one + leaf earlier so `TreeT` can still retract to exactly that size. This one-leaf + overlap is enforced inside `TiledTreeT::compact`. +- **Rollback vs. immutable tiles.** Tiles are write-once, so rolling the tree + back (`retract_to`) over a range that a flush may have published would leave + stale, never-rewritten tiles. `TiledTreeT::retract_to` therefore throws if the + resulting size is below `immutable_size()`. A failed flush may advance + `immutable_size()` without advancing `flushed_size()`; retry with the same + tree state. Retracting the underlying tree directly via `tree_ref()` bypasses + this guard, can make the size boundaries inconsistent or non-monotonic, and + must be avoided. Files written through `store_ref()` are trusted without + checking that they match the tree and can invalidate proofs after compaction. +- **No internal synchronization.** Every tiled-storage object and shared store + prefix requires external serialization. This includes `const` proof reads, + which update the tile cache. +- **Very large indices.** Index math uses `uint64_t`; encoding handles + multi-group indices. Level bound `<= 63` per spec (8 suffices for `2^64`). + Resume scans are bounded by the requested tree size and cannot follow sparse + files beyond that range. diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 6957da3..c1a7e82 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -256,11 +256,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::vector bytes = read_file(path, expected); if (bytes.size() != expected) { - throw std::runtime_error(std::format( - "unexpected tile size for {}: expected {} bytes, got {}", - path.string(), - expected, - bytes.size())); + throw std::runtime_error( + std::format( + "unexpected tile size for {}: expected {} bytes, got {}", + path.string(), + expected, + bytes.size())); } std::vector hashes; @@ -323,8 +324,9 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } catch (const std::runtime_error& error) { - throw std::runtime_error(std::format( - "invalid entry bundle {}: {}", path.string(), error.what())); + throw std::runtime_error( + std::format( + "invalid entry bundle {}: {}", path.string(), error.what())); } } @@ -485,6 +487,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) throw std::runtime_error( "hash algorithm short name does not match hash size"); } + prefix = std::filesystem::absolute(prefix); prefix /= storage_directory_name(hash_algorithm_short_name); return prefix; } @@ -604,10 +607,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const bool exists = std::filesystem::exists(directory, ec); if (ec) { - throw std::runtime_error(std::format( - "cannot inspect directory {}: {}", - directory.string(), - ec.message())); + throw std::runtime_error( + std::format( + "cannot inspect directory {}: {}", + directory.string(), + ec.message())); } if (exists) { @@ -615,16 +619,19 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::filesystem::is_directory(directory, ec); if (ec) { - throw std::runtime_error(std::format( - "cannot inspect directory {}: {}", - directory.string(), - ec.message())); + throw std::runtime_error( + std::format( + "cannot inspect directory {}: {}", + directory.string(), + ec.message())); } if (!is_directory) { - throw std::runtime_error(std::format( - "cannot create directory {}: path exists and is not a directory", - directory.string())); + throw std::runtime_error( + std::format( + "cannot create directory {}: path exists and is not a " + "directory", + directory.string())); } } else @@ -632,10 +639,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const bool created = std::filesystem::create_directory(directory, ec); if (ec) { - throw std::runtime_error(std::format( - "cannot create directory {}: {}", - directory.string(), - ec.message())); + throw std::runtime_error( + std::format( + "cannot create directory {}: {}", + directory.string(), + ec.message())); } if (!created && !std::filesystem::is_directory(directory, ec)) { @@ -1041,8 +1049,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Resolves a complete subtree known to lie within the full-tile /// prefix, reading the highest-level full tile that holds it (and rolling - /// up); descends to lower full tiles when a higher-level full tile has not - /// completed. Terminates because full level-0 tiles always cover the + /// up); descends to lower full tiles when a higher-level full tile has + /// not completed. Terminates because full level-0 tiles always cover the /// prefix. void resolve(uint8_t level, uint64_t index, Hash& out) const { @@ -1061,8 +1069,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const uint8_t r = level % TILE_HEIGHT; const uint64_t first = index << r; // first level-L entry const uint64_t n = first / TILE_WIDTH; // level-L tile index - const unsigned full_shift = static_cast(TILE_HEIGHT) * - (static_cast(L) + 1U); + const unsigned full_shift = + static_cast(TILE_HEIGHT) * (static_cast(L) + 1U); const uint64_t full_tiles = full_shift >= 64 ? 0 : (available_size >> full_shift); @@ -1118,8 +1126,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// HASH_FUNCTION, so an inclusion proof is byte-identical to the one /// produced by merkle::TreeT::path()/past_path() and verifies with /// PathT::verify(). - /// @warning Thread safety is inherited from the supplied HashSourceT. Callers - /// must serialize operations when the source is shared. + /// @warning Thread safety is inherited from the supplied HashSourceT. + /// Callers must serialize operations when the source is shared. template < size_t HASH_SIZE, void HASH_FUNCTION( @@ -1486,6 +1494,322 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const Source& secondary; }; + /// @brief A merkle tree backed by tlog-tiles storage. + /// @note Appends grow an in-memory tree; flush() durably writes only full + /// (balanced) tiles, so the incomplete frontier is never tiled and stays + /// resident in memory. Compaction (dropping from memory the leaves already + /// covered by a full tile) is optional: enable it per flush with + /// Config::compact_on_flush, or call compact() explicitly; it never drops + /// the un-tiled frontier. Proofs are served from the combination of the + /// resident tree (frontier) and the full tiles (compacted past). + /// @note TiledTree creates a new tiled tree and cannot reopen one from tile + /// files alone. Construction atomically claims a previously absent tile + /// namespace because the files do not identify their tree or record enough + /// state to restore it. Use TileWriter directly only when the caller owns + /// and restores that state. + /// @warning No internal synchronization is provided. Callers must serialize + /// all access to a shared tree, including proof operations. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TiledTreeT + { + public: + using Hash = HashT; + using Tree = TreeT; + using Path = PathT; + using Store = TileStoreT; + using Writer = TileWriterT; + using Stats = typename Writer::Stats; + + /// @brief Configuration for a tiled tree. + struct Config + { + /// @brief Root directory for a new tiled tree. + /// @note The directory itself may exist, but its algorithm-qualified + /// tile subdirectory must be absent. + std::filesystem::path prefix; + + /// @brief Number of most-recent leaves to keep resident when + /// compacting (i.e. a minimum never dropped from memory). + /// @note Tile alignment may retain up to TILE_WIDTH - 1 additional + /// tiled leaves. A zero margin retains one tiled boundary leaf so + /// rollback to exactly immutable_size() remains possible. + uint64_t retention_margin = 0; + + /// @brief If set, flush() compacts after writing tiles, dropping + /// from memory the leaves already covered by a full tile. Off by + /// default: tiles are written but the tree keeps every leaf resident. + bool compact_on_flush = false; + }; + + explicit TiledTreeT(Config config) : + config(std::move(config)), store(this->config.prefix), writer(store) + { + claim_tile_namespace(); + } + + /// @brief Constructs a tiled tree with an explicit storage namespace. + /// @note Required for custom hash functions whose algorithm name cannot + /// be inferred by TileStoreT. + TiledTreeT(Config config, const std::string& hash_algorithm_short_name) : + config(std::move(config)), + store(this->config.prefix, hash_algorithm_short_name), + writer(store) + { + claim_tile_namespace(); + } + + TiledTreeT(const TiledTreeT&) = delete; + TiledTreeT& operator=(const TiledTreeT&) = delete; + + /// @brief Moves a tiled tree, rebinding its writer to the moved store. + TiledTreeT(TiledTreeT&& other) noexcept : + config(std::move(other.config)), + store(std::move(other.store)), + writer(store), + tree(std::move(other.tree)), + tiles_size(std::exchange(other.tiles_size, 0)), + sealed_size(std::exchange(other.sealed_size, 0)) + {} + + TiledTreeT& operator=(TiledTreeT&&) = delete; + + /// @brief Appends a leaf hash. + void append(const Hash& leaf_hash) + { + tree.insert(leaf_hash); + } + + /// @brief The number of leaves (including flushed ones). + [[nodiscard]] uint64_t size() const + { + return tree.num_leaves(); + } + + /// @brief The current Merkle root. + Hash root() + { + return tree.root(); + } + + /// @brief The number of leaves covered by the last fully successful + /// flush. + /// @note This is always a multiple of TILE_WIDTH. It advances only after + /// every required tile level has been written successfully, and controls + /// proof reads and compaction. + [[nodiscard]] uint64_t flushed_size() const + { + return tiles_size; + } + + /// @brief The rollback seal for ranges a flush may have published. + /// @note A flush seals its full-tile boundary before writing. If the + /// write fails, this may exceed flushed_size(); keep the same tree + /// contents and retry the flush. + [[nodiscard]] uint64_t immutable_size() const + { + return sealed_size; + } + + /// @brief Access to the underlying tree. + /// @warning Mutating the tree directly bypasses tiled-tree bookkeeping. + /// In particular, direct retraction can make flushed_size() and + /// immutable_size() exceed size() and can make flushed_size() regress. + /// Use TiledTreeT operations whenever they are available. + Tree& tree_ref() + { + return tree; + } + + /// @brief Access to the underlying tile store. + /// @warning Files written or changed through this reference are trusted + /// by later flushes without checking that their hashes match this tree. + /// Mismatched files can silently invalidate proofs after compaction. + Store& store_ref() + { + return store; + } + + /// @brief Writes newly-complete full tiles to disk; compacts only if + /// Config::compact_on_flush is set. + /// @return Counts of the full tiles written by this flush + /// @note The full-tile boundary is made immutable before any tile write. + /// Only after every required tile level succeeds does flushed_size() + /// advance to that boundary. On failure, immutable_size() may advance + /// while flushed_size() does not; the tree remains resident and the flush + /// can be retried without rewriting finalized tiles. + Stats flush() + { + Stats stats; + const uint64_t n = tree.num_leaves(); + if (n == 0) + { + return stats; + } + + const uint64_t covered = (n / TILE_WIDTH) * TILE_WIDTH; + if (covered > sealed_size) + { + sealed_size = covered; + } + + stats = writer.write_up_to(n, [this](uint64_t i) -> const Hash& { + return tree.leaf((size_t)i); + }); + tiles_size = covered; + + if (config.compact_on_flush) + { + compact(); + } + return stats; + } + + /// @brief Drops old leaves covered by durably-written full tiles, keeping + /// at least retention_margin recent leaves and a tiled boundary leaf. + /// @return The new minimum (smallest still-resident) leaf index + /// @note Only leaves covered by a full tile are dropped, so the un-tiled + /// frontier is always retained in memory and inclusion/consistency proofs + /// remain available (the past from tiles, the frontier from memory). The + /// leaf at flushed_size() - 1 also remains resident so retract_to() can + /// represent a tree whose size is exactly immutable_size(). Has no effect + /// until tiling has produced full tiles. + uint64_t compact() + { + const uint64_t covered = (tiles_size / TILE_WIDTH) * TILE_WIDTH; + uint64_t target = covered > config.retention_margin ? + covered - config.retention_margin : + 0; + target = (target / TILE_WIDTH) * TILE_WIDTH; + // TreeT cannot retract below min_index(). Keep the final tiled leaf + // resident so rollback to a size of exactly immutable_size() remains + // representable after compaction. + if (covered > 0 && target == covered) + { + target--; + } + if (target > tree.min_index()) + { + tree.flush_to((size_t)target); + } + return tree.min_index(); + } + + /// @brief Rolls the tree back so that @p index becomes the last leaf, + /// removing all leaves after it (same semantics as TreeT::retract_to). + /// @note Only full tiles are immutable: this throws if the resulting size + /// would be smaller than immutable_size(). A failed flush may advance + /// immutable_size() without advancing flushed_size(). + void retract_to(size_t index) + { + if (sealed_size > 0 && (uint64_t)index < sealed_size - 1) + { + throw std::runtime_error( + "TiledTree::retract_to: cannot roll back entries sealed for " + "immutable tiles (resulting size < immutable size)"); + } + tree.retract_to(index); + } + + /// @brief Inclusion proof for @p index in a tree of @p proof_size leaves. + /// @note Served from tiles (flushed past) combined with the resident tree + /// (recent frontier); @p proof_size may exceed flushed_size(). + std::shared_ptr inclusion_proof(uint64_t index, uint64_t proof_size) + { + if (proof_size > size()) + { + throw std::runtime_error( + "inclusion proof size exceeds current tree size"); + } + return with_engine([&](const auto& engine) { + return engine.inclusion_proof(index, proof_size); + }); + } + + /// @brief Consistency proof between tree sizes @p m and @p n. + std::vector consistency_proof(uint64_t m, uint64_t n) + { + if (n > size()) + { + throw std::runtime_error( + "consistency proof size exceeds current tree size"); + } + return with_engine( + [&](const auto& engine) { return engine.consistency_proof(m, n); }); + } + + /// @brief Consistency proof between the trees whose last leaves are at + /// indices @p first_index and @p second_index (first_index <= + /// second_index). + /// @note Equivalent to consistency_proof(first_index + 1, + /// second_index + 1). + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) + { + if (first_index >= size() || second_index >= size()) + { + throw std::runtime_error( + "consistency proof index exceeds current tree size"); + } + return with_engine([&](const auto& engine) { + return engine.consistency_proof_from_indices( + first_index, second_index); + }); + } + + protected: + Config config; + Store store; + Writer writer; + Tree tree; + uint64_t tiles_size = 0; + uint64_t sealed_size = 0; + + void claim_tile_namespace() const + { + const auto tile_root = store.root() / "tile"; + std::error_code ec; + std::filesystem::create_directories(tile_root.parent_path(), ec); + if (ec) + { + throw std::runtime_error( + "TiledTree: cannot create tile namespace parent " + + tile_root.parent_path().string() + ": " + ec.message()); + } + const bool claimed = std::filesystem::create_directory(tile_root, ec); + if (ec) + { + throw std::runtime_error( + "TiledTree: cannot claim tile namespace " + tile_root.string() + + ": " + ec.message()); + } + if (!claimed) + { + throw std::runtime_error( + "TiledTree: tile namespace already exists; reopening or sharing " + "a tiled tree is not supported"); + } + } + + /// @brief Builds a proof engine over the combined resident-tree + /// (frontier) and full-tile (flushed past) source, and invokes @p fn with + /// it. + /// @note The sources and engine are stack-local; @p fn must consume the + /// engine before returning (proofs are returned by value, holding hash + /// copies, so the result outlives the engine). + template + auto with_engine(Fn fn) + { + MemoryHashSourceT mem(tree); + TileHashSourceT tile_src(store, tiles_size); + CombinedHashSourceT combined(mem, tile_src); + ProofEngineT engine(combined); + return fn(engine); + } + }; + /// @brief Writes tlog-tiles entry bundles (raw log entries) for a growing /// log. /// @note Entry bundles are level-0 only and application-owned: merklecpp @@ -1613,10 +1937,51 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) merkle::Tree::Hash::size_bytes, merkle::Tree::hash_function>; + /// @brief Default tiled tree (SHA256, default hash function). + using TiledTree = + TiledTreeT; + /// @brief Default entry-bundle writer (SHA256, default hash function). using EntryBundleWriter = EntryBundleWriterT< merkle::Tree::Hash::size_bytes, merkle::Tree::hash_function>; +#ifdef HAVE_OPENSSL + /// @brief SHA384 tile store. + using TileStore384 = TileStoreT<48, sha384_openssl>; + + /// @brief SHA512 tile store. + using TileStore512 = TileStoreT<64, sha512_openssl>; + + /// @brief SHA384 tile writer. + using TileWriter384 = TileWriterT<48, sha384_openssl>; + + /// @brief SHA512 tile writer. + using TileWriter512 = TileWriterT<64, sha512_openssl>; + + /// @brief SHA384 hash source, tile-backed source and proof engine. + using HashSource384 = HashSourceT<48, sha384_openssl>; + using TileHashSource384 = TileHashSourceT<48, sha384_openssl>; + using ProofEngine384 = ProofEngineT<48, sha384_openssl>; + + /// @brief SHA512 hash source, tile-backed source and proof engine. + using HashSource512 = HashSourceT<64, sha512_openssl>; + using TileHashSource512 = TileHashSourceT<64, sha512_openssl>; + using ProofEngine512 = ProofEngineT<64, sha512_openssl>; + + /// @brief SHA384 memory/combined sources and tiled tree. + using MemoryHashSource384 = MemoryHashSourceT<48, sha384_openssl>; + using CombinedHashSource384 = CombinedHashSourceT<48, sha384_openssl>; + using TiledTree384 = TiledTreeT<48, sha384_openssl>; + + /// @brief SHA512 memory/combined sources and tiled tree. + using MemoryHashSource512 = MemoryHashSourceT<64, sha512_openssl>; + using CombinedHashSource512 = CombinedHashSourceT<64, sha512_openssl>; + using TiledTree512 = TiledTreeT<64, sha512_openssl>; + + /// @brief SHA384/512 entry-bundle writers. + using EntryBundleWriter384 = EntryBundleWriterT<48, sha384_openssl>; + using EntryBundleWriter512 = EntryBundleWriterT<64, sha512_openssl>; +#endif } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e555eee..7648f61 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -40,8 +40,13 @@ add_merklecpp_test(coverage coverage.cpp) add_merklecpp_test(tiles_store tiles_store.cpp) add_merklecpp_test(tiles_writer tiles_writer.cpp) add_merklecpp_test(tiles_proofs tiles_proofs.cpp) +add_merklecpp_test(tiles_tree tiles_tree.cpp) add_merklecpp_test(tiles_entries tiles_entries.cpp) +if(OPENSSL) + add_merklecpp_test(tiles_hashes tiles_hashes.cpp) +endif() + if(LONG_TESTS) add_merklecpp_test(tiles_level2 tiles_level2.cpp) set(TILES_LEVEL2_TIMEOUT 900) diff --git a/test/tiles_hashes.cpp b/test/tiles_hashes.cpp new file mode 100644 index 0000000..8ebb6a5 --- /dev/null +++ b/test/tiles_hashes.cpp @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +template < + size_t HASH_SIZE, + typename Tree, + typename TiledTree, + typename ProofEngine> +static void exercise_tiled_hash(const fs::path& prefix, const std::string& name) +{ + const auto hashes = make_hashesT(300); + Tree reference; + typename TiledTree::Config config; + config.prefix = prefix; + config.compact_on_flush = true; + TiledTree tree(config); + + for (const auto& hash : hashes) + { + reference.insert(hash); + tree.append(hash); + } + + expect(tree.flush().full_written == 1, name + " full tile written"); + expect(tree.flushed_size() == 256, name + " flushed size"); + expect(tree.root() == reference.root(), name + " root"); + expect( + fs::file_size(tree.store_ref().root() / "tile" / "0" / "000") == + (uintmax_t)merkle::tiles::TILE_WIDTH * HASH_SIZE, + name + " tile byte size"); + + const auto inclusion = tree.inclusion_proof(0, tree.size()); + expect(inclusion->verify(reference.root()), name + " inclusion proof"); + + const auto consistency = tree.consistency_proof(256, tree.size()); + expect( + ProofEngine::verify_consistency( + 256, + tree.size(), + *reference.past_root(255), + reference.root(), + consistency), + name + " consistency proof"); +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_hashes"); + const fs::path& base = temporary_directory.path(); + + try + { + exercise_tiled_hash< + 48, + merkle::Tree384, + merkle::tiles::TiledTree384, + merkle::tiles::ProofEngine384>(base / "sha384", "SHA384"); + exercise_tiled_hash< + 64, + merkle::Tree512, + merkle::tiles::TiledTree512, + merkle::tiles::ProofEngine512>(base / "sha512", "SHA512"); + + std::cout << "tiles_hashes: OK" << '\n'; + } + catch (const std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + return 1; + } + + return 0; +} diff --git a/test/tiles_tree.cpp b/test/tiles_tree.cpp new file mode 100644 index 0000000..5ec17ae --- /dev/null +++ b/test/tiles_tree.cpp @@ -0,0 +1,805 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::ProofEngine; +using merkle::tiles::TiledTree; +using merkle::tiles::TileStore; + +static void custom_hash(const Hash& left, const Hash& right, Hash& out) +{ + merkle::Tree::hash_function(left, right, out); +} + +using CustomTiledTree = + merkle::tiles::TiledTreeT; + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(!std::is_move_assignable_v); + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static fs::path store_root(const fs::path& prefix) +{ + return prefix / TileStore::storage_directory_name("sha256"); +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_tree"); + const fs::path& base = temporary_directory.path(); + + try + { + const auto hashes = make_hashes(1500); + + // ---- Part 0: the empty tree. flush()/compact() are no-ops and there is no + // root. + { + TiledTree::Config ecfg; + ecfg.prefix = base / "tt_empty"; + TiledTree ett(ecfg); + expect(ett.size() == 0, "empty: size 0"); + expect(ett.flush().full_written == 0, "empty: flush writes nothing"); + expect(ett.flushed_size() == 0, "empty: flushed size 0"); + expect(ett.immutable_size() == 0, "empty: immutable size 0"); + expect(ett.compact() == 0, "empty: compact no-op"); + bool ethrew = false; + try + { + (void)ett.root(); + } + catch (const std::exception&) + { + ethrew = true; + } + expect(ethrew, "empty: root throws"); + std::cout << "empty tree: OK" << '\n'; + } + + // ---- Part 0b: moving a tree rebinds its writer to the destination store. + { + const fs::path previous_cwd = fs::current_path(); + try + { + fs::create_directories(base); + fs::current_path(base); + TiledTree::Config move_cfg; + move_cfg.prefix = "tt_move_expected"; + const fs::path expected_prefix = fs::absolute(move_cfg.prefix); + TiledTree source(move_cfg); + for (uint64_t i = 0; i < 300; i++) + { + source.append(hashes[i]); + } + source.flush(); + + const fs::path work = base / "tt_move_cwd"; + fs::create_directories(work); + fs::current_path(work); + TiledTree moved(std::move(source)); + expect(moved.flushed_size() == 256, "move: flushed size retained"); + expect(moved.immutable_size() == 256, "move: immutable size retained"); + expect( + moved.store_ref().root() == store_root(expected_prefix), + "move: destination store retained"); + for (uint64_t i = 300; i < 512; i++) + { + moved.append(hashes[i]); + } + expect(moved.flush().full_written == 1, "move: next tile written"); + expect( + fs::is_regular_file(store_root(expected_prefix) / "tile/0/001"), + "move: tile written to configured store"); + expect( + !fs::exists(store_root(work) / "tile/0/001"), + "move: no tile written relative to current directory"); + } + catch (...) + { + fs::current_path(previous_cwd); + throw; + } + fs::current_path(previous_cwd); + std::cout << "move construction: OK" << '\n'; + } + + // ---- Part 0c: a TiledTree atomically claims a fresh namespace, so another + // tree cannot pass construction before the first one writes tiles. + { + TiledTree::Config existing_cfg; + existing_cfg.prefix = base / "tt_existing"; + TiledTree first(existing_cfg); + + bool construction_threw = false; + try + { + const TiledTree second(existing_cfg); + } + catch (const std::exception&) + { + construction_threw = true; + } + expect( + construction_threw, + "existing prefix: second tree rejects claimed namespace"); + for (uint64_t i = 0; i < 256; i++) + { + first.append(hashes[i]); + } + expect( + first.flush().full_written == 1, + "existing prefix: owning tree writes tile"); + std::cout << "exclusive tile namespace: OK" << '\n'; + } + + // ---- Part 0d: generic tiled trees accept the explicit storage namespace + // required by custom hash functions. + { + CustomTiledTree::Config custom_cfg; + custom_cfg.prefix = base / "tt_custom"; + CustomTiledTree custom(custom_cfg, "custom-sha256"); + merkle::Tree reference; + for (uint64_t i = 0; i < 300; i++) + { + custom.append(hashes[i]); + reference.insert(hashes[i]); + } + expect( + custom.flush().full_written == 1, "custom hash: full tile written"); + expect(custom.root() == reference.root(), "custom hash: root matches"); + expect( + custom.store_ref().root() == + fs::absolute(custom_cfg.prefix) / + TileStore::storage_directory_name("custom-sha256"), + "custom hash: explicit namespace used"); + std::cout << "custom hash namespace: OK" << '\n'; + } + + // ---- Part 0e: proof requests cannot describe a state beyond the current + // tree, even when the lower proof engine would not read its source. + { + TiledTree::Config bounds_cfg; + bounds_cfg.prefix = base / "tt_bounds"; + TiledTree bounded(bounds_cfg); + bounded.append(hashes[0]); + + bool threw = false; + try + { + (void)bounded.consistency_proof(2, 2); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "proof bounds: reject future equal sizes"); + + threw = false; + try + { + (void)bounded.inclusion_proof(0, 2); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "proof bounds: reject future inclusion size"); + + threw = false; + try + { + const auto maximum = std::numeric_limits::max(); + (void)bounded.consistency_proof_from_indices(maximum, maximum); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "proof bounds: reject overflowing indices"); + std::cout << "proof bounds: OK" << '\n'; + } + + // ---- Part 2: TiledTree flush, proofs over tiles + memory. + const uint64_t n1 = 1000; // first flush size + const uint64_t N = 1500; // final size + + // Reference: a plain (never flushed) tree with the same leaves. + merkle::Tree ref; + for (uint64_t i = 0; i < N; i++) + { + ref.insert(hashes[i]); + } + const Hash ref_root = ref.root(); + + // Default mode: flush() writes tiles but drops nothing from memory; + // an explicit compact() drops only the leaves already covered by a tile. + { + TiledTree::Config dcfg; + dcfg.prefix = base / "tt_default"; + TiledTree dtt(dcfg); + for (uint64_t i = 0; i < n1; i++) + { + dtt.append(hashes[i]); + } + dtt.flush(); + expect(dtt.tree_ref().min_index() == 0, "default: nothing dropped"); + dtt.compact(); + expect( + dtt.tree_ref().min_index() == 767, "compact() keeps boundary leaf"); + } + + TiledTree::Config cfg; + cfg.prefix = base / "tt"; + cfg.retention_margin = 0; + cfg.compact_on_flush = true; + TiledTree tt(cfg); + + for (uint64_t i = 0; i < n1; i++) + { + tt.append(hashes[i]); + } + tt.flush(); // flushes full tiles; flushed_size = covered = 768 + expect(tt.flushed_size() == 768, "flushed size"); + expect(tt.tree_ref().min_index() == 767, "compacted with boundary overlap"); + + for (uint64_t i = n1; i < N; i++) + { + tt.append(hashes[i]); + } + expect(tt.size() == N, "size after appends"); + expect(tt.root() == ref_root, "tiled root == reference root"); + + // Indices that are: flushed (tiles only), in the flushed-but-resident + // overlap, and on the un-flushed resident frontier. + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)767, + (uint64_t)800, + (uint64_t)999, + (uint64_t)1000, + (uint64_t)1499}) + { + const auto p = tt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "combined inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "combined inclusion verify i=" + std::to_string(i)); + } + + // The resident tree alone cannot prove a flushed index. + bool threw = false; + try + { + (void)tt.tree_ref().path(0); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "memory-only path throws for flushed index"); + + // Consistency across the flush boundary: flushed size -> current size, + // and a flushed-era size -> current size. + { + const auto cp = tt.consistency_proof(n1, N); + expect( + ProofEngine::verify_consistency( + n1, N, *ref.past_root(n1 - 1), ref_root, cp), + "consistency n1->N across flush"); + + const uint64_t m = 500; // flushed era + const auto cp2 = tt.consistency_proof(m, N); + expect( + ProofEngine::verify_consistency( + m, N, *ref.past_root(m - 1), ref_root, cp2), + "consistency m->N across flush"); + + // Index-based variant: consistency_proof_from_indices(i, j) == + // consistency_proof(i + 1, j + 1), spanning tiles and the live tree. + const auto cpi = tt.consistency_proof_from_indices(m - 1, N - 1); + expect(cpi == cp2, "consistency index variant across flush"); + expect( + ProofEngine::verify_consistency( + m, N, *ref.past_root(m - 1), ref_root, cpi), + "consistency index variant verifies"); + } + + // A second flush writes further tiles; proofs for now-flushed indices still + // work (this also confirms the writer never reads a flushed leaf). + tt.flush(); // flushes full tiles; flushed_size = covered = 1280 + expect(tt.flushed_size() == 1280, "second flushed size"); + expect(tt.tree_ref().min_index() == 1279, "second boundary leaf retained"); + + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)1000, + (uint64_t)1279, + (uint64_t)1280, + (uint64_t)1499}) + { + const auto p = tt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "post-2nd inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "post-2nd inclusion verify i=" + std::to_string(i)); + } + + std::cout << "tiled tree (flush + combination): OK" << '\n'; + + // ---- Part 2b: compaction when the size is an exact multiple of TILE_WIDTH + // (here 512). flush_to cannot drain the whole tree, so compact() keeps + // one resident leaf rather than throwing; proofs still resolve. + { + TiledTree::Config mcfg; + mcfg.prefix = base / "tt_multiple"; + mcfg.compact_on_flush = true; + TiledTree mtt(mcfg); + for (uint64_t i = 0; i < 512; i++) + { + mtt.append(hashes[i]); + } + mtt.flush(); // covered == size == 512; compaction must not throw + expect(mtt.flushed_size() == 512, "multiple: flushed size 512"); + expect(mtt.tree_ref().min_index() == 511, "multiple: one leaf retained"); + + merkle::Tree mref; + for (uint64_t i = 0; i < 512; i++) + { + mref.insert(hashes[i]); + } + const Hash mroot = mref.root(); + expect(mtt.root() == mroot, "multiple: root matches reference"); + for (const uint64_t i : {(uint64_t)0, (uint64_t)256, (uint64_t)511}) + { + const auto p = mtt.inclusion_proof(i, 512); + expect(*p == *mref.path(i), "multiple: inclusion==ref"); + expect(p->verify(mroot), "multiple: inclusion verify"); + } + std::cout << "tiled tree (exact-multiple compaction): OK" << '\n'; + } + + // ---- Part 2c: compaction with a non-zero retention_margin keeps the most + // recent leaves resident while flushed indices are still served from + // tiles. The immutable prefix remains the full-tile prefix, regardless + // of the margin. + { + TiledTree::Config rcfg; + rcfg.prefix = base / "tt_margin"; + rcfg.retention_margin = 300; + rcfg.compact_on_flush = true; + TiledTree rtt(rcfg); + for (uint64_t i = 0; i < n1; i++) // n1 == 1000 + { + rtt.append(hashes[i]); + } + rtt.flush(); // covered = 768; target = floor((768 - 300) / 256) * 256 = + // 256 + expect(rtt.flushed_size() == 768, "margin: flushed size 768"); + expect( + rtt.tree_ref().min_index() == 256, + "margin: retained >= 300 recent leaves"); + + for (uint64_t i = n1; i < N; i++) + { + rtt.append(hashes[i]); + } + expect(rtt.root() == ref_root, "margin: root matches reference"); + + // Flushed-only (0, 255), flushed-but-resident overlap (256, 767), and the + // un-flushed frontier (1000, 1499). + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)255, + (uint64_t)256, + (uint64_t)767, + (uint64_t)1000, + (uint64_t)1499}) + { + const auto p = rtt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "margin inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "margin inclusion verify i=" + std::to_string(i)); + } + + bool threw = false; + try + { + rtt.retract_to(700); // size 701 < immutable_size 768 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "margin: retract below flushed prefix throws"); + + std::cout << "tiled tree (retention margin): OK" << '\n'; + } + + // ---- Part 3: rollback. Tiles are immutable, so only un-tiled + // (post-flush) entries may be rolled back. + { + // 3a. Before any flush nothing is tiled, so rollback is + // unrestricted. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_pre"; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 50; i++) + { + rb.append(hashes[i]); + } + rb.retract_to(29); // tiles_size == 0 -> allowed + expect(rb.size() == 30, "rb pre-flush retract allowed"); + } + + // 3b. After a flush, the un-tiled frontier can be rolled back and + // the + // tiled region stays consistent and provable; committed entries + // can't. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb"; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 300; i++) + { + rb.append(hashes[i]); + } + rb.flush(); // flushed_size = covered = 256 + for (uint64_t i = 300; i < 400; i++) + { + rb.append(hashes[i]); + } + rb.retract_to(349); // keep [0,349] + expect(rb.size() == 350, "rb retracted to 350"); + for (uint64_t i = 350; i < 400; i++) + { + rb.append(hashes[1000 + i]); // re-append DIFFERENT leaves + } + rb.flush(); // flushed_size = covered = 256 (tile [0,256) already + // written) + + // Reference tree of the exact post-rollback state. + merkle::Tree exp_tree; + for (uint64_t i = 0; i < 350; i++) + { + exp_tree.insert(hashes[i]); + } + for (uint64_t i = 350; i < 400; i++) + { + exp_tree.insert(hashes[1000 + i]); + } + const Hash exp_root = exp_tree.root(); + expect(rb.root() == exp_root, "rb root matches reference"); + + // Proofs for a tiled index and a frontier index match the reference. + for (const uint64_t i : {(uint64_t)100, (uint64_t)299, (uint64_t)399}) + { + const auto p = rb.inclusion_proof(i, 400); + expect( + *p == *exp_tree.path(i), + "rb inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(exp_root), "rb inclusion verify i=" + std::to_string(i)); + } + const auto cp = rb.consistency_proof(300, 400); + expect( + ProofEngine::verify_consistency( + 300, 400, *exp_tree.past_root(299), exp_root, cp), + "rb consistency 300->400"); + + // Only the immutable full-tile prefix [0,256) is protected: rolling + // back into it is refused, while rolling back within the un-tiled + // frontier (>= immutable_size()) is allowed. + expect(rb.flushed_size() == 256, "rb flushed to full-tile prefix"); + + rb.retract_to(std::numeric_limits::max()); + expect(rb.size() == 400, "rb past-current index is a no-op"); + + bool threw = false; + try + { + rb.retract_to(100); // size 101 < 256 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb retract below tiled prefix throws"); + threw = false; + try + { + rb.retract_to(254); // size 255 < 256 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb retract just below tiled prefix throws"); + rb.retract_to(255); // size 256 == immutable_size(): drops only frontier + expect(rb.size() == 256, "rb retract to tiled prefix allowed"); + } + + // 3c. Compaction keeps the final tiled leaf resident, so rollback to a + // tree size of exactly immutable_size() remains representable. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_exact_boundary"; + cfg.compact_on_flush = true; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 1000; i++) + { + rb.append(hashes[i]); + } + rb.flush(); + expect(rb.flushed_size() == 768, "rb boundary flushed size"); + expect(rb.immutable_size() == 768, "rb boundary immutable size"); + expect( + rb.tree_ref().min_index() == 767, + "rb boundary final tiled leaf retained"); + + rb.retract_to(767); + expect(rb.size() == 768, "rb exact immutable boundary allowed"); + + merkle::Tree expected; + for (uint64_t i = 0; i < 768; i++) + { + expected.insert(hashes[i]); + } + const Hash expected_root = expected.root(); + expect( + rb.root() == expected_root, "rb boundary root matches reference"); + const auto proof = rb.inclusion_proof(0, 768); + expect( + *proof == *expected.path(0), + "rb boundary inclusion matches reference"); + expect(proof->verify(expected_root), "rb boundary inclusion verifies"); + } + + // 3d. Rollback interacts correctly with compaction (flushed + tiled + // past). + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_compact"; + cfg.compact_on_flush = true; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 1000; i++) + { + rb.append(hashes[i]); + } + rb.flush(); // flushed_size = covered = 768; retain boundary leaf 767 + expect( + rb.tree_ref().min_index() == 767, + "rb compact boundary leaf retained"); + for (uint64_t i = 1000; i < 1200; i++) + { + rb.append(hashes[i]); + } + // Frontier rollback (>= immutable_size()) is allowed. + rb.retract_to(1099); + expect(rb.size() == 1100, "rb compact frontier retract ok"); + + merkle::Tree exp_tree; + for (uint64_t i = 0; i < 1100; i++) + { + exp_tree.insert(hashes[i]); + } + const Hash exp_root = exp_tree.root(); + const auto p = rb.inclusion_proof(0, 1100); // leaf 0 is flushed + tiled + expect(*p == *exp_tree.path(0), "rb compact flushed inclusion==ref"); + expect(p->verify(exp_root), "rb compact flushed inclusion verify"); + + bool threw = false; + try + { + rb.retract_to(500); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb compact retract below tiled throws"); + } + + // 3e. A failed flush may publish an immutable full tile before a later + // write fails. The attempted full-tile boundary is sealed against + // rollback, while flushed_size advances only after a successful retry. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_interrupted"; + TiledTree interrupted(cfg); + for (uint64_t i = 0; i < 512; i++) + { + interrupted.append(hashes[i]); + } + + const fs::path blocker = store_root(cfg.prefix) / "tile/0/001"; + fs::create_directories(blocker); + bool flush_threw = false; + try + { + interrupted.flush(); + } + catch (const std::exception&) + { + flush_threw = true; + } + + expect(flush_threw, "interrupted flush throws"); + expect( + fs::is_regular_file(store_root(cfg.prefix) / "tile/0/000"), + "interrupted flush published first full tile"); + expect( + interrupted.flushed_size() == 0, + "interrupted flush does not advance flushed size"); + expect( + interrupted.immutable_size() == 512, + "interrupted flush seals attempted boundary"); + expect( + interrupted.compact() == 0, + "interrupted flush does not permit compaction"); + + TiledTree recovered(std::move(interrupted)); + expect( + recovered.immutable_size() == 512, + "interrupted flush seal survives move"); + + bool retract_threw = false; + try + { + recovered.retract_to(0); + } + catch (const std::exception&) + { + retract_threw = true; + } + expect( + retract_threw, + "interrupted flush rejects rollback across attempted tiles"); + expect(recovered.size() == 512, "interrupted rollback changes nothing"); + + fs::remove(blocker); + expect( + recovered.flush().full_written == 1, + "interrupted flush retry writes only missing tile"); + expect( + recovered.flushed_size() == 512, + "interrupted flush retry advances flushed size"); + expect( + recovered.immutable_size() == 512, + "interrupted flush retry preserves immutable size"); + recovered.compact(); + + merkle::Tree expected; + for (uint64_t i = 0; i < 512; i++) + { + expected.insert(hashes[i]); + } + const Hash expected_root = expected.root(); + expect( + recovered.root() == expected_root, + "interrupted flush root matches reference"); + for (const uint64_t i : + {(uint64_t)0, (uint64_t)255, (uint64_t)256, (uint64_t)511}) + { + const auto proof = recovered.inclusion_proof(i, 512); + expect( + *proof == *expected.path(i), + "interrupted flush inclusion matches reference"); + expect( + proof->verify(expected_root), + "interrupted flush inclusion verifies"); + } + } + + // 3f. flushed_size advances only after every required level succeeds. + // Here all 256 level-0 tiles publish before the level-1 tile is blocked. + { + constexpr uint64_t level1_size = + (uint64_t)merkle::tiles::TILE_WIDTH * merkle::tiles::TILE_WIDTH; + const auto level1_hashes = make_hashes((size_t)level1_size); + + TiledTree::Config cfg; + cfg.prefix = base / "rb_interrupted_level1"; + TiledTree interrupted(cfg); + for (const auto& hash : level1_hashes) + { + interrupted.append(hash); + } + + const fs::path blocker = store_root(cfg.prefix) / "tile/1/000"; + fs::create_directories(blocker); + bool flush_threw = false; + try + { + interrupted.flush(); + } + catch (const std::exception&) + { + flush_threw = true; + } + + expect(flush_threw, "level-1 interrupted flush throws"); + expect( + interrupted.store_ref().has_full_tile(0, 255), + "level-1 interrupted flush publishes all level-0 tiles"); + expect( + interrupted.flushed_size() == 0, + "level-1 interrupted flush does not advance flushed size"); + expect( + interrupted.immutable_size() == level1_size, + "level-1 interrupted flush seals attempted boundary"); + + fs::remove(blocker); + expect( + interrupted.flush().full_written == 1, + "level-1 interrupted retry writes only the missing roll-up"); + expect( + interrupted.flushed_size() == level1_size, + "level-1 interrupted retry advances flushed size"); + expect( + interrupted.store_ref().has_full_tile(1, 0), + "level-1 interrupted retry publishes roll-up"); + + merkle::Tree expected; + for (const auto& hash : level1_hashes) + { + expected.insert(hash); + } + const Hash expected_root = expected.root(); + expect( + interrupted.root() == expected_root, + "level-1 interrupted root matches reference"); + const auto proof = interrupted.inclusion_proof(0, level1_size); + expect( + *proof == *expected.path(0), + "level-1 interrupted inclusion matches reference"); + expect( + proof->verify(expected_root), + "level-1 interrupted inclusion verifies"); + } + } + std::cout << "rollback: OK" << '\n'; + + std::cout << "tiles_tree: OK" << '\n'; + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + return 1; + } + + return 0; +} From 1abe9528c6c18ddb7c60e6658368c0c5269bfed7 Mon Sep 17 00:00:00 2001 From: achamayou Date: Mon, 13 Jul 2026 11:51:27 +0000 Subject: [PATCH 12/12] Add tiled-storage user docs and performance benchmark Add the README 'Tiled storage' and 'Building and testing' sections covering the TiledTree usage snippet, the fresh-only/rejection contract, LONG_TESTS, and links to the guide and design doc. Add doc/tiles-guide.md (a how-to for flushing, compaction, rollback, proofs, and the lower-level building blocks) and doc/tiles-illustrated.md (a worked, diagram-led walkthrough of the tiled tree lifecycle). No new storage semantics: these are documentation-only additions over the tiled-storage API completed in earlier branches. Add time_tiles, benchmarking append, flush, in-memory and tile-served inclusion/consistency proofs, and compaction against a plain in-memory Tree baseline; wire it into the existing LONG_TESTS group alongside tiles_level2. Enable the documentation workflow on pull requests (building, but not publishing, the site) so doc changes get functional CI coverage. Finalize the design doc's testing-strategy and build/backwards- compatibility-impact sections, completing the document that has grown incrementally across the previous branches. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build-docs.yml | 5 +- README.md | 63 +++ doc/tiles-guide.md | 325 +++++++++++++ doc/tiles-illustrated.md | 764 +++++++++++++++++++++++++++++++ test/CMakeLists.txt | 2 + test/time_tiles.cpp | 192 ++++++++ 6 files changed, 1350 insertions(+), 1 deletion(-) create mode 100644 doc/tiles-guide.md create mode 100644 doc/tiles-illustrated.md create mode 100644 test/time_tiles.cpp diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 8e089b7..70a4d14 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -4,10 +4,11 @@ on: push: branches: - main + pull_request: workflow_dispatch: concurrency: - group: pages + group: pages-${{ github.ref }} cancel-in-progress: false permissions: read-all @@ -24,6 +25,7 @@ jobs: - name: Setup Pages id: pages uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + if: github.event_name != 'pull_request' - name: Install dependencies run: | @@ -42,6 +44,7 @@ jobs: uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: doc/build + if: github.event_name != 'pull_request' deploy: name: Deploy diff --git a/README.md b/README.md index db01d1c..1009d55 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,69 @@ merklecpp requires C++20. assert(path->verify(root)); +## Tiled storage (tlog-tiles) + +The companion header `merklecpp_tiles.h` adds optional, header-only support for +persisting a tree as [tlog-tiles](https://c2sp.org/tlog-tiles) tile files +*progressively* (optionally dropping already-tiled leaves from memory) and for +retrieving inclusion and consistency proofs from those tiles, from the in-memory +tree, or from a combination of the two. The hashing is unchanged: tiles and tile-derived proofs are templated on +the tree's existing hash function, so a tile-derived inclusion proof is +byte-identical to one from `merkle::Tree::path()` and verifies with the same +`merkle::Path::verify()`. + + #include + + merkle::tiles::TiledTree::Config cfg; + cfg.prefix = "/var/log/mylog"; // tile files live here + cfg.retention_margin = 1024; // keep the most recent leaves in memory + cfg.compact_on_flush = true; // opt in to dropping already-tiled leaves + + merkle::tiles::TiledTree log(cfg); + for (const auto& leaf_hash : batch) + log.append(leaf_hash); + + // Write newly-complete tiles. With compaction enabled + // this also drops from memory the leaves already covered by a full tile; + // otherwise the tree keeps every leaf and you can call log.compact() later. + log.flush(); + + // Proofs are served from tiles + the resident tree, even for flushed leaves. + auto inclusion = log.inclusion_proof(/*index=*/0, log.size()); + assert(inclusion->verify(log.root())); + + auto consistency = log.consistency_proof(/*m=*/100, /*n=*/log.size()); + +`TiledTree` creates a new tiled tree: the configured directory may exist, but +its `tile` subdirectory must be absent or empty. It deliberately rejects +existing tile data because tile files alone do not identify or restore the +tree that produced them. Applications with externally persisted tree state can +use the lower-level `TileStore` and `TileWriter` APIs to resume a store. + +See the [tiled storage guide](doc/tiles-guide.md) for a how-to covering +flushing, compaction, rollback, proofs, and the lower-level building blocks, +and [doc/design/tlog-tiles.md](doc/design/tlog-tiles.md) for the full design, +file/directory layout, and the proof algorithms. + + +## Building and testing + +Enable the test suite with CMake's `TESTS` option: + + cmake -S . -B build -DTESTS=ON + cmake --build build + ctest --test-dir build + +Some tile coverage is intentionally long-running. `LONG_TESTS` is off by +default for local builds; turn it on when you want the full tile stress suite, +including level-2 tile coverage and tile proof timing: + + cmake -S . -B build -DTESTS=ON -DLONG_TESTS=ON + +The repository CI enables `LONG_TESTS` so pull requests continue to exercise the +full tiled-storage matrix. + + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/doc/tiles-guide.md b/doc/tiles-guide.md new file mode 100644 index 0000000..396dbaa --- /dev/null +++ b/doc/tiles-guide.md @@ -0,0 +1,325 @@ +# Tiled storage and proofs — a guide to `merklecpp_tiles.h` + +`merklecpp_tiles.h` is an optional, header-only companion to `merklecpp.h`. It +lets you persist a Merkle tree as a set of immutable **tile files** on disk and +serve **inclusion** and **consistency** proofs from those tiles, from the +in-memory tree, or from a combination of the two — so proofs stay available even +after old entries are dropped from memory. + +It builds on the [tlog-tiles](https://c2sp.org/tlog-tiles) file/directory layout +but is **not** trying to be wire-compatible with external tlog-tiles clients. See +[`design/tlog-tiles.md`](design/tlog-tiles.md) for the design and internals; this +page is a practical how-to. + +## Contents + +- [Requirements and a note on hashing](#requirements-and-a-note-on-hashing) +- [Thread safety](#thread-safety) +- [Quick start: `TiledTree`](#quick-start-tiledtree) +- [Flushing and compaction](#flushing-and-compaction) +- [Rollback](#rollback) +- [Proofs](#proofs) +- [Lower-level building blocks](#lower-level-building-blocks) +- [Entry bundles (optional)](#entry-bundles-optional) +- [On-disk layout](#on-disk-layout) + +## Requirements and a note on hashing + +- C++17 (the header uses `` and small platform-specific file-sync + calls for durable tile writes). +- Include the companion header; it pulls in `merklecpp.h` for you: + + ```cpp + #include + ``` + +- Everything lives in `namespace merkle::tiles` and is templated on the same + `` as your tree. The default aliases + (`merkle::tiles::TiledTree`, `TileStore`, `TileWriter`, `ProofEngine`, …) use + the **same** SHA-256 as `merkle::Tree`, so a tile-derived inclusion proof is + byte-identical to one from `merkle::Tree::path()` and verifies with the usual + `merkle::Path::verify()`. +- You insert **leaf hashes**, not raw entries — exactly like `merkle::Tree`. + Deriving a leaf hash from an entry (e.g. `leaf = H(entry)`) is your + application's job. The tile hash values are whatever your `HASH_FUNCTION` + produces; they are not RFC 6962 unless you instantiate your tree with an + RFC 6962 hash function (not required, and not the goal here). + +## Thread safety + +The tiled-storage API provides no internal synchronization. Treat every +`TileStore`, `TileWriter`, `TileHashSource`, `ProofEngine`, `TiledTree`, and +`EntryBundleWriter` instance as single-threaded. + +If an object or store prefix is shared between threads, the caller must +serialize every operation. This includes methods declared `const`: proof +generation updates the `TileHashSource` LRU cache. The library deliberately +does not add locks or otherwise coordinate concurrent readers and writers. + +## Quick start: `TiledTree` + +`TiledTree` is the high-level wrapper: append leaf hashes, flush them to disk +(which writes tiles), and ask for proofs. + +```cpp +#include + +merkle::tiles::TiledTree::Config cfg; +cfg.prefix = "/var/log/mylog"; // directory for tile files + +merkle::tiles::TiledTree log(cfg); + +// Append leaf hashes (compute these from your entries however you like). +for (const merkle::Hash& leaf : batch) + log.append(leaf); + +// Persist newly-complete tiles to disk. +log.flush(); + +merkle::Hash root = log.root(); // current Merkle root +uint64_t n = log.size(); // number of leaves + +// Inclusion proof for leaf 0 in the tree of `n` leaves. +auto inclusion = log.inclusion_proof(/*index=*/0, /*size=*/n); +assert(inclusion->verify(root)); + +// Consistency proof that size 100 is a prefix of size n. +auto consistency = log.consistency_proof(/*m=*/100, /*n=*/n); +``` + +`TiledTree` can be move-constructed, but it cannot be copied or assigned. Move +construction keeps its writer bound to the destination tree's tile store. + +`TiledTree` always creates a new tiled tree. The configured directory may +already exist, but its `tile` subdirectory must be absent or empty. Construction +throws rather than adopting existing tiles because those files do not identify +the tree that produced them or contain enough state to restore its size and +root. If your application persists and validates that state separately, use +the lower-level `TileStore` and `TileWriter` APIs; `TileWriter` intentionally +resumes existing full tiles and therefore trusts the caller to supply the same +tree and hash function. A fresh writer scans the requested range in order, +stopping at the first missing or malformed file, so an interior hole is +rewritten rather than hidden by later files. + +`flush()` is incremental: each call writes only the full tiles that became +complete since the previous call. Full tiles are immutable: written once after +all 256 entries are final and never rewritten. The remaining frontier stays in +memory until it crosses the next full-tile boundary. + +Tile files are written through unique temporary files, synced, then published +with an atomic replace. On POSIX systems, each newly created directory is made +durable by syncing its parent, and the destination directory is synced after +the rename. Before reusing a visible file, a writer also re-confirms its +directory chain and destination directory. This makes a retry repeat a failed +directory sync even when the rename or directory creation is already visible. +A wrong-size file at a tile path is not a published tile and is rewritten. + +## Flushing and compaction + +By default `flush()` only *writes* tiles; it keeps every leaf resident in +memory. Dropping already-tiled leaves from memory ("compaction") is **opt-in**, +because once you drop them you can only prove them from the tiles. + +```cpp +merkle::tiles::TiledTree::Config cfg; +cfg.prefix = "/var/log/mylog"; +cfg.compact_on_flush = true; // drop tiled leaves after each flush +cfg.retention_margin = 4096; // ...but keep the most recent 4096 resident + +merkle::tiles::TiledTree log(cfg); +``` + +- `compact_on_flush` (default `false`): when set, `flush()` calls + `compact()` for you. +- `compact()` can also be called explicitly at any time. It drops from memory + only leaves already covered by a **durably written full tile**, keeping at + least `retention_margin` recent leaves resident. It also retains the final + tiled leaf so rollback to exactly `immutable_size()` remains representable. + It returns the new minimum (smallest still-resident) leaf index. +- Proofs for dropped leaves are still produced — they are served from the tiles + and transparently combined with the resident frontier. + +`flushed_size()` is the boundary completed successfully at every required tile +level, and it is the only boundary used for proof reads and compaction. +`immutable_size()` is the rollback boundary. A flush seals that boundary before +it starts writing, because an error can occur after a full tile becomes visible. +If a flush throws, `immutable_size()` may advance while `flushed_size()` does +not. Keep the same tree contents, correct the I/O failure, and retry `flush()`; +finalized tiles are reused rather than rewritten. + +```cpp +log.compact(); // free memory now +uint64_t resident_from = log.tree_ref().min_index(); +``` + +## Rollback + +Tiles are immutable, so you may only roll back entries beyond the boundary +returned by `immutable_size()`. `retract_to` enforces this: + +```cpp +log.retract_to(index); // keep leaves [0, index], drop the rest +``` + +- Allowed when the resulting size is `>= immutable_size()`. +- Throws otherwise, because a flush may already have published an immutable + full tile for that range. +- The exact `immutable_size()` boundary remains available after compaction; + compaction retains the final tiled leaf needed by the in-memory tree. +- After a successful flush, `immutable_size() == flushed_size()`. After an + interrupted flush, `immutable_size()` may be larger until the same tree state + is flushed successfully. +- `retract_to` mirrors `merkle::Tree::retract_to`: `index` is the new *last* + leaf, so the resulting size is `index + 1`. + +> **Warning:** Treat `tree_ref()` as an inspection escape hatch unless you also +> maintain every tiled-tree invariant yourself. Direct retraction bypasses the +> guard, can make `flushed_size()` and `immutable_size()` exceed `size()`, and +> can make `flushed_size()` regress. Use `TiledTree::retract_to` instead. + +`store_ref()` is similarly unsafe for mutation. A later flush trusts any +correctly sized tile written through it without checking that the hashes match +the in-memory tree. A mismatched tile can silently invalidate proofs after +compaction. + +## Proofs + +Both proof types come from `TiledTree` (or, at a lower level, from +`ProofEngine`). They are produced with your tree's hash function, so they match +what `merkle::Tree` would produce. Requests outside the current tree (e.g. a +size greater than `size()`, or an out-of-range index) throw `std::runtime_error` +rather than returning an incorrect proof. + +### Inclusion proofs + +```cpp +// Prove leaf `index` in a tree of `size` leaves. +std::shared_ptr p = log.inclusion_proof(index, size); +bool ok = p->verify(root_at_size); +``` + +`size` is the tree size you are proving against: + +- `size == log.size()` ⇒ equivalent to `merkle::Tree::path(index)`; verify + against `log.root()`. +- a past `size` ⇒ equivalent to `merkle::Tree::past_path(index, size - 1)`; + verify against the root at that size (e.g. a past root you are + auditing). + +`size` may even exceed `flushed_size()`: the recent, not-yet-tiled frontier is +taken from the resident tree while the older part comes from tiles. + +### Consistency proofs + +```cpp +std::vector proof = log.consistency_proof(m, n); // m <= n +bool ok = merkle::tiles::ProofEngine::verify_consistency( + m, n, old_root /* root at size m */, new_root /* root at size n */, proof); +``` + +`verify_consistency` is a static helper, so you can verify on a client that only +has the two roots and the proof. + +The arguments are tree **sizes** (leaf counts). If you have leaf **indices** +instead, use the variant that maps index `i` to the tree of size `i + 1` (the +"last leaf" convention, matching `past_path`/`retract_to`): + +```cpp +// Equivalent to consistency_proof(i + 1, j + 1). +auto proof = log.consistency_proof_from_indices(i, j); // i <= j +``` + +Both `TiledTree` and the lower-level `ProofEngine` provide +`consistency_proof_from_indices`. + +## Lower-level building blocks + +If you manage your own tree/storage you can use the pieces directly instead of +`TiledTree`. + +### Writing tiles from your own tree + +```cpp +merkle::Tree tree; +for (auto& leaf : batch) tree.insert(leaf); + +merkle::tiles::TileStore store("/var/log/mylog"); +merkle::tiles::TileWriter writer(store); + +// Write all newly-complete full tiles; keep the remaining frontier in memory. +auto stats = writer.write_up_to( + tree.num_leaves(), + [&](uint64_t i) -> const merkle::Hash& { return tree.leaf(i); }); +// stats.full_written +``` + +`TileWriter` keeps an in-memory next-file cursor. A new writer reconstructs it +by checking the contiguous prefix only up to the number of full files relevant +to the requested tree size. Existing files are re-confirmed as durably +published before reuse; malformed files and holes are rewritten. + +### Reading tiles and computing proofs + +A `HashSource` resolves the root of a complete subtree; pick where it reads from: + +- `TileHashSource(store, available_size)` — from full tile files; resolves the + full-tile-covered prefix only (the frontier needs a memory source). +- `MemoryHashSource(tree)` — from a resident `merkle::Tree`. +- `CombinedHashSource(primary, secondary)` — try `primary` first, then + `secondary` (e.g. memory then tiles). + +```cpp +merkle::tiles::TileHashSource src(store, /*available_size=*/tree.num_leaves()); +merkle::tiles::ProofEngine engine(src); + +merkle::Hash root = engine.root(size); +auto inclusion = engine.inclusion_proof(index, size); +auto consistency = engine.consistency_proof(m, n); +``` + +A tile-only source can resolve proofs whose subtrees all lie within the +full-tile-covered prefix (`available_size` is rounded down to a whole number of +tiles). For the live frontier, combine it with a `MemoryHashSource` — which is +exactly what `TiledTree` does for you. + +`TiledTree` simply wires a `CombinedHashSource(MemoryHashSource, TileHashSource)` +into a `ProofEngine` for you. It creates these sources for each proof call, so +its tile cache is per-call. A long-lived lower-level `TileHashSource` retains +its cache across calls. + +## Entry bundles (optional) + +If you also want to store the raw log entries (tlog-tiles "entry bundles"), use +`EntryBundleWriter`. Bundles are level-0 only and application-owned — merklecpp +stores leaf hashes; you supply the raw bytes and decide how an entry maps to its +leaf hash. Only full bundles (256 entries) are written; the incomplete tail +stays with your application until it completes a bundle. + +```cpp +merkle::tiles::EntryBundleWriter bundles(store); +bundles.write_up_to(num_entries, + [&](uint64_t i) -> std::vector { return raw_entry_bytes(i); }); + +// Read a full bundle back (256 entries). +std::vector> e = store.read_entry_bundle(/*index=*/0); +``` + +Entries are encoded as big-endian `uint16` length-prefixed byte strings. + +## On-disk layout + +Under the configured `prefix`: + +``` +/ + tile/0/000, tile/0/001 … # level-0 tiles (leaf hashes), 256 hashes each + tile/1/… # higher levels (roll-ups of full tiles below) + tile/entries/… # optional raw entry bundles +``` + +Tile indices use the tlog-tiles path encoding: zero-padded 3-digit groups with +all but the last prefixed by `x` (e.g. index `1234067` -> `x001/x234/067`). Every +tile is full (256-wide), final, and immutable. Entries beyond the last full-tile +boundary remain in memory. See +[`design/tlog-tiles.md`](design/tlog-tiles.md) for the full specification of the +geometry and proof algorithms. diff --git a/doc/tiles-illustrated.md b/doc/tiles-illustrated.md new file mode 100644 index 0000000..30bf65f --- /dev/null +++ b/doc/tiles-illustrated.md @@ -0,0 +1,764 @@ +# Tiled Merkle trees: an illustrated walkthrough + +This page builds a visual model of how an append-only Merkle tree moves from +memory into immutable tile files, and how proofs continue to work across both +places. + +> [!IMPORTANT] +> **Every example on this page uses an imaginary tile width of 32 entries +> purely to keep the diagrams readable.** +> +> The merklecpp implementation remains fixed at `TILE_WIDTH = 256` and +> `TILE_HEIGHT = 8`. A width of 32 is not a configuration option, and this page +> does not propose changing the code, file format, defaults, or examples +> elsewhere. Unless a section explicitly says "illustrative", use 256. + +## The scaled-down model + +A production tile contains 256 entries and spans 8 binary tree levels because +`256 = 2^8`. This page scales that geometry down to 32 entries and 5 levels +because `32 = 2^5`. + +| Property | This page only | Production merklecpp | +|---|---:|---:| +| Tile width | 32 entries | 256 entries | +| Tree levels spanned by one tile | 5 | 8 | +| Leaves covered by one full level-0 tile | 32 | 256 | +| Leaves covered by one level-1 entry | 32 | 256 | +| Leaves covered by one full level-1 tile | 1,024 | 65,536 | + +The scaling changes only the numbers in the drawings. The rules are the same: + +1. Only full tiles are written. +2. A level-0 tile contains leaf hashes. +3. A higher-level tile contains roots of complete tiles from the level below. +4. The incomplete right-hand frontier remains in memory. +5. Published tiles are immutable. +6. Proofs can resolve subtree roots from memory, tiles, or both. + +### Notation + +- `h7` is the hash of leaf 7. +- `R[a, b)` is the Merkle root of the half-open leaf range `[a, b)`. +- `tile/L/NNN` is tile index `NNN` at tile level `L`. +- "Resident" means the in-memory tree can still expand that range to answer + proof requests. +- "Compacted" means the in-memory tree retains enough summary hashes to keep + its root correct, but no longer retains all detail below that range. + +### Colors used below + +```mermaid +flowchart TB + T["Tile-backed hash or range"]:::tile + M["Resident in-memory hash or range"]:::memory + B["Available from both tiles and memory"]:::both + S["Compacted in-memory summary"]:::summary + X["Leaf being proved"]:::target + PT["Hash emitted in a proof
blue outline: from tiles"]:::proofTile + PM["Hash emitted in a proof
green outline: from memory"]:::proofMemory + + T ~~~ M + M ~~~ B + B ~~~ S + S ~~~ X + X ~~~ PT + PT ~~~ PM + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +## What `flush()` and `compact()` each do + +Appending, flushing, and compacting are separate operations: + +```mermaid +flowchart TB + A["append(h)
Add a leaf hash to the in-memory tree"]:::memory + B["A complete 32-entry range now exists
but no file is written automatically"]:::memory + C["flush()
Write every newly complete full tile"]:::tile + D["The same range exists on disk and in memory
(the default after flush)"]:::both + E["compact()
Optionally discard old resident detail"]:::summary + F["Old detail is served from tiles;
the incomplete frontier stays in memory"]:::both + + A --> B + B --> C + C --> D + D --> E + E --> F + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 +``` + +`flush()` does not compact by default. Setting `compact_on_flush = true` makes +the final two steps happen in one call, but the durability rule is unchanged: +compaction happens only after all required tile writes succeed. + +If a tile write fails, `immutable_size()` may advance past `flushed_size()` +because a published tile cannot be rolled back. Keep the same tree contents and +retry the flush. See +[Flushing and compaction](tiles-guide.md#flushing-and-compaction) for the full +interrupted-write contract. + +## What is inside a tile file? + +In the illustrative model, `tile/0/000` is the concatenation of 32 leaf hashes: + +```mermaid +flowchart TB + F["tile/0/000
32 serialized hashes"]:::tile + A["entries 0..7
h0 ... h7"]:::tile + B["entries 8..15
h8 ... h15"]:::tile + C["entries 16..23
h16 ... h23"]:::tile + D["entries 24..31
h24 ... h31"]:::tile + R["R[0, 32)
reconstructed by hashing the entries"]:::computed + N["Internal binary-tree nodes are reconstructed;
they are not separately stored in the file"]:::note + + F -->|first bytes| A + A -->|followed by| B + B -->|followed by| C + C -->|followed by| D + D --> R + R --> N + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef note fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +At level 1, each entry is already the root of 32 leaves: + +```mermaid +flowchart TB + L1["tile/1/000
32 serialized subtree roots"]:::tile + L0A["entry 0
root(tile/0/000) = R[0, 32)"]:::tile + L0B["entry 1
root(tile/0/001) = R[32, 64)"]:::tile + L0C["entries 2..30
..."]:::tile + L0Z["entry 31
root(tile/0/031) = R[992, 1024)"]:::tile + ROOT["R[0, 1024)
reconstructed from tile/1/000"]:::computed + + L1 --> L0A + L0A --> L0B + L0B --> L0C + L0C --> L0Z + L0Z --> ROOT + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +The production version of the second diagram needs 256 level-0 tile roots, so +its first full level-1 tile appears at 65,536 leaves rather than 1,024. + +## On-disk file layout + +After an illustrative 1,030-leaf tree is flushed, the full-tile boundary is +1,024: + +```text +prefix/ + tile/ + 0/ + 000 # h0 ... h31 + 001 # h32 ... h63 + ... + 031 # h992 ... h1023 + 1/ + 000 # R[0,32), R[32,64), ... R[992,1024) +``` + +Leaves `[1024, 1030)` do not appear in a tile file because they do not complete +another 32-entry tile. They remain in memory. + +```mermaid +flowchart TB + N["n = 1,030 leaves"]:::computed + C["covered = floor(1,030 / 32) * 32 = 1,024"]:::computed + L0["32 full level-0 files
tile/0/000 through tile/0/031"]:::tile + L1["1 full level-1 file
tile/1/000"]:::tile + M["6-leaf frontier
[1024, 1030) in memory"]:::memory + + N --> C + C --> L0 + L0 --> L1 + L1 --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +The optional `tile/entries/` bundles are omitted here. They store raw +application entries, not Merkle tree nodes, and do not change proof generation. + +## Tree growth, one snapshot at a time + +The next snapshots assume `retention_margin = 0`. Where compaction is shown, +merklecpp still retains the final tiled leaf as a boundary leaf. This is why +the "both" range below is one leaf wide. + +### Snapshot A: 20 leaves + +No full 32-entry tile exists: + +```mermaid +flowchart TB + N["n = 20"]:::computed + C["full-tile boundary = 0"]:::computed + M["Memory only
[0, 20)"]:::memory + D["Disk
no tile files"]:::empty + + N --> C + C --> M + M --> D + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef empty fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +Calling `flush()` at this point writes nothing. Every root and proof is served +from the in-memory tree. + +### Snapshot B: 40 leaves, before the first flush + +The first 32 leaves form a complete tile, but tile creation is explicit: + +```mermaid +flowchart TB + N["n = 40"]:::computed + M0["[0, 32)
complete and eligible, still memory only"]:::memory + M1["[32, 40)
incomplete frontier, memory only"]:::memory + D["Disk
still empty until flush()"]:::empty + + N --> M0 + M0 --> M1 + M1 --> D + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef empty fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +### Snapshot B: 40 leaves, after `flush()` + +The default `flush()` writes the full prefix but does not remove it from +memory: + +```mermaid +flowchart TB + F["flush() succeeds
flushed_size() = 32"]:::computed + B["[0, 32)
tile/0/000 + resident memory"]:::both + M["[32, 40)
resident memory only"]:::memory + + F --> B + B --> M + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +At this point a proof may be answered entirely from memory even though a tile +copy exists. + +### Snapshot B: 40 leaves, after compaction + +With zero retention, compaction drops old leaf detail while preserving leaf 31 +as the rollback boundary: + +```mermaid +flowchart TB + R["R[0, 40)
current in-memory root"]:::computed + P["R[0, 32)
prefix represented by compacted summaries"]:::summary + C["[0, 31)
not leaf-addressable in memory"]:::summary + B["h31
retained boundary leaf"]:::both + M["R[32, 40)
fully resident frontier"]:::memory + T["tile/0/000
proof detail for [0, 32)"]:::tile + + R --> P + P --> C + P --> B + R --> M + P -.->|subtree and leaf detail| T + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +There are now three logical ownership ranges: + +| Leaf range | Proof detail available from | +|---|---| +| `[0, 31)` | tiles only | +| `[31, 32)` | tiles and memory | +| `[32, 40)` | memory only | + +The compacted in-memory summaries still contribute to `root()`. "Tiles only" +means that a request for a leaf or complete subtree in that range must use the +tile source; it does not mean the in-memory root forgot the prefix hash. + +### Snapshot C: grow from 40 to 72 leaves + +Assume the tree was flushed and compacted at size 40, then 32 more leaves were +appended. + +Before the second flush: + +```mermaid +flowchart TB + N["n = 72
flushed_size() is still 32"]:::computed + T["[0, 31)
tiles only"]:::tile + B["[31, 32)
boundary leaf in both"]:::both + E["[32, 64)
complete and eligible, but still memory only"]:::memory + F["[64, 72)
incomplete memory frontier"]:::memory + + N --> T + T --> B + B --> E + E --> F + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +After the second flush and compaction: + +```mermaid +flowchart TB + F["flush() writes tile/0/001
flushed_size() = 64"]:::computed + T0["tile/0/000 covers [0, 32)"]:::tile + T1["tile/0/001 covers [32, 64)"]:::tile + C["[0, 63)
tiles only after compaction"]:::tile + B["[63, 64)
new boundary leaf in both"]:::both + M["[64, 72)
memory-only frontier"]:::memory + + F --> T0 + T0 --> T1 + T1 --> C + C --> B + B --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +### Snapshot D: 1,030 leaves + +This is the first snapshot with a full illustrative level-1 tile: + +```mermaid +flowchart TB + N["n = 1,030"]:::computed + T0["Level 0
32 files cover [0, 1024)"]:::tile + T1["Level 1
tile/1/000 contains their 32 roots"]:::tile + C["After compaction
[0, 1023) uses tiles for proof detail"]:::tile + B["h1023
boundary leaf in both"]:::both + M["[1024, 1030)
memory-only frontier"]:::memory + + N --> T0 + T0 --> T1 + T1 --> C + C --> B + B --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +### Snapshot summary + +This table assumes each snapshot has just completed a successful flush and +compaction with zero retention: + +| Tree size | `flushed_size()` | Files written | Tiles only | Tiles + memory | Memory only | +|---:|---:|---|---|---|---| +| 20 | 0 | none | none | none | `[0, 20)` | +| 40 | 32 | `tile/0/000` | `[0, 31)` | `[31, 32)` | `[32, 40)` | +| 72 | 64 | `tile/0/000..001` | `[0, 63)` | `[63, 64)` | `[64, 72)` | +| 1,030 | 1,024 | 32 level-0 tiles + `tile/1/000` | `[0, 1023)` | `[1023, 1024)` | `[1024, 1030)` | + +Again, multiply the tile geometry back to 256 for production. In particular, +the production level-1 example starts at 65,536 leaves, not 1,024. + +## How a proof finds a subtree root + +`TiledTree` gives `ProofEngine` a combined source. It tries the resident tree +first because that avoids I/O, then falls back to tiles: + +```mermaid +flowchart TB + Q["ProofEngine requests R[a, b)"]:::computed + L{"Is the range one leaf?"}:::decision + A{"Otherwise, is its width a power of two
and is the range aligned to that width?"}:::decision + P{"Is this complete subtree
fully resident in memory?"}:::decision + M["Return the in-memory hash"]:::memory + T{"Can the tile source resolve it
inside flushed_size()?"}:::decision + D["Read the appropriate tile entries
and roll them up"]:::tile + S["Split the range into smaller subtrees
and resolve each one"]:::computed + E["Fail if no source can resolve a required leaf"]:::error + + Q --> L + L -->|yes| P + L -->|no| A + A -->|yes| P + A -->|no| S + P -->|yes| M + P -->|no| T + T -->|yes| D + T -->|no, and range has multiple leaves| S + T -->|no, and range is one leaf| E + S -->|smaller range| Q + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef decision fill:#fff7ed,stroke:#c2410c,color:#111827 + classDef error fill:#fee2e2,stroke:#dc2626,color:#111827 +``` + +For example, after compacting the 40-leaf tree: + +- `R[0, 32)` is not fully resident, so memory declines it and tiles return it. +- `R[32, 36)` is resident, so memory returns it without touching disk. +- `R[24, 40)` crosses the boundary and is not one complete aligned subtree. + The proof engine splits it into resolvable pieces. + +## Inclusion proof 1: entirely from one tile + +Consider a proof against tree size 32 after `tile/0/000` has been written and +the old leaves have been compacted. This may be the current size or a historical +prefix of a larger tree. We want to prove leaf 5. + +Every required hash is reconstructed from `tile/0/000`: + +```mermaid +flowchart TB + R032["R[0, 32)"]:::tile + R016["R[0, 16)"]:::tile + P1632["R[16, 32)
proof"]:::proofTile + R08["R[0, 8)"]:::tile + P816["R[8, 16)
proof"]:::proofTile + P04["R[0, 4)
proof"]:::proofTile + R48["R[4, 8)"]:::tile + R46["R[4, 6)"]:::tile + P68["R[6, 8)
proof"]:::proofTile + P4["h4
proof"]:::proofTile + X5["h5
target leaf"]:::target + + R032 --> R016 + R032 --> P1632 + R016 --> R08 + R016 --> P816 + R08 --> P04 + R08 --> R48 + R48 --> R46 + R48 --> P68 + R46 --> P4 + R46 --> X5 + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 +``` + +The proof payload is ordered from the leaf toward the root: + +| Order | Proof hash | Position relative to the running hash | Source | +|---:|---|---|---| +| 1 | `h4` | left | `tile/0/000` | +| 2 | `R[6, 8)` | right | `tile/0/000` | +| 3 | `R[0, 4)` | left | `tile/0/000` | +| 4 | `R[8, 16)` | right | `tile/0/000` | +| 5 | `R[16, 32)` | right | `tile/0/000` | + +The internal roots in this table are computed on demand from the tile's leaf +hashes. They are not additional files. + +Verification starts with `h5`, combines the five proof hashes in order, and +arrives at `R[0, 32)`. + +## Inclusion proof 2: tiles and memory together + +Return to the compacted 40-leaf tree and prove leaf 36 against the current root +`R[0, 40)`. + +The target and its nearby siblings are in the resident frontier. The old +32-leaf prefix is supplied as one tile-backed subtree root: + +```mermaid +flowchart TB + R040["R[0, 40)"]:::computed + P032["R[0, 32)
proof from tile"]:::proofTile + R3240["R[32, 40)
resident frontier"]:::memory + P3236["R[32, 36)
proof from memory"]:::proofMemory + R3640["R[36, 40)"]:::memory + R3638["R[36, 38)"]:::memory + P3840["R[38, 40)
proof from memory"]:::proofMemory + X36["h36
target leaf"]:::target + P37["h37
proof from memory"]:::proofMemory + + R040 --> P032 + R040 --> R3240 + R3240 --> P3236 + R3240 --> R3640 + R3640 --> R3638 + R3640 --> P3840 + R3638 --> X36 + R3638 --> P37 + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +The mixed proof payload is: + +| Order | Proof hash | Position | Source | +|---:|---|---|---| +| 1 | `h37` | right | memory | +| 2 | `R[38, 40)` | right | memory | +| 3 | `R[32, 36)` | left | memory | +| 4 | `R[0, 32)` | left | `tile/0/000` | + +The caller sees one ordinary `merkle::Path`. Source selection is internal; the +proof format does not mark some hashes as "tile" and others as "memory". + +Proving an old leaf in the current tree is mixed in the opposite direction. +For example, a proof for leaf 5 at size 40 gets its target and lower siblings +from `tile/0/000`, then gets the final sibling `R[32, 40)` from memory. + +## Consistency proofs: the idea + +An inclusion proof answers: + +> Is this leaf part of this tree root? + +A consistency proof answers: + +> Can the tree with `m` leaves be extended, without changing its first `m` +> leaves, to produce the tree with `n` leaves? + +The verifier already knows: + +- `m` and the old root `R[0, m)`; +- `n` and the new root `R[0, n)`. + +The proof supplies enough complete subtree roots to reconstruct both roots +through a shared history. + +The producer recursively follows the part of the new tree that contains the +old boundary and emits the sibling subtree at each split: + +```mermaid +flowchart TB + A["Start with [0, n) and old size m"]:::computed + B["Split at the largest power of two
smaller than the current range"]:::computed + C{"Which side contains
the old boundary?"}:::decision + D["Recurse into that side"]:::computed + E["Emit the other side's root
as a proof hash"]:::proof + F{"Reached exactly
the old boundary?"}:::decision + G["Return proof hashes
from deepest to highest"]:::proof + + A --> B + B --> C + C --> D + D --> E + E --> F + F -->|no| B + F -->|yes| G + + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef decision fill:#fff7ed,stroke:#c2410c,color:#111827 + classDef proof fill:#fef3c7,stroke:#b45309,color:#111827 +``` + +Each emitted range is resolved through the same memory-first, tile-second +source used by inclusion proofs. + +## Consistency proof 1: a perfect old tree + +First prove that the 32-leaf tree is a prefix of the 40-leaf tree: + +```cpp +auto proof = log.consistency_proof(32, 40); +``` + +Because 32 is a power of two, the old root is already one complete left +subtree. The proof needs only the new right-hand range: + +```mermaid +flowchart TB + OLD["Known old root
R[0, 32)"]:::tile + EXT["proof[0]
R[32, 40) from memory"]:::proofMemory + JOIN["H(R[0, 32), R[32, 40))"]:::computed + NEW["Expected new root
R[0, 40)"]:::result + + OLD --> JOIN + EXT --> JOIN + JOIN --> NEW + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +The old root may have been calculated from `tile/0/000`; the extension is +resident in memory. Verification combines the known old root with the single +proof hash and compares the result with the known new root. + +## Consistency proof 2: a non-perfect old tree + +Now prove that the 20-leaf tree is a prefix of the 40-leaf tree: + +```cpp +auto proof = log.consistency_proof(20, 40); +``` + +Size 20 is not a power of two, so the old root does not line up with a single +node in the 40-leaf tree. The proof decomposes the relevant ranges: + +```mermaid +flowchart TB + R040["R[0, 40)"]:::computed + R032["R[0, 32)"]:::tile + P3240["P4 = R[32, 40)
memory"]:::proofMemory + P016["P3 = R[0, 16)
tile"]:::proofTile + R1632["R[16, 32)"]:::tile + R1624["R[16, 24)"]:::tile + P2432["P2 = R[24, 32)
tile"]:::proofTile + P1620["P0 = R[16, 20)
tile seed"]:::proofTile + P2024["P1 = R[20, 24)
tile"]:::proofTile + + R040 --> R032 + R040 --> P3240 + R032 --> P016 + R032 --> R1632 + R1632 --> R1624 + R1632 --> P2432 + R1624 --> P1620 + R1624 --> P2024 + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +The proof vector contains hashes only; the range labels are shown here to make +the algorithm visible. Given `m = 20` and `n = 40`, the verifier derives where +each hash belongs. + +| Order | Illustrative range | Source | Why it is needed | +|---:|---|---|---| +| `P0` | `R[16, 20)` | tile | Seed shared by old and new reconstructions | +| `P1` | `R[20, 24)` | tile | Extend only the new reconstruction | +| `P2` | `R[24, 32)` | tile | Extend only the new reconstruction | +| `P3` | `R[0, 16)` | tile | Complete both the old and new left sides | +| `P4` | `R[32, 40)` | memory | Extend the new reconstruction to size 40 | + +Verification evolves two accumulators: + +The verifier uses the bit structure of `m` and `n` to decide which accumulator +each proof hash updates. Intuitively, `P0` seeds a subtree shared by both +histories and `P3` completes that shared old-tree boundary. `P1`, `P2`, and +`P4` cover leaves at or beyond the old size, so they extend only the new +accumulator. + +```mermaid +flowchart TB + S["Seed both accumulators with P0
old = new = R[16, 20)"]:::proofTile + A["Combine P1 on the right
new = R[16, 24)"]:::tile + B["Combine P2 on the right
new = R[16, 32)"]:::tile + C["Combine P3 on the left
old = R[0, 20)
new = R[0, 32)"]:::tile + D["Combine P4 on the right
new = R[0, 40)"]:::memory + V["Compare both reconstructed roots
with the caller's old and new roots"]:::result + + S --> A + A --> B + B --> C + C --> D + D --> V + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +This example is mixed in a useful way: + +- The old 20-leaf state can be reconstructed from the first tile even though + the live in-memory tree has compacted those leaves. +- The newly appended range `[32, 40)` comes from memory. +- The proof is still an ordinary vector of hashes, independent of where each + hash was found. + +## The complete mental model + +```mermaid +flowchart TB + A["Append leaf hashes"]:::memory + B["In-memory left-balanced Merkle tree"]:::memory + C["A 32-entry range becomes complete
(256 entries in production)"]:::both + D["flush() publishes immutable full tiles"]:::tile + E["compact() optionally drops old resident detail"]:::summary + F["MemoryHashSource
serves whatever remains resident"]:::memory + G["TileHashSource
serves the flushed prefix"]:::tile + H["CombinedHashSource
tries memory, then tiles"]:::both + I["ProofEngine"]:::computed + J["Current or historical root"]:::result + K["Inclusion proof"]:::result + L["Consistency proof"]:::result + + A --> B + B --> C + C --> D + D -.->|optional| E + D --> F + D --> G + E --> F + F --> H + G --> H + H --> I + I --> J + I --> K + I --> L + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +The important boundary is always the last successfully flushed full tile: + +- Below it, immutable tiles can preserve proof detail after compaction. +- Above it, the incomplete frontier must remain resident in memory. +- A proof may resolve several component subtrees from either side of the + boundary, but the caller receives one normal proof. +- None of these rules depends on the illustrative width of 32. Production uses + the same model with 256-entry tiles. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7648f61..e5fd16f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -49,12 +49,14 @@ endif() if(LONG_TESTS) add_merklecpp_test(tiles_level2 tiles_level2.cpp) + add_merklecpp_test(time_tiles time_tiles.cpp) set(TILES_LEVEL2_TIMEOUT 900) if(WIN32) set(TILES_LEVEL2_TIMEOUT 3600) endif() set_tests_properties( ${MERKLECPP_TEST_PREFIX}tiles_level2 + ${MERKLECPP_TEST_PREFIX}time_tiles PROPERTIES TIMEOUT ${TILES_LEVEL2_TIMEOUT} ) endif() diff --git a/test/time_tiles.cpp b/test/time_tiles.cpp new file mode 100644 index 0000000..dce7d8d --- /dev/null +++ b/test/time_tiles.cpp @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TiledTree; + +static double secs_since( + const std::chrono::high_resolution_clock::time_point& start) +{ + const auto stop = std::chrono::high_resolution_clock::now(); + return (double)std::chrono::duration_cast( + stop - start) + .count() / + 1e9; +} + +// Uniform random in [0, bound). +static uint64_t rand_below(uint64_t bound) +{ + if (bound == 0) + { + return 0; + } + return (uint64_t)((std::rand() / (RAND_MAX + 1.0)) * (double)bound); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + +#ifndef NDEBUG + const uint64_t num_leaves = 50000; + const uint64_t num_proofs = 100; +#else + const uint64_t num_leaves = 1000000; + const uint64_t num_proofs = 10000; +#endif + + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_time_tiles_" + std::to_string((unsigned long long)seed)); + + // Accumulator that consumes each proof, so the work is not optimised away. + volatile uint64_t sink = 0; + + std::cout << std::fixed << std::setprecision(3); + + try + { + const auto hashes = make_hashes(num_leaves); + + TiledTree::Config cfg; + cfg.prefix = dir; + TiledTree log(cfg); + + // 1. Append: grow the in-memory tree. + auto t = std::chrono::high_resolution_clock::now(); + for (uint64_t i = 0; i < num_leaves; i++) + { + log.append(hashes[i]); + } + log.root(); + const double append_s = secs_since(t); + std::cout << "append : " << num_leaves << " leaves in " + << append_s << " sec (" + << (uint64_t)((double)num_leaves / append_s) << " leaves/sec)\n"; + + // 2. Flush: write newly-complete full tiles to disk. + t = std::chrono::high_resolution_clock::now(); + const auto stats = log.flush(); + const double flush_s = secs_since(t); + std::cout << "flush (to disk) : " << stats.full_written + << " full tiles in " << flush_s << " sec (" + << (uint64_t)((double)stats.full_written / flush_s) + << " tiles/sec)\n"; + + const uint64_t n = log.size(); + + // 3. Inclusion proofs while everything is still resident (memory path). + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.inclusion_proof(rand_below(n), n)->size(); + } + const double inc_mem_s = secs_since(t); + std::cout << "inclusion (memory) : " << num_proofs << " proofs in " + << inc_mem_s << " sec (" + << (uint64_t)((double)num_proofs / inc_mem_s) << " proofs/sec)\n"; + + // 4. Consistency proofs while resident (memory path). + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.consistency_proof(1 + rand_below(n - 1), n).size(); + } + const double con_mem_s = secs_since(t); + std::cout << "consistency(memory): " << num_proofs << " proofs in " + << con_mem_s << " sec (" + << (uint64_t)((double)num_proofs / con_mem_s) << " proofs/sec)\n"; + + // 5. Compact: drop tiled leaves from memory. + t = std::chrono::high_resolution_clock::now(); + const uint64_t min_idx = log.compact(); + const double compact_s = secs_since(t); + std::cout << "compact : dropped " << min_idx + << " leaves from memory in " << compact_s << " sec\n"; + + // 6. Inclusion proofs for evicted leaves: served from the on-disk tiles. + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.inclusion_proof(rand_below(min_idx), n)->size(); + } + const double inc_tile_s = secs_since(t); + std::cout << "inclusion (tiles) : " << num_proofs << " proofs in " + << inc_tile_s << " sec (" + << (uint64_t)((double)num_proofs / inc_tile_s) + << " proofs/sec)\n"; + + // 7. Consistency proofs spanning the tiled (evicted) past. + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.consistency_proof(1 + rand_below(min_idx - 1), n).size(); + } + const double con_tile_s = secs_since(t); + std::cout << "consistency(tiles) : " << num_proofs << " proofs in " + << con_tile_s << " sec (" + << (uint64_t)((double)num_proofs / con_tile_s) + << " proofs/sec)\n"; + + // 8. Baseline: identical inclusion proofs from a plain in-memory Tree. + merkle::Tree ref; + for (uint64_t i = 0; i < num_leaves; i++) + { + ref.insert(hashes[i]); + } + ref.root(); + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += ref.path(rand_below(n))->size(); + } + const double inc_ref_s = secs_since(t); + std::cout << "inclusion (Tree) : " << num_proofs << " proofs in " + << inc_ref_s << " sec (" + << (uint64_t)((double)num_proofs / inc_ref_s) << " proofs/sec)\n"; + + // Sanity: a tile-served proof still verifies against the reference root. + if (!log.inclusion_proof(0, n)->verify(ref.root())) + { + throw std::runtime_error("benchmark proof failed to verify"); + } + + std::cout << "time_tiles: OK (checksum " << sink << ")\n"; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +}