diff --git a/.github/processor_count.cmake b/.github/processor_count.cmake new file mode 100644 index 0000000..3993f24 --- /dev/null +++ b/.github/processor_count.cmake @@ -0,0 +1,16 @@ +include(ProcessorCount) + +ProcessorCount(PARALLELISM) +if(PARALLELISM EQUAL 0) + set(PARALLELISM 1) +endif() + +message( + STATUS + "Setting CMAKE_BUILD_PARALLEL_LEVEL and CTEST_PARALLEL_LEVEL to ${PARALLELISM}" +) +file( + APPEND "$ENV{GITHUB_ENV}" + "CMAKE_BUILD_PARALLEL_LEVEL=${PARALLELISM}\n" + "CTEST_PARALLEL_LEVEL=${PARALLELISM}\n" +) \ No newline at end of file diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 8e089b7..70a4d14 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -4,10 +4,11 @@ on: push: branches: - main + pull_request: workflow_dispatch: concurrency: - group: pages + group: pages-${{ github.ref }} cancel-in-progress: false permissions: read-all @@ -24,6 +25,7 @@ jobs: - name: Setup Pages id: pages uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + if: github.event_name != 'pull_request' - name: Install dependencies run: | @@ -42,6 +44,7 @@ jobs: uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: path: doc/build + if: github.event_name != 'pull_request' deploy: name: Deploy diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b301f9a..0027c96 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,18 +47,36 @@ jobs: - name: Create Build Environment run: cmake -E make_directory ${{github.workspace}}/build/${{ matrix.build_type }} + - name: Detect Runner Parallelism + shell: bash + run: cmake -P "$GITHUB_WORKSPACE/.github/processor_count.cmake" + - name: Configure CMake # Use a bash shell so we can use the same syntax for environment variable # access regardless of the host operating system shell: bash working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} run: | + long_tests=OFF + if [ "${{ matrix.build_type }}" == "Release" ]; then + long_tests=ON + fi + + linux_options=() if [ "$RUNNER_OS" == "Linux" ]; then - cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - else - cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} + linux_options=( + "-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}" + -DCLANG_TIDY=ON + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + ) fi + cmake "$GITHUB_WORKSPACE" \ + "-DCMAKE_BUILD_TYPE=${{ matrix.build_type }}" \ + "-DLONG_TESTS=$long_tests" \ + "-DOPENSSL=${{ matrix.openssl }}" \ + "${linux_options[@]}" + - name: Build working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash @@ -76,5 +94,3 @@ jobs: working-directory: ${{github.workspace}}/build/${{ matrix.build_type }} shell: bash run: ctest -VV -C ${{ matrix.build_type }} --timeout 300 - env: - ASAN_OPTIONS: use_sigaltstack=false # To avoid SetAlternateSignalStack with clang-11 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d052011..88c9f6e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: - name: Configure merklecpp working-directory: ${{github.workspace}}/build - run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON $GITHUB_WORKSPACE + run: cmake -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON $GITHUB_WORKSPACE - name: Build merklecpp working-directory: ${{github.workspace}}/build diff --git a/CMakeLists.txt b/CMakeLists.txt index b6f5ad2..6252f35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ cmake_minimum_required(VERSION 3.14) project(merklecpp LANGUAGES CXX) include(GNUInstallDirs) +include(CTest) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -13,10 +14,10 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(MERKLECPP_DIR ${CMAKE_CURRENT_SOURCE_DIR}) option(PROFILE "enable profiling" OFF) -option(TESTS "enable testing" OFF) option(OPENSSL "enable OpenSSL" OFF) option(TRACE "enable debug traces" OFF) option(CLANG_TIDY "enable clang-tidy checks during build" OFF) +option(LONG_TESTS "enable long-running tests" OFF) if(CLANG_TIDY) find_program(CLANG_TIDY_PROGRAM clang-tidy) @@ -45,9 +46,7 @@ if(OPENSSL) target_link_libraries(merklecpp INTERFACE OpenSSL::Crypto) endif() -if(TESTS) - enable_testing() - +if(BUILD_TESTING) function(add_unit_test NAME SRC) add_executable(${NAME} ${SRC}) target_link_libraries(${NAME} PRIVATE $) diff --git a/README.md b/README.md index db01d1c..1009d55 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,69 @@ merklecpp requires C++20. assert(path->verify(root)); +## Tiled storage (tlog-tiles) + +The companion header `merklecpp_tiles.h` adds optional, header-only support for +persisting a tree as [tlog-tiles](https://c2sp.org/tlog-tiles) tile files +*progressively* (optionally dropping already-tiled leaves from memory) and for +retrieving inclusion and consistency proofs from those tiles, from the in-memory +tree, or from a combination of the two. The hashing is unchanged: tiles and tile-derived proofs are templated on +the tree's existing hash function, so a tile-derived inclusion proof is +byte-identical to one from `merkle::Tree::path()` and verifies with the same +`merkle::Path::verify()`. + + #include + + merkle::tiles::TiledTree::Config cfg; + cfg.prefix = "/var/log/mylog"; // tile files live here + cfg.retention_margin = 1024; // keep the most recent leaves in memory + cfg.compact_on_flush = true; // opt in to dropping already-tiled leaves + + merkle::tiles::TiledTree log(cfg); + for (const auto& leaf_hash : batch) + log.append(leaf_hash); + + // Write newly-complete tiles. With compaction enabled + // this also drops from memory the leaves already covered by a full tile; + // otherwise the tree keeps every leaf and you can call log.compact() later. + log.flush(); + + // Proofs are served from tiles + the resident tree, even for flushed leaves. + auto inclusion = log.inclusion_proof(/*index=*/0, log.size()); + assert(inclusion->verify(log.root())); + + auto consistency = log.consistency_proof(/*m=*/100, /*n=*/log.size()); + +`TiledTree` creates a new tiled tree: the configured directory may exist, but +its `tile` subdirectory must be absent or empty. It deliberately rejects +existing tile data because tile files alone do not identify or restore the +tree that produced them. Applications with externally persisted tree state can +use the lower-level `TileStore` and `TileWriter` APIs to resume a store. + +See the [tiled storage guide](doc/tiles-guide.md) for a how-to covering +flushing, compaction, rollback, proofs, and the lower-level building blocks, +and [doc/design/tlog-tiles.md](doc/design/tlog-tiles.md) for the full design, +file/directory layout, and the proof algorithms. + + +## Building and testing + +Enable the test suite with CMake's `TESTS` option: + + cmake -S . -B build -DTESTS=ON + cmake --build build + ctest --test-dir build + +Some tile coverage is intentionally long-running. `LONG_TESTS` is off by +default for local builds; turn it on when you want the full tile stress suite, +including level-2 tile coverage and tile proof timing: + + cmake -S . -B build -DTESTS=ON -DLONG_TESTS=ON + +The repository CI enables `LONG_TESTS` so pull requests continue to exercise the +full tiled-storage matrix. + + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 5219ef2..b2aa05b 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -31,7 +31,7 @@ serving the static resources. |---|---| | Tile | `/-256w/tile//`, `application/octet-stream`; `L` is decimal `0..63` without leading zeros, and `N` uses the grouped encoding in [section 4](#4-storage-layout-and-publication). | | Full tile | 256 hashes. Level 0 stores leaf hashes; each level-`L` entry for `L >= 1` is the Merkle Tree Hash of one complete level-`L-1` tile. | -| Entry bundle | `/-256w/tile/entries/`; 256 raw entries encoded as big-endian `uint16` length-prefixed values. See [section 5.3](#53-entry-bundles-optional). | +| Entry bundle | `/-256w/tile/entries/`; 256 raw entries encoded as big-endian `uint16` length-prefixed values. See [section 5.4](#54-entry-bundles-optional). | | Pruning | Tiles or bundles ending at or before a log's minimum index may be denied, matching `flush_to()` / `min_index()`. | A tile spans eight tree levels. Tile `n` at level `l` contains, for @@ -174,10 +174,10 @@ hashes, not bytes. │ owns Tree ▲ inclusion / consistency proofs │ tiles ▼ │ - ProofEngine ──▶ HashSource ◀──────────┘ - ├─ MemoryHashSource - ├─ TileHashSource - └─ CombinedHashSource + ProofEngineT ──▶ HashSourceT ◀─────────┘ + ├─ MemoryHashSourceT + ├─ TileHashSourceT + └─ CombinedHashSourceT ``` `merklecpp_tiles.h` contains the public `merkle::tiles` API and includes @@ -194,9 +194,8 @@ hashes, not bytes. | `ProofEngineT` | Roots, inclusion/consistency proofs, and verification | | `TiledTreeT` | `append`, `flush`, proof APIs, and compaction | -The planned `TileHashSource` owns the proof-read LRU cache; `TileStoreT` does -not cache. A combined source may require one read-only, non-hashing core -`subtree_root` accessor. +`TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. +`MemoryHashSourceT` uses the `TreeT::subtree_root` accessor. ### 5.1 Types and aliases @@ -217,7 +216,8 @@ using TileStore = TileStoreT; using TiledTree = TiledTreeT; -// Equivalent SHA-384 and SHA-512 aliases. +// Writer, hash-source, proof-engine, and entry-bundle aliases follow the same +// template pattern. } ``` @@ -297,21 +297,280 @@ stores the supplied leaf hash unchanged. `EntryBundleWriterT` mirrors `TileWriterT`: it writes only complete 256-entry bundles, confirms existing bundles before reusing them, and leaves the incomplete tail with the application. -## 6. Delivery plan +### 5.5 `TreeT::subtree_root` -Phases 0-2 now deliver the storage primitives plus incremental tile and -entry-bundle writers. Later PRs deliver the remaining independently testable -phases; phase 3 needs no further core changes, while phase 4 may add one -non-hashing accessor. +Proofs over the resident frontier use one core accessor: + +```cpp +std::optional subtree_root(uint8_t level, size_t index); +``` + +It returns the existing root of the complete subtree spanning +`[index << level, (index + 1) << level)`. The method rejects overflow, flushed +or out-of-range leaves, and non-perfect frontier nodes by returning +`std::nullopt`. Like `root()` and `path()`, it may materialize pending nodes and +compute dirty hashes, but does not change logical leaf contents or hashing +semantics. + +### 5.6 Hash sources + +```cpp +struct HashSourceT { + // MTH(D[index<; + explicit ProofEngineT(const Source& source); + + Hash root(uint64_t size) const; // = mth_range(0, size) + + // Inclusion path for leaf `index` in the tree of `size` leaves. + // Equivalent to Tree::path(index) when size==num_leaves(), + // and to Tree::past_path(index, size-1) otherwise. + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size) const; + + // RFC 6962 consistency proof that size `m` is a prefix of size `n` (m<=n). + std::vector consistency_proof(uint64_t m, uint64_t n) const; + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) const; + + // Verifier (consistency is new to merklecpp; inclusion reuses PathT::verify). + static bool verify_consistency(uint64_t m, uint64_t n, + const Hash& old_root, const Hash& new_root, + const std::vector& proof); +}; +``` + +Inclusion (top-down; element order/`direction` chosen to match `Tree::path`): + +``` +elements = [] # leaf→root order via push_front +lo = 0, hi = size, idx = index +while hi - lo > 1: + k = largest_pow2_lt(hi - lo) # split at lo+k + if idx - lo < k: # target in left ⇒ sibling on the RIGHT + sib = mth_range(lo+k, hi); dir = PATH_RIGHT; hi = lo + k + else: # target in right ⇒ sibling on the LEFT + sib = mth_range(lo, lo+k); dir = PATH_LEFT; lo = lo + k + elements.push_front({sib, dir}) +leaf = src.leaf(index) +return Path(leaf, index, elements, max_index = size - 1) +``` + +Consistency (RFC 6962 `SUBPROOF`): + +``` +consistency_proof(m, n): # 0 < m <= n + if m == n: return [] + subproof(m, lo=0, hi=n, complete=true) + +subproof(m, lo, hi, complete): + if m == hi - lo: + if not complete: proof.push_back(mth_range(lo, hi)) + return + k = largest_pow2_lt(hi - lo) + if m <= k: + subproof(m, lo, lo+k, complete) + proof.push_back(mth_range(lo+k, hi)) + else: + subproof(m-k, lo+k, hi, false) + proof.push_back(mth_range(lo, lo+k)) +``` + +Because every emitted hash is an `mth_range` computed with `HASH_FUNCTION`, the +consistency proof reconciles `Tree::past_root(m-1)` with +`Tree::past_root(n-1)` — i.e. it is consistent with the existing library. + +### 5.8 `TiledTreeT` - convenience wrapper + +```cpp +class TiledTreeT { +public: + struct Config { + std::filesystem::path prefix; + uint64_t retention_margin = 0; // keep at least this many resident + bool compact_on_flush = false; // opt in to dropping tiled leaves + }; + explicit TiledTreeT(Config); + TiledTreeT(Config, const std::string& hash_algorithm_short_name); + + void append(const Hash& leaf_hash); // tree.insert + uint64_t size() const; // tree.num_leaves + Hash root(); // tree.root + uint64_t flushed_size() const; // successful tile boundary + uint64_t immutable_size() const; // rollback boundary + + // Write newly-complete full tiles. Compaction (dropping already-tiled + // leaves from memory) happens only if compact_on_flush. + Stats flush(); + + // Drop from memory the leaves already covered by a full tile (opt-in); the + // un-tiled frontier is always retained, and proofs for dropped leaves remain + // available from the tiles. + uint64_t compact(); + + // Roll back only beyond immutable_size(). + void retract_to(size_t index); + + // Proofs over tiles ∪ resident tree (works for flushed indices). + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size); + std::vector consistency_proof(uint64_t m, uint64_t n); + std::vector consistency_proof_from_indices(uint64_t i, uint64_t j); + + Tree& tree_ref(); // mutable escape hatch + Store& store_ref(); // mutable escape hatch +}; +``` + +`TiledTreeT` performs no internal locking. The caller must serialize every +operation on a shared instance, including proof calls. + +Callers with their own storage can construct a `TileWriterT` and call +`write_up_to`, then build a `ProofEngineT` on a `CombinedHashSource`, so the +wrapper is optional sugar. + +## 6. Progressive production and compaction + +The pairing of tile writing with `flush_to` gives two central correctness +invariants: + +> **Compaction invariant.** Retain the final leaf of the last fully successful +> flush: `flushed_size == 0 || min_index() < flushed_size`. +> +> **Immutability invariant.** Never roll back below a full-tile boundary that a +> flush may have published: `size >= immutable_size`. + +`TiledTreeT` is fresh-only: its configured directory may exist, but the +algorithm-qualified `tile` subdirectory must be absent. Construction atomically +creates that directory to claim exclusive ownership. Tile files do not carry +the size, root, hash identity, or ownership information needed to reopen or +share a tree safely. The lower-level `TileWriterT` supports resume for +applications that persist and validate the matching tree state themselves. + +Per flush: + +1. Append new leaf hashes and compute the root as needed. +2. Compute `covered = floor(size / 256) * 256` and advance `immutable_size` to + `covered` before any write can publish a full tile. +3. Call `write_up_to(size, leaf_at)` to persist newly complete full tiles at all + levels. +4. After every level succeeds, set `flushed_size = covered`. +5. Optionally, `compact()` computes an aligned retention target capped below + nonzero `covered`, then calls `flush_to(target)`. This reclaims memory only + when `compact_on_flush` is set or `compact()` is called explicitly. It keeps + at least `retention_margin` recent leaves; alignment can retain up to 255 + additional tiled leaves, and a zero margin keeps the final tiled leaf. The + entire un-tiled frontier always remains resident. + +If tile writing fails, the final two steps do not run. `immutable_size` stays +advanced to prevent stale-tile rollback, while `flushed_size` stays at the last +complete all-level write so proofs and compaction do not trust an incomplete +flush. + +Given these invariants, every leaf and every perfect subtree is resolvable: + +- A leaf below `covered` is in a full level-0 tile; a leaf at or above + `min_index` is resident. Since `min_index <= covered`, every leaf is in tiles, + memory, or both. The frontier `[covered, size)` is always resident because + compaction never flushes past `covered`. +- `mth_range` resolves a perfect subtree directly when it lies wholly in tiles + (`end <= covered`) or wholly in memory (`start >= min_index`). Otherwise it + splits and recurses until each piece is resolvable. A subtree within + `covered` whose level has no completed full tile descends to the highest + available full tile. + +Inclusion and consistency proofs therefore remain available after compaction, +from full tiles for the tiled prefix, memory for the resident frontier, or the +combination. A flush costs `O(new full tiles)`; higher-level tiles are roll-ups +of 256 child hashes. Proof generation performs `O(log(size))` range operations, +with repeated tile reads served from the per-source cache. + +## 7. Pruning and minimum index + +tlog-tiles pruning maps directly onto merklecpp: + +- The log's minimum index is `tree.min_index()` (equal to `num_flushed`). +- A serving layer can deny tiles or bundles whose end index is at or below the + minimum index. On-disk tiles may instead be retained so historical proofs + remain producible. +- The unpruned default is `min_index() == 0` (no `flush_to`). + +`flush_to` provides the mechanism; the application owns the retention policy, +as in the tlog-tiles ecosystem. + +## 8. Delivery plan + +Phases 0-4 now deliver the storage primitives, incremental tile and entry-bundle +writers, hash sources, proof engine, the only required core accessor, and the +lifecycle wrapper. Later PRs deliver user documentation and performance +coverage; no further core changes are planned. | Phase | Scope | Key tests | |---|---|---| | 0. Scaffolding | Headers, PAL, namespace, geometry, aliases, and CMake test wiring | Public-header and build integration | | 1. Coordinates/store | `TileRef`, index/path encoding, `TileStoreT`, durable atomic I/O, entry-bundle primitives | Encoding vectors; algorithm roots; 256-hash SHA-256/384 tiles; round trips; file/symlink collisions | | 2. Writers | Incremental `TileWriterT::write_up_to` from `leaf_at`, roll-ups, and `EntryBundleWriterT`; full resources only | Sizes 256 and 70,000 produce the exact tile set; repeated writes preserve immutability | -| 3. Proof engine | `TileHashSource`, `mth_range`, roots, inclusion/consistency proofs, and verification | Tile roots equal tree roots; inclusion equals `path()` / `past_path()` and verifies; consistency reconciles `past_root()` values | -| 4. Combined tree | Optional `subtree_root`; memory/combined sources; `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | +| 3. Proof engine | `TreeT::subtree_root`; tile, memory, and combined hash sources; roots; inclusion/consistency proofs and verification | Tile roots equal tree roots; inclusion equals `path()` / `past_path()` and verifies; consistency reconciles `past_root()` values | +| 4. Combined tree | `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | | 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | -Deliverables are `merklecpp_tiles.h`, `merklecpp_pal.h`, `test/tiles_*.cpp`, -CMake wiring, the optional core accessor, and README/design updates. +Delivered through phase 4 are `merklecpp_tiles.h`, `merklecpp_pal.h`, +`test/tiles_*.cpp`, CMake wiring, the core accessor, the lifecycle wrapper, and +design updates. + +## 9. Risks and edge cases + +- **External interop (by design, no).** With the default combiner the tiles are + not byte-compatible with RFC 6962 tooling. See + [section 2.3](#23-algorithms-and-namespaces); opting into a compatible + `HASH_FUNCTION` is the consumer's choice and out of scope. +- **Filesystem dependency.** Tile I/O needs ``/``; isolated + in the companion header so the core stays dependency-free. +- **Immutable full tiles.** A tile is emitted only after all of its entries are + final, and every emitted tile is write-once. A stand-alone tile reader cannot + serve the frontier; that is the in-memory tree's job (or the application must + keep it elsewhere). +- **`flush_to` alignment.** Compaction normally flushes to a 256-multiple + derived from retention. When that target equals `flushed_size`, it stops one + leaf earlier so `TreeT` can still retract to exactly that size. This one-leaf + overlap is enforced inside `TiledTreeT::compact`. +- **Rollback vs. immutable tiles.** Tiles are write-once, so rolling the tree + back (`retract_to`) over a range that a flush may have published would leave + stale, never-rewritten tiles. `TiledTreeT::retract_to` therefore throws if the + resulting size is below `immutable_size()`. A failed flush may advance + `immutable_size()` without advancing `flushed_size()`; retry with the same + tree state. Retracting the underlying tree directly via `tree_ref()` bypasses + this guard, can make the size boundaries inconsistent or non-monotonic, and + must be avoided. Files written through `store_ref()` are trusted without + checking that they match the tree and can invalidate proofs after compaction. +- **No internal synchronization.** Every tiled-storage object and shared store + prefix requires external serialization. This includes `const` proof reads, + which update the tile cache. +- **Very large indices.** Index math uses `uint64_t`; encoding handles + multi-group indices. Level bound `<= 63` per spec (8 suffices for `2^64`). + Resume scans are bounded by the requested tree size and cannot follow sparse + files beyond that range. diff --git a/doc/tiles-guide.md b/doc/tiles-guide.md new file mode 100644 index 0000000..396dbaa --- /dev/null +++ b/doc/tiles-guide.md @@ -0,0 +1,325 @@ +# Tiled storage and proofs — a guide to `merklecpp_tiles.h` + +`merklecpp_tiles.h` is an optional, header-only companion to `merklecpp.h`. It +lets you persist a Merkle tree as a set of immutable **tile files** on disk and +serve **inclusion** and **consistency** proofs from those tiles, from the +in-memory tree, or from a combination of the two — so proofs stay available even +after old entries are dropped from memory. + +It builds on the [tlog-tiles](https://c2sp.org/tlog-tiles) file/directory layout +but is **not** trying to be wire-compatible with external tlog-tiles clients. See +[`design/tlog-tiles.md`](design/tlog-tiles.md) for the design and internals; this +page is a practical how-to. + +## Contents + +- [Requirements and a note on hashing](#requirements-and-a-note-on-hashing) +- [Thread safety](#thread-safety) +- [Quick start: `TiledTree`](#quick-start-tiledtree) +- [Flushing and compaction](#flushing-and-compaction) +- [Rollback](#rollback) +- [Proofs](#proofs) +- [Lower-level building blocks](#lower-level-building-blocks) +- [Entry bundles (optional)](#entry-bundles-optional) +- [On-disk layout](#on-disk-layout) + +## Requirements and a note on hashing + +- C++17 (the header uses `` and small platform-specific file-sync + calls for durable tile writes). +- Include the companion header; it pulls in `merklecpp.h` for you: + + ```cpp + #include + ``` + +- Everything lives in `namespace merkle::tiles` and is templated on the same + `` as your tree. The default aliases + (`merkle::tiles::TiledTree`, `TileStore`, `TileWriter`, `ProofEngine`, …) use + the **same** SHA-256 as `merkle::Tree`, so a tile-derived inclusion proof is + byte-identical to one from `merkle::Tree::path()` and verifies with the usual + `merkle::Path::verify()`. +- You insert **leaf hashes**, not raw entries — exactly like `merkle::Tree`. + Deriving a leaf hash from an entry (e.g. `leaf = H(entry)`) is your + application's job. The tile hash values are whatever your `HASH_FUNCTION` + produces; they are not RFC 6962 unless you instantiate your tree with an + RFC 6962 hash function (not required, and not the goal here). + +## Thread safety + +The tiled-storage API provides no internal synchronization. Treat every +`TileStore`, `TileWriter`, `TileHashSource`, `ProofEngine`, `TiledTree`, and +`EntryBundleWriter` instance as single-threaded. + +If an object or store prefix is shared between threads, the caller must +serialize every operation. This includes methods declared `const`: proof +generation updates the `TileHashSource` LRU cache. The library deliberately +does not add locks or otherwise coordinate concurrent readers and writers. + +## Quick start: `TiledTree` + +`TiledTree` is the high-level wrapper: append leaf hashes, flush them to disk +(which writes tiles), and ask for proofs. + +```cpp +#include + +merkle::tiles::TiledTree::Config cfg; +cfg.prefix = "/var/log/mylog"; // directory for tile files + +merkle::tiles::TiledTree log(cfg); + +// Append leaf hashes (compute these from your entries however you like). +for (const merkle::Hash& leaf : batch) + log.append(leaf); + +// Persist newly-complete tiles to disk. +log.flush(); + +merkle::Hash root = log.root(); // current Merkle root +uint64_t n = log.size(); // number of leaves + +// Inclusion proof for leaf 0 in the tree of `n` leaves. +auto inclusion = log.inclusion_proof(/*index=*/0, /*size=*/n); +assert(inclusion->verify(root)); + +// Consistency proof that size 100 is a prefix of size n. +auto consistency = log.consistency_proof(/*m=*/100, /*n=*/n); +``` + +`TiledTree` can be move-constructed, but it cannot be copied or assigned. Move +construction keeps its writer bound to the destination tree's tile store. + +`TiledTree` always creates a new tiled tree. The configured directory may +already exist, but its `tile` subdirectory must be absent or empty. Construction +throws rather than adopting existing tiles because those files do not identify +the tree that produced them or contain enough state to restore its size and +root. If your application persists and validates that state separately, use +the lower-level `TileStore` and `TileWriter` APIs; `TileWriter` intentionally +resumes existing full tiles and therefore trusts the caller to supply the same +tree and hash function. A fresh writer scans the requested range in order, +stopping at the first missing or malformed file, so an interior hole is +rewritten rather than hidden by later files. + +`flush()` is incremental: each call writes only the full tiles that became +complete since the previous call. Full tiles are immutable: written once after +all 256 entries are final and never rewritten. The remaining frontier stays in +memory until it crosses the next full-tile boundary. + +Tile files are written through unique temporary files, synced, then published +with an atomic replace. On POSIX systems, each newly created directory is made +durable by syncing its parent, and the destination directory is synced after +the rename. Before reusing a visible file, a writer also re-confirms its +directory chain and destination directory. This makes a retry repeat a failed +directory sync even when the rename or directory creation is already visible. +A wrong-size file at a tile path is not a published tile and is rewritten. + +## Flushing and compaction + +By default `flush()` only *writes* tiles; it keeps every leaf resident in +memory. Dropping already-tiled leaves from memory ("compaction") is **opt-in**, +because once you drop them you can only prove them from the tiles. + +```cpp +merkle::tiles::TiledTree::Config cfg; +cfg.prefix = "/var/log/mylog"; +cfg.compact_on_flush = true; // drop tiled leaves after each flush +cfg.retention_margin = 4096; // ...but keep the most recent 4096 resident + +merkle::tiles::TiledTree log(cfg); +``` + +- `compact_on_flush` (default `false`): when set, `flush()` calls + `compact()` for you. +- `compact()` can also be called explicitly at any time. It drops from memory + only leaves already covered by a **durably written full tile**, keeping at + least `retention_margin` recent leaves resident. It also retains the final + tiled leaf so rollback to exactly `immutable_size()` remains representable. + It returns the new minimum (smallest still-resident) leaf index. +- Proofs for dropped leaves are still produced — they are served from the tiles + and transparently combined with the resident frontier. + +`flushed_size()` is the boundary completed successfully at every required tile +level, and it is the only boundary used for proof reads and compaction. +`immutable_size()` is the rollback boundary. A flush seals that boundary before +it starts writing, because an error can occur after a full tile becomes visible. +If a flush throws, `immutable_size()` may advance while `flushed_size()` does +not. Keep the same tree contents, correct the I/O failure, and retry `flush()`; +finalized tiles are reused rather than rewritten. + +```cpp +log.compact(); // free memory now +uint64_t resident_from = log.tree_ref().min_index(); +``` + +## Rollback + +Tiles are immutable, so you may only roll back entries beyond the boundary +returned by `immutable_size()`. `retract_to` enforces this: + +```cpp +log.retract_to(index); // keep leaves [0, index], drop the rest +``` + +- Allowed when the resulting size is `>= immutable_size()`. +- Throws otherwise, because a flush may already have published an immutable + full tile for that range. +- The exact `immutable_size()` boundary remains available after compaction; + compaction retains the final tiled leaf needed by the in-memory tree. +- After a successful flush, `immutable_size() == flushed_size()`. After an + interrupted flush, `immutable_size()` may be larger until the same tree state + is flushed successfully. +- `retract_to` mirrors `merkle::Tree::retract_to`: `index` is the new *last* + leaf, so the resulting size is `index + 1`. + +> **Warning:** Treat `tree_ref()` as an inspection escape hatch unless you also +> maintain every tiled-tree invariant yourself. Direct retraction bypasses the +> guard, can make `flushed_size()` and `immutable_size()` exceed `size()`, and +> can make `flushed_size()` regress. Use `TiledTree::retract_to` instead. + +`store_ref()` is similarly unsafe for mutation. A later flush trusts any +correctly sized tile written through it without checking that the hashes match +the in-memory tree. A mismatched tile can silently invalidate proofs after +compaction. + +## Proofs + +Both proof types come from `TiledTree` (or, at a lower level, from +`ProofEngine`). They are produced with your tree's hash function, so they match +what `merkle::Tree` would produce. Requests outside the current tree (e.g. a +size greater than `size()`, or an out-of-range index) throw `std::runtime_error` +rather than returning an incorrect proof. + +### Inclusion proofs + +```cpp +// Prove leaf `index` in a tree of `size` leaves. +std::shared_ptr p = log.inclusion_proof(index, size); +bool ok = p->verify(root_at_size); +``` + +`size` is the tree size you are proving against: + +- `size == log.size()` ⇒ equivalent to `merkle::Tree::path(index)`; verify + against `log.root()`. +- a past `size` ⇒ equivalent to `merkle::Tree::past_path(index, size - 1)`; + verify against the root at that size (e.g. a past root you are + auditing). + +`size` may even exceed `flushed_size()`: the recent, not-yet-tiled frontier is +taken from the resident tree while the older part comes from tiles. + +### Consistency proofs + +```cpp +std::vector proof = log.consistency_proof(m, n); // m <= n +bool ok = merkle::tiles::ProofEngine::verify_consistency( + m, n, old_root /* root at size m */, new_root /* root at size n */, proof); +``` + +`verify_consistency` is a static helper, so you can verify on a client that only +has the two roots and the proof. + +The arguments are tree **sizes** (leaf counts). If you have leaf **indices** +instead, use the variant that maps index `i` to the tree of size `i + 1` (the +"last leaf" convention, matching `past_path`/`retract_to`): + +```cpp +// Equivalent to consistency_proof(i + 1, j + 1). +auto proof = log.consistency_proof_from_indices(i, j); // i <= j +``` + +Both `TiledTree` and the lower-level `ProofEngine` provide +`consistency_proof_from_indices`. + +## Lower-level building blocks + +If you manage your own tree/storage you can use the pieces directly instead of +`TiledTree`. + +### Writing tiles from your own tree + +```cpp +merkle::Tree tree; +for (auto& leaf : batch) tree.insert(leaf); + +merkle::tiles::TileStore store("/var/log/mylog"); +merkle::tiles::TileWriter writer(store); + +// Write all newly-complete full tiles; keep the remaining frontier in memory. +auto stats = writer.write_up_to( + tree.num_leaves(), + [&](uint64_t i) -> const merkle::Hash& { return tree.leaf(i); }); +// stats.full_written +``` + +`TileWriter` keeps an in-memory next-file cursor. A new writer reconstructs it +by checking the contiguous prefix only up to the number of full files relevant +to the requested tree size. Existing files are re-confirmed as durably +published before reuse; malformed files and holes are rewritten. + +### Reading tiles and computing proofs + +A `HashSource` resolves the root of a complete subtree; pick where it reads from: + +- `TileHashSource(store, available_size)` — from full tile files; resolves the + full-tile-covered prefix only (the frontier needs a memory source). +- `MemoryHashSource(tree)` — from a resident `merkle::Tree`. +- `CombinedHashSource(primary, secondary)` — try `primary` first, then + `secondary` (e.g. memory then tiles). + +```cpp +merkle::tiles::TileHashSource src(store, /*available_size=*/tree.num_leaves()); +merkle::tiles::ProofEngine engine(src); + +merkle::Hash root = engine.root(size); +auto inclusion = engine.inclusion_proof(index, size); +auto consistency = engine.consistency_proof(m, n); +``` + +A tile-only source can resolve proofs whose subtrees all lie within the +full-tile-covered prefix (`available_size` is rounded down to a whole number of +tiles). For the live frontier, combine it with a `MemoryHashSource` — which is +exactly what `TiledTree` does for you. + +`TiledTree` simply wires a `CombinedHashSource(MemoryHashSource, TileHashSource)` +into a `ProofEngine` for you. It creates these sources for each proof call, so +its tile cache is per-call. A long-lived lower-level `TileHashSource` retains +its cache across calls. + +## Entry bundles (optional) + +If you also want to store the raw log entries (tlog-tiles "entry bundles"), use +`EntryBundleWriter`. Bundles are level-0 only and application-owned — merklecpp +stores leaf hashes; you supply the raw bytes and decide how an entry maps to its +leaf hash. Only full bundles (256 entries) are written; the incomplete tail +stays with your application until it completes a bundle. + +```cpp +merkle::tiles::EntryBundleWriter bundles(store); +bundles.write_up_to(num_entries, + [&](uint64_t i) -> std::vector { return raw_entry_bytes(i); }); + +// Read a full bundle back (256 entries). +std::vector> e = store.read_entry_bundle(/*index=*/0); +``` + +Entries are encoded as big-endian `uint16` length-prefixed byte strings. + +## On-disk layout + +Under the configured `prefix`: + +``` +/ + tile/0/000, tile/0/001 … # level-0 tiles (leaf hashes), 256 hashes each + tile/1/… # higher levels (roll-ups of full tiles below) + tile/entries/… # optional raw entry bundles +``` + +Tile indices use the tlog-tiles path encoding: zero-padded 3-digit groups with +all but the last prefixed by `x` (e.g. index `1234067` -> `x001/x234/067`). Every +tile is full (256-wide), final, and immutable. Entries beyond the last full-tile +boundary remain in memory. See +[`design/tlog-tiles.md`](design/tlog-tiles.md) for the full specification of the +geometry and proof algorithms. diff --git a/doc/tiles-illustrated.md b/doc/tiles-illustrated.md new file mode 100644 index 0000000..30bf65f --- /dev/null +++ b/doc/tiles-illustrated.md @@ -0,0 +1,764 @@ +# Tiled Merkle trees: an illustrated walkthrough + +This page builds a visual model of how an append-only Merkle tree moves from +memory into immutable tile files, and how proofs continue to work across both +places. + +> [!IMPORTANT] +> **Every example on this page uses an imaginary tile width of 32 entries +> purely to keep the diagrams readable.** +> +> The merklecpp implementation remains fixed at `TILE_WIDTH = 256` and +> `TILE_HEIGHT = 8`. A width of 32 is not a configuration option, and this page +> does not propose changing the code, file format, defaults, or examples +> elsewhere. Unless a section explicitly says "illustrative", use 256. + +## The scaled-down model + +A production tile contains 256 entries and spans 8 binary tree levels because +`256 = 2^8`. This page scales that geometry down to 32 entries and 5 levels +because `32 = 2^5`. + +| Property | This page only | Production merklecpp | +|---|---:|---:| +| Tile width | 32 entries | 256 entries | +| Tree levels spanned by one tile | 5 | 8 | +| Leaves covered by one full level-0 tile | 32 | 256 | +| Leaves covered by one level-1 entry | 32 | 256 | +| Leaves covered by one full level-1 tile | 1,024 | 65,536 | + +The scaling changes only the numbers in the drawings. The rules are the same: + +1. Only full tiles are written. +2. A level-0 tile contains leaf hashes. +3. A higher-level tile contains roots of complete tiles from the level below. +4. The incomplete right-hand frontier remains in memory. +5. Published tiles are immutable. +6. Proofs can resolve subtree roots from memory, tiles, or both. + +### Notation + +- `h7` is the hash of leaf 7. +- `R[a, b)` is the Merkle root of the half-open leaf range `[a, b)`. +- `tile/L/NNN` is tile index `NNN` at tile level `L`. +- "Resident" means the in-memory tree can still expand that range to answer + proof requests. +- "Compacted" means the in-memory tree retains enough summary hashes to keep + its root correct, but no longer retains all detail below that range. + +### Colors used below + +```mermaid +flowchart TB + T["Tile-backed hash or range"]:::tile + M["Resident in-memory hash or range"]:::memory + B["Available from both tiles and memory"]:::both + S["Compacted in-memory summary"]:::summary + X["Leaf being proved"]:::target + PT["Hash emitted in a proof
blue outline: from tiles"]:::proofTile + PM["Hash emitted in a proof
green outline: from memory"]:::proofMemory + + T ~~~ M + M ~~~ B + B ~~~ S + S ~~~ X + X ~~~ PT + PT ~~~ PM + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +## What `flush()` and `compact()` each do + +Appending, flushing, and compacting are separate operations: + +```mermaid +flowchart TB + A["append(h)
Add a leaf hash to the in-memory tree"]:::memory + B["A complete 32-entry range now exists
but no file is written automatically"]:::memory + C["flush()
Write every newly complete full tile"]:::tile + D["The same range exists on disk and in memory
(the default after flush)"]:::both + E["compact()
Optionally discard old resident detail"]:::summary + F["Old detail is served from tiles;
the incomplete frontier stays in memory"]:::both + + A --> B + B --> C + C --> D + D --> E + E --> F + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 +``` + +`flush()` does not compact by default. Setting `compact_on_flush = true` makes +the final two steps happen in one call, but the durability rule is unchanged: +compaction happens only after all required tile writes succeed. + +If a tile write fails, `immutable_size()` may advance past `flushed_size()` +because a published tile cannot be rolled back. Keep the same tree contents and +retry the flush. See +[Flushing and compaction](tiles-guide.md#flushing-and-compaction) for the full +interrupted-write contract. + +## What is inside a tile file? + +In the illustrative model, `tile/0/000` is the concatenation of 32 leaf hashes: + +```mermaid +flowchart TB + F["tile/0/000
32 serialized hashes"]:::tile + A["entries 0..7
h0 ... h7"]:::tile + B["entries 8..15
h8 ... h15"]:::tile + C["entries 16..23
h16 ... h23"]:::tile + D["entries 24..31
h24 ... h31"]:::tile + R["R[0, 32)
reconstructed by hashing the entries"]:::computed + N["Internal binary-tree nodes are reconstructed;
they are not separately stored in the file"]:::note + + F -->|first bytes| A + A -->|followed by| B + B -->|followed by| C + C -->|followed by| D + D --> R + R --> N + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef note fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +At level 1, each entry is already the root of 32 leaves: + +```mermaid +flowchart TB + L1["tile/1/000
32 serialized subtree roots"]:::tile + L0A["entry 0
root(tile/0/000) = R[0, 32)"]:::tile + L0B["entry 1
root(tile/0/001) = R[32, 64)"]:::tile + L0C["entries 2..30
..."]:::tile + L0Z["entry 31
root(tile/0/031) = R[992, 1024)"]:::tile + ROOT["R[0, 1024)
reconstructed from tile/1/000"]:::computed + + L1 --> L0A + L0A --> L0B + L0B --> L0C + L0C --> L0Z + L0Z --> ROOT + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +The production version of the second diagram needs 256 level-0 tile roots, so +its first full level-1 tile appears at 65,536 leaves rather than 1,024. + +## On-disk file layout + +After an illustrative 1,030-leaf tree is flushed, the full-tile boundary is +1,024: + +```text +prefix/ + tile/ + 0/ + 000 # h0 ... h31 + 001 # h32 ... h63 + ... + 031 # h992 ... h1023 + 1/ + 000 # R[0,32), R[32,64), ... R[992,1024) +``` + +Leaves `[1024, 1030)` do not appear in a tile file because they do not complete +another 32-entry tile. They remain in memory. + +```mermaid +flowchart TB + N["n = 1,030 leaves"]:::computed + C["covered = floor(1,030 / 32) * 32 = 1,024"]:::computed + L0["32 full level-0 files
tile/0/000 through tile/0/031"]:::tile + L1["1 full level-1 file
tile/1/000"]:::tile + M["6-leaf frontier
[1024, 1030) in memory"]:::memory + + N --> C + C --> L0 + L0 --> L1 + L1 --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +The optional `tile/entries/` bundles are omitted here. They store raw +application entries, not Merkle tree nodes, and do not change proof generation. + +## Tree growth, one snapshot at a time + +The next snapshots assume `retention_margin = 0`. Where compaction is shown, +merklecpp still retains the final tiled leaf as a boundary leaf. This is why +the "both" range below is one leaf wide. + +### Snapshot A: 20 leaves + +No full 32-entry tile exists: + +```mermaid +flowchart TB + N["n = 20"]:::computed + C["full-tile boundary = 0"]:::computed + M["Memory only
[0, 20)"]:::memory + D["Disk
no tile files"]:::empty + + N --> C + C --> M + M --> D + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef empty fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +Calling `flush()` at this point writes nothing. Every root and proof is served +from the in-memory tree. + +### Snapshot B: 40 leaves, before the first flush + +The first 32 leaves form a complete tile, but tile creation is explicit: + +```mermaid +flowchart TB + N["n = 40"]:::computed + M0["[0, 32)
complete and eligible, still memory only"]:::memory + M1["[32, 40)
incomplete frontier, memory only"]:::memory + D["Disk
still empty until flush()"]:::empty + + N --> M0 + M0 --> M1 + M1 --> D + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef empty fill:#f9fafb,stroke:#9ca3af,color:#374151 +``` + +### Snapshot B: 40 leaves, after `flush()` + +The default `flush()` writes the full prefix but does not remove it from +memory: + +```mermaid +flowchart TB + F["flush() succeeds
flushed_size() = 32"]:::computed + B["[0, 32)
tile/0/000 + resident memory"]:::both + M["[32, 40)
resident memory only"]:::memory + + F --> B + B --> M + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +At this point a proof may be answered entirely from memory even though a tile +copy exists. + +### Snapshot B: 40 leaves, after compaction + +With zero retention, compaction drops old leaf detail while preserving leaf 31 +as the rollback boundary: + +```mermaid +flowchart TB + R["R[0, 40)
current in-memory root"]:::computed + P["R[0, 32)
prefix represented by compacted summaries"]:::summary + C["[0, 31)
not leaf-addressable in memory"]:::summary + B["h31
retained boundary leaf"]:::both + M["R[32, 40)
fully resident frontier"]:::memory + T["tile/0/000
proof detail for [0, 32)"]:::tile + + R --> P + P --> C + P --> B + R --> M + P -.->|subtree and leaf detail| T + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +There are now three logical ownership ranges: + +| Leaf range | Proof detail available from | +|---|---| +| `[0, 31)` | tiles only | +| `[31, 32)` | tiles and memory | +| `[32, 40)` | memory only | + +The compacted in-memory summaries still contribute to `root()`. "Tiles only" +means that a request for a leaf or complete subtree in that range must use the +tile source; it does not mean the in-memory root forgot the prefix hash. + +### Snapshot C: grow from 40 to 72 leaves + +Assume the tree was flushed and compacted at size 40, then 32 more leaves were +appended. + +Before the second flush: + +```mermaid +flowchart TB + N["n = 72
flushed_size() is still 32"]:::computed + T["[0, 31)
tiles only"]:::tile + B["[31, 32)
boundary leaf in both"]:::both + E["[32, 64)
complete and eligible, but still memory only"]:::memory + F["[64, 72)
incomplete memory frontier"]:::memory + + N --> T + T --> B + B --> E + E --> F + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +After the second flush and compaction: + +```mermaid +flowchart TB + F["flush() writes tile/0/001
flushed_size() = 64"]:::computed + T0["tile/0/000 covers [0, 32)"]:::tile + T1["tile/0/001 covers [32, 64)"]:::tile + C["[0, 63)
tiles only after compaction"]:::tile + B["[63, 64)
new boundary leaf in both"]:::both + M["[64, 72)
memory-only frontier"]:::memory + + F --> T0 + T0 --> T1 + T1 --> C + C --> B + B --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +### Snapshot D: 1,030 leaves + +This is the first snapshot with a full illustrative level-1 tile: + +```mermaid +flowchart TB + N["n = 1,030"]:::computed + T0["Level 0
32 files cover [0, 1024)"]:::tile + T1["Level 1
tile/1/000 contains their 32 roots"]:::tile + C["After compaction
[0, 1023) uses tiles for proof detail"]:::tile + B["h1023
boundary leaf in both"]:::both + M["[1024, 1030)
memory-only frontier"]:::memory + + N --> T0 + T0 --> T1 + T1 --> C + C --> B + B --> M + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 +``` + +### Snapshot summary + +This table assumes each snapshot has just completed a successful flush and +compaction with zero retention: + +| Tree size | `flushed_size()` | Files written | Tiles only | Tiles + memory | Memory only | +|---:|---:|---|---|---|---| +| 20 | 0 | none | none | none | `[0, 20)` | +| 40 | 32 | `tile/0/000` | `[0, 31)` | `[31, 32)` | `[32, 40)` | +| 72 | 64 | `tile/0/000..001` | `[0, 63)` | `[63, 64)` | `[64, 72)` | +| 1,030 | 1,024 | 32 level-0 tiles + `tile/1/000` | `[0, 1023)` | `[1023, 1024)` | `[1024, 1030)` | + +Again, multiply the tile geometry back to 256 for production. In particular, +the production level-1 example starts at 65,536 leaves, not 1,024. + +## How a proof finds a subtree root + +`TiledTree` gives `ProofEngine` a combined source. It tries the resident tree +first because that avoids I/O, then falls back to tiles: + +```mermaid +flowchart TB + Q["ProofEngine requests R[a, b)"]:::computed + L{"Is the range one leaf?"}:::decision + A{"Otherwise, is its width a power of two
and is the range aligned to that width?"}:::decision + P{"Is this complete subtree
fully resident in memory?"}:::decision + M["Return the in-memory hash"]:::memory + T{"Can the tile source resolve it
inside flushed_size()?"}:::decision + D["Read the appropriate tile entries
and roll them up"]:::tile + S["Split the range into smaller subtrees
and resolve each one"]:::computed + E["Fail if no source can resolve a required leaf"]:::error + + Q --> L + L -->|yes| P + L -->|no| A + A -->|yes| P + A -->|no| S + P -->|yes| M + P -->|no| T + T -->|yes| D + T -->|no, and range has multiple leaves| S + T -->|no, and range is one leaf| E + S -->|smaller range| Q + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef decision fill:#fff7ed,stroke:#c2410c,color:#111827 + classDef error fill:#fee2e2,stroke:#dc2626,color:#111827 +``` + +For example, after compacting the 40-leaf tree: + +- `R[0, 32)` is not fully resident, so memory declines it and tiles return it. +- `R[32, 36)` is resident, so memory returns it without touching disk. +- `R[24, 40)` crosses the boundary and is not one complete aligned subtree. + The proof engine splits it into resolvable pieces. + +## Inclusion proof 1: entirely from one tile + +Consider a proof against tree size 32 after `tile/0/000` has been written and +the old leaves have been compacted. This may be the current size or a historical +prefix of a larger tree. We want to prove leaf 5. + +Every required hash is reconstructed from `tile/0/000`: + +```mermaid +flowchart TB + R032["R[0, 32)"]:::tile + R016["R[0, 16)"]:::tile + P1632["R[16, 32)
proof"]:::proofTile + R08["R[0, 8)"]:::tile + P816["R[8, 16)
proof"]:::proofTile + P04["R[0, 4)
proof"]:::proofTile + R48["R[4, 8)"]:::tile + R46["R[4, 6)"]:::tile + P68["R[6, 8)
proof"]:::proofTile + P4["h4
proof"]:::proofTile + X5["h5
target leaf"]:::target + + R032 --> R016 + R032 --> P1632 + R016 --> R08 + R016 --> P816 + R08 --> P04 + R08 --> R48 + R48 --> R46 + R48 --> P68 + R46 --> P4 + R46 --> X5 + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 +``` + +The proof payload is ordered from the leaf toward the root: + +| Order | Proof hash | Position relative to the running hash | Source | +|---:|---|---|---| +| 1 | `h4` | left | `tile/0/000` | +| 2 | `R[6, 8)` | right | `tile/0/000` | +| 3 | `R[0, 4)` | left | `tile/0/000` | +| 4 | `R[8, 16)` | right | `tile/0/000` | +| 5 | `R[16, 32)` | right | `tile/0/000` | + +The internal roots in this table are computed on demand from the tile's leaf +hashes. They are not additional files. + +Verification starts with `h5`, combines the five proof hashes in order, and +arrives at `R[0, 32)`. + +## Inclusion proof 2: tiles and memory together + +Return to the compacted 40-leaf tree and prove leaf 36 against the current root +`R[0, 40)`. + +The target and its nearby siblings are in the resident frontier. The old +32-leaf prefix is supplied as one tile-backed subtree root: + +```mermaid +flowchart TB + R040["R[0, 40)"]:::computed + P032["R[0, 32)
proof from tile"]:::proofTile + R3240["R[32, 40)
resident frontier"]:::memory + P3236["R[32, 36)
proof from memory"]:::proofMemory + R3640["R[36, 40)"]:::memory + R3638["R[36, 38)"]:::memory + P3840["R[38, 40)
proof from memory"]:::proofMemory + X36["h36
target leaf"]:::target + P37["h37
proof from memory"]:::proofMemory + + R040 --> P032 + R040 --> R3240 + R3240 --> P3236 + R3240 --> R3640 + R3640 --> R3638 + R3640 --> P3840 + R3638 --> X36 + R3638 --> P37 + + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef target fill:#fed7aa,stroke:#ea580c,stroke-width:3px,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +The mixed proof payload is: + +| Order | Proof hash | Position | Source | +|---:|---|---|---| +| 1 | `h37` | right | memory | +| 2 | `R[38, 40)` | right | memory | +| 3 | `R[32, 36)` | left | memory | +| 4 | `R[0, 32)` | left | `tile/0/000` | + +The caller sees one ordinary `merkle::Path`. Source selection is internal; the +proof format does not mark some hashes as "tile" and others as "memory". + +Proving an old leaf in the current tree is mixed in the opposite direction. +For example, a proof for leaf 5 at size 40 gets its target and lower siblings +from `tile/0/000`, then gets the final sibling `R[32, 40)` from memory. + +## Consistency proofs: the idea + +An inclusion proof answers: + +> Is this leaf part of this tree root? + +A consistency proof answers: + +> Can the tree with `m` leaves be extended, without changing its first `m` +> leaves, to produce the tree with `n` leaves? + +The verifier already knows: + +- `m` and the old root `R[0, m)`; +- `n` and the new root `R[0, n)`. + +The proof supplies enough complete subtree roots to reconstruct both roots +through a shared history. + +The producer recursively follows the part of the new tree that contains the +old boundary and emits the sibling subtree at each split: + +```mermaid +flowchart TB + A["Start with [0, n) and old size m"]:::computed + B["Split at the largest power of two
smaller than the current range"]:::computed + C{"Which side contains
the old boundary?"}:::decision + D["Recurse into that side"]:::computed + E["Emit the other side's root
as a proof hash"]:::proof + F{"Reached exactly
the old boundary?"}:::decision + G["Return proof hashes
from deepest to highest"]:::proof + + A --> B + B --> C + C --> D + D --> E + E --> F + F -->|no| B + F -->|yes| G + + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef decision fill:#fff7ed,stroke:#c2410c,color:#111827 + classDef proof fill:#fef3c7,stroke:#b45309,color:#111827 +``` + +Each emitted range is resolved through the same memory-first, tile-second +source used by inclusion proofs. + +## Consistency proof 1: a perfect old tree + +First prove that the 32-leaf tree is a prefix of the 40-leaf tree: + +```cpp +auto proof = log.consistency_proof(32, 40); +``` + +Because 32 is a power of two, the old root is already one complete left +subtree. The proof needs only the new right-hand range: + +```mermaid +flowchart TB + OLD["Known old root
R[0, 32)"]:::tile + EXT["proof[0]
R[32, 40) from memory"]:::proofMemory + JOIN["H(R[0, 32), R[32, 40))"]:::computed + NEW["Expected new root
R[0, 40)"]:::result + + OLD --> JOIN + EXT --> JOIN + JOIN --> NEW + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +The old root may have been calculated from `tile/0/000`; the extension is +resident in memory. Verification combines the known old root with the single +proof hash and compares the result with the known new root. + +## Consistency proof 2: a non-perfect old tree + +Now prove that the 20-leaf tree is a prefix of the 40-leaf tree: + +```cpp +auto proof = log.consistency_proof(20, 40); +``` + +Size 20 is not a power of two, so the old root does not line up with a single +node in the 40-leaf tree. The proof decomposes the relevant ranges: + +```mermaid +flowchart TB + R040["R[0, 40)"]:::computed + R032["R[0, 32)"]:::tile + P3240["P4 = R[32, 40)
memory"]:::proofMemory + P016["P3 = R[0, 16)
tile"]:::proofTile + R1632["R[16, 32)"]:::tile + R1624["R[16, 24)"]:::tile + P2432["P2 = R[24, 32)
tile"]:::proofTile + P1620["P0 = R[16, 20)
tile seed"]:::proofTile + P2024["P1 = R[20, 24)
tile"]:::proofTile + + R040 --> R032 + R040 --> P3240 + R032 --> P016 + R032 --> R1632 + R1632 --> R1624 + R1632 --> P2432 + R1624 --> P1620 + R1624 --> P2024 + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef proofMemory fill:#fef3c7,stroke:#16a34a,stroke-width:3px,color:#111827 +``` + +The proof vector contains hashes only; the range labels are shown here to make +the algorithm visible. Given `m = 20` and `n = 40`, the verifier derives where +each hash belongs. + +| Order | Illustrative range | Source | Why it is needed | +|---:|---|---|---| +| `P0` | `R[16, 20)` | tile | Seed shared by old and new reconstructions | +| `P1` | `R[20, 24)` | tile | Extend only the new reconstruction | +| `P2` | `R[24, 32)` | tile | Extend only the new reconstruction | +| `P3` | `R[0, 16)` | tile | Complete both the old and new left sides | +| `P4` | `R[32, 40)` | memory | Extend the new reconstruction to size 40 | + +Verification evolves two accumulators: + +The verifier uses the bit structure of `m` and `n` to decide which accumulator +each proof hash updates. Intuitively, `P0` seeds a subtree shared by both +histories and `P3` completes that shared old-tree boundary. `P1`, `P2`, and +`P4` cover leaves at or beyond the old size, so they extend only the new +accumulator. + +```mermaid +flowchart TB + S["Seed both accumulators with P0
old = new = R[16, 20)"]:::proofTile + A["Combine P1 on the right
new = R[16, 24)"]:::tile + B["Combine P2 on the right
new = R[16, 32)"]:::tile + C["Combine P3 on the left
old = R[0, 20)
new = R[0, 32)"]:::tile + D["Combine P4 on the right
new = R[0, 40)"]:::memory + V["Compare both reconstructed roots
with the caller's old and new roots"]:::result + + S --> A + A --> B + B --> C + C --> D + D --> V + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef proofTile fill:#fef3c7,stroke:#2563eb,stroke-width:3px,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +This example is mixed in a useful way: + +- The old 20-leaf state can be reconstructed from the first tile even though + the live in-memory tree has compacted those leaves. +- The newly appended range `[32, 40)` comes from memory. +- The proof is still an ordinary vector of hashes, independent of where each + hash was found. + +## The complete mental model + +```mermaid +flowchart TB + A["Append leaf hashes"]:::memory + B["In-memory left-balanced Merkle tree"]:::memory + C["A 32-entry range becomes complete
(256 entries in production)"]:::both + D["flush() publishes immutable full tiles"]:::tile + E["compact() optionally drops old resident detail"]:::summary + F["MemoryHashSource
serves whatever remains resident"]:::memory + G["TileHashSource
serves the flushed prefix"]:::tile + H["CombinedHashSource
tries memory, then tiles"]:::both + I["ProofEngine"]:::computed + J["Current or historical root"]:::result + K["Inclusion proof"]:::result + L["Consistency proof"]:::result + + A --> B + B --> C + C --> D + D -.->|optional| E + D --> F + D --> G + E --> F + F --> H + G --> H + H --> I + I --> J + I --> K + I --> L + + classDef tile fill:#dbeafe,stroke:#2563eb,color:#111827 + classDef memory fill:#dcfce7,stroke:#16a34a,color:#111827 + classDef both fill:#f3e8ff,stroke:#9333ea,color:#111827 + classDef summary fill:#e5e7eb,stroke:#6b7280,color:#111827 + classDef computed fill:#ede9fe,stroke:#7c3aed,color:#111827 + classDef result fill:#dcfce7,stroke:#15803d,stroke-width:3px,color:#111827 +``` + +The important boundary is always the last successfully flushed full tile: + +- Below it, immutable tiles can preserve proof detail after compaction. +- Above it, the incomplete frontier must remain resident in memory. +- A proof may resolve several component subtrees from either side of the + boundary, but the caller receives one normal proof. +- None of these rules depends on the illustrative width of 32. Production uses + the same model with 256-entry tiles. diff --git a/merklecpp.h b/merklecpp.h index 5709da7..0306f2e 100644 --- a/merklecpp.h +++ b/merklecpp.h @@ -13,8 +13,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -703,7 +705,14 @@ namespace merkle /// 2**height-1. [[nodiscard]] bool is_full() const { - size_t max_size = (1 << height) - 1; + constexpr size_t size_digits = std::numeric_limits::digits; + if (height > size_digits) + { + return false; + } + const size_t max_size = height == size_digits ? + std::numeric_limits::max() : + (size_t{1} << height) - 1; assert(size <= max_size); return size == max_size; } @@ -1336,6 +1345,80 @@ namespace merkle leaf_node(index)->hash, index, std::move(path), as_of); } + /// @brief Extracts the root hash of a complete subtree resident in memory + /// @param level The height of the subtree (it spans 2**level leaves) + /// @param index The index of the subtree at that height + /// @return The subtree root hash if the subtree is complete (balanced) and + /// fully resident in memory; otherwise, std::nullopt + /// @note Like root() and path(), it may + /// materialize pending nodes and compute dirty hashes, but does not change + /// logical leaf contents or hashing semantics. It returns std::nullopt if + /// any leaf of the subtree has been flushed, if the subtree extends past + /// the last leaf, or if the node at that position is not a full subtree. + /// The subtree spans leaf indices + /// [index << level, (index + 1) << level). + std::optional subtree_root(uint8_t level, size_t index) + { + const size_t leaves = num_leaves(); + if (leaves == 0 || level >= std::numeric_limits::digits) + { + return std::nullopt; + } + if (index > (std::numeric_limits::max() >> level)) + { + return std::nullopt; + } + + const size_t lo = index << level; + const size_t count = (size_t)1 << level; + + if (lo < min_index() || count > leaves || lo > leaves - count) + { + return std::nullopt; + } + + if (level == 0) + { + return leaf(lo); + } + + compute_root(); + + const uint8_t target_height = level + 1; + if (!_root || _root->height < target_height) + { + return std::nullopt; + } + + Node* cur = _root; + size_t it = lo << (sizeof(lo) * 8 - _root->height + 1); + for (uint8_t height = _root->height; height > target_height;) + { + const bool go_right = ((it >> (8 * sizeof(it) - 1)) & 0x01) != 0U; + if (cur->height == height) + { + Node* next = go_right ? cur->right : cur->left; + if (!next) + { + return std::nullopt; // conflated/flushed: not resident + } + cur = next; + } + it <<= 1; + height--; + } + + if (cur->height != target_height || !cur->is_full()) + { + return std::nullopt; + } + if (cur->dirty) + { + hash(cur); + } + return cur->hash; + } + /// @brief Serialises the tree /// @param bytes The vector of bytes to serialise to void serialise(std::vector& bytes) diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 5169d14..c1a7e82 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -255,11 +256,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::vector bytes = read_file(path, expected); if (bytes.size() != expected) { - throw std::runtime_error(std::format( - "unexpected tile size for {}: expected {} bytes, got {}", - path.string(), - expected, - bytes.size())); + throw std::runtime_error( + std::format( + "unexpected tile size for {}: expected {} bytes, got {}", + path.string(), + expected, + bytes.size())); } std::vector hashes; @@ -322,8 +324,9 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } catch (const std::runtime_error& error) { - throw std::runtime_error(std::format( - "invalid entry bundle {}: {}", path.string(), error.what())); + throw std::runtime_error( + std::format( + "invalid entry bundle {}: {}", path.string(), error.what())); } } @@ -484,6 +487,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) throw std::runtime_error( "hash algorithm short name does not match hash size"); } + prefix = std::filesystem::absolute(prefix); prefix /= storage_directory_name(hash_algorithm_short_name); return prefix; } @@ -603,10 +607,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const bool exists = std::filesystem::exists(directory, ec); if (ec) { - throw std::runtime_error(std::format( - "cannot inspect directory {}: {}", - directory.string(), - ec.message())); + throw std::runtime_error( + std::format( + "cannot inspect directory {}: {}", + directory.string(), + ec.message())); } if (exists) { @@ -614,16 +619,19 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::filesystem::is_directory(directory, ec); if (ec) { - throw std::runtime_error(std::format( - "cannot inspect directory {}: {}", - directory.string(), - ec.message())); + throw std::runtime_error( + std::format( + "cannot inspect directory {}: {}", + directory.string(), + ec.message())); } if (!is_directory) { - throw std::runtime_error(std::format( - "cannot create directory {}: path exists and is not a directory", - directory.string())); + throw std::runtime_error( + std::format( + "cannot create directory {}: path exists and is not a " + "directory", + directory.string())); } } else @@ -631,10 +639,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const bool created = std::filesystem::create_directory(directory, ec); if (ec) { - throw std::runtime_error(std::format( - "cannot create directory {}: {}", - directory.string(), - ec.message())); + throw std::runtime_error( + std::format( + "cannot create directory {}: {}", + directory.string(), + ec.message())); } if (!created && !std::filesystem::is_directory(directory, ec)) { @@ -731,6 +740,32 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @endcond }; + namespace detail + { + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + HashT perfect_root_range( + const std::vector>& hashes, + size_t offset, + size_t count) + { + if (count == 1) + { + return hashes[offset]; + } + const size_t half = count / 2; + const auto left = + perfect_root_range(hashes, offset, half); + const auto right = perfect_root_range( + hashes, offset + half, half); + HashT out; + HASH_FUNCTION(left, right, out); + return out; + } + } + /// @brief Computes the Merkle Tree Hash of a perfect (balanced) subtree. /// @param leaves The subtree's leaves; the count MUST be a power of two. /// @return The subtree root, computed with the tree's HASH_FUNCTION. @@ -754,20 +789,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) "perfect_root requires a power-of-two number of leaves"); } - std::vector> level = leaves; - while (level.size() > 1) - { - std::vector> next; - next.reserve(level.size() / 2); - for (size_t i = 0; i + 1 < level.size(); i += 2) - { - HashT h; - HASH_FUNCTION(level[i], level[i + 1], h); - next.push_back(h); - } - level.swap(next); - } - return level.front(); + return detail::perfect_root_range( + leaves, 0, leaves.size()); } /// @brief Computes and persists tlog-tiles tiles for a growing tree. @@ -935,6 +958,858 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } }; + /// @brief Abstract source of Merkle subtree roots for proof generation. + /// @note Implementations resolve the root of a complete (balanced) subtree + /// from tiles, from an in-memory tree, or from a combination of the two. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + struct HashSourceT + { + /// @brief The type of hashes resolved. + using Hash = HashT; + + virtual ~HashSourceT() = default; + + /// @brief Resolves MTH(D[index << level : (index + 1) << level]). + /// @param level The subtree height (the subtree spans 2**level leaves) + /// @param index The subtree index at that height + /// @param out Set to the subtree root on success + /// @return Whether the complete, balanced subtree could be resolved + virtual bool subtree_root( + uint8_t level, uint64_t index, Hash& out) const = 0; + + /// @brief Resolves the level-0 leaf hash at @p index. + virtual bool leaf(uint64_t index, Hash& out) const + { + return subtree_root(0, index, out); + } + }; + + /// @brief Resolves subtree roots from tlog-tiles tile files. + /// @note @p available_size is rounded down to a whole number of full tiles: + /// only complete, durably-written full tiles are read. A complete subtree + /// within that full-tile prefix is resolvable; anything reaching into the + /// incomplete frontier yields false so that a proof builder can fall back + /// to another source (e.g. an in-memory tree). + /// @warning No internal synchronization is provided. Even const operations + /// update the internal LRU cache, so callers must serialize all access to a + /// shared source. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TileHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Store = TileStoreT; + + /// @brief Constructs a source over @p store for trees up to + /// @p available_size leaves. @p available_size is rounded down to a whole + /// number of full tiles, since only full tiles are durable. + TileHashSourceT(const Store& store, uint64_t available_size) : + store(store), available_size((available_size / TILE_WIDTH) * TILE_WIDTH) + { + tile_cache.reserve(TILE_CACHE_SIZE); + } + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + // The subtree covers leaves [index << level, (index + 1) << level). It + // is resolvable only when it lies entirely within the full-tile-covered + // prefix; the incomplete frontier is served from another source. + if (level >= 64 || index >= (available_size >> level)) + { + return false; + } + resolve(level, index, out); + return true; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Store& store; + uint64_t available_size; // full-tile prefix length (a multiple of WIDTH) + + /// @brief Combines the @p span entries at @p off of @p tile into a root. + static Hash roll_up( + const std::vector& tile, uint64_t off, uint64_t span) + { + if ( + span == 0 || span > TILE_WIDTH || (span & (span - 1)) != 0 || + off > tile.size() || span > tile.size() - off) + { + throw std::runtime_error("invalid tile roll-up range"); + } + return detail::perfect_root_range( + tile, static_cast(off), static_cast(span)); + } + + /// @brief Resolves a complete subtree known to lie within the full-tile + /// prefix, reading the highest-level full tile that holds it (and rolling + /// up); descends to lower full tiles when a higher-level full tile has + /// not completed. Terminates because full level-0 tiles always cover the + /// prefix. + void resolve(uint8_t level, uint64_t index, Hash& out) const + { + if (level <= TILE_HEIGHT) + { + // Spans 2**level <= TILE_WIDTH leaves: held by one level-0 tile. + const uint64_t span = (uint64_t)1 << level; + const uint64_t start = index << level; + const std::vector& tile = + read_tile(TileRef{0, start / TILE_WIDTH}); + out = roll_up(tile, start % TILE_WIDTH, span); + return; + } + + const uint8_t L = level / TILE_HEIGHT; + const uint8_t r = level % TILE_HEIGHT; + const uint64_t first = index << r; // first level-L entry + const uint64_t n = first / TILE_WIDTH; // level-L tile index + const unsigned full_shift = + static_cast(TILE_HEIGHT) * (static_cast(L) + 1U); + const uint64_t full_tiles = + full_shift >= 64 ? 0 : (available_size >> full_shift); + + if (n < full_tiles) + { + // One full level-L tile holds all 2**r entries of this subtree. + const std::vector& tile = read_tile(TileRef{L, n}); + out = roll_up(tile, first % TILE_WIDTH, (uint64_t)1 << r); + return; + } + + // No full level-L tile here: split into two level-(level-1) subtrees. + Hash lo; + Hash hi; + resolve((uint8_t)(level - 1), index * 2, lo); + resolve((uint8_t)(level - 1), index * 2 + 1, hi); + HASH_FUNCTION(lo, hi, out); + } + + struct TileCacheEntry + { + TileRef ref; + std::vector hashes; + }; + + static constexpr size_t TILE_CACHE_SIZE = 64; + mutable std::vector tile_cache; + + const std::vector& read_tile(const TileRef& ref) const + { + for (auto it = tile_cache.begin(); it != tile_cache.end(); it++) + { + if (it->ref.level == ref.level && it->ref.index == ref.index) + { + TileCacheEntry entry = std::move(*it); + tile_cache.erase(it); + tile_cache.push_back(std::move(entry)); + return tile_cache.back().hashes; + } + } + + if (tile_cache.size() >= TILE_CACHE_SIZE) + { + tile_cache.erase(tile_cache.begin()); + } + tile_cache.push_back(TileCacheEntry{ref, store.read_tile(ref)}); + return tile_cache.back().hashes; + } + }; + + /// @brief Builds and verifies inclusion and consistency proofs. + /// @note Proofs are assembled from a HashSourceT using the tree's + /// HASH_FUNCTION, so an inclusion proof is byte-identical to the one + /// produced by merkle::TreeT::path()/past_path() and verifies with + /// PathT::verify(). + /// @warning Thread safety is inherited from the supplied HashSourceT. + /// Callers must serialize operations when the source is shared. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class ProofEngineT + { + public: + using Hash = HashT; + using Path = PathT; + using Source = HashSourceT; + + explicit ProofEngineT(const Source& source) : source(source) {} + + /// @brief The Merkle root of a tree of @p size leaves. + Hash root(uint64_t size) const + { + if (size == 0) + { + throw std::runtime_error("empty tree has no root"); + } + Hash out; + if (!mth_range(0, size, out)) + { + throw std::runtime_error("unresolved subtree while computing root"); + } + return out; + } + + /// @brief Inclusion proof for leaf @p index in a tree of @p size leaves. + /// @note Equivalent to TreeT::path(index) when size == num_leaves(), and + /// to TreeT::past_path(index, size - 1) otherwise. + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size) const + { + if (index >= size) + { + throw std::runtime_error("leaf index out of bounds"); + } + if ( + index > std::numeric_limits::max() || + size - 1 > std::numeric_limits::max()) + { + throw std::runtime_error("inclusion proof exceeds PathT index range"); + } + + std::list elements; // leaf -> root order + uint64_t lo = 0; + uint64_t hi = size; + while (hi - lo > 1) + { + const uint64_t k = largest_pow2_lt(hi - lo); + typename Path::Element e; + if (index - lo < k) + { + if (!mth_range(lo + k, hi, e.hash)) + { + throw std::runtime_error("unresolved subtree in inclusion proof"); + } + e.direction = Path::PATH_RIGHT; + hi = lo + k; + } + else + { + if (!mth_range(lo, lo + k, e.hash)) + { + throw std::runtime_error("unresolved subtree in inclusion proof"); + } + e.direction = Path::PATH_LEFT; + lo = lo + k; + } + elements.push_front(std::move(e)); + } + + Hash leaf; + if (!source.leaf(index, leaf)) + { + throw std::runtime_error("unresolved leaf in inclusion proof"); + } + return std::make_shared( + leaf, + static_cast(index), + std::move(elements), + static_cast(size - 1)); + } + + /// @brief Consistency proof that a tree of @p m leaves is a prefix of a + /// tree of @p n leaves (RFC 6962). + std::vector consistency_proof(uint64_t m, uint64_t n) const + { + if (m == 0 || m > n) + { + throw std::runtime_error("invalid consistency proof sizes"); + } + std::vector proof; + if (m == n) + { + return proof; + } + subproof(m, 0, n, true, proof); + return proof; + } + + /// @brief Consistency proof between the trees whose last leaves are at + /// indices @p first_index and @p second_index (first_index <= + /// second_index). + /// @note Equivalent to consistency_proof(first_index + 1, + /// second_index + 1): it proves the tree of the first first_index + 1 + /// leaves is a prefix of the tree of the first second_index + 1 leaves. + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) const + { + if ( + first_index == std::numeric_limits::max() || + second_index == std::numeric_limits::max()) + { + throw std::runtime_error("consistency proof index out of bounds"); + } + return consistency_proof(first_index + 1, second_index + 1); + } + + /// @brief Verifies an RFC 6962 consistency proof reconciling the roots of + /// trees of @p m and @p n leaves. + static bool verify_consistency( + uint64_t m, + uint64_t n, + const Hash& first_hash, + const Hash& second_hash, + const std::vector& proof) + { + if (m > n) + { + return false; + } + if (m == n) + { + return proof.empty() && first_hash == second_hash; + } + if (m == 0) + { + return proof.empty(); + } + + size_t proof_index = 0; + Hash fr = first_hash; + Hash sr = first_hash; + if (!is_pow2(m)) + { + if (proof.empty()) + { + return false; + } + fr = proof[0]; + sr = proof[0]; + proof_index = 1; + } + + uint64_t fn = m - 1; + uint64_t sn = n - 1; + while ((fn & 1) != 0) + { + fn >>= 1; + sn >>= 1; + } + + for (size_t i = proof_index; i < proof.size(); i++) + { + if (sn == 0) + { + return false; + } + const Hash& c = proof[i]; + if ((fn & 1) != 0 || fn == sn) + { + HASH_FUNCTION(c, fr, fr); + HASH_FUNCTION(c, sr, sr); + if ((fn & 1) == 0) + { + while ((fn & 1) == 0 && fn != 0) + { + fn >>= 1; + sn >>= 1; + } + } + } + else + { + HASH_FUNCTION(sr, c, sr); + } + fn >>= 1; + sn >>= 1; + } + + return fr == first_hash && sr == second_hash && sn == 0; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& source; + + static bool is_pow2(uint64_t n) + { + return n != 0 && (n & (n - 1)) == 0; + } + + static uint64_t largest_pow2_lt(uint64_t n) + { + uint64_t k = 1; + while (k <= (n - 1) / 2) + { + k <<= 1; + } + return k; + } + + static uint8_t log2_exact(uint64_t n) + { + uint8_t r = 0; + while (n > 1) + { + n >>= 1; + r++; + } + return r; + } + + /// @brief MTH(D[a:b]) via the source; falls back to splitting when a + /// perfect subtree cannot be resolved directly. + bool mth_range(uint64_t a, uint64_t b, Hash& out) const + { + const uint64_t w = b - a; + if (w == 0) + { + return false; + } + if (w == 1) + { + return source.leaf(a, out); + } + if (is_pow2(w) && (a % w == 0)) + { + if (source.subtree_root(log2_exact(w), a / w, out)) + { + return true; + } + } + const uint64_t k = largest_pow2_lt(w); + Hash left; + Hash right; + if (!mth_range(a, a + k, left) || !mth_range(a + k, b, right)) + { + return false; + } + HASH_FUNCTION(left, right, out); + return true; + } + + void subproof( + uint64_t m, + uint64_t lo, + uint64_t hi, + bool complete, + std::vector& proof) const + { + if (m == hi - lo) + { + if (!complete) + { + Hash h; + if (!mth_range(lo, hi, h)) + { + throw std::runtime_error( + "unresolved subtree in consistency proof"); + } + proof.push_back(h); + } + return; + } + const uint64_t k = largest_pow2_lt(hi - lo); + Hash h; + if (m <= k) + { + subproof(m, lo, lo + k, complete, proof); + if (!mth_range(lo + k, hi, h)) + { + throw std::runtime_error("unresolved subtree in consistency proof"); + } + } + else + { + subproof(m - k, lo + k, hi, false, proof); + if (!mth_range(lo, lo + k, h)) + { + throw std::runtime_error("unresolved subtree in consistency proof"); + } + } + proof.push_back(h); + } + }; + + /// @brief Resolves subtree roots from an in-memory merkle::TreeT. + /// @note Resolves only complete subtrees that are fully resident (not + /// flushed), returning false otherwise so that a builder can fall back to + /// another source. It may materialize pending nodes and compute dirty + /// hashes but does not change logical contents or hashing semantics. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class MemoryHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Tree = TreeT; + + explicit MemoryHashSourceT(Tree& tree) : tree(tree) {} + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + if (index > std::numeric_limits::max()) + { + return false; + } + const auto root = tree.subtree_root(level, static_cast(index)); + if (!root) + { + return false; + } + out = *root; + return true; + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + Tree& tree; + }; + + /// @brief Resolves subtree roots from a primary source, falling back to a + /// secondary source. + /// @note Used to combine an in-memory tree (primary: no I/O, serves the + /// resident frontier) with tile files (secondary: serve the flushed past). + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class CombinedHashSourceT : public HashSourceT + { + public: + using Hash = HashT; + using Source = HashSourceT; + + CombinedHashSourceT(const Source& primary, const Source& secondary) : + primary(primary), secondary(secondary) + {} + + bool subtree_root(uint8_t level, uint64_t index, Hash& out) const override + { + return primary.subtree_root(level, index, out) || + secondary.subtree_root(level, index, out); + } + + protected: + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& primary; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + const Source& secondary; + }; + + /// @brief A merkle tree backed by tlog-tiles storage. + /// @note Appends grow an in-memory tree; flush() durably writes only full + /// (balanced) tiles, so the incomplete frontier is never tiled and stays + /// resident in memory. Compaction (dropping from memory the leaves already + /// covered by a full tile) is optional: enable it per flush with + /// Config::compact_on_flush, or call compact() explicitly; it never drops + /// the un-tiled frontier. Proofs are served from the combination of the + /// resident tree (frontier) and the full tiles (compacted past). + /// @note TiledTree creates a new tiled tree and cannot reopen one from tile + /// files alone. Construction atomically claims a previously absent tile + /// namespace because the files do not identify their tree or record enough + /// state to restore it. Use TileWriter directly only when the caller owns + /// and restores that state. + /// @warning No internal synchronization is provided. Callers must serialize + /// all access to a shared tree, including proof operations. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TiledTreeT + { + public: + using Hash = HashT; + using Tree = TreeT; + using Path = PathT; + using Store = TileStoreT; + using Writer = TileWriterT; + using Stats = typename Writer::Stats; + + /// @brief Configuration for a tiled tree. + struct Config + { + /// @brief Root directory for a new tiled tree. + /// @note The directory itself may exist, but its algorithm-qualified + /// tile subdirectory must be absent. + std::filesystem::path prefix; + + /// @brief Number of most-recent leaves to keep resident when + /// compacting (i.e. a minimum never dropped from memory). + /// @note Tile alignment may retain up to TILE_WIDTH - 1 additional + /// tiled leaves. A zero margin retains one tiled boundary leaf so + /// rollback to exactly immutable_size() remains possible. + uint64_t retention_margin = 0; + + /// @brief If set, flush() compacts after writing tiles, dropping + /// from memory the leaves already covered by a full tile. Off by + /// default: tiles are written but the tree keeps every leaf resident. + bool compact_on_flush = false; + }; + + explicit TiledTreeT(Config config) : + config(std::move(config)), store(this->config.prefix), writer(store) + { + claim_tile_namespace(); + } + + /// @brief Constructs a tiled tree with an explicit storage namespace. + /// @note Required for custom hash functions whose algorithm name cannot + /// be inferred by TileStoreT. + TiledTreeT(Config config, const std::string& hash_algorithm_short_name) : + config(std::move(config)), + store(this->config.prefix, hash_algorithm_short_name), + writer(store) + { + claim_tile_namespace(); + } + + TiledTreeT(const TiledTreeT&) = delete; + TiledTreeT& operator=(const TiledTreeT&) = delete; + + /// @brief Moves a tiled tree, rebinding its writer to the moved store. + TiledTreeT(TiledTreeT&& other) noexcept : + config(std::move(other.config)), + store(std::move(other.store)), + writer(store), + tree(std::move(other.tree)), + tiles_size(std::exchange(other.tiles_size, 0)), + sealed_size(std::exchange(other.sealed_size, 0)) + {} + + TiledTreeT& operator=(TiledTreeT&&) = delete; + + /// @brief Appends a leaf hash. + void append(const Hash& leaf_hash) + { + tree.insert(leaf_hash); + } + + /// @brief The number of leaves (including flushed ones). + [[nodiscard]] uint64_t size() const + { + return tree.num_leaves(); + } + + /// @brief The current Merkle root. + Hash root() + { + return tree.root(); + } + + /// @brief The number of leaves covered by the last fully successful + /// flush. + /// @note This is always a multiple of TILE_WIDTH. It advances only after + /// every required tile level has been written successfully, and controls + /// proof reads and compaction. + [[nodiscard]] uint64_t flushed_size() const + { + return tiles_size; + } + + /// @brief The rollback seal for ranges a flush may have published. + /// @note A flush seals its full-tile boundary before writing. If the + /// write fails, this may exceed flushed_size(); keep the same tree + /// contents and retry the flush. + [[nodiscard]] uint64_t immutable_size() const + { + return sealed_size; + } + + /// @brief Access to the underlying tree. + /// @warning Mutating the tree directly bypasses tiled-tree bookkeeping. + /// In particular, direct retraction can make flushed_size() and + /// immutable_size() exceed size() and can make flushed_size() regress. + /// Use TiledTreeT operations whenever they are available. + Tree& tree_ref() + { + return tree; + } + + /// @brief Access to the underlying tile store. + /// @warning Files written or changed through this reference are trusted + /// by later flushes without checking that their hashes match this tree. + /// Mismatched files can silently invalidate proofs after compaction. + Store& store_ref() + { + return store; + } + + /// @brief Writes newly-complete full tiles to disk; compacts only if + /// Config::compact_on_flush is set. + /// @return Counts of the full tiles written by this flush + /// @note The full-tile boundary is made immutable before any tile write. + /// Only after every required tile level succeeds does flushed_size() + /// advance to that boundary. On failure, immutable_size() may advance + /// while flushed_size() does not; the tree remains resident and the flush + /// can be retried without rewriting finalized tiles. + Stats flush() + { + Stats stats; + const uint64_t n = tree.num_leaves(); + if (n == 0) + { + return stats; + } + + const uint64_t covered = (n / TILE_WIDTH) * TILE_WIDTH; + if (covered > sealed_size) + { + sealed_size = covered; + } + + stats = writer.write_up_to(n, [this](uint64_t i) -> const Hash& { + return tree.leaf((size_t)i); + }); + tiles_size = covered; + + if (config.compact_on_flush) + { + compact(); + } + return stats; + } + + /// @brief Drops old leaves covered by durably-written full tiles, keeping + /// at least retention_margin recent leaves and a tiled boundary leaf. + /// @return The new minimum (smallest still-resident) leaf index + /// @note Only leaves covered by a full tile are dropped, so the un-tiled + /// frontier is always retained in memory and inclusion/consistency proofs + /// remain available (the past from tiles, the frontier from memory). The + /// leaf at flushed_size() - 1 also remains resident so retract_to() can + /// represent a tree whose size is exactly immutable_size(). Has no effect + /// until tiling has produced full tiles. + uint64_t compact() + { + const uint64_t covered = (tiles_size / TILE_WIDTH) * TILE_WIDTH; + uint64_t target = covered > config.retention_margin ? + covered - config.retention_margin : + 0; + target = (target / TILE_WIDTH) * TILE_WIDTH; + // TreeT cannot retract below min_index(). Keep the final tiled leaf + // resident so rollback to a size of exactly immutable_size() remains + // representable after compaction. + if (covered > 0 && target == covered) + { + target--; + } + if (target > tree.min_index()) + { + tree.flush_to((size_t)target); + } + return tree.min_index(); + } + + /// @brief Rolls the tree back so that @p index becomes the last leaf, + /// removing all leaves after it (same semantics as TreeT::retract_to). + /// @note Only full tiles are immutable: this throws if the resulting size + /// would be smaller than immutable_size(). A failed flush may advance + /// immutable_size() without advancing flushed_size(). + void retract_to(size_t index) + { + if (sealed_size > 0 && (uint64_t)index < sealed_size - 1) + { + throw std::runtime_error( + "TiledTree::retract_to: cannot roll back entries sealed for " + "immutable tiles (resulting size < immutable size)"); + } + tree.retract_to(index); + } + + /// @brief Inclusion proof for @p index in a tree of @p proof_size leaves. + /// @note Served from tiles (flushed past) combined with the resident tree + /// (recent frontier); @p proof_size may exceed flushed_size(). + std::shared_ptr inclusion_proof(uint64_t index, uint64_t proof_size) + { + if (proof_size > size()) + { + throw std::runtime_error( + "inclusion proof size exceeds current tree size"); + } + return with_engine([&](const auto& engine) { + return engine.inclusion_proof(index, proof_size); + }); + } + + /// @brief Consistency proof between tree sizes @p m and @p n. + std::vector consistency_proof(uint64_t m, uint64_t n) + { + if (n > size()) + { + throw std::runtime_error( + "consistency proof size exceeds current tree size"); + } + return with_engine( + [&](const auto& engine) { return engine.consistency_proof(m, n); }); + } + + /// @brief Consistency proof between the trees whose last leaves are at + /// indices @p first_index and @p second_index (first_index <= + /// second_index). + /// @note Equivalent to consistency_proof(first_index + 1, + /// second_index + 1). + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) + { + if (first_index >= size() || second_index >= size()) + { + throw std::runtime_error( + "consistency proof index exceeds current tree size"); + } + return with_engine([&](const auto& engine) { + return engine.consistency_proof_from_indices( + first_index, second_index); + }); + } + + protected: + Config config; + Store store; + Writer writer; + Tree tree; + uint64_t tiles_size = 0; + uint64_t sealed_size = 0; + + void claim_tile_namespace() const + { + const auto tile_root = store.root() / "tile"; + std::error_code ec; + std::filesystem::create_directories(tile_root.parent_path(), ec); + if (ec) + { + throw std::runtime_error( + "TiledTree: cannot create tile namespace parent " + + tile_root.parent_path().string() + ": " + ec.message()); + } + const bool claimed = std::filesystem::create_directory(tile_root, ec); + if (ec) + { + throw std::runtime_error( + "TiledTree: cannot claim tile namespace " + tile_root.string() + + ": " + ec.message()); + } + if (!claimed) + { + throw std::runtime_error( + "TiledTree: tile namespace already exists; reopening or sharing " + "a tiled tree is not supported"); + } + } + + /// @brief Builds a proof engine over the combined resident-tree + /// (frontier) and full-tile (flushed past) source, and invokes @p fn with + /// it. + /// @note The sources and engine are stack-local; @p fn must consume the + /// engine before returning (proofs are returned by value, holding hash + /// copies, so the result outlives the engine). + template + auto with_engine(Fn fn) + { + MemoryHashSourceT mem(tree); + TileHashSourceT tile_src(store, tiles_size); + CombinedHashSourceT combined(mem, tile_src); + ProofEngineT engine(combined); + return fn(engine); + } + }; + /// @brief Writes tlog-tiles entry bundles (raw log entries) for a growing /// log. /// @note Entry bundles are level-0 only and application-owned: merklecpp @@ -1039,10 +1914,74 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) using TileWriter = TileWriterT; + /// @brief Default abstract hash source (SHA256, default hash function). + using HashSource = + HashSourceT; + + /// @brief Default tile-backed hash source (SHA256, default hash function). + using TileHashSource = TileHashSourceT< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>; + + /// @brief Default proof engine (SHA256, default hash function). + using ProofEngine = + ProofEngineT; + + /// @brief Default in-memory hash source (SHA256, default hash function). + using MemoryHashSource = MemoryHashSourceT< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>; + + /// @brief Default combined hash source (SHA256, default hash function). + using CombinedHashSource = CombinedHashSourceT< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>; + + /// @brief Default tiled tree (SHA256, default hash function). + using TiledTree = + TiledTreeT; + /// @brief Default entry-bundle writer (SHA256, default hash function). using EntryBundleWriter = EntryBundleWriterT< merkle::Tree::Hash::size_bytes, merkle::Tree::hash_function>; +#ifdef HAVE_OPENSSL + /// @brief SHA384 tile store. + using TileStore384 = TileStoreT<48, sha384_openssl>; + + /// @brief SHA512 tile store. + using TileStore512 = TileStoreT<64, sha512_openssl>; + + /// @brief SHA384 tile writer. + using TileWriter384 = TileWriterT<48, sha384_openssl>; + + /// @brief SHA512 tile writer. + using TileWriter512 = TileWriterT<64, sha512_openssl>; + + /// @brief SHA384 hash source, tile-backed source and proof engine. + using HashSource384 = HashSourceT<48, sha384_openssl>; + using TileHashSource384 = TileHashSourceT<48, sha384_openssl>; + using ProofEngine384 = ProofEngineT<48, sha384_openssl>; + + /// @brief SHA512 hash source, tile-backed source and proof engine. + using HashSource512 = HashSourceT<64, sha512_openssl>; + using TileHashSource512 = TileHashSourceT<64, sha512_openssl>; + using ProofEngine512 = ProofEngineT<64, sha512_openssl>; + + /// @brief SHA384 memory/combined sources and tiled tree. + using MemoryHashSource384 = MemoryHashSourceT<48, sha384_openssl>; + using CombinedHashSource384 = CombinedHashSourceT<48, sha384_openssl>; + using TiledTree384 = TiledTreeT<48, sha384_openssl>; + + /// @brief SHA512 memory/combined sources and tiled tree. + using MemoryHashSource512 = MemoryHashSourceT<64, sha512_openssl>; + using CombinedHashSource512 = CombinedHashSourceT<64, sha512_openssl>; + using TiledTree512 = TiledTreeT<64, sha512_openssl>; + + /// @brief SHA384/512 entry-bundle writers. + using EntryBundleWriter384 = EntryBundleWriterT<48, sha384_openssl>; + using EntryBundleWriter512 = EntryBundleWriterT<64, sha512_openssl>; +#endif } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9e61e03..e5fd16f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -37,12 +37,32 @@ add_merklecpp_test(serialisation serialisation.cpp) add_merklecpp_test(partial_serialisation partial_serialisation.cpp) add_merklecpp_test(serialise_to_file serialise_to_file.cpp) add_merklecpp_test(coverage coverage.cpp) +add_merklecpp_test(tiles_store tiles_store.cpp) +add_merklecpp_test(tiles_writer tiles_writer.cpp) +add_merklecpp_test(tiles_proofs tiles_proofs.cpp) +add_merklecpp_test(tiles_tree tiles_tree.cpp) +add_merklecpp_test(tiles_entries tiles_entries.cpp) + +if(OPENSSL) + add_merklecpp_test(tiles_hashes tiles_hashes.cpp) +endif() + +if(LONG_TESTS) + add_merklecpp_test(tiles_level2 tiles_level2.cpp) + add_merklecpp_test(time_tiles time_tiles.cpp) + set(TILES_LEVEL2_TIMEOUT 900) + if(WIN32) + set(TILES_LEVEL2_TIMEOUT 3600) + endif() + set_tests_properties( + ${MERKLECPP_TEST_PREFIX}tiles_level2 + ${MERKLECPP_TEST_PREFIX}time_tiles + PROPERTIES TIMEOUT ${TILES_LEVEL2_TIMEOUT} + ) +endif() if(OPENSSL) add_merklecpp_test(compare_hash_functions compare_hash_functions.cpp) endif() -add_merklecpp_test(tiles_store tiles_store.cpp) -add_merklecpp_test(tiles_writer tiles_writer.cpp) -add_merklecpp_test(tiles_entries tiles_entries.cpp) add_merklecpp_test(unit_tests unit_tests.cpp) diff --git a/test/tiles_hashes.cpp b/test/tiles_hashes.cpp new file mode 100644 index 0000000..8ebb6a5 --- /dev/null +++ b/test/tiles_hashes.cpp @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +template < + size_t HASH_SIZE, + typename Tree, + typename TiledTree, + typename ProofEngine> +static void exercise_tiled_hash(const fs::path& prefix, const std::string& name) +{ + const auto hashes = make_hashesT(300); + Tree reference; + typename TiledTree::Config config; + config.prefix = prefix; + config.compact_on_flush = true; + TiledTree tree(config); + + for (const auto& hash : hashes) + { + reference.insert(hash); + tree.append(hash); + } + + expect(tree.flush().full_written == 1, name + " full tile written"); + expect(tree.flushed_size() == 256, name + " flushed size"); + expect(tree.root() == reference.root(), name + " root"); + expect( + fs::file_size(tree.store_ref().root() / "tile" / "0" / "000") == + (uintmax_t)merkle::tiles::TILE_WIDTH * HASH_SIZE, + name + " tile byte size"); + + const auto inclusion = tree.inclusion_proof(0, tree.size()); + expect(inclusion->verify(reference.root()), name + " inclusion proof"); + + const auto consistency = tree.consistency_proof(256, tree.size()); + expect( + ProofEngine::verify_consistency( + 256, + tree.size(), + *reference.past_root(255), + reference.root(), + consistency), + name + " consistency proof"); +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_hashes"); + const fs::path& base = temporary_directory.path(); + + try + { + exercise_tiled_hash< + 48, + merkle::Tree384, + merkle::tiles::TiledTree384, + merkle::tiles::ProofEngine384>(base / "sha384", "SHA384"); + exercise_tiled_hash< + 64, + merkle::Tree512, + merkle::tiles::TiledTree512, + merkle::tiles::ProofEngine512>(base / "sha512", "SHA512"); + + std::cout << "tiles_hashes: OK" << '\n'; + } + catch (const std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + return 1; + } + + return 0; +} diff --git a/test/tiles_level2.cpp b/test/tiles_level2.cpp new file mode 100644 index 0000000..2936ec6 --- /dev/null +++ b/test/tiles_level2.cpp @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// End-to-end coverage of the level-2 tile paths. A full level-2 tile requires +// 256^3 == 16,777,216 leaves, so this is a deliberately large test: it writes +// the tiles for that many leaves (~65k tile files) from a deterministic leaf +// source -- no in-memory tree is built -- and cross-checks the writer's +// level-by-level roll-up against TileHashSourceT::resolve (which reads the +// level-2 tile) and against the underlying leaves. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TILE_WIDTH; +using merkle::tiles::TileHashSource; +using merkle::tiles::TileRef; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Roll up a perfect (power-of-two) set of hashes with the default combiner. +static Hash rollup(const std::vector& hashes) +{ + return merkle::tiles::perfect_root< + merkle::Tree::Hash::size_bytes, + merkle::Tree::hash_function>(hashes); +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_level2"); + const fs::path& dir = temporary_directory.path(); + + try + { + // 256^3 leaves == exactly one full level-2 tile. + const uint64_t n = + (uint64_t)TILE_WIDTH * (uint64_t)TILE_WIDTH * (uint64_t)TILE_WIDTH; + + TileStore store(dir); + TileWriter writer(store); + + // Deterministic leaf hash derived from the index (low 8 bytes); avoids + // materialising a 16.7M-entry vector. + Hash leaf; + const auto leaf_at = [&](uint64_t i) -> const Hash& { + leaf.zero(); + for (int b = 0; b < 8; b++) + { + leaf.bytes[b] = (uint8_t)(i >> (8 * b)); + } + return leaf; + }; + + const auto stats = writer.write_up_to(n, leaf_at); + + // 65536 full L0 tiles + 256 full L1 tiles + 1 full L2 tile. + expect( + stats.full_written == (uint64_t)TILE_WIDTH * TILE_WIDTH + TILE_WIDTH + 1, + "level2: tile counts"); + expect(store.has_full_tile(2, 0), "level2: L2 tile present"); + expect(!store.has_full_tile(2, 1), "level2: no second L2 tile"); + expect(!store.has_full_tile(3, 0), "level2: no L3 tile"); + + const auto l2 = store.read_tile(TileRef{2, 0}); + expect(l2.size() == TILE_WIDTH, "level2: L2 tile width"); + + const TileHashSource src(store, n); + + // Each level-2 entry j is the root of level-1 tile j, which rolls up + // level-0 tiles, which are the leaves verbatim. Cross-check the writer's + // roll-up, resolve's level-2 read, and the leaf chain on a sample of + // indices. + for (const uint64_t j : + {(uint64_t)0, (uint64_t)1, (uint64_t)200, (uint64_t)255}) + { + const auto l1j = store.read_tile(TileRef{1, j}); + expect(l2[j] == rollup(l1j), "level2: L2[j] == rollup(L1 tile j)"); + + // resolve reads the level-2 tile for the 2^16-leaf subtree at index j. + Hash r16; + expect( + src.subtree_root(16, j, r16), "level2: subtree_root(16,j) resolves"); + expect(r16 == l2[j], "level2: resolve(16,j) == L2[j]"); + + // Anchor to leaves: L1 tile j entry 0 == root of L0 tile (j*256), whose + // first entry is leaf (j * 65536). + const auto l0 = store.read_tile(TileRef{0, j * TILE_WIDTH}); + expect(l1j[0] == rollup(l0), "level2: L1[j][0] == rollup(L0 tile)"); + expect( + l0[0] == leaf_at(j * (uint64_t)TILE_WIDTH * TILE_WIDTH), + "level2: L0 entry == leaf"); + } + + // Intra-tile roll-up: the 2^17-leaf subtree at 0 hashes L2[0] and L2[1]. + Hash r17; + expect(src.subtree_root(17, 0, r17), "level2: subtree_root(17,0) resolves"); + expect( + r17 == rollup({l2[0], l2[1]}), "level2: resolve(17,0) == H(L2[0],L2[1])"); + + std::cout << "tiles_level2: OK" << '\n'; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +} diff --git a/test/tiles_proofs.cpp b/test/tiles_proofs.cpp new file mode 100644 index 0000000..1f7d453 --- /dev/null +++ b/test/tiles_proofs.cpp @@ -0,0 +1,422 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::CombinedHashSource; +using merkle::tiles::MemoryHashSource; +using merkle::tiles::ProofEngine; +using merkle::tiles::TILE_WIDTH; +using merkle::tiles::TileHashSource; +using merkle::tiles::TileStore; +using merkle::tiles::TileWriter; + +class ProofEngineProbe : public ProofEngine +{ +public: + using ProofEngine::largest_pow2_lt; +}; + +class TreeProbe : public merkle::Tree +{ +public: + static bool node_is_full(uint8_t height, size_t size) + { + Node node{}; + node.height = height; + node.size = size; + return node.is_full(); + } +}; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +// Validates the TreeT::subtree_root accessor via memory-only proofs: they must +// match the library exactly. +static void check_memory_source(uint64_t n, const std::vector& hashes) +{ + const std::string at = " @n=" + std::to_string(n); + + merkle::Tree tree; + for (uint64_t i = 0; i < n; i++) + { + tree.insert(hashes[i]); + } + const Hash root = tree.root(); + + const MemoryHashSource source(tree); + const ProofEngine engine(source); + + expect(engine.root(n) == root, "mem root" + at); + + std::vector indices; + if (n <= 16) + { + for (uint64_t i = 0; i < n; i++) + { + indices.push_back(i); + } + } + else + { + for (const uint64_t i : {(uint64_t)0, (uint64_t)1, n / 2, n - 1}) + { + indices.push_back(i); + } + } + + for (const uint64_t i : indices) + { + const auto p = engine.inclusion_proof(i, n); + expect( + *p == *tree.path(i), "mem inclusion==path i=" + std::to_string(i) + at); + expect(p->verify(root), "mem inclusion verify i=" + std::to_string(i) + at); + } + + std::vector> pairs; + if (n <= 16) + { + for (uint64_t m = 1; m < n; m++) + { + for (uint64_t k = m + 1; k <= n; k++) + { + pairs.emplace_back(m, k); + } + } + } + else + { + pairs = {{1, n}, {n / 2, n}, {n - 1, n}}; + } + + for (const auto& pr : pairs) + { + const uint64_t m = pr.first; + const uint64_t k = pr.second; + const Hash rm = engine.root(m); + const Hash rk = engine.root(k); + expect(rm == *tree.past_root(m - 1), "mem past_root m" + at); + + const auto pp = engine.inclusion_proof(m / 2, m); + expect( + *pp == *tree.past_path(m / 2, m - 1), "mem inclusion(m)==past_path" + at); + + const auto cp = engine.consistency_proof(m, k); + expect( + ProofEngine::verify_consistency(m, k, rm, rk, cp), + "mem consistency" + at); + } +} + +// Exercises tile-derived proofs for a tree of `n` leaves against the existing +// library (which acts as the oracle: proofs must be byte-identical). Full tiles +// serve the covered prefix and an in-memory tree serves the un-tiled frontier, +// exactly as TiledTree combines them. +static void check_size( + const fs::path& dir, uint64_t n, const std::vector& hashes) +{ + const std::string at = " @n=" + std::to_string(n); + + TileStore store(dir); + TileWriter writer(store); + const auto leaf_at = [&](uint64_t i) -> const Hash& { return hashes[i]; }; + writer.write_up_to(n, leaf_at); + + // Oracle: a full, never-flushed tree with the same leaves. + merkle::Tree tree; + for (uint64_t i = 0; i < n; i++) + { + tree.insert(hashes[i]); + } + const Hash root = tree.root(); + + // Production-shaped source: full tiles serve the covered prefix, an in-memory + // tree serves the un-tiled frontier. Drop the tiled past from the frontier + // tree so proofs over it are genuinely served from the tiles. merklecpp keeps + // at least one resident leaf, so never flush the whole tree. + const uint64_t covered = (n / TILE_WIDTH) * TILE_WIDTH; + merkle::Tree frontier; + for (uint64_t i = 0; i < n; i++) + { + frontier.insert(hashes[i]); + } + uint64_t drop_to = covered; + if (n > 0 && drop_to >= n) + { + drop_to = n - 1; + } + if (drop_to > 0) + { + frontier.flush_to((size_t)drop_to); + } + const MemoryHashSource mem(frontier); + const TileHashSource tiles(store, covered); + const CombinedHashSource source(mem, tiles); + const ProofEngine engine(source); + + // Root recomputed from tiles equals the library root. + expect(engine.root(n) == root, "root" + at); + + // Indices to probe: all of them for small trees, else a spread. + std::vector indices; + if (n <= 16) + { + for (uint64_t i = 0; i < n; i++) + { + indices.push_back(i); + } + } + else + { + for (const uint64_t i : + {(uint64_t)0, (uint64_t)1, n / 3, n / 2, n - 2, n - 1}) + { + indices.push_back(i); + } + } + + // Inclusion proofs are identical to TreeT::path and verify. + for (const uint64_t i : indices) + { + if (i >= n) + { + continue; + } + const auto p = engine.inclusion_proof(i, n); + expect(*p == *tree.path(i), "inclusion==path i=" + std::to_string(i) + at); + expect(p->verify(root), "inclusion verify i=" + std::to_string(i) + at); + } + + // Consistency pairs: exhaustive for small trees, else a fixed spread. + std::vector> pairs; + if (n <= 16) + { + for (uint64_t m = 1; m < n; m++) + { + for (uint64_t k = m + 1; k <= n; k++) + { + pairs.emplace_back(m, k); + } + } + } + else + { + pairs = {{1, n}, {n / 2, n}, {n - 1, n}, {1, 2}}; + // Tile-boundary crossings. + if (n > TILE_WIDTH) + { + pairs.emplace_back(TILE_WIDTH, n); + pairs.emplace_back((uint64_t)TILE_WIDTH + 1, n); + } + } + + for (const auto& pr : pairs) + { + const uint64_t m = pr.first; + const uint64_t k = pr.second; + if (m == 0 || m >= k || k > n) + { + continue; + } + + const Hash rm = engine.root(m); + const Hash rk = engine.root(k); + expect(rm == *tree.past_root(m - 1), "past_root m" + at); + expect(rk == *tree.past_root(k - 1), "past_root k" + at); + + // Past inclusion proof matches TreeT::past_path. + const uint64_t i = m / 2; + const auto pp = engine.inclusion_proof(i, m); + expect( + *pp == *tree.past_path(i, m - 1), + "inclusion(m)==past_path i=" + std::to_string(i) + at); + expect(pp->verify(rm), "inclusion(m) verify" + at); + + // Consistency proof reconciles the two roots. + const auto cp = engine.consistency_proof(m, k); + expect( + ProofEngine::verify_consistency(m, k, rm, rk, cp), + "consistency " + std::to_string(m) + "->" + std::to_string(k) + at); + + // The index-based variant is consistency_proof(i+1, j+1). + expect( + engine.consistency_proof_from_indices(m - 1, k - 1) == cp, + "consistency index variant" + at); + + // Tampering with a proof element or a root is rejected. + auto bad = cp; + bad[0].bytes[0] ^= 0xFFU; + expect( + !ProofEngine::verify_consistency(m, k, rm, rk, bad), + "consistency tamper rejected" + at); + + Hash wrong = rk; + wrong.bytes[0] ^= 0xFFU; + expect( + !ProofEngine::verify_consistency(m, k, rm, wrong, cp), + "consistency wrong root rejected" + at); + } + + std::error_code ec; + fs::remove_all(dir, ec); +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_proofs"); + const fs::path& base = temporary_directory.path(); + const uint64_t tile_width = TILE_WIDTH; + const uint64_t level1_width = tile_width * tile_width; + + try + { + const auto hashes = make_hashes(300000); + + // ---- Memory-only proofs (exercises TreeT::subtree_root). + for (const uint64_t n : + {(uint64_t)1, + (uint64_t)2, + (uint64_t)3, + (uint64_t)5, + (uint64_t)8, + (uint64_t)13, + (uint64_t)16, + tile_width, + tile_width + 1, + (uint64_t)1000}) + { + check_memory_source(n, hashes); + } + std::cout << "memory source: OK" << '\n'; + + // ---- Hostile arithmetic inputs are rejected without UB or overflow + // loops. + { + merkle::Tree tree; + tree.insert(hashes[0]); + const MemoryHashSource source(tree); + const ProofEngine engine(source); + expect( + tree.subtree_root(0, 0) == hashes[0], + "subtree_root returns a resident leaf hash"); + expect(!tree.subtree_root(64, 0), "subtree_root rejects level 64"); + expect(!tree.subtree_root(100, 0), "subtree_root rejects level 100"); + expect( + !tree.subtree_root(1, std::numeric_limits::max()), + "subtree_root rejects overflowing index"); + + const auto signed_shift_boundary = + static_cast(std::numeric_limits::digits); + if ( + signed_shift_boundary < std::numeric_limits::digits) + { + const size_t full_size = + (size_t{1} << signed_shift_boundary) - 1; + expect( + TreeProbe::node_is_full(signed_shift_boundary, full_size), + "is_full handles signed-shift boundary"); + } + expect( + TreeProbe::node_is_full( + static_cast(std::numeric_limits::digits), + std::numeric_limits::max()), + "is_full handles maximum size_t height"); + + expect(ProofEngineProbe::largest_pow2_lt(2) == 1, "pow2_lt 2"); + expect( + ProofEngineProbe::largest_pow2_lt((uint64_t)1 << 63) == + ((uint64_t)1 << 62), + "pow2_lt 2^63"); + expect( + ProofEngineProbe::largest_pow2_lt(((uint64_t)1 << 63) + 1) == + ((uint64_t)1 << 63), + "pow2_lt 2^63+1"); + expect( + ProofEngineProbe::largest_pow2_lt( + std::numeric_limits::max()) == ((uint64_t)1 << 63), + "pow2_lt uint64 max"); + bool rejected = false; + try + { + engine.consistency_proof_from_indices( + 0, std::numeric_limits::max()); + } + catch (const std::runtime_error&) + { + rejected = true; + } + expect(rejected, "consistency index rejects overflow"); + std::cout << "hostile arithmetic inputs: OK" << '\n'; + } + + for (const uint64_t n : + {(uint64_t)1, + (uint64_t)2, + (uint64_t)3, + (uint64_t)4, + (uint64_t)5, + (uint64_t)7, + (uint64_t)8, + (uint64_t)13, + (uint64_t)16, + tile_width - 1, + tile_width, + tile_width + 1, + (uint64_t)1000}) + { + check_size(base / ("n" + std::to_string(n)), n, hashes); + } + std::cout << "small/medium sizes: OK" << '\n'; + + // Large trees. TILE_WIDTH * TILE_WIDTH is one full L1 tile (the exact L1 + // boundary); the next size is one past it. 70000 exercises a full L1 tile + // plus an in-memory frontier; 300000 forces proofs over height >= 16 + // subtrees, so TileHashSource::resolve descends through level-2 logic + // (full_shift = 24) before falling back to level-1 tiles. tiles_level2 + // separately covers a completed level-2 tile. + for (const uint64_t n : + {level1_width, + level1_width + 1, + (uint64_t)70000, + (uint64_t)300000}) + { + check_size(base / ("big" + std::to_string(n)), n, hashes); + std::cout << "size " << n << ": OK" << '\n'; + } + + std::cout << "tiles_proofs: OK" << '\n'; + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + return 1; + } + + return 0; +} diff --git a/test/tiles_tree.cpp b/test/tiles_tree.cpp new file mode 100644 index 0000000..5ec17ae --- /dev/null +++ b/test/tiles_tree.cpp @@ -0,0 +1,805 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::ProofEngine; +using merkle::tiles::TiledTree; +using merkle::tiles::TileStore; + +static void custom_hash(const Hash& left, const Hash& right, Hash& out) +{ + merkle::Tree::hash_function(left, right, out); +} + +using CustomTiledTree = + merkle::tiles::TiledTreeT; + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(!std::is_move_assignable_v); + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static fs::path store_root(const fs::path& prefix) +{ + return prefix / TileStore::storage_directory_name("sha256"); +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_tree"); + const fs::path& base = temporary_directory.path(); + + try + { + const auto hashes = make_hashes(1500); + + // ---- Part 0: the empty tree. flush()/compact() are no-ops and there is no + // root. + { + TiledTree::Config ecfg; + ecfg.prefix = base / "tt_empty"; + TiledTree ett(ecfg); + expect(ett.size() == 0, "empty: size 0"); + expect(ett.flush().full_written == 0, "empty: flush writes nothing"); + expect(ett.flushed_size() == 0, "empty: flushed size 0"); + expect(ett.immutable_size() == 0, "empty: immutable size 0"); + expect(ett.compact() == 0, "empty: compact no-op"); + bool ethrew = false; + try + { + (void)ett.root(); + } + catch (const std::exception&) + { + ethrew = true; + } + expect(ethrew, "empty: root throws"); + std::cout << "empty tree: OK" << '\n'; + } + + // ---- Part 0b: moving a tree rebinds its writer to the destination store. + { + const fs::path previous_cwd = fs::current_path(); + try + { + fs::create_directories(base); + fs::current_path(base); + TiledTree::Config move_cfg; + move_cfg.prefix = "tt_move_expected"; + const fs::path expected_prefix = fs::absolute(move_cfg.prefix); + TiledTree source(move_cfg); + for (uint64_t i = 0; i < 300; i++) + { + source.append(hashes[i]); + } + source.flush(); + + const fs::path work = base / "tt_move_cwd"; + fs::create_directories(work); + fs::current_path(work); + TiledTree moved(std::move(source)); + expect(moved.flushed_size() == 256, "move: flushed size retained"); + expect(moved.immutable_size() == 256, "move: immutable size retained"); + expect( + moved.store_ref().root() == store_root(expected_prefix), + "move: destination store retained"); + for (uint64_t i = 300; i < 512; i++) + { + moved.append(hashes[i]); + } + expect(moved.flush().full_written == 1, "move: next tile written"); + expect( + fs::is_regular_file(store_root(expected_prefix) / "tile/0/001"), + "move: tile written to configured store"); + expect( + !fs::exists(store_root(work) / "tile/0/001"), + "move: no tile written relative to current directory"); + } + catch (...) + { + fs::current_path(previous_cwd); + throw; + } + fs::current_path(previous_cwd); + std::cout << "move construction: OK" << '\n'; + } + + // ---- Part 0c: a TiledTree atomically claims a fresh namespace, so another + // tree cannot pass construction before the first one writes tiles. + { + TiledTree::Config existing_cfg; + existing_cfg.prefix = base / "tt_existing"; + TiledTree first(existing_cfg); + + bool construction_threw = false; + try + { + const TiledTree second(existing_cfg); + } + catch (const std::exception&) + { + construction_threw = true; + } + expect( + construction_threw, + "existing prefix: second tree rejects claimed namespace"); + for (uint64_t i = 0; i < 256; i++) + { + first.append(hashes[i]); + } + expect( + first.flush().full_written == 1, + "existing prefix: owning tree writes tile"); + std::cout << "exclusive tile namespace: OK" << '\n'; + } + + // ---- Part 0d: generic tiled trees accept the explicit storage namespace + // required by custom hash functions. + { + CustomTiledTree::Config custom_cfg; + custom_cfg.prefix = base / "tt_custom"; + CustomTiledTree custom(custom_cfg, "custom-sha256"); + merkle::Tree reference; + for (uint64_t i = 0; i < 300; i++) + { + custom.append(hashes[i]); + reference.insert(hashes[i]); + } + expect( + custom.flush().full_written == 1, "custom hash: full tile written"); + expect(custom.root() == reference.root(), "custom hash: root matches"); + expect( + custom.store_ref().root() == + fs::absolute(custom_cfg.prefix) / + TileStore::storage_directory_name("custom-sha256"), + "custom hash: explicit namespace used"); + std::cout << "custom hash namespace: OK" << '\n'; + } + + // ---- Part 0e: proof requests cannot describe a state beyond the current + // tree, even when the lower proof engine would not read its source. + { + TiledTree::Config bounds_cfg; + bounds_cfg.prefix = base / "tt_bounds"; + TiledTree bounded(bounds_cfg); + bounded.append(hashes[0]); + + bool threw = false; + try + { + (void)bounded.consistency_proof(2, 2); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "proof bounds: reject future equal sizes"); + + threw = false; + try + { + (void)bounded.inclusion_proof(0, 2); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "proof bounds: reject future inclusion size"); + + threw = false; + try + { + const auto maximum = std::numeric_limits::max(); + (void)bounded.consistency_proof_from_indices(maximum, maximum); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "proof bounds: reject overflowing indices"); + std::cout << "proof bounds: OK" << '\n'; + } + + // ---- Part 2: TiledTree flush, proofs over tiles + memory. + const uint64_t n1 = 1000; // first flush size + const uint64_t N = 1500; // final size + + // Reference: a plain (never flushed) tree with the same leaves. + merkle::Tree ref; + for (uint64_t i = 0; i < N; i++) + { + ref.insert(hashes[i]); + } + const Hash ref_root = ref.root(); + + // Default mode: flush() writes tiles but drops nothing from memory; + // an explicit compact() drops only the leaves already covered by a tile. + { + TiledTree::Config dcfg; + dcfg.prefix = base / "tt_default"; + TiledTree dtt(dcfg); + for (uint64_t i = 0; i < n1; i++) + { + dtt.append(hashes[i]); + } + dtt.flush(); + expect(dtt.tree_ref().min_index() == 0, "default: nothing dropped"); + dtt.compact(); + expect( + dtt.tree_ref().min_index() == 767, "compact() keeps boundary leaf"); + } + + TiledTree::Config cfg; + cfg.prefix = base / "tt"; + cfg.retention_margin = 0; + cfg.compact_on_flush = true; + TiledTree tt(cfg); + + for (uint64_t i = 0; i < n1; i++) + { + tt.append(hashes[i]); + } + tt.flush(); // flushes full tiles; flushed_size = covered = 768 + expect(tt.flushed_size() == 768, "flushed size"); + expect(tt.tree_ref().min_index() == 767, "compacted with boundary overlap"); + + for (uint64_t i = n1; i < N; i++) + { + tt.append(hashes[i]); + } + expect(tt.size() == N, "size after appends"); + expect(tt.root() == ref_root, "tiled root == reference root"); + + // Indices that are: flushed (tiles only), in the flushed-but-resident + // overlap, and on the un-flushed resident frontier. + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)767, + (uint64_t)800, + (uint64_t)999, + (uint64_t)1000, + (uint64_t)1499}) + { + const auto p = tt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "combined inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "combined inclusion verify i=" + std::to_string(i)); + } + + // The resident tree alone cannot prove a flushed index. + bool threw = false; + try + { + (void)tt.tree_ref().path(0); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "memory-only path throws for flushed index"); + + // Consistency across the flush boundary: flushed size -> current size, + // and a flushed-era size -> current size. + { + const auto cp = tt.consistency_proof(n1, N); + expect( + ProofEngine::verify_consistency( + n1, N, *ref.past_root(n1 - 1), ref_root, cp), + "consistency n1->N across flush"); + + const uint64_t m = 500; // flushed era + const auto cp2 = tt.consistency_proof(m, N); + expect( + ProofEngine::verify_consistency( + m, N, *ref.past_root(m - 1), ref_root, cp2), + "consistency m->N across flush"); + + // Index-based variant: consistency_proof_from_indices(i, j) == + // consistency_proof(i + 1, j + 1), spanning tiles and the live tree. + const auto cpi = tt.consistency_proof_from_indices(m - 1, N - 1); + expect(cpi == cp2, "consistency index variant across flush"); + expect( + ProofEngine::verify_consistency( + m, N, *ref.past_root(m - 1), ref_root, cpi), + "consistency index variant verifies"); + } + + // A second flush writes further tiles; proofs for now-flushed indices still + // work (this also confirms the writer never reads a flushed leaf). + tt.flush(); // flushes full tiles; flushed_size = covered = 1280 + expect(tt.flushed_size() == 1280, "second flushed size"); + expect(tt.tree_ref().min_index() == 1279, "second boundary leaf retained"); + + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)1000, + (uint64_t)1279, + (uint64_t)1280, + (uint64_t)1499}) + { + const auto p = tt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "post-2nd inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "post-2nd inclusion verify i=" + std::to_string(i)); + } + + std::cout << "tiled tree (flush + combination): OK" << '\n'; + + // ---- Part 2b: compaction when the size is an exact multiple of TILE_WIDTH + // (here 512). flush_to cannot drain the whole tree, so compact() keeps + // one resident leaf rather than throwing; proofs still resolve. + { + TiledTree::Config mcfg; + mcfg.prefix = base / "tt_multiple"; + mcfg.compact_on_flush = true; + TiledTree mtt(mcfg); + for (uint64_t i = 0; i < 512; i++) + { + mtt.append(hashes[i]); + } + mtt.flush(); // covered == size == 512; compaction must not throw + expect(mtt.flushed_size() == 512, "multiple: flushed size 512"); + expect(mtt.tree_ref().min_index() == 511, "multiple: one leaf retained"); + + merkle::Tree mref; + for (uint64_t i = 0; i < 512; i++) + { + mref.insert(hashes[i]); + } + const Hash mroot = mref.root(); + expect(mtt.root() == mroot, "multiple: root matches reference"); + for (const uint64_t i : {(uint64_t)0, (uint64_t)256, (uint64_t)511}) + { + const auto p = mtt.inclusion_proof(i, 512); + expect(*p == *mref.path(i), "multiple: inclusion==ref"); + expect(p->verify(mroot), "multiple: inclusion verify"); + } + std::cout << "tiled tree (exact-multiple compaction): OK" << '\n'; + } + + // ---- Part 2c: compaction with a non-zero retention_margin keeps the most + // recent leaves resident while flushed indices are still served from + // tiles. The immutable prefix remains the full-tile prefix, regardless + // of the margin. + { + TiledTree::Config rcfg; + rcfg.prefix = base / "tt_margin"; + rcfg.retention_margin = 300; + rcfg.compact_on_flush = true; + TiledTree rtt(rcfg); + for (uint64_t i = 0; i < n1; i++) // n1 == 1000 + { + rtt.append(hashes[i]); + } + rtt.flush(); // covered = 768; target = floor((768 - 300) / 256) * 256 = + // 256 + expect(rtt.flushed_size() == 768, "margin: flushed size 768"); + expect( + rtt.tree_ref().min_index() == 256, + "margin: retained >= 300 recent leaves"); + + for (uint64_t i = n1; i < N; i++) + { + rtt.append(hashes[i]); + } + expect(rtt.root() == ref_root, "margin: root matches reference"); + + // Flushed-only (0, 255), flushed-but-resident overlap (256, 767), and the + // un-flushed frontier (1000, 1499). + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)255, + (uint64_t)256, + (uint64_t)767, + (uint64_t)1000, + (uint64_t)1499}) + { + const auto p = rtt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "margin inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "margin inclusion verify i=" + std::to_string(i)); + } + + bool threw = false; + try + { + rtt.retract_to(700); // size 701 < immutable_size 768 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "margin: retract below flushed prefix throws"); + + std::cout << "tiled tree (retention margin): OK" << '\n'; + } + + // ---- Part 3: rollback. Tiles are immutable, so only un-tiled + // (post-flush) entries may be rolled back. + { + // 3a. Before any flush nothing is tiled, so rollback is + // unrestricted. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_pre"; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 50; i++) + { + rb.append(hashes[i]); + } + rb.retract_to(29); // tiles_size == 0 -> allowed + expect(rb.size() == 30, "rb pre-flush retract allowed"); + } + + // 3b. After a flush, the un-tiled frontier can be rolled back and + // the + // tiled region stays consistent and provable; committed entries + // can't. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb"; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 300; i++) + { + rb.append(hashes[i]); + } + rb.flush(); // flushed_size = covered = 256 + for (uint64_t i = 300; i < 400; i++) + { + rb.append(hashes[i]); + } + rb.retract_to(349); // keep [0,349] + expect(rb.size() == 350, "rb retracted to 350"); + for (uint64_t i = 350; i < 400; i++) + { + rb.append(hashes[1000 + i]); // re-append DIFFERENT leaves + } + rb.flush(); // flushed_size = covered = 256 (tile [0,256) already + // written) + + // Reference tree of the exact post-rollback state. + merkle::Tree exp_tree; + for (uint64_t i = 0; i < 350; i++) + { + exp_tree.insert(hashes[i]); + } + for (uint64_t i = 350; i < 400; i++) + { + exp_tree.insert(hashes[1000 + i]); + } + const Hash exp_root = exp_tree.root(); + expect(rb.root() == exp_root, "rb root matches reference"); + + // Proofs for a tiled index and a frontier index match the reference. + for (const uint64_t i : {(uint64_t)100, (uint64_t)299, (uint64_t)399}) + { + const auto p = rb.inclusion_proof(i, 400); + expect( + *p == *exp_tree.path(i), + "rb inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(exp_root), "rb inclusion verify i=" + std::to_string(i)); + } + const auto cp = rb.consistency_proof(300, 400); + expect( + ProofEngine::verify_consistency( + 300, 400, *exp_tree.past_root(299), exp_root, cp), + "rb consistency 300->400"); + + // Only the immutable full-tile prefix [0,256) is protected: rolling + // back into it is refused, while rolling back within the un-tiled + // frontier (>= immutable_size()) is allowed. + expect(rb.flushed_size() == 256, "rb flushed to full-tile prefix"); + + rb.retract_to(std::numeric_limits::max()); + expect(rb.size() == 400, "rb past-current index is a no-op"); + + bool threw = false; + try + { + rb.retract_to(100); // size 101 < 256 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb retract below tiled prefix throws"); + threw = false; + try + { + rb.retract_to(254); // size 255 < 256 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb retract just below tiled prefix throws"); + rb.retract_to(255); // size 256 == immutable_size(): drops only frontier + expect(rb.size() == 256, "rb retract to tiled prefix allowed"); + } + + // 3c. Compaction keeps the final tiled leaf resident, so rollback to a + // tree size of exactly immutable_size() remains representable. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_exact_boundary"; + cfg.compact_on_flush = true; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 1000; i++) + { + rb.append(hashes[i]); + } + rb.flush(); + expect(rb.flushed_size() == 768, "rb boundary flushed size"); + expect(rb.immutable_size() == 768, "rb boundary immutable size"); + expect( + rb.tree_ref().min_index() == 767, + "rb boundary final tiled leaf retained"); + + rb.retract_to(767); + expect(rb.size() == 768, "rb exact immutable boundary allowed"); + + merkle::Tree expected; + for (uint64_t i = 0; i < 768; i++) + { + expected.insert(hashes[i]); + } + const Hash expected_root = expected.root(); + expect( + rb.root() == expected_root, "rb boundary root matches reference"); + const auto proof = rb.inclusion_proof(0, 768); + expect( + *proof == *expected.path(0), + "rb boundary inclusion matches reference"); + expect(proof->verify(expected_root), "rb boundary inclusion verifies"); + } + + // 3d. Rollback interacts correctly with compaction (flushed + tiled + // past). + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_compact"; + cfg.compact_on_flush = true; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 1000; i++) + { + rb.append(hashes[i]); + } + rb.flush(); // flushed_size = covered = 768; retain boundary leaf 767 + expect( + rb.tree_ref().min_index() == 767, + "rb compact boundary leaf retained"); + for (uint64_t i = 1000; i < 1200; i++) + { + rb.append(hashes[i]); + } + // Frontier rollback (>= immutable_size()) is allowed. + rb.retract_to(1099); + expect(rb.size() == 1100, "rb compact frontier retract ok"); + + merkle::Tree exp_tree; + for (uint64_t i = 0; i < 1100; i++) + { + exp_tree.insert(hashes[i]); + } + const Hash exp_root = exp_tree.root(); + const auto p = rb.inclusion_proof(0, 1100); // leaf 0 is flushed + tiled + expect(*p == *exp_tree.path(0), "rb compact flushed inclusion==ref"); + expect(p->verify(exp_root), "rb compact flushed inclusion verify"); + + bool threw = false; + try + { + rb.retract_to(500); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb compact retract below tiled throws"); + } + + // 3e. A failed flush may publish an immutable full tile before a later + // write fails. The attempted full-tile boundary is sealed against + // rollback, while flushed_size advances only after a successful retry. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_interrupted"; + TiledTree interrupted(cfg); + for (uint64_t i = 0; i < 512; i++) + { + interrupted.append(hashes[i]); + } + + const fs::path blocker = store_root(cfg.prefix) / "tile/0/001"; + fs::create_directories(blocker); + bool flush_threw = false; + try + { + interrupted.flush(); + } + catch (const std::exception&) + { + flush_threw = true; + } + + expect(flush_threw, "interrupted flush throws"); + expect( + fs::is_regular_file(store_root(cfg.prefix) / "tile/0/000"), + "interrupted flush published first full tile"); + expect( + interrupted.flushed_size() == 0, + "interrupted flush does not advance flushed size"); + expect( + interrupted.immutable_size() == 512, + "interrupted flush seals attempted boundary"); + expect( + interrupted.compact() == 0, + "interrupted flush does not permit compaction"); + + TiledTree recovered(std::move(interrupted)); + expect( + recovered.immutable_size() == 512, + "interrupted flush seal survives move"); + + bool retract_threw = false; + try + { + recovered.retract_to(0); + } + catch (const std::exception&) + { + retract_threw = true; + } + expect( + retract_threw, + "interrupted flush rejects rollback across attempted tiles"); + expect(recovered.size() == 512, "interrupted rollback changes nothing"); + + fs::remove(blocker); + expect( + recovered.flush().full_written == 1, + "interrupted flush retry writes only missing tile"); + expect( + recovered.flushed_size() == 512, + "interrupted flush retry advances flushed size"); + expect( + recovered.immutable_size() == 512, + "interrupted flush retry preserves immutable size"); + recovered.compact(); + + merkle::Tree expected; + for (uint64_t i = 0; i < 512; i++) + { + expected.insert(hashes[i]); + } + const Hash expected_root = expected.root(); + expect( + recovered.root() == expected_root, + "interrupted flush root matches reference"); + for (const uint64_t i : + {(uint64_t)0, (uint64_t)255, (uint64_t)256, (uint64_t)511}) + { + const auto proof = recovered.inclusion_proof(i, 512); + expect( + *proof == *expected.path(i), + "interrupted flush inclusion matches reference"); + expect( + proof->verify(expected_root), + "interrupted flush inclusion verifies"); + } + } + + // 3f. flushed_size advances only after every required level succeeds. + // Here all 256 level-0 tiles publish before the level-1 tile is blocked. + { + constexpr uint64_t level1_size = + (uint64_t)merkle::tiles::TILE_WIDTH * merkle::tiles::TILE_WIDTH; + const auto level1_hashes = make_hashes((size_t)level1_size); + + TiledTree::Config cfg; + cfg.prefix = base / "rb_interrupted_level1"; + TiledTree interrupted(cfg); + for (const auto& hash : level1_hashes) + { + interrupted.append(hash); + } + + const fs::path blocker = store_root(cfg.prefix) / "tile/1/000"; + fs::create_directories(blocker); + bool flush_threw = false; + try + { + interrupted.flush(); + } + catch (const std::exception&) + { + flush_threw = true; + } + + expect(flush_threw, "level-1 interrupted flush throws"); + expect( + interrupted.store_ref().has_full_tile(0, 255), + "level-1 interrupted flush publishes all level-0 tiles"); + expect( + interrupted.flushed_size() == 0, + "level-1 interrupted flush does not advance flushed size"); + expect( + interrupted.immutable_size() == level1_size, + "level-1 interrupted flush seals attempted boundary"); + + fs::remove(blocker); + expect( + interrupted.flush().full_written == 1, + "level-1 interrupted retry writes only the missing roll-up"); + expect( + interrupted.flushed_size() == level1_size, + "level-1 interrupted retry advances flushed size"); + expect( + interrupted.store_ref().has_full_tile(1, 0), + "level-1 interrupted retry publishes roll-up"); + + merkle::Tree expected; + for (const auto& hash : level1_hashes) + { + expected.insert(hash); + } + const Hash expected_root = expected.root(); + expect( + interrupted.root() == expected_root, + "level-1 interrupted root matches reference"); + const auto proof = interrupted.inclusion_proof(0, level1_size); + expect( + *proof == *expected.path(0), + "level-1 interrupted inclusion matches reference"); + expect( + proof->verify(expected_root), + "level-1 interrupted inclusion verifies"); + } + } + std::cout << "rollback: OK" << '\n'; + + std::cout << "tiles_tree: OK" << '\n'; + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + return 1; + } + + return 0; +} diff --git a/test/time_tiles.cpp b/test/time_tiles.cpp new file mode 100644 index 0000000..dce7d8d --- /dev/null +++ b/test/time_tiles.cpp @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::TiledTree; + +static double secs_since( + const std::chrono::high_resolution_clock::time_point& start) +{ + const auto stop = std::chrono::high_resolution_clock::now(); + return (double)std::chrono::duration_cast( + stop - start) + .count() / + 1e9; +} + +// Uniform random in [0, bound). +static uint64_t rand_below(uint64_t bound) +{ + if (bound == 0) + { + return 0; + } + return (uint64_t)((std::rand() / (RAND_MAX + 1.0)) * (double)bound); +} + +int main() +{ + const auto seed = std::time(nullptr); + std::srand((unsigned)seed); + std::cout << "seed=" << seed << '\n'; + +#ifndef NDEBUG + const uint64_t num_leaves = 50000; + const uint64_t num_proofs = 100; +#else + const uint64_t num_leaves = 1000000; + const uint64_t num_proofs = 10000; +#endif + + const fs::path dir = fs::temp_directory_path() / + ("merklecpp_time_tiles_" + std::to_string((unsigned long long)seed)); + + // Accumulator that consumes each proof, so the work is not optimised away. + volatile uint64_t sink = 0; + + std::cout << std::fixed << std::setprecision(3); + + try + { + const auto hashes = make_hashes(num_leaves); + + TiledTree::Config cfg; + cfg.prefix = dir; + TiledTree log(cfg); + + // 1. Append: grow the in-memory tree. + auto t = std::chrono::high_resolution_clock::now(); + for (uint64_t i = 0; i < num_leaves; i++) + { + log.append(hashes[i]); + } + log.root(); + const double append_s = secs_since(t); + std::cout << "append : " << num_leaves << " leaves in " + << append_s << " sec (" + << (uint64_t)((double)num_leaves / append_s) << " leaves/sec)\n"; + + // 2. Flush: write newly-complete full tiles to disk. + t = std::chrono::high_resolution_clock::now(); + const auto stats = log.flush(); + const double flush_s = secs_since(t); + std::cout << "flush (to disk) : " << stats.full_written + << " full tiles in " << flush_s << " sec (" + << (uint64_t)((double)stats.full_written / flush_s) + << " tiles/sec)\n"; + + const uint64_t n = log.size(); + + // 3. Inclusion proofs while everything is still resident (memory path). + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.inclusion_proof(rand_below(n), n)->size(); + } + const double inc_mem_s = secs_since(t); + std::cout << "inclusion (memory) : " << num_proofs << " proofs in " + << inc_mem_s << " sec (" + << (uint64_t)((double)num_proofs / inc_mem_s) << " proofs/sec)\n"; + + // 4. Consistency proofs while resident (memory path). + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.consistency_proof(1 + rand_below(n - 1), n).size(); + } + const double con_mem_s = secs_since(t); + std::cout << "consistency(memory): " << num_proofs << " proofs in " + << con_mem_s << " sec (" + << (uint64_t)((double)num_proofs / con_mem_s) << " proofs/sec)\n"; + + // 5. Compact: drop tiled leaves from memory. + t = std::chrono::high_resolution_clock::now(); + const uint64_t min_idx = log.compact(); + const double compact_s = secs_since(t); + std::cout << "compact : dropped " << min_idx + << " leaves from memory in " << compact_s << " sec\n"; + + // 6. Inclusion proofs for evicted leaves: served from the on-disk tiles. + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.inclusion_proof(rand_below(min_idx), n)->size(); + } + const double inc_tile_s = secs_since(t); + std::cout << "inclusion (tiles) : " << num_proofs << " proofs in " + << inc_tile_s << " sec (" + << (uint64_t)((double)num_proofs / inc_tile_s) + << " proofs/sec)\n"; + + // 7. Consistency proofs spanning the tiled (evicted) past. + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += log.consistency_proof(1 + rand_below(min_idx - 1), n).size(); + } + const double con_tile_s = secs_since(t); + std::cout << "consistency(tiles) : " << num_proofs << " proofs in " + << con_tile_s << " sec (" + << (uint64_t)((double)num_proofs / con_tile_s) + << " proofs/sec)\n"; + + // 8. Baseline: identical inclusion proofs from a plain in-memory Tree. + merkle::Tree ref; + for (uint64_t i = 0; i < num_leaves; i++) + { + ref.insert(hashes[i]); + } + ref.root(); + t = std::chrono::high_resolution_clock::now(); + for (uint64_t k = 0; k < num_proofs; k++) + { + sink += ref.path(rand_below(n))->size(); + } + const double inc_ref_s = secs_since(t); + std::cout << "inclusion (Tree) : " << num_proofs << " proofs in " + << inc_ref_s << " sec (" + << (uint64_t)((double)num_proofs / inc_ref_s) << " proofs/sec)\n"; + + // Sanity: a tile-served proof still verifies against the reference root. + if (!log.inclusion_proof(0, n)->verify(ref.root())) + { + throw std::runtime_error("benchmark proof failed to verify"); + } + + std::cout << "time_tiles: OK (checksum " << sink << ")\n"; + + std::error_code ec; + fs::remove_all(dir, ec); + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + std::error_code ec; + fs::remove_all(dir, ec); + return 1; + } + + return 0; +}