diff --git a/doc/design/tlog-tiles.md b/doc/design/tlog-tiles.md index 583ccf2..b2aa05b 100644 --- a/doc/design/tlog-tiles.md +++ b/doc/design/tlog-tiles.md @@ -31,7 +31,7 @@ serving the static resources. |---|---| | Tile | `/-256w/tile//`, `application/octet-stream`; `L` is decimal `0..63` without leading zeros, and `N` uses the grouped encoding in [section 4](#4-storage-layout-and-publication). | | Full tile | 256 hashes. Level 0 stores leaf hashes; each level-`L` entry for `L >= 1` is the Merkle Tree Hash of one complete level-`L-1` tile. | -| Entry bundle | `/-256w/tile/entries/`; 256 raw entries encoded as big-endian `uint16` length-prefixed values. See [section 5.3](#53-entry-bundles-optional). | +| Entry bundle | `/-256w/tile/entries/`; 256 raw entries encoded as big-endian `uint16` length-prefixed values. See [section 5.4](#54-entry-bundles-optional). | | Pruning | Tiles or bundles ending at or before a log's minimum index may be denied, matching `flush_to()` / `min_index()`. | A tile spans eight tree levels. Tile `n` at level `l` contains, for @@ -192,7 +192,7 @@ hashes, not bytes. | `TileWriterT` | Persist newly completed full tiles | | Hash sources | Resolve subtree roots from memory, tiles, or both | | `ProofEngineT` | Roots, inclusion/consistency proofs, and verification | -| `TiledTreeT` (planned) | `append`, `flush`, proof APIs, and compaction | +| `TiledTreeT` | `append`, `flush`, proof APIs, and compaction | `TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache. `MemoryHashSourceT` uses the `TreeT::subtree_root` accessor. @@ -214,8 +214,10 @@ class TileStoreT; using TileStore = TileStoreT; -// Hash-source and proof-engine aliases follow the same pattern. -// The TiledTree alias arrives with the phase-4 lifecycle wrapper. +using TiledTree = + TiledTreeT; +// Writer, hash-source, proof-engine, and entry-bundle aliases follow the same +// template pattern. } ``` @@ -403,12 +405,129 @@ Because every emitted hash is an `mth_range` computed with `HASH_FUNCTION`, the consistency proof reconciles `Tree::past_root(m-1)` with `Tree::past_root(n-1)` — i.e. it is consistent with the existing library. -## 6. Delivery plan +### 5.8 `TiledTreeT` - convenience wrapper -Phases 0-3 now deliver the storage primitives, incremental tile and entry-bundle -writers, hash sources, proof engine, and the only required core accessor. Later -PRs deliver the lifecycle wrapper, user documentation, and performance coverage; -no further core changes are planned. +```cpp +class TiledTreeT { +public: + struct Config { + std::filesystem::path prefix; + uint64_t retention_margin = 0; // keep at least this many resident + bool compact_on_flush = false; // opt in to dropping tiled leaves + }; + explicit TiledTreeT(Config); + TiledTreeT(Config, const std::string& hash_algorithm_short_name); + + void append(const Hash& leaf_hash); // tree.insert + uint64_t size() const; // tree.num_leaves + Hash root(); // tree.root + uint64_t flushed_size() const; // successful tile boundary + uint64_t immutable_size() const; // rollback boundary + + // Write newly-complete full tiles. Compaction (dropping already-tiled + // leaves from memory) happens only if compact_on_flush. + Stats flush(); + + // Drop from memory the leaves already covered by a full tile (opt-in); the + // un-tiled frontier is always retained, and proofs for dropped leaves remain + // available from the tiles. + uint64_t compact(); + + // Roll back only beyond immutable_size(). + void retract_to(size_t index); + + // Proofs over tiles ∪ resident tree (works for flushed indices). + std::shared_ptr inclusion_proof(uint64_t index, uint64_t size); + std::vector consistency_proof(uint64_t m, uint64_t n); + std::vector consistency_proof_from_indices(uint64_t i, uint64_t j); + + Tree& tree_ref(); // mutable escape hatch + Store& store_ref(); // mutable escape hatch +}; +``` + +`TiledTreeT` performs no internal locking. The caller must serialize every +operation on a shared instance, including proof calls. + +Callers with their own storage can construct a `TileWriterT` and call +`write_up_to`, then build a `ProofEngineT` on a `CombinedHashSource`, so the +wrapper is optional sugar. + +## 6. Progressive production and compaction + +The pairing of tile writing with `flush_to` gives two central correctness +invariants: + +> **Compaction invariant.** Retain the final leaf of the last fully successful +> flush: `flushed_size == 0 || min_index() < flushed_size`. +> +> **Immutability invariant.** Never roll back below a full-tile boundary that a +> flush may have published: `size >= immutable_size`. + +`TiledTreeT` is fresh-only: its configured directory may exist, but the +algorithm-qualified `tile` subdirectory must be absent. Construction atomically +creates that directory to claim exclusive ownership. Tile files do not carry +the size, root, hash identity, or ownership information needed to reopen or +share a tree safely. The lower-level `TileWriterT` supports resume for +applications that persist and validate the matching tree state themselves. + +Per flush: + +1. Append new leaf hashes and compute the root as needed. +2. Compute `covered = floor(size / 256) * 256` and advance `immutable_size` to + `covered` before any write can publish a full tile. +3. Call `write_up_to(size, leaf_at)` to persist newly complete full tiles at all + levels. +4. After every level succeeds, set `flushed_size = covered`. +5. Optionally, `compact()` computes an aligned retention target capped below + nonzero `covered`, then calls `flush_to(target)`. This reclaims memory only + when `compact_on_flush` is set or `compact()` is called explicitly. It keeps + at least `retention_margin` recent leaves; alignment can retain up to 255 + additional tiled leaves, and a zero margin keeps the final tiled leaf. The + entire un-tiled frontier always remains resident. + +If tile writing fails, the final two steps do not run. `immutable_size` stays +advanced to prevent stale-tile rollback, while `flushed_size` stays at the last +complete all-level write so proofs and compaction do not trust an incomplete +flush. + +Given these invariants, every leaf and every perfect subtree is resolvable: + +- A leaf below `covered` is in a full level-0 tile; a leaf at or above + `min_index` is resident. Since `min_index <= covered`, every leaf is in tiles, + memory, or both. The frontier `[covered, size)` is always resident because + compaction never flushes past `covered`. +- `mth_range` resolves a perfect subtree directly when it lies wholly in tiles + (`end <= covered`) or wholly in memory (`start >= min_index`). Otherwise it + splits and recurses until each piece is resolvable. A subtree within + `covered` whose level has no completed full tile descends to the highest + available full tile. + +Inclusion and consistency proofs therefore remain available after compaction, +from full tiles for the tiled prefix, memory for the resident frontier, or the +combination. A flush costs `O(new full tiles)`; higher-level tiles are roll-ups +of 256 child hashes. Proof generation performs `O(log(size))` range operations, +with repeated tile reads served from the per-source cache. + +## 7. Pruning and minimum index + +tlog-tiles pruning maps directly onto merklecpp: + +- The log's minimum index is `tree.min_index()` (equal to `num_flushed`). +- A serving layer can deny tiles or bundles whose end index is at or below the + minimum index. On-disk tiles may instead be retained so historical proofs + remain producible. +- The unpruned default is `min_index() == 0` (no `flush_to`). + +`flush_to` provides the mechanism; the application owns the retention policy, +as in the tlog-tiles ecosystem. + +## 8. Delivery plan + +Phases 0-4 now deliver the storage primitives, incremental tile and entry-bundle +writers, hash sources, proof engine, the only required core accessor, and the +lifecycle wrapper. Later PRs deliver user documentation and performance +coverage; no further core changes are planned. | Phase | Scope | Key tests | |---|---|---| @@ -419,5 +538,39 @@ no further core changes are planned. | 4. Combined tree | `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary | | 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage | -Delivered through phase 3 are `merklecpp_tiles.h`, `merklecpp_pal.h`, -`test/tiles_*.cpp`, CMake wiring, the core accessor, and design updates. +Delivered through phase 4 are `merklecpp_tiles.h`, `merklecpp_pal.h`, +`test/tiles_*.cpp`, CMake wiring, the core accessor, the lifecycle wrapper, and +design updates. + +## 9. Risks and edge cases + +- **External interop (by design, no).** With the default combiner the tiles are + not byte-compatible with RFC 6962 tooling. See + [section 2.3](#23-algorithms-and-namespaces); opting into a compatible + `HASH_FUNCTION` is the consumer's choice and out of scope. +- **Filesystem dependency.** Tile I/O needs ``/``; isolated + in the companion header so the core stays dependency-free. +- **Immutable full tiles.** A tile is emitted only after all of its entries are + final, and every emitted tile is write-once. A stand-alone tile reader cannot + serve the frontier; that is the in-memory tree's job (or the application must + keep it elsewhere). +- **`flush_to` alignment.** Compaction normally flushes to a 256-multiple + derived from retention. When that target equals `flushed_size`, it stops one + leaf earlier so `TreeT` can still retract to exactly that size. This one-leaf + overlap is enforced inside `TiledTreeT::compact`. +- **Rollback vs. immutable tiles.** Tiles are write-once, so rolling the tree + back (`retract_to`) over a range that a flush may have published would leave + stale, never-rewritten tiles. `TiledTreeT::retract_to` therefore throws if the + resulting size is below `immutable_size()`. A failed flush may advance + `immutable_size()` without advancing `flushed_size()`; retry with the same + tree state. Retracting the underlying tree directly via `tree_ref()` bypasses + this guard, can make the size boundaries inconsistent or non-monotonic, and + must be avoided. Files written through `store_ref()` are trusted without + checking that they match the tree and can invalidate proofs after compaction. +- **No internal synchronization.** Every tiled-storage object and shared store + prefix requires external serialization. This includes `const` proof reads, + which update the tile cache. +- **Very large indices.** Index math uses `uint64_t`; encoding handles + multi-group indices. Level bound `<= 63` per spec (8 suffices for `2^64`). + Resume scans are bounded by the requested tree size and cannot follow sparse + files beyond that range. diff --git a/merklecpp_tiles.h b/merklecpp_tiles.h index 6957da3..a15a800 100644 --- a/merklecpp_tiles.h +++ b/merklecpp_tiles.h @@ -256,11 +256,12 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::vector bytes = read_file(path, expected); if (bytes.size() != expected) { - throw std::runtime_error(std::format( - "unexpected tile size for {}: expected {} bytes, got {}", - path.string(), - expected, - bytes.size())); + throw std::runtime_error( + std::format( + "unexpected tile size for {}: expected {} bytes, got {}", + path.string(), + expected, + bytes.size())); } std::vector hashes; @@ -323,8 +324,9 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) } catch (const std::runtime_error& error) { - throw std::runtime_error(std::format( - "invalid entry bundle {}: {}", path.string(), error.what())); + throw std::runtime_error( + std::format( + "invalid entry bundle {}: {}", path.string(), error.what())); } } @@ -485,6 +487,7 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) throw std::runtime_error( "hash algorithm short name does not match hash size"); } + prefix = std::filesystem::absolute(prefix); prefix /= storage_directory_name(hash_algorithm_short_name); return prefix; } @@ -604,10 +607,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const bool exists = std::filesystem::exists(directory, ec); if (ec) { - throw std::runtime_error(std::format( - "cannot inspect directory {}: {}", - directory.string(), - ec.message())); + throw std::runtime_error( + std::format( + "cannot inspect directory {}: {}", + directory.string(), + ec.message())); } if (exists) { @@ -615,16 +619,19 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) std::filesystem::is_directory(directory, ec); if (ec) { - throw std::runtime_error(std::format( - "cannot inspect directory {}: {}", - directory.string(), - ec.message())); + throw std::runtime_error( + std::format( + "cannot inspect directory {}: {}", + directory.string(), + ec.message())); } if (!is_directory) { - throw std::runtime_error(std::format( - "cannot create directory {}: path exists and is not a directory", - directory.string())); + throw std::runtime_error( + std::format( + "cannot create directory {}: path exists and is not a " + "directory", + directory.string())); } } else @@ -632,10 +639,11 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const bool created = std::filesystem::create_directory(directory, ec); if (ec) { - throw std::runtime_error(std::format( - "cannot create directory {}: {}", - directory.string(), - ec.message())); + throw std::runtime_error( + std::format( + "cannot create directory {}: {}", + directory.string(), + ec.message())); } if (!created && !std::filesystem::is_directory(directory, ec)) { @@ -1041,8 +1049,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// @brief Resolves a complete subtree known to lie within the full-tile /// prefix, reading the highest-level full tile that holds it (and rolling - /// up); descends to lower full tiles when a higher-level full tile has not - /// completed. Terminates because full level-0 tiles always cover the + /// up); descends to lower full tiles when a higher-level full tile has + /// not completed. Terminates because full level-0 tiles always cover the /// prefix. void resolve(uint8_t level, uint64_t index, Hash& out) const { @@ -1061,8 +1069,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const uint8_t r = level % TILE_HEIGHT; const uint64_t first = index << r; // first level-L entry const uint64_t n = first / TILE_WIDTH; // level-L tile index - const unsigned full_shift = static_cast(TILE_HEIGHT) * - (static_cast(L) + 1U); + const unsigned full_shift = + static_cast(TILE_HEIGHT) * (static_cast(L) + 1U); const uint64_t full_tiles = full_shift >= 64 ? 0 : (available_size >> full_shift); @@ -1118,8 +1126,8 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) /// HASH_FUNCTION, so an inclusion proof is byte-identical to the one /// produced by merkle::TreeT::path()/past_path() and verifies with /// PathT::verify(). - /// @warning Thread safety is inherited from the supplied HashSourceT. Callers - /// must serialize operations when the source is shared. + /// @warning Thread safety is inherited from the supplied HashSourceT. + /// Callers must serialize operations when the source is shared. template < size_t HASH_SIZE, void HASH_FUNCTION( @@ -1486,6 +1494,336 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) const Source& secondary; }; + /// @brief A merkle tree backed by tlog-tiles storage. + /// @note Appends grow an in-memory tree; flush() durably writes only full + /// (balanced) tiles, so the incomplete frontier is never tiled and stays + /// resident in memory. Compaction (dropping from memory the leaves already + /// covered by a full tile) is optional: enable it per flush with + /// Config::compact_on_flush, or call compact() explicitly; it never drops + /// the un-tiled frontier. Proofs are served from the combination of the + /// resident tree (frontier) and the full tiles (compacted past). + /// @note TiledTree creates a new tiled tree and cannot reopen one from tile + /// files alone. Construction atomically claims a previously absent tile + /// namespace because the files do not identify their tree or record enough + /// state to restore it. Use TileWriter directly only when the caller owns + /// and restores that state. + /// @warning No internal synchronization is provided. Callers must serialize + /// all access to a shared tree, including proof operations. + template < + size_t HASH_SIZE, + void HASH_FUNCTION( + const HashT&, const HashT&, HashT&)> + class TiledTreeT + { + public: + using Hash = HashT; + using Tree = TreeT; + using Path = PathT; + using Store = TileStoreT; + using Writer = TileWriterT; + using Stats = typename Writer::Stats; + + /// @brief Configuration for a tiled tree. + struct Config + { + /// @brief Root directory for a new tiled tree. + /// @note The directory itself may exist, but its algorithm-qualified + /// tile subdirectory must be absent. + std::filesystem::path prefix; + + /// @brief Number of most-recent leaves to keep resident when + /// compacting (i.e. a minimum never dropped from memory). + /// @note Tile alignment may retain up to TILE_WIDTH - 1 additional + /// tiled leaves. A zero margin retains one tiled boundary leaf so + /// rollback to exactly immutable_size() remains possible. + uint64_t retention_margin = 0; + + /// @brief If set, flush() compacts after writing tiles, dropping + /// from memory the leaves already covered by a full tile. Off by + /// default: tiles are written but the tree keeps every leaf resident. + bool compact_on_flush = false; + }; + + explicit TiledTreeT(Config config) : + config(std::move(config)), store(this->config.prefix), writer(store) + { + claim_tile_namespace(); + } + + /// @brief Constructs a tiled tree with an explicit storage namespace. + /// @note Required for custom hash functions whose algorithm name cannot + /// be inferred by TileStoreT. + TiledTreeT(Config config, const std::string& hash_algorithm_short_name) : + config(std::move(config)), + store(this->config.prefix, hash_algorithm_short_name), + writer(store) + { + claim_tile_namespace(); + } + + TiledTreeT(const TiledTreeT&) = delete; + TiledTreeT& operator=(const TiledTreeT&) = delete; + + /// @brief Moves a tiled tree, rebinding its writer to the moved store. + TiledTreeT(TiledTreeT&& other) noexcept : + config(std::move(other.config)), + store(std::move(other.store)), + writer(store), + tree(std::move(other.tree)), + tiles_size(std::exchange(other.tiles_size, 0)), + sealed_size(std::exchange(other.sealed_size, 0)) + {} + + TiledTreeT& operator=(TiledTreeT&&) = delete; + + /// @brief Appends a leaf hash. + void append(const Hash& leaf_hash) + { + tree.insert(leaf_hash); + } + + /// @brief The number of leaves (including flushed ones). + [[nodiscard]] uint64_t size() const + { + return tree.num_leaves(); + } + + /// @brief The current Merkle root. + Hash root() + { + return tree.root(); + } + + /// @brief The number of leaves covered by the last fully successful + /// flush. + /// @note This is always a multiple of TILE_WIDTH. It advances only after + /// every required tile level has been written successfully, and controls + /// proof reads and compaction. + [[nodiscard]] uint64_t flushed_size() const + { + return tiles_size; + } + + /// @brief The rollback seal for ranges a flush may have published. + /// @note A flush seals its full-tile boundary before writing. If the + /// write fails, this may exceed flushed_size(); keep the same tree + /// contents and retry the flush. + [[nodiscard]] uint64_t immutable_size() const + { + return sealed_size; + } + + /// @brief Access to the underlying tree. + /// @warning Mutating the tree directly bypasses tiled-tree bookkeeping. + /// In particular, direct retraction can make flushed_size() and + /// immutable_size() exceed size() and can make flushed_size() regress. + /// Use TiledTreeT operations whenever they are available. + Tree& tree_ref() + { + return tree; + } + + /// @brief Access to the underlying tile store. + /// @warning Files written or changed through this reference are trusted + /// by later flushes without checking that their hashes match this tree. + /// Mismatched files can silently invalidate proofs after compaction. + Store& store_ref() + { + return store; + } + + /// @brief Writes newly-complete full tiles to disk; compacts only if + /// Config::compact_on_flush is set. + /// @return Counts of the full tiles written by this flush + /// @note The full-tile boundary is made immutable before any tile write. + /// Only after every required tile level succeeds does flushed_size() + /// advance to that boundary. On failure, immutable_size() may advance + /// while flushed_size() does not; the tree remains resident and the flush + /// can be retried without rewriting finalized tiles. + Stats flush() + { + Stats stats; + const uint64_t n = tree.num_leaves(); + if (n == 0) + { + return stats; + } + + const uint64_t covered = (n / TILE_WIDTH) * TILE_WIDTH; + if (covered > sealed_size) + { + sealed_size = covered; + } + + stats = writer.write_up_to(n, [this](uint64_t i) -> const Hash& { + if (i < tree.min_index()) + { + throw std::runtime_error(std::format( + "TiledTree::flush: cannot regenerate a missing or malformed " + "tile from non-resident leaf {}", + i)); + } + return tree.leaf((size_t)i); + }); + tiles_size = covered; + + if (config.compact_on_flush) + { + compact(); + } + return stats; + } + + /// @brief Drops old leaves covered by durably-written full tiles, keeping + /// at least retention_margin recent leaves and a tiled boundary leaf. + /// @return The new minimum (smallest still-resident) leaf index + /// @note Only leaves covered by a full tile are dropped, so the un-tiled + /// frontier is always retained in memory and inclusion/consistency proofs + /// remain available (the past from tiles, the frontier from memory). The + /// leaf at flushed_size() - 1 also remains resident so retract_to() can + /// represent a tree whose size is exactly immutable_size(). Has no effect + /// until tiling has produced full tiles. + uint64_t compact() + { + const uint64_t covered = (tiles_size / TILE_WIDTH) * TILE_WIDTH; + uint64_t target = covered > config.retention_margin ? + covered - config.retention_margin : + 0; + target = (target / TILE_WIDTH) * TILE_WIDTH; + // TreeT cannot retract below min_index(). Keep the final tiled leaf + // resident so rollback to a size of exactly immutable_size() remains + // representable after compaction. + if (covered > 0 && target == covered) + { + target--; + } + if (target > tree.min_index()) + { + tree.flush_to((size_t)target); + } + return tree.min_index(); + } + + /// @brief Rolls the tree back so that @p index becomes the last leaf, + /// removing all leaves after it (same semantics as TreeT::retract_to). + /// @note Only full tiles are immutable: this throws if the resulting size + /// would be smaller than immutable_size(). A failed flush may advance + /// immutable_size() without advancing flushed_size(). + void retract_to(size_t index) + { + if (sealed_size > 0 && (uint64_t)index < sealed_size - 1) + { + throw std::runtime_error( + "TiledTree::retract_to: cannot roll back entries sealed for " + "immutable tiles (resulting size < immutable size)"); + } + tree.retract_to(index); + } + + /// @brief Inclusion proof for @p index in a tree of @p proof_size leaves. + /// @note Served from tiles (flushed past) combined with the resident tree + /// (recent frontier); @p proof_size may exceed flushed_size(). + std::shared_ptr inclusion_proof(uint64_t index, uint64_t proof_size) + { + if (proof_size > size()) + { + throw std::runtime_error( + "inclusion proof size exceeds current tree size"); + } + return with_engine([&](const auto& engine) { + return engine.inclusion_proof(index, proof_size); + }); + } + + /// @brief Consistency proof between tree sizes @p m and @p n. + std::vector consistency_proof(uint64_t m, uint64_t n) + { + if (n > size()) + { + throw std::runtime_error( + "consistency proof size exceeds current tree size"); + } + return with_engine( + [&](const auto& engine) { return engine.consistency_proof(m, n); }); + } + + /// @brief Consistency proof between the trees whose last leaves are at + /// indices @p first_index and @p second_index (first_index <= + /// second_index). + /// @note Equivalent to consistency_proof(first_index + 1, + /// second_index + 1). + std::vector consistency_proof_from_indices( + uint64_t first_index, uint64_t second_index) + { + if (first_index >= size() || second_index >= size()) + { + throw std::runtime_error( + "consistency proof index exceeds current tree size"); + } + if (first_index > second_index) + { + throw std::runtime_error( + "first consistency proof index exceeds second index"); + } + return with_engine([&](const auto& engine) { + return engine.consistency_proof_from_indices( + first_index, second_index); + }); + } + + protected: + Config config; + Store store; + Writer writer; + Tree tree; + uint64_t tiles_size = 0; + uint64_t sealed_size = 0; + + void claim_tile_namespace() const + { + const auto tile_root = store.root() / "tile"; + std::error_code ec; + std::filesystem::create_directories(tile_root.parent_path(), ec); + if (ec) + { + throw std::runtime_error(std::format( + "TiledTree: cannot create tile namespace parent {}: {}", + tile_root.parent_path().string(), + ec.message())); + } + const bool claimed = std::filesystem::create_directory(tile_root, ec); + if (ec) + { + throw std::runtime_error(std::format( + "TiledTree: cannot claim tile namespace {}: {}", + tile_root.string(), + ec.message())); + } + if (!claimed) + { + throw std::runtime_error( + "TiledTree: tile namespace already exists; reopening or sharing " + "a tiled tree is not supported"); + } + } + + /// @brief Builds a proof engine over the combined resident-tree + /// (frontier) and full-tile (flushed past) source, and invokes @p fn with + /// it. + /// @note The sources and engine are stack-local; @p fn must consume the + /// engine before returning (proofs are returned by value, holding hash + /// copies, so the result outlives the engine). + template + auto with_engine(Fn fn) + { + MemoryHashSourceT mem(tree); + TileHashSourceT tile_src(store, tiles_size); + CombinedHashSourceT combined(mem, tile_src); + ProofEngineT engine(combined); + return fn(engine); + } + }; + /// @brief Writes tlog-tiles entry bundles (raw log entries) for a growing /// log. /// @note Entry bundles are level-0 only and application-owned: merklecpp @@ -1613,10 +1951,51 @@ namespace merkle // NOLINT(modernize-concat-nested-namespaces) merkle::Tree::Hash::size_bytes, merkle::Tree::hash_function>; + /// @brief Default tiled tree (SHA256, default hash function). + using TiledTree = + TiledTreeT; + /// @brief Default entry-bundle writer (SHA256, default hash function). using EntryBundleWriter = EntryBundleWriterT< merkle::Tree::Hash::size_bytes, merkle::Tree::hash_function>; +#ifdef HAVE_OPENSSL + /// @brief SHA384 tile store. + using TileStore384 = TileStoreT<48, sha384_openssl>; + + /// @brief SHA512 tile store. + using TileStore512 = TileStoreT<64, sha512_openssl>; + + /// @brief SHA384 tile writer. + using TileWriter384 = TileWriterT<48, sha384_openssl>; + + /// @brief SHA512 tile writer. + using TileWriter512 = TileWriterT<64, sha512_openssl>; + + /// @brief SHA384 hash source, tile-backed source and proof engine. + using HashSource384 = HashSourceT<48, sha384_openssl>; + using TileHashSource384 = TileHashSourceT<48, sha384_openssl>; + using ProofEngine384 = ProofEngineT<48, sha384_openssl>; + + /// @brief SHA512 hash source, tile-backed source and proof engine. + using HashSource512 = HashSourceT<64, sha512_openssl>; + using TileHashSource512 = TileHashSourceT<64, sha512_openssl>; + using ProofEngine512 = ProofEngineT<64, sha512_openssl>; + + /// @brief SHA384 memory/combined sources and tiled tree. + using MemoryHashSource384 = MemoryHashSourceT<48, sha384_openssl>; + using CombinedHashSource384 = CombinedHashSourceT<48, sha384_openssl>; + using TiledTree384 = TiledTreeT<48, sha384_openssl>; + + /// @brief SHA512 memory/combined sources and tiled tree. + using MemoryHashSource512 = MemoryHashSourceT<64, sha512_openssl>; + using CombinedHashSource512 = CombinedHashSourceT<64, sha512_openssl>; + using TiledTree512 = TiledTreeT<64, sha512_openssl>; + + /// @brief SHA384/512 entry-bundle writers. + using EntryBundleWriter384 = EntryBundleWriterT<48, sha384_openssl>; + using EntryBundleWriter512 = EntryBundleWriterT<64, sha512_openssl>; +#endif } } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e555eee..7648f61 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -40,8 +40,13 @@ add_merklecpp_test(coverage coverage.cpp) add_merklecpp_test(tiles_store tiles_store.cpp) add_merklecpp_test(tiles_writer tiles_writer.cpp) add_merklecpp_test(tiles_proofs tiles_proofs.cpp) +add_merklecpp_test(tiles_tree tiles_tree.cpp) add_merklecpp_test(tiles_entries tiles_entries.cpp) +if(OPENSSL) + add_merklecpp_test(tiles_hashes tiles_hashes.cpp) +endif() + if(LONG_TESTS) add_merklecpp_test(tiles_level2 tiles_level2.cpp) set(TILES_LEVEL2_TIMEOUT 900) diff --git a/test/tiles_hashes.cpp b/test/tiles_hashes.cpp new file mode 100644 index 0000000..d05b816 --- /dev/null +++ b/test/tiles_hashes.cpp @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +template < + size_t HASH_SIZE, + typename Tree, + typename TiledTree, + typename ProofEngine> +static void exercise_tiled_hash(const fs::path& prefix, const std::string& name) +{ + const auto hashes = make_hashesT(300); + Tree reference; + typename TiledTree::Config config; + config.prefix = prefix; + config.compact_on_flush = true; + TiledTree tree(config); + + for (const auto& hash : hashes) + { + reference.insert(hash); + tree.append(hash); + } + + expect(tree.flush().full_written == 1, name + " full tile written"); + expect(tree.flushed_size() == 256, name + " flushed size"); + expect(tree.root() == reference.root(), name + " root"); + expect( + fs::file_size(tree.store_ref().root() / "tile" / "0" / "000") == + (uintmax_t)merkle::tiles::TILE_WIDTH * HASH_SIZE, + name + " tile byte size"); + + const auto inclusion = tree.inclusion_proof(0, tree.size()); + expect(inclusion->verify(reference.root()), name + " inclusion proof"); + + const auto consistency = tree.consistency_proof(256, tree.size()); + expect( + ProofEngine::verify_consistency( + 256, + tree.size(), + *reference.past_root(255), + reference.root(), + consistency), + name + " consistency proof"); +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_hashes"); + const fs::path& base = temporary_directory.path(); + + try + { + exercise_tiled_hash< + 48, + merkle::Tree384, + merkle::tiles::TiledTree384, + merkle::tiles::ProofEngine384>(base / "sha384", "SHA384"); + exercise_tiled_hash< + 64, + merkle::Tree512, + merkle::tiles::TiledTree512, + merkle::tiles::ProofEngine512>(base / "sha512", "SHA512"); + + std::cout << "tiles_hashes: OK" << '\n'; + } + catch (const std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + return 1; + } + + return 0; +} diff --git a/test/tiles_tree.cpp b/test/tiles_tree.cpp new file mode 100644 index 0000000..6d56afa --- /dev/null +++ b/test/tiles_tree.cpp @@ -0,0 +1,852 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#include "tiles_test_util.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using merkle::Hash; +using merkle::tiles::ProofEngine; +using merkle::tiles::TiledTree; +using merkle::tiles::TileStore; + +static void custom_hash(const Hash& left, const Hash& right, Hash& out) +{ + merkle::Tree::hash_function(left, right, out); +} + +using CustomTiledTree = + merkle::tiles::TiledTreeT; + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(!std::is_move_assignable_v); + +static void expect(bool cond, const std::string& what) +{ + if (!cond) + { + throw std::runtime_error("check failed: " + what); + } +} + +static fs::path store_root(const fs::path& prefix) +{ + return prefix / TileStore::storage_directory_name("sha256"); +} + +int main() +{ + const TemporaryDirectory temporary_directory("merklecpp_tiles_tree"); + const fs::path& base = temporary_directory.path(); + + try + { + const auto hashes = make_hashes(1500); + + // ---- Part 0: the empty tree. flush()/compact() are no-ops and there is no + // root. + { + TiledTree::Config ecfg; + ecfg.prefix = base / "tt_empty"; + TiledTree ett(ecfg); + expect(ett.size() == 0, "empty: size 0"); + expect(ett.flush().full_written == 0, "empty: flush writes nothing"); + expect(ett.flushed_size() == 0, "empty: flushed size 0"); + expect(ett.immutable_size() == 0, "empty: immutable size 0"); + expect(ett.compact() == 0, "empty: compact no-op"); + bool ethrew = false; + try + { + (void)ett.root(); + } + catch (const std::exception&) + { + ethrew = true; + } + expect(ethrew, "empty: root throws"); + std::cout << "empty tree: OK" << '\n'; + } + + // ---- Part 0b: moving a tree rebinds its writer to the destination store. + { + const fs::path previous_cwd = fs::current_path(); + try + { + fs::create_directories(base); + fs::current_path(base); + TiledTree::Config move_cfg; + move_cfg.prefix = "tt_move_expected"; + const fs::path expected_prefix = fs::absolute(move_cfg.prefix); + TiledTree source(move_cfg); + for (uint64_t i = 0; i < 300; i++) + { + source.append(hashes[i]); + } + source.flush(); + + const fs::path work = base / "tt_move_cwd"; + fs::create_directories(work); + fs::current_path(work); + TiledTree moved(std::move(source)); + expect(moved.flushed_size() == 256, "move: flushed size retained"); + expect(moved.immutable_size() == 256, "move: immutable size retained"); + expect( + moved.store_ref().root() == store_root(expected_prefix), + "move: destination store retained"); + for (uint64_t i = 300; i < 512; i++) + { + moved.append(hashes[i]); + } + expect(moved.flush().full_written == 1, "move: next tile written"); + expect( + fs::is_regular_file(store_root(expected_prefix) / "tile/0/001"), + "move: tile written to configured store"); + expect( + !fs::exists(store_root(work) / "tile/0/001"), + "move: no tile written relative to current directory"); + } + catch (...) + { + fs::current_path(previous_cwd); + throw; + } + fs::current_path(previous_cwd); + std::cout << "move construction: OK" << '\n'; + } + + // ---- Part 0c: a TiledTree atomically claims a fresh namespace, so another + // tree cannot pass construction before the first one writes tiles. + { + TiledTree::Config existing_cfg; + existing_cfg.prefix = base / "tt_existing"; + TiledTree first(existing_cfg); + + bool construction_threw = false; + try + { + const TiledTree second(existing_cfg); + } + catch (const std::exception&) + { + construction_threw = true; + } + expect( + construction_threw, + "existing prefix: second tree rejects claimed namespace"); + for (uint64_t i = 0; i < 256; i++) + { + first.append(hashes[i]); + } + expect( + first.flush().full_written == 1, + "existing prefix: owning tree writes tile"); + std::cout << "exclusive tile namespace: OK" << '\n'; + } + + // ---- Part 0d: generic tiled trees accept the explicit storage namespace + // required by custom hash functions. + { + CustomTiledTree::Config custom_cfg; + custom_cfg.prefix = base / "tt_custom"; + CustomTiledTree custom(custom_cfg, "custom-sha256"); + merkle::Tree reference; + for (uint64_t i = 0; i < 300; i++) + { + custom.append(hashes[i]); + reference.insert(hashes[i]); + } + expect( + custom.flush().full_written == 1, "custom hash: full tile written"); + expect(custom.root() == reference.root(), "custom hash: root matches"); + expect( + custom.store_ref().root() == + fs::absolute(custom_cfg.prefix) / + TileStore::storage_directory_name("custom-sha256"), + "custom hash: explicit namespace used"); + std::cout << "custom hash namespace: OK" << '\n'; + } + + // ---- Part 0e: proof requests cannot describe a state beyond the current + // tree, even when the lower proof engine would not read its source. + { + TiledTree::Config bounds_cfg; + bounds_cfg.prefix = base / "tt_bounds"; + TiledTree bounded(bounds_cfg); + bounded.append(hashes[0]); + + bool threw = false; + try + { + (void)bounded.consistency_proof(2, 2); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "proof bounds: reject future equal sizes"); + + threw = false; + try + { + (void)bounded.inclusion_proof(0, 2); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "proof bounds: reject future inclusion size"); + + threw = false; + try + { + const auto maximum = std::numeric_limits::max(); + (void)bounded.consistency_proof_from_indices(maximum, maximum); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "proof bounds: reject overflowing indices"); + + bounded.append(hashes[1]); + threw = false; + try + { + (void)bounded.consistency_proof_from_indices(1, 0); + } + catch (const std::runtime_error& error) + { + threw = std::string(error.what()) == + "first consistency proof index exceeds second index"; + } + expect(threw, "proof bounds: reject reversed indices clearly"); + std::cout << "proof bounds: OK" << '\n'; + } + + { + TiledTree::Config missing_cfg; + missing_cfg.prefix = base / "tt_missing_compacted_tile"; + missing_cfg.compact_on_flush = true; + TiledTree compacted(missing_cfg); + for (uint64_t i = 0; i < 300; i++) + { + compacted.append(hashes[i]); + } + compacted.flush(); + expect( + compacted.tree_ref().min_index() > 0, + "missing tile: old leaves compacted"); + fs::remove(store_root(missing_cfg.prefix) / "tile/0/000"); + + TiledTree moved(std::move(compacted)); + bool threw_clear_error = false; + try + { + moved.flush(); + } + catch (const std::runtime_error& error) + { + threw_clear_error = + std::string(error.what()).find("non-resident leaf") != + std::string::npos; + } + expect( + threw_clear_error, + "missing tile: regeneration reports non-resident leaf"); + std::cout << "missing compacted tile: OK" << '\n'; + } + + // ---- Part 2: TiledTree flush, proofs over tiles + memory. + const uint64_t n1 = 1000; // first flush size + const uint64_t N = 1500; // final size + + // Reference: a plain (never flushed) tree with the same leaves. + merkle::Tree ref; + for (uint64_t i = 0; i < N; i++) + { + ref.insert(hashes[i]); + } + const Hash ref_root = ref.root(); + + // Default mode: flush() writes tiles but drops nothing from memory; + // an explicit compact() drops only the leaves already covered by a tile. + { + TiledTree::Config dcfg; + dcfg.prefix = base / "tt_default"; + TiledTree dtt(dcfg); + for (uint64_t i = 0; i < n1; i++) + { + dtt.append(hashes[i]); + } + dtt.flush(); + expect(dtt.tree_ref().min_index() == 0, "default: nothing dropped"); + dtt.compact(); + expect( + dtt.tree_ref().min_index() == 767, "compact() keeps boundary leaf"); + } + + TiledTree::Config cfg; + cfg.prefix = base / "tt"; + cfg.retention_margin = 0; + cfg.compact_on_flush = true; + TiledTree tt(cfg); + + for (uint64_t i = 0; i < n1; i++) + { + tt.append(hashes[i]); + } + tt.flush(); // flushes full tiles; flushed_size = covered = 768 + expect(tt.flushed_size() == 768, "flushed size"); + expect(tt.tree_ref().min_index() == 767, "compacted with boundary overlap"); + + for (uint64_t i = n1; i < N; i++) + { + tt.append(hashes[i]); + } + expect(tt.size() == N, "size after appends"); + expect(tt.root() == ref_root, "tiled root == reference root"); + + // Indices that are: flushed (tiles only), in the flushed-but-resident + // overlap, and on the un-flushed resident frontier. + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)767, + (uint64_t)800, + (uint64_t)999, + (uint64_t)1000, + (uint64_t)1499}) + { + const auto p = tt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "combined inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "combined inclusion verify i=" + std::to_string(i)); + } + + // The resident tree alone cannot prove a flushed index. + bool threw = false; + try + { + (void)tt.tree_ref().path(0); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "memory-only path throws for flushed index"); + + // Consistency across the flush boundary: flushed size -> current size, + // and a flushed-era size -> current size. + { + const auto cp = tt.consistency_proof(n1, N); + expect( + ProofEngine::verify_consistency( + n1, N, *ref.past_root(n1 - 1), ref_root, cp), + "consistency n1->N across flush"); + + const uint64_t m = 500; // flushed era + const auto cp2 = tt.consistency_proof(m, N); + expect( + ProofEngine::verify_consistency( + m, N, *ref.past_root(m - 1), ref_root, cp2), + "consistency m->N across flush"); + + // Index-based variant: consistency_proof_from_indices(i, j) == + // consistency_proof(i + 1, j + 1), spanning tiles and the live tree. + const auto cpi = tt.consistency_proof_from_indices(m - 1, N - 1); + expect(cpi == cp2, "consistency index variant across flush"); + expect( + ProofEngine::verify_consistency( + m, N, *ref.past_root(m - 1), ref_root, cpi), + "consistency index variant verifies"); + } + + // A second flush writes further tiles; proofs for now-flushed indices still + // work (this also confirms the writer never reads a flushed leaf). + tt.flush(); // flushes full tiles; flushed_size = covered = 1280 + expect(tt.flushed_size() == 1280, "second flushed size"); + expect(tt.tree_ref().min_index() == 1279, "second boundary leaf retained"); + + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)1000, + (uint64_t)1279, + (uint64_t)1280, + (uint64_t)1499}) + { + const auto p = tt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "post-2nd inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "post-2nd inclusion verify i=" + std::to_string(i)); + } + + std::cout << "tiled tree (flush + combination): OK" << '\n'; + + // ---- Part 2b: compaction when the size is an exact multiple of TILE_WIDTH + // (here 512). flush_to cannot drain the whole tree, so compact() keeps + // one resident leaf rather than throwing; proofs still resolve. + { + TiledTree::Config mcfg; + mcfg.prefix = base / "tt_multiple"; + mcfg.compact_on_flush = true; + TiledTree mtt(mcfg); + for (uint64_t i = 0; i < 512; i++) + { + mtt.append(hashes[i]); + } + mtt.flush(); // covered == size == 512; compaction must not throw + expect(mtt.flushed_size() == 512, "multiple: flushed size 512"); + expect(mtt.tree_ref().min_index() == 511, "multiple: one leaf retained"); + + merkle::Tree mref; + for (uint64_t i = 0; i < 512; i++) + { + mref.insert(hashes[i]); + } + const Hash mroot = mref.root(); + expect(mtt.root() == mroot, "multiple: root matches reference"); + for (const uint64_t i : {(uint64_t)0, (uint64_t)256, (uint64_t)511}) + { + const auto p = mtt.inclusion_proof(i, 512); + expect(*p == *mref.path(i), "multiple: inclusion==ref"); + expect(p->verify(mroot), "multiple: inclusion verify"); + } + std::cout << "tiled tree (exact-multiple compaction): OK" << '\n'; + } + + // ---- Part 2c: compaction with a non-zero retention_margin keeps the most + // recent leaves resident while flushed indices are still served from + // tiles. The immutable prefix remains the full-tile prefix, regardless + // of the margin. + { + TiledTree::Config rcfg; + rcfg.prefix = base / "tt_margin"; + rcfg.retention_margin = 300; + rcfg.compact_on_flush = true; + TiledTree rtt(rcfg); + for (uint64_t i = 0; i < n1; i++) // n1 == 1000 + { + rtt.append(hashes[i]); + } + rtt.flush(); // covered = 768; target = floor((768 - 300) / 256) * 256 = + // 256 + expect(rtt.flushed_size() == 768, "margin: flushed size 768"); + expect( + rtt.tree_ref().min_index() == 256, + "margin: retained >= 300 recent leaves"); + + for (uint64_t i = n1; i < N; i++) + { + rtt.append(hashes[i]); + } + expect(rtt.root() == ref_root, "margin: root matches reference"); + + // Flushed-only (0, 255), flushed-but-resident overlap (256, 767), and the + // un-flushed frontier (1000, 1499). + for (const uint64_t i : + {(uint64_t)0, + (uint64_t)255, + (uint64_t)256, + (uint64_t)767, + (uint64_t)1000, + (uint64_t)1499}) + { + const auto p = rtt.inclusion_proof(i, N); + expect( + *p == *ref.path(i), "margin inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(ref_root), + "margin inclusion verify i=" + std::to_string(i)); + } + + bool threw = false; + try + { + rtt.retract_to(700); // size 701 < immutable_size 768 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "margin: retract below flushed prefix throws"); + + std::cout << "tiled tree (retention margin): OK" << '\n'; + } + + // ---- Part 3: rollback. Tiles are immutable, so only un-tiled + // (post-flush) entries may be rolled back. + { + // 3a. Before any flush nothing is tiled, so rollback is + // unrestricted. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_pre"; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 50; i++) + { + rb.append(hashes[i]); + } + rb.retract_to(29); // tiles_size == 0 -> allowed + expect(rb.size() == 30, "rb pre-flush retract allowed"); + } + + // 3b. After a flush, the un-tiled frontier can be rolled back and + // the + // tiled region stays consistent and provable; committed entries + // can't. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb"; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 300; i++) + { + rb.append(hashes[i]); + } + rb.flush(); // flushed_size = covered = 256 + for (uint64_t i = 300; i < 400; i++) + { + rb.append(hashes[i]); + } + rb.retract_to(349); // keep [0,349] + expect(rb.size() == 350, "rb retracted to 350"); + for (uint64_t i = 350; i < 400; i++) + { + rb.append(hashes[1000 + i]); // re-append DIFFERENT leaves + } + rb.flush(); // flushed_size = covered = 256 (tile [0,256) already + // written) + + // Reference tree of the exact post-rollback state. + merkle::Tree exp_tree; + for (uint64_t i = 0; i < 350; i++) + { + exp_tree.insert(hashes[i]); + } + for (uint64_t i = 350; i < 400; i++) + { + exp_tree.insert(hashes[1000 + i]); + } + const Hash exp_root = exp_tree.root(); + expect(rb.root() == exp_root, "rb root matches reference"); + + // Proofs for a tiled index and a frontier index match the reference. + for (const uint64_t i : {(uint64_t)100, (uint64_t)299, (uint64_t)399}) + { + const auto p = rb.inclusion_proof(i, 400); + expect( + *p == *exp_tree.path(i), + "rb inclusion==ref i=" + std::to_string(i)); + expect( + p->verify(exp_root), "rb inclusion verify i=" + std::to_string(i)); + } + const auto cp = rb.consistency_proof(300, 400); + expect( + ProofEngine::verify_consistency( + 300, 400, *exp_tree.past_root(299), exp_root, cp), + "rb consistency 300->400"); + + // Only the immutable full-tile prefix [0,256) is protected: rolling + // back into it is refused, while rolling back within the un-tiled + // frontier (>= immutable_size()) is allowed. + expect(rb.flushed_size() == 256, "rb flushed to full-tile prefix"); + + rb.retract_to(std::numeric_limits::max()); + expect(rb.size() == 400, "rb past-current index is a no-op"); + + bool threw = false; + try + { + rb.retract_to(100); // size 101 < 256 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb retract below tiled prefix throws"); + threw = false; + try + { + rb.retract_to(254); // size 255 < 256 + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb retract just below tiled prefix throws"); + rb.retract_to(255); // size 256 == immutable_size(): drops only frontier + expect(rb.size() == 256, "rb retract to tiled prefix allowed"); + } + + // 3c. Compaction keeps the final tiled leaf resident, so rollback to a + // tree size of exactly immutable_size() remains representable. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_exact_boundary"; + cfg.compact_on_flush = true; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 1000; i++) + { + rb.append(hashes[i]); + } + rb.flush(); + expect(rb.flushed_size() == 768, "rb boundary flushed size"); + expect(rb.immutable_size() == 768, "rb boundary immutable size"); + expect( + rb.tree_ref().min_index() == 767, + "rb boundary final tiled leaf retained"); + + rb.retract_to(767); + expect(rb.size() == 768, "rb exact immutable boundary allowed"); + + merkle::Tree expected; + for (uint64_t i = 0; i < 768; i++) + { + expected.insert(hashes[i]); + } + const Hash expected_root = expected.root(); + expect( + rb.root() == expected_root, "rb boundary root matches reference"); + const auto proof = rb.inclusion_proof(0, 768); + expect( + *proof == *expected.path(0), + "rb boundary inclusion matches reference"); + expect(proof->verify(expected_root), "rb boundary inclusion verifies"); + } + + // 3d. Rollback interacts correctly with compaction (flushed + tiled + // past). + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_compact"; + cfg.compact_on_flush = true; + TiledTree rb(cfg); + for (uint64_t i = 0; i < 1000; i++) + { + rb.append(hashes[i]); + } + rb.flush(); // flushed_size = covered = 768; retain boundary leaf 767 + expect( + rb.tree_ref().min_index() == 767, + "rb compact boundary leaf retained"); + for (uint64_t i = 1000; i < 1200; i++) + { + rb.append(hashes[i]); + } + // Frontier rollback (>= immutable_size()) is allowed. + rb.retract_to(1099); + expect(rb.size() == 1100, "rb compact frontier retract ok"); + + merkle::Tree exp_tree; + for (uint64_t i = 0; i < 1100; i++) + { + exp_tree.insert(hashes[i]); + } + const Hash exp_root = exp_tree.root(); + const auto p = rb.inclusion_proof(0, 1100); // leaf 0 is flushed + tiled + expect(*p == *exp_tree.path(0), "rb compact flushed inclusion==ref"); + expect(p->verify(exp_root), "rb compact flushed inclusion verify"); + + bool threw = false; + try + { + rb.retract_to(500); + } + catch (const std::exception&) + { + threw = true; + } + expect(threw, "rb compact retract below tiled throws"); + } + + // 3e. A failed flush may publish an immutable full tile before a later + // write fails. The attempted full-tile boundary is sealed against + // rollback, while flushed_size advances only after a successful retry. + { + TiledTree::Config cfg; + cfg.prefix = base / "rb_interrupted"; + TiledTree interrupted(cfg); + for (uint64_t i = 0; i < 512; i++) + { + interrupted.append(hashes[i]); + } + + const fs::path blocker = store_root(cfg.prefix) / "tile/0/001"; + fs::create_directories(blocker); + bool flush_threw = false; + try + { + interrupted.flush(); + } + catch (const std::exception&) + { + flush_threw = true; + } + + expect(flush_threw, "interrupted flush throws"); + expect( + fs::is_regular_file(store_root(cfg.prefix) / "tile/0/000"), + "interrupted flush published first full tile"); + expect( + interrupted.flushed_size() == 0, + "interrupted flush does not advance flushed size"); + expect( + interrupted.immutable_size() == 512, + "interrupted flush seals attempted boundary"); + expect( + interrupted.compact() == 0, + "interrupted flush does not permit compaction"); + + TiledTree recovered(std::move(interrupted)); + expect( + recovered.immutable_size() == 512, + "interrupted flush seal survives move"); + + bool retract_threw = false; + try + { + recovered.retract_to(0); + } + catch (const std::exception&) + { + retract_threw = true; + } + expect( + retract_threw, + "interrupted flush rejects rollback across attempted tiles"); + expect(recovered.size() == 512, "interrupted rollback changes nothing"); + + fs::remove(blocker); + expect( + recovered.flush().full_written == 1, + "interrupted flush retry writes only missing tile"); + expect( + recovered.flushed_size() == 512, + "interrupted flush retry advances flushed size"); + expect( + recovered.immutable_size() == 512, + "interrupted flush retry preserves immutable size"); + recovered.compact(); + + merkle::Tree expected; + for (uint64_t i = 0; i < 512; i++) + { + expected.insert(hashes[i]); + } + const Hash expected_root = expected.root(); + expect( + recovered.root() == expected_root, + "interrupted flush root matches reference"); + for (const uint64_t i : + {(uint64_t)0, (uint64_t)255, (uint64_t)256, (uint64_t)511}) + { + const auto proof = recovered.inclusion_proof(i, 512); + expect( + *proof == *expected.path(i), + "interrupted flush inclusion matches reference"); + expect( + proof->verify(expected_root), + "interrupted flush inclusion verifies"); + } + } + + // 3f. flushed_size advances only after every required level succeeds. + // Here all 256 level-0 tiles publish before the level-1 tile is blocked. + { + constexpr uint64_t level1_size = + (uint64_t)merkle::tiles::TILE_WIDTH * merkle::tiles::TILE_WIDTH; + const auto level1_hashes = make_hashes((size_t)level1_size); + + TiledTree::Config cfg; + cfg.prefix = base / "rb_interrupted_level1"; + TiledTree interrupted(cfg); + for (const auto& hash : level1_hashes) + { + interrupted.append(hash); + } + + const fs::path blocker = store_root(cfg.prefix) / "tile/1/000"; + fs::create_directories(blocker); + bool flush_threw = false; + try + { + interrupted.flush(); + } + catch (const std::exception&) + { + flush_threw = true; + } + + expect(flush_threw, "level-1 interrupted flush throws"); + expect( + interrupted.store_ref().has_full_tile(0, 255), + "level-1 interrupted flush publishes all level-0 tiles"); + expect( + interrupted.flushed_size() == 0, + "level-1 interrupted flush does not advance flushed size"); + expect( + interrupted.immutable_size() == level1_size, + "level-1 interrupted flush seals attempted boundary"); + + fs::remove(blocker); + expect( + interrupted.flush().full_written == 1, + "level-1 interrupted retry writes only the missing roll-up"); + expect( + interrupted.flushed_size() == level1_size, + "level-1 interrupted retry advances flushed size"); + expect( + interrupted.store_ref().has_full_tile(1, 0), + "level-1 interrupted retry publishes roll-up"); + + merkle::Tree expected; + for (const auto& hash : level1_hashes) + { + expected.insert(hash); + } + const Hash expected_root = expected.root(); + expect( + interrupted.root() == expected_root, + "level-1 interrupted root matches reference"); + const auto proof = interrupted.inclusion_proof(0, level1_size); + expect( + *proof == *expected.path(0), + "level-1 interrupted inclusion matches reference"); + expect( + proof->verify(expected_root), + "level-1 interrupted inclusion verifies"); + } + } + std::cout << "rollback: OK" << '\n'; + + std::cout << "tiles_tree: OK" << '\n'; + } + catch (std::exception& ex) + { + std::cout << "Error: " << ex.what() << '\n'; + return 1; + } + catch (...) + { + std::cout << "Error" << '\n'; + return 1; + } + + return 0; +}