Add tile-backed proof generation

.github/processor_count.cmake

@@ -0,0 +1,16 @@
+include(ProcessorCount)
+
+ProcessorCount(PARALLELISM)
+if(PARALLELISM EQUAL 0)
+  set(PARALLELISM 1)
+endif()
+
+message(
+  STATUS
+    "Setting CMAKE_BUILD_PARALLEL_LEVEL and CTEST_PARALLEL_LEVEL to ${PARALLELISM}"
+)
+file(
+  APPEND "$ENV{GITHUB_ENV}"
+  "CMAKE_BUILD_PARALLEL_LEVEL=${PARALLELISM}\n"
+  "CTEST_PARALLEL_LEVEL=${PARALLELISM}\n"
+)

.github/workflows/ci.yml

@@ -47,18 +47,36 @@ jobs:
     - name: Create Build Environment
       run: cmake -E make_directory ${{github.workspace}}/build/${{ matrix.build_type }}
 
+    - name: Detect Runner Parallelism
+      shell: bash
+      run: cmake -P "$GITHUB_WORKSPACE/.github/processor_count.cmake"
+
     - name: Configure CMake
       # Use a bash shell so we can use the same syntax for environment variable
       # access regardless of the host operating system
       shell: bash
       working-directory: ${{github.workspace}}/build/${{ matrix.build_type }}
       run: |
+        long_tests=OFF
+        if [ "${{ matrix.build_type }}" == "Release" ]; then
+          long_tests=ON
+        fi
+
+        linux_options=()
         if [ "$RUNNER_OS" == "Linux" ]; then
-          cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-        else
-          cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }}
+          linux_options=(
+            "-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}"
+            -DCLANG_TIDY=ON
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+          )
         fi
 
+        cmake "$GITHUB_WORKSPACE" \
+          "-DCMAKE_BUILD_TYPE=${{ matrix.build_type }}" \
+          "-DLONG_TESTS=$long_tests" \
+          "-DOPENSSL=${{ matrix.openssl }}" \
+          "${linux_options[@]}"
+
     - name: Build
       working-directory: ${{github.workspace}}/build/${{ matrix.build_type }}
       shell: bash
@@ -76,5 +94,3 @@ jobs:
       working-directory: ${{github.workspace}}/build/${{ matrix.build_type }}
       shell: bash
       run: ctest -VV -C ${{ matrix.build_type }} --timeout 300
-      env: 
-        ASAN_OPTIONS: use_sigaltstack=false # To avoid SetAlternateSignalStack with clang-11 

.github/workflows/codeql-analysis.yml

@@ -43,7 +43,7 @@ jobs:
 
     - name: Configure merklecpp
       working-directory: ${{github.workspace}}/build
-      run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON $GITHUB_WORKSPACE
+      run: cmake -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON $GITHUB_WORKSPACE
 
     - name: Build merklecpp
       working-directory: ${{github.workspace}}/build

CMakeLists.txt

@@ -6,17 +6,18 @@ cmake_minimum_required(VERSION 3.14)
 project(merklecpp LANGUAGES CXX)
 
 include(GNUInstallDirs)
+include(CTest)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(MERKLECPP_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 option(PROFILE "enable profiling" OFF)
-option(TESTS "enable testing" OFF)
 option(OPENSSL "enable OpenSSL" OFF)
 option(TRACE "enable debug traces" OFF)
 option(CLANG_TIDY "enable clang-tidy checks during build" OFF)
+option(LONG_TESTS "enable long-running tests" OFF)
 
 if(CLANG_TIDY)
   find_program(CLANG_TIDY_PROGRAM clang-tidy)
@@ -45,9 +46,7 @@ if(OPENSSL)
   target_link_libraries(merklecpp INTERFACE OpenSSL::Crypto)
 endif()
 
-if(TESTS)
-  enable_testing()
-
+if(BUILD_TESTING)
   function(add_unit_test NAME SRC)
     add_executable(${NAME} ${SRC})
     target_link_libraries(${NAME} PRIVATE $<BUILD_INTERFACE:merklecpp>)

doc/design/tlog-tiles.md

@@ -174,10 +174,10 @@ hashes, not bytes.
                               │ owns Tree                    ▲
               inclusion / consistency proofs                │ tiles
                               ▼                              │
-                       ProofEngine ──▶ HashSource ◀──────────┘
-                                        ├─ MemoryHashSource
-                                        ├─ TileHashSource
-                                        └─ CombinedHashSource
+                      ProofEngineT ──▶ HashSourceT ◀─────────┘
+                                         ├─ MemoryHashSourceT
+                                         ├─ TileHashSourceT
+                                         └─ CombinedHashSourceT
 ```
 
 `merklecpp_tiles.h` contains the public `merkle::tiles` API and includes
@@ -192,11 +192,10 @@ hashes, not bytes.
 | `TileWriterT` | Persist newly completed full tiles |
 | Hash sources | Resolve subtree roots from memory, tiles, or both |
 | `ProofEngineT` | Roots, inclusion/consistency proofs, and verification |
-| `TiledTreeT` | `append`, `flush`, proof APIs, and compaction |
+| `TiledTreeT` (planned) | `append`, `flush`, proof APIs, and compaction |
 
-The planned `TileHashSource` owns the proof-read LRU cache; `TileStoreT` does
-not cache. A combined source may require one read-only, non-hashing core
-`subtree_root` accessor.
+`TileHashSourceT` owns the proof-read LRU cache; `TileStoreT` does not cache.
+`MemoryHashSourceT` uses the `TreeT::subtree_root` accessor.
 
 ### 5.1 Types and aliases
 
@@ -215,9 +214,8 @@ class TileStoreT;
 
 using TileStore =
   TileStoreT<Tree::Hash::size_bytes, Tree::hash_function>;
-using TiledTree =
-  TiledTreeT<Tree::Hash::size_bytes, Tree::hash_function>;
-// Equivalent SHA-384 and SHA-512 aliases.
+// Hash-source and proof-engine aliases follow the same pattern.
+// The TiledTree alias arrives with the phase-4 lifecycle wrapper.
 
 }
 ```
@@ -297,21 +295,129 @@ stores the supplied leaf hash unchanged. `EntryBundleWriterT` mirrors
 `TileWriterT`: it writes only complete 256-entry bundles, confirms existing
 bundles before reusing them, and leaves the incomplete tail with the application.
 
+### 5.5 `TreeT::subtree_root`
+
+Proofs over the resident frontier use one core accessor:
+
+```cpp
+std::optional<Hash> subtree_root(uint8_t level, size_t index);
+```
+
+It returns the existing root of the complete subtree spanning
+`[index << level, (index + 1) << level)`. The method rejects overflow, flushed
+or out-of-range leaves, and non-perfect frontier nodes by returning
+`std::nullopt`. Like `root()` and `path()`, it may materialize pending nodes and
+compute dirty hashes, but does not change logical leaf contents or hashing
+semantics.
+
+### 5.6 Hash sources
+
+```cpp
+struct HashSourceT {
+  // MTH(D[index<<level : (index+1)<<level]) for a perfect, aligned subtree.
+  virtual bool subtree_root(uint8_t level, uint64_t index, Hash& out) const = 0;
+  virtual bool leaf(uint64_t i, Hash& out) const { return subtree_root(0,i,out); }
+};
+```
+
+- `TileHashSourceT{store, size}` resolves from **full tiles** using the
+  `subtree_root` formula in [section 3](#3-tile-mapping-and-proof-math) (`size` is
+  rounded down to a whole number of full tiles); returns `false` when the
+  requested subtree reaches into the un-tiled frontier.
+- `MemoryHashSourceT{tree}` resolves only resident, full subtrees through
+  `TreeT::subtree_root`.
+- `CombinedHashSourceT{primary, secondary}` tries the primary source first,
+  then falls back to the secondary. Using memory first avoids I/O for the
+  resident frontier.
+- `TileHashSourceT` mutates its LRU cache during `const` reads. It and every
+  `ProofEngineT` that refers to it require caller-provided synchronization when
+  shared between threads.
+
+### 5.7 `ProofEngineT`
+
+All three proof building blocks reduce to `mth_range` over a `HashSourceT`.
+Returned `PathT` objects are byte-identical to `Tree::path` / `Tree::past_path`.
+
+```cpp
+class ProofEngineT {
+public:
+  using Source = HashSourceT<HASH_SIZE, HASH_FUNCTION>;
+  explicit ProofEngineT(const Source& source);
+
+  Hash root(uint64_t size) const;                 // = mth_range(0, size)
+
+  // Inclusion path for leaf `index` in the tree of `size` leaves.
+  // Equivalent to Tree::path(index) when size==num_leaves(),
+  // and to Tree::past_path(index, size-1) otherwise.
+  std::shared_ptr<Path> inclusion_proof(uint64_t index, uint64_t size) const;
+
+  // RFC 6962 consistency proof that size `m` is a prefix of size `n` (m<=n).
+  std::vector<Hash> consistency_proof(uint64_t m, uint64_t n) const;
+  std::vector<Hash> consistency_proof_from_indices(
+    uint64_t first_index, uint64_t second_index) const;
+
+  // Verifier (consistency is new to merklecpp; inclusion reuses PathT::verify).
+  static bool verify_consistency(uint64_t m, uint64_t n,
+                                 const Hash& old_root, const Hash& new_root,
+                                 const std::vector<Hash>& proof);
+};
+```
+
+Inclusion (top-down; element order/`direction` chosen to match `Tree::path`):
+
+```
+elements = []                         # leaf→root order via push_front
+lo = 0, hi = size, idx = index
+while hi - lo > 1:
+    k = largest_pow2_lt(hi - lo)      # split at lo+k
+    if idx - lo < k:                  # target in left ⇒ sibling on the RIGHT
+        sib = mth_range(lo+k, hi);  dir = PATH_RIGHT;  hi = lo + k
+    else:                             # target in right ⇒ sibling on the LEFT
+        sib = mth_range(lo, lo+k);  dir = PATH_LEFT;   lo = lo + k
+    elements.push_front({sib, dir})
+leaf = src.leaf(index)
+return Path(leaf, index, elements, max_index = size - 1)
+```
+
+Consistency (RFC 6962 `SUBPROOF`):
+
+```
+consistency_proof(m, n):              # 0 < m <= n
+    if m == n: return []
+    subproof(m, lo=0, hi=n, complete=true)
+
+subproof(m, lo, hi, complete):
+    if m == hi - lo:
+        if not complete: proof.push_back(mth_range(lo, hi))
+        return
+    k = largest_pow2_lt(hi - lo)
+    if m <= k:
+        subproof(m, lo, lo+k, complete)
+        proof.push_back(mth_range(lo+k, hi))
+    else:
+        subproof(m-k, lo+k, hi, false)
+        proof.push_back(mth_range(lo, lo+k))
+```
+
+Because every emitted hash is an `mth_range` computed with `HASH_FUNCTION`, the
+consistency proof reconciles `Tree::past_root(m-1)` with
+`Tree::past_root(n-1)` — i.e. it is consistent with the existing library.
+
 ## 6. Delivery plan
 
-Phases 0-2 now deliver the storage primitives plus incremental tile and
-entry-bundle writers. Later PRs deliver the remaining independently testable
-phases; phase 3 needs no further core changes, while phase 4 may add one
-non-hashing accessor.
+Phases 0-3 now deliver the storage primitives, incremental tile and entry-bundle
+writers, hash sources, proof engine, and the only required core accessor. Later
+PRs deliver the lifecycle wrapper, user documentation, and performance coverage;
+no further core changes are planned.
 
 | Phase | Scope | Key tests |
 |---|---|---|
 | 0. Scaffolding | Headers, PAL, namespace, geometry, aliases, and CMake test wiring | Public-header and build integration |
 | 1. Coordinates/store | `TileRef`, index/path encoding, `TileStoreT`, durable atomic I/O, entry-bundle primitives | Encoding vectors; algorithm roots; 256-hash SHA-256/384 tiles; round trips; file/symlink collisions |
 | 2. Writers | Incremental `TileWriterT::write_up_to` from `leaf_at`, roll-ups, and `EntryBundleWriterT`; full resources only | Sizes 256 and 70,000 produce the exact tile set; repeated writes preserve immutability |
-| 3. Proof engine | `TileHashSource`, `mth_range`, roots, inclusion/consistency proofs, and verification | Tile roots equal tree roots; inclusion equals `path()` / `past_path()` and verifies; consistency reconciles `past_root()` values |
-| 4. Combined tree | Optional `subtree_root`; memory/combined sources; `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary |
+| 3. Proof engine | `TreeT::subtree_root`; tile, memory, and combined hash sources; roots; inclusion/consistency proofs and verification | Tile roots equal tree roots; inclusion equals `path()` / `past_path()` and verifies; consistency reconciles `past_root()` values |
+| 4. Combined tree | `TiledTreeT` append, flush, proof, and compaction APIs | Prove flushed and resident leaves against a non-flushed reference; consistency across a flush boundary |
 | 5. Documentation/performance | README usage, design link, and tile-backed benchmarks | Documentation and benchmark coverage |
 
-Deliverables are `merklecpp_tiles.h`, `merklecpp_pal.h`, `test/tiles_*.cpp`,
-CMake wiring, the optional core accessor, and README/design updates.
+Delivered through phase 3 are `merklecpp_tiles.h`, `merklecpp_pal.h`,
+`test/tiles_*.cpp`, CMake wiring, the core accessor, and design updates.

merklecpp.h

@@ -13,8 +13,10 @@
 #include <format>
 #include <functional>
 #include <iterator>
+#include <limits>
 #include <list>
 #include <memory>
+#include <optional>
 #include <sstream>
 #include <stack>
 #include <stdexcept>
@@ -703,7 +705,14 @@ namespace merkle
       /// 2**height-1.
       [[nodiscard]] bool is_full() const
       {
-        size_t max_size = (1 << height) - 1;
+        constexpr size_t size_digits = std::numeric_limits<size_t>::digits;
+        if (height > size_digits)
+        {
+          return false;
+        }
+        const size_t max_size = height == size_digits ?
+          std::numeric_limits<size_t>::max() :
+          (size_t{1} << height) - 1;
         assert(size <= max_size);
         return size == max_size;
       }
@@ -1336,6 +1345,80 @@ namespace merkle
         leaf_node(index)->hash, index, std::move(path), as_of);
     }
 
+    /// @brief Extracts the root hash of a complete subtree resident in memory
+    /// @param level The height of the subtree (it spans 2**level leaves)
+    /// @param index The index of the subtree at that height
+    /// @return The subtree root hash if the subtree is complete (balanced) and
+    /// fully resident in memory; otherwise, std::nullopt
+    /// @note Like root() and path(), it may
+    /// materialize pending nodes and compute dirty hashes, but does not change
+    /// logical leaf contents or hashing semantics. It returns std::nullopt if
+    /// any leaf of the subtree has been flushed, if the subtree extends past
+    /// the last leaf, or if the node at that position is not a full subtree.
+    /// The subtree spans leaf indices
+    /// [index << level, (index + 1) << level).
+    std::optional<Hash> subtree_root(uint8_t level, size_t index)
+    {
+      const size_t leaves = num_leaves();
+      if (leaves == 0 || level >= std::numeric_limits<size_t>::digits)
+      {
+        return std::nullopt;
+      }
+      if (index > (std::numeric_limits<size_t>::max() >> level))
+      {
+        return std::nullopt;
+      }
+
+      const size_t lo = index << level;
+      const size_t count = (size_t)1 << level;
+
+      if (lo < min_index() || count > leaves || lo > leaves - count)
+      {
+        return std::nullopt;
+      }
+
+      if (level == 0)
+      {
+        return leaf(lo);
+      }
+
+      compute_root();
+
+      const uint8_t target_height = level + 1;
+      if (!_root || _root->height < target_height)
+      {
+        return std::nullopt;
+      }
+
+      Node* cur = _root;
+      size_t it = lo << (sizeof(lo) * 8 - _root->height + 1);
+      for (uint8_t height = _root->height; height > target_height;)
+      {
+        const bool go_right = ((it >> (8 * sizeof(it) - 1)) & 0x01) != 0U;
+        if (cur->height == height)
+        {
+          Node* next = go_right ? cur->right : cur->left;
+          if (!next)
+          {
+            return std::nullopt; // conflated/flushed: not resident
+          }
+          cur = next;
+        }
+        it <<= 1;
+        height--;
+      }
+
+      if (cur->height != target_height || !cur->is_full())
+      {
+        return std::nullopt;
+      }
+      if (cur->dirty)
+      {
+        hash(cur);
+      }
+      return cur->hash;
+    }
+
     /// @brief Serialises the tree
     /// @param bytes The vector of bytes to serialise to
     void serialise(std::vector<uint8_t>& bytes)