Document tiled storage and add performance coverage

.github/processor_count.cmake

@@ -0,0 +1,16 @@
+include(ProcessorCount)
+
+ProcessorCount(PARALLELISM)
+if(PARALLELISM EQUAL 0)
+  set(PARALLELISM 1)
+endif()
+
+message(
+  STATUS
+    "Setting CMAKE_BUILD_PARALLEL_LEVEL and CTEST_PARALLEL_LEVEL to ${PARALLELISM}"
+)
+file(
+  APPEND "$ENV{GITHUB_ENV}"
+  "CMAKE_BUILD_PARALLEL_LEVEL=${PARALLELISM}\n"
+  "CTEST_PARALLEL_LEVEL=${PARALLELISM}\n"
+)

.github/workflows/build-docs.yml

@@ -4,10 +4,11 @@ on:
   push:
     branches:
       - main
+  pull_request:
   workflow_dispatch:
 
 concurrency:
-  group: pages
+  group: pages-${{ github.ref }}
   cancel-in-progress: false
 
 permissions: read-all
@@ -24,6 +25,7 @@ jobs:
       - name: Setup Pages
         id: pages
         uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
+        if: github.event_name != 'pull_request'
 
       - name: Install dependencies
         run: |
@@ -42,6 +44,7 @@ jobs:
         uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
         with:
           path: doc/build
+        if: github.event_name != 'pull_request'
 
   deploy:
     name: Deploy

.github/workflows/ci.yml

@@ -47,18 +47,36 @@ jobs:
     - name: Create Build Environment
       run: cmake -E make_directory ${{github.workspace}}/build/${{ matrix.build_type }}
 
+    - name: Detect Runner Parallelism
+      shell: bash
+      run: cmake -P "$GITHUB_WORKSPACE/.github/processor_count.cmake"
+
     - name: Configure CMake
       # Use a bash shell so we can use the same syntax for environment variable
       # access regardless of the host operating system
       shell: bash
       working-directory: ${{github.workspace}}/build/${{ matrix.build_type }}
       run: |
+        long_tests=OFF
+        if [ "${{ matrix.build_type }}" == "Release" ]; then
+          long_tests=ON
+        fi
+
+        linux_options=()
         if [ "$RUNNER_OS" == "Linux" ]; then
-          cmake $GITHUB_WORKSPACE -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }} -DCLANG_TIDY=ON -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-        else
-          cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTESTS=ON -DOPENSSL=${{ matrix.openssl }}
+          linux_options=(
+            "-DCMAKE_CXX_COMPILER=${{ matrix.compiler }}"
+            -DCLANG_TIDY=ON
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+          )
         fi
 
+        cmake "$GITHUB_WORKSPACE" \
+          "-DCMAKE_BUILD_TYPE=${{ matrix.build_type }}" \
+          "-DLONG_TESTS=$long_tests" \
+          "-DOPENSSL=${{ matrix.openssl }}" \
+          "${linux_options[@]}"
+
     - name: Build
       working-directory: ${{github.workspace}}/build/${{ matrix.build_type }}
       shell: bash
@@ -76,5 +94,3 @@ jobs:
       working-directory: ${{github.workspace}}/build/${{ matrix.build_type }}
       shell: bash
       run: ctest -VV -C ${{ matrix.build_type }} --timeout 300
-      env: 
-        ASAN_OPTIONS: use_sigaltstack=false # To avoid SetAlternateSignalStack with clang-11 

.github/workflows/codeql-analysis.yml

@@ -43,7 +43,7 @@ jobs:
 
     - name: Configure merklecpp
       working-directory: ${{github.workspace}}/build
-      run: cmake -DCMAKE_BUILD_TYPE=Debug -DTESTS=ON $GITHUB_WORKSPACE
+      run: cmake -DCMAKE_BUILD_TYPE=Debug -DLONG_TESTS=ON $GITHUB_WORKSPACE
 
     - name: Build merklecpp
       working-directory: ${{github.workspace}}/build

CMakeLists.txt

@@ -6,17 +6,18 @@ cmake_minimum_required(VERSION 3.14)
 project(merklecpp LANGUAGES CXX)
 
 include(GNUInstallDirs)
+include(CTest)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 set(MERKLECPP_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 
 option(PROFILE "enable profiling" OFF)
-option(TESTS "enable testing" OFF)
 option(OPENSSL "enable OpenSSL" OFF)
 option(TRACE "enable debug traces" OFF)
 option(CLANG_TIDY "enable clang-tidy checks during build" OFF)
+option(LONG_TESTS "enable long-running tests" OFF)
 
 if(CLANG_TIDY)
   find_program(CLANG_TIDY_PROGRAM clang-tidy)
@@ -45,9 +46,7 @@ if(OPENSSL)
   target_link_libraries(merklecpp INTERFACE OpenSSL::Crypto)
 endif()
 
-if(TESTS)
-  enable_testing()
-
+if(BUILD_TESTING)
   function(add_unit_test NAME SRC)
     add_executable(${NAME} ${SRC})
     target_link_libraries(${NAME} PRIVATE $<BUILD_INTERFACE:merklecpp>)

README.md

@@ -23,6 +23,69 @@ merklecpp requires C++20.
     assert(path->verify(root));
 
 
+## Tiled storage (tlog-tiles)
+
+The companion header `merklecpp_tiles.h` adds optional, header-only support for
+persisting a tree as [tlog-tiles](https://c2sp.org/tlog-tiles) tile files
+*progressively* (optionally dropping already-tiled leaves from memory) and for
+retrieving inclusion and consistency proofs from those tiles, from the in-memory
+tree, or from a combination of the two. The hashing is unchanged: tiles and tile-derived proofs are templated on
+the tree's existing hash function, so a tile-derived inclusion proof is
+byte-identical to one from `merkle::Tree::path()` and verifies with the same
+`merkle::Path::verify()`.
+
+    #include <merklecpp_tiles.h>
+
+    merkle::tiles::TiledTree::Config cfg;
+    cfg.prefix = "/var/log/mylog";       // tile files live here
+    cfg.retention_margin = 1024;         // keep the most recent leaves in memory
+    cfg.compact_on_flush = true;         // opt in to dropping already-tiled leaves
+
+    merkle::tiles::TiledTree log(cfg);
+    for (const auto& leaf_hash : batch)
+      log.append(leaf_hash);
+
+    // Write newly-complete tiles. With compaction enabled
+    // this also drops from memory the leaves already covered by a full tile;
+    // otherwise the tree keeps every leaf and you can call log.compact() later.
+    log.flush();
+
+    // Proofs are served from tiles + the resident tree, even for flushed leaves.
+    auto inclusion = log.inclusion_proof(/*index=*/0, log.size());
+    assert(inclusion->verify(log.root()));
+
+    auto consistency = log.consistency_proof(/*m=*/100, /*n=*/log.size());
+
+`TiledTree` creates a new tiled tree: the configured directory may exist, but
+its `tile` subdirectory must be absent or empty. It deliberately rejects
+existing tile data because tile files alone do not identify or restore the
+tree that produced them. Applications with externally persisted tree state can
+use the lower-level `TileStore` and `TileWriter` APIs to resume a store.
+
+See the [tiled storage guide](doc/tiles-guide.md) for a how-to covering
+flushing, compaction, rollback, proofs, and the lower-level building blocks,
+and [doc/design/tlog-tiles.md](doc/design/tlog-tiles.md) for the full design,
+file/directory layout, and the proof algorithms.
+
+
+## Building and testing
+
+Enable the test suite with CMake's `TESTS` option:
+
+    cmake -S . -B build -DTESTS=ON
+    cmake --build build
+    ctest --test-dir build
+
+Some tile coverage is intentionally long-running. `LONG_TESTS` is off by
+default for local builds; turn it on when you want the full tile stress suite,
+including level-2 tile coverage and tile proof timing:
+
+    cmake -S . -B build -DTESTS=ON -DLONG_TESTS=ON
+
+The repository CI enables `LONG_TESTS` so pull requests continue to exercise the
+full tiled-storage matrix.
+
+
 ## Contributing
 
 This project welcomes contributions and suggestions.  Most contributions require you to agree to a