[stack 28/40] Switch to consolidated benchmark corpus

apps/landing-page-astro/src/pages/benchmark.astro

@@ -5,7 +5,7 @@ import Footer from '@/components/Footer.astro';
 import dataset from '@/../public/benchmark/codevetter-benchmark-v1.json';
 import results from '@/data/benchmark-results.json';
 
-// Real numbers, sourced from benchmark/cases + benchmark/reviews scored by
+// Real numbers, sourced from benchmarks/public-catch-rate/cases + .../reviews scored by
 // scripts/run-public-benchmark.mjs. Regenerate with `node /tmp/gen-benchmark.mjs`
 // (or the committed generator) after any case/review change.
 const cv = results.codevetter;
@@ -185,7 +185,7 @@ const datasetJsonLd = {
 
       <h2 class="mt-12 text-xl font-semibold text-white">Methodology</h2>
       <ol class="mt-3 list-decimal space-y-2 pl-5 text-sm text-[--color-text-dim] marker:text-[--color-accent]">
-        <li>Each case lives in <code class="text-[--color-text]">benchmark/cases/&lt;id&gt;/</code> with a <code class="text-[--color-text]">source.&lt;ext&gt;</code> and a hand-written <code class="text-[--color-text]">label.json</code> ground truth (type, severity, line range, description).</li>
+        <li>Each case lives in <code class="text-[--color-text]">benchmarks/public-catch-rate/cases/&lt;id&gt;/</code> with a <code class="text-[--color-text]">source.&lt;ext&gt;</code> and a hand-written <code class="text-[--color-text]">label.json</code> ground truth (type, severity, line range, description).</li>
         <li>A reviewer's output is normalized into <code class="text-[--color-text]">reviews/&lt;case-id&gt;.&lt;reviewer&gt;.json</code> with a <code class="text-[--color-text]">matched_ground_truth</code> array per finding.</li>
         <li>The scorer (<code class="text-[--color-text]">scripts/run-public-benchmark.mjs</code>) computes catch rate, precision, F1, false positives, and redundant matches.</li>
         <li><strong class="text-white">Catch rate</strong> = matched ground-truth issues ÷ total expected. <strong class="text-white">Precision</strong> = matched ÷ (matched + false positives + redundant). <strong class="text-white">F1</strong> = harmonic mean.</li>

apps/macos/CodeVetterPackage/Sources/CodeVetterFeature/VerificationRunner.swift

@@ -3511,28 +3511,29 @@ public final class CodeVetterProcessRunner: @unchecked Sendable {
   private func runReadOnly(executable: URL, arguments: [String]) async throws -> Data {
     try await withCheckedThrowingContinuation { continuation in
       let process = Process()
-      let stdout = LockedData()
-      let stderr = LockedData()
+      let stdout = LockedPipeCapture()
+      let stderr = LockedPipeCapture()
       let stdoutPipe = Pipe()
       let stderrPipe = Pipe()
-      stdoutPipe.fileHandleForReading.readabilityHandler = { stdout.append($0.availableData) }
-      stderrPipe.fileHandleForReading.readabilityHandler = { stderr.append($0.availableData) }
+      stdoutPipe.fileHandleForReading.readabilityHandler = { stdout.consume(from: $0) }
+      stderrPipe.fileHandleForReading.readabilityHandler = { stderr.consume(from: $0) }
       process.executableURL = executable
       process.arguments = arguments
       process.standardOutput = stdoutPipe
       process.standardError = stderrPipe
       process.terminationHandler = { completed in
         stdoutPipe.fileHandleForReading.readabilityHandler = nil
         stderrPipe.fileHandleForReading.readabilityHandler = nil
-        stdout.append(stdoutPipe.fileHandleForReading.readDataToEndOfFile())
-        stderr.append(stderrPipe.fileHandleForReading.readDataToEndOfFile())
+        let output = stdout.finish(from: stdoutPipe.fileHandleForReading)
+        let errors = stderr.finish(from: stderrPipe.fileHandleForReading)
         guard completed.terminationStatus == 0 else {
           continuation.resume(
             throwing: VerificationRunnerError.launchFailed(
-              stderr.string.trimmingCharacters(in: .whitespacesAndNewlines)))
+              String(decoding: errors, as: UTF8.self)
+                .trimmingCharacters(in: .whitespacesAndNewlines)))
           return
         }
-        continuation.resume(returning: stdout.value)
+        continuation.resume(returning: output)
       }
       do {
         try process.run()
@@ -3549,18 +3550,13 @@ public final class CodeVetterProcessRunner: @unchecked Sendable {
     requestID: String? = nil,
     onStderrLine: (@Sendable (String) -> Void)? = nil
   ) async throws -> TrackedProcessOutput {
-    let stdout = LockedData()
-    let stderr = LockedData()
+    let stdout = LockedPipeCapture()
+    let stderr = LockedPipeCapture()
     let stdoutPipe = Pipe()
     let stderrPipe = Pipe()
-    let stderrLines = LockedLineBuffer()
-    stdoutPipe.fileHandleForReading.readabilityHandler = { stdout.append($0.availableData) }
+    stdoutPipe.fileHandleForReading.readabilityHandler = { stdout.consume(from: $0) }
     stderrPipe.fileHandleForReading.readabilityHandler = { handle in
-      let data = handle.availableData
-      stderr.append(data)
-      for line in stderrLines.append(data) {
-        onStderrLine?(line)
-      }
+      stderr.consume(from: handle, onLine: onStderrLine)
     }
 
     let process = Process()
@@ -3574,20 +3570,19 @@ public final class CodeVetterProcessRunner: @unchecked Sendable {
         process.terminationHandler = { [weak self] completed in
           stdoutPipe.fileHandleForReading.readabilityHandler = nil
           stderrPipe.fileHandleForReading.readabilityHandler = nil
-          stdout.append(stdoutPipe.fileHandleForReading.readDataToEndOfFile())
-          let trailingErrors = stderrPipe.fileHandleForReading.readDataToEndOfFile()
-          stderr.append(trailingErrors)
-          for line in stderrLines.append(trailingErrors) + stderrLines.finish() {
-            onStderrLine?(line)
-          }
+          let output = stdout.finish(from: stdoutPipe.fileHandleForReading)
+          let errors = stderr.finish(
+            from: stderrPipe.fileHandleForReading,
+            onLine: onStderrLine
+          )
           let wasCancelled = self?.clear(completed) ?? false
           if wasCancelled {
             continuation.resume(throwing: CancellationError())
           } else {
             continuation.resume(
               returning: TrackedProcessOutput(
-                stdout: stdout.value,
-                stderr: stderr.value,
+                stdout: output,
+                stderr: errors,
                 status: completed.terminationStatus
               ))
           }
@@ -3661,60 +3656,51 @@ private func decodeProgress(
   onProgress(progress)
 }
 
-private final class LockedData: @unchecked Sendable {
+private final class LockedPipeCapture: @unchecked Sendable {
   private let lock = NSLock()
   private var data = Data()
+  private var pending = Data()
+  private var finished = false
 
-  var string: String {
+  func consume(
+    from handle: FileHandle,
+    onLine: (@Sendable (String) -> Void)? = nil
+  ) {
     lock.lock()
     defer { lock.unlock() }
-    return String(data: data, encoding: .utf8) ?? ""
+    guard !finished else { return }
+    append(handle.availableData, onLine: onLine)
   }
 
-  var value: Data {
+  func finish(
+    from handle: FileHandle,
+    onLine: (@Sendable (String) -> Void)? = nil
+  ) -> Data {
     lock.lock()
     defer { lock.unlock() }
+    guard !finished else { return data }
+    append(handle.readDataToEndOfFile(), onLine: onLine)
+    if let onLine, !pending.isEmpty,
+      let line = String(data: pending, encoding: .utf8)
+    {
+      onLine(line)
+    }
+    pending.removeAll()
+    finished = true
     return data
   }
 
-  func append(_ chunk: Data) {
+  private func append(_ chunk: Data, onLine: (@Sendable (String) -> Void)?) {
     guard !chunk.isEmpty else { return }
-    lock.lock()
     data.append(chunk)
-    lock.unlock()
-  }
-}
-
-private final class LockedLineBuffer: @unchecked Sendable {
-  private let lock = NSLock()
-  private var pending = Data()
-
-  func append(_ chunk: Data) -> [String] {
-    guard !chunk.isEmpty else { return [] }
-    lock.lock()
-    defer { lock.unlock() }
+    guard let onLine else { return }
     pending.append(chunk)
-    return drainCompleteLines()
-  }
-
-  func finish() -> [String] {
-    lock.lock()
-    defer { lock.unlock() }
-    guard !pending.isEmpty else { return [] }
-    defer { pending.removeAll() }
-    guard let line = String(data: pending, encoding: .utf8) else { return [] }
-    return [line]
-  }
-
-  private func drainCompleteLines() -> [String] {
-    var lines: [String] = []
     while let newline = pending.firstIndex(of: 0x0A) {
       let line = pending[..<newline]
       if let text = String(data: line, encoding: .utf8), !text.isEmpty {
-        lines.append(text)
+        onLine(text)
       }
       pending.removeSubrange(...newline)
     }
-    return lines
   }
 }

apps/macos/CodeVetterPackage/Tests/CodeVetterFeatureTests/CodeVetterFeatureTests.swift

@@ -960,28 +960,30 @@ func supervisedRunnerStreamsStructuredProgressAndCancelsWithoutAReceipt() async
   try script.write(to: executable, atomically: true, encoding: .utf8)
   try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: executable.path)
 
-  let observed = LockedProgress()
-  let result = try await CodeVetterProcessRunner(executableURL: executable).run(
-    VerificationRequest(
-      requestID: requestID,
-      repositoryPath: "/fixture/repo",
-      change: "main...HEAD",
-      task: "Prove output"
-    ),
-    preflight: false,
-    onProgress: { observed.append($0) }
-  )
-  #expect(result.processStatus == 0)
-  #expect(
-    observed.values == [
-      VerificationProgress(
-        schemaVersion: "codevetter.progress/v2",
+  for _ in 0..<20 {
+    let observed = LockedProgress()
+    let result = try await CodeVetterProcessRunner(executableURL: executable).run(
+      VerificationRequest(
         requestID: requestID,
-        sequence: 0,
-        stage: "correctness",
-        state: "running"
-      )
-    ])
+        repositoryPath: "/fixture/repo",
+        change: "main...HEAD",
+        task: "Prove output"
+      ),
+      preflight: false,
+      onProgress: { observed.append($0) }
+    )
+    #expect(result.processStatus == 0)
+    #expect(
+      observed.values == [
+        VerificationProgress(
+          schemaVersion: "codevetter.progress/v2",
+          requestID: requestID,
+          sequence: 0,
+          stage: "correctness",
+          state: "running"
+        )
+      ])
+  }
 
   let sleeper = fixtureDirectory.appending(path: "codevetter-sleeper")
   try "#!/bin/sh\nexec sleep 30\n".write(to: sleeper, atomically: true, encoding: .utf8)

docs/architecture/data-model.md

@@ -65,8 +65,10 @@ repairs run as idempotent migrations guarded by feature flags. The groups:
 
 ## What is not persisted
 
-- **LLM API keys** — stored in user settings via Tauri preferences, not in
-  SQLite review tables.
+- **Direct LLM API keys** — not persisted by the active review/standards
+configuration. Legacy provider fields are allowlist-scrubbed from localStorage
+on the next read; installed agent CLI credentials remain external to
+CodeVetter.
 - **Raw CLI agent transcripts** — read from disk on demand; only parsed
   summaries land in SQLite.
 - **Structural graph for unopened repos** — built on demand and persisted per

docs/architecture/mcp-sidecar.md

@@ -67,6 +67,8 @@ limits, stable links, and structured data.
 | Tool | Purpose |
 |---|---|
 | `prepare_review` | Compose a bounded `codevetter.review-packet/v1` for one exact change from existing graph, history, prior-review, and verification-candidate evidence. It does not run a reviewer or execute a check. |
+| `resolve_evidence_scope` | Resolve a bounded flow, exact change, or codebase portfolio into canonical testing or performance candidates. It never executes the candidates and therefore returns planning evidence, not runtime proof. |
+| `verification_get_receipt` | Read one persisted canonical local-check receipt by bounded run ID inside the authorized repository scope. It cannot start, cancel, or mutate verification. |
 | `graph_query` | Search the structural graph or return a compact overview. |
 | `graph_get_node` | Explain one stable node and its source-backed relationships. |
 | `graph_get_neighbors` | Read bounded incoming, outgoing, or bidirectional neighbors. |
@@ -96,6 +98,43 @@ or the business-rule catalog. Follow stable IDs into explanation, lineage,
 trace, or hydration calls, and request only citations the agent actually needs.
 Normal execution never makes a model or provider call.
 
+### Resolve verification evidence
+
+`resolve_evidence_scope` is the direct agent projection of the same Rust planner
+used by native Testing/Performance and `codevetter scope`. Choose `testing` or
+`performance`, then provide a `flow`, `change`, or `codebase` scope. Flow and
+change scopes require `scope_value`; codebase scope omits it.
+
+```json
+{
+  "consumer": "performance",
+  "scope_kind": "change",
+  "scope_value": "main...feature"
+}
+```
+
+The response preserves candidate confidence, source leads, uncovered paths,
+dirty state, and limitations. MCP remains read-only: use the UI or CLI to admit
+and execute a selected workload.
+
+### Read a verification receipt
+
+`verification_get_receipt` is the read-only agent projection of a local check
+that already ran through native Review, the CLI, or another explicit local CLI
+consent boundary. It requires the exact persisted `run_id`, verifies that the
+receipt belongs to the MCP server's authorized repository, decodes the
+canonical `codevetter.local-check/v1` contract, and applies the normal MCP
+redaction and response-size policy.
+
+```json
+{
+  "run_id": "local-check-01234567-89ab-cdef-0123-456789abcdef"
+}
+```
+
+The tool never executes verification. Agents that need execution must use the
+separate local CLI consent boundary; enabling MCP does not grant that authority.
+
 ### Prepare a review
 
 `prepare_review` is the task-level entry point for a review agent. Use the