From b1cebcb86a23136d35741c8fd2406399cf90f2b7 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Wed, 2 Sep 2026 22:11:45 +0530 Subject: [PATCH 1/3] refactor: switch verification tooling to consolidated corpus 1/1 --- PROJECT_STATUS.md | 266 +++++++++++- .../src/pages/benchmark.astro | 4 +- docs/architecture/data-model.md | 6 +- docs/architecture/mcp-sidecar.md | 39 ++ docs/architecture/native-migration-matrix.md | 98 +++++ docs/architecture/native-rust-boundary.md | 93 +++++ docs/architecture/review-pipeline.md | 71 +++- docs/development/benchmark.md | 24 +- docs/development/configuration.md | 3 +- docs/development/docs.md | 2 +- docs/development/native-macos.md | 394 ++++++++++++++++++ docs/development/performance.md | 34 +- docs/development/testing.md | 31 +- docs/development/verification-receipts.md | 44 +- docs/index.md | 6 + .../codebase-context-tools-landscape.md | 4 +- docs/knowledge/performance-lab-publication.md | 2 +- docs/knowledge/tooling-decisions.md | 85 +++- docs/knowledge/tooling-documentation.md | 25 +- docs/knowledge/tooling-sandboxing.md | 53 ++- .../tooling-secrets-and-supply-chain.md | 85 +++- docs/operations/automation-contract.md | 2 +- docs/operations/ci.md | 46 +- docs/operations/release-pipeline.md | 15 +- docs/operations/signing-and-updates.md | 20 + docs/product/overview.md | 2 +- docs/product/surfaces.md | 81 +++- 27 files changed, 1441 insertions(+), 94 deletions(-) create mode 100644 docs/architecture/native-migration-matrix.md create mode 100644 docs/architecture/native-rust-boundary.md create mode 100644 docs/development/native-macos.md diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index 2275663f..809baf10 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -1,6 +1,6 @@ # Project Status -Last updated: 2026-08-24 +Last updated: 2026-09-02 ## Why / What @@ -39,7 +39,6 @@ separately approved safety design justify renewed investment. External: - Bundled `ccusage` 20.0.20 sidecar — local, offline Claude/Codex/Grok usage accounting; exact updates are opened weekly and remain qualification-gated. -- User-supplied LLM API keys (Anthropic / OpenAI / OpenRouter) stored in user settings — no server-side auth. - Installed and authenticated Codex or Claude CLI for Work conversations; provider account policy remains external to CodeVetter. - GitHub Releases + GitHub Actions — `auto-release.yml` cuts a `v` release on `tauri.conf.json` version bumps, dispatching `release.yml` to build/sign/upload Tauri binaries; `@tauri-apps/plugin-updater` consumes the `latest.json` manifest. - Cloudflare Pages — hosts the landing page (`codevetter` project, codevetter.com). @@ -56,6 +55,263 @@ Internal (fleet): ## Timeline +- **2026-09-02 — Independent Claude and Codex review (unreleased source):** + native Review and `codevetter check --agent cross` now request two sequential, + independent passes against the same immutable Rust-owned target and context. + The second reviewer receives no first-review output. Deterministic + reconciliation uses exact source-qualified path, line, and anchor identity; + title similarity never merges findings, a unique high-risk finding remains + actionable, and severity disagreement remains visible. Missing executors, + target drift, incomplete coverage, cancellation, or an unqualified candidate + fail closed without a composite finding claim. Persisted receipts preserve + reviewer-specific manifests, qualified candidates, readiness, duration, and + explicit usage/raw-candidate availability limits; repository-scoped MCP can + inspect the same receipt but cannot execute or cancel it. The 35-state native + packet includes dark/light cross-review evidence. Provider-backed caught-bug + 27-case provider-backed corpus run found 29/29 labels with Claude, 28/29 with + Codex, and 29/29 with the union. Cross-review recovered one low-severity + unused-helper label over Codex, but emitted 99 findings versus 46 and took + 187.5 seconds per case versus 99.1 seconds; usage/cost remained unavailable. + Cross-review is therefore qualified as an optional high-recall strategy and + is not the default. + +- **2026-09-02 — Native owner-quality refinement (unreleased source):** the + Evidence Workbench preserves a true-black canvas and chrome while restoring + restrained 1--4% near-black separation across working planes. Standalone + Review proof-map and intent renders now own an opaque canvas, and the owner + packet deterministically captures both search-only and rich repository-query + states, plus light-appearance Review, Testing, Performance, Runs, history + recovery, memory inspection, Agent Island configuration, and read-only Ops + status. All 35 + current-tree image hashes match the manifest. The same pass + reproduced and fixed a repository-query cancellation hang by closing stdin, + granting a bounded 200 ms termination grace, and using a final kill only for + the exclusively owned read-only worker; its test requires settlement within + one second. Appearance-aware amber and semantic evidence foregrounds now + meet a checked 4.5:1 normal-text contrast floor in dark and light modes. The + native Usage settings and CLI now share a bounded Rust history-root receipt: + selected Codex session folders normalize to their canonical home, unrelated + directories fail closed, and add/remove never reads or deletes transcripts. + Agent and MCP authority remain unavailable. Native Settings and + `codevetter memories` now share a read-only `codevetter.memories/v1` receipt: + Rust discovers bounded known locations, exposes only existing sources through + opaque identities and non-absolute display paths, caps reads/output, and + redacts secret-like content and Git-diff lines heuristically. Memory editing + and agent/MCP projections remain unavailable. The same Rust settings receipt + now preserves all 12 non-secret Agent Island preferences across native UI, + CLI, and the retained helper. Native labels configuration as live and the + supervised runtime as pending; the helper remains off by default and is not + launched by the Evidence Workbench. Native Settings and `codevetter ops` + now share a fixed-window `codevetter.ops-status/v1` receipt for local + configuration presence and aggregate run evidence. It excludes credentials, + webhook URLs, provider calls, webhook sends, writes, and agent/MCP authority. + The retained Tauri shell now attempts the sanitized one-time custom-rubric + transfer on every startup until Rust owns a canonical preference. Existing + Rust state wins, invalid legacy state writes nothing, browser-only mode does + not invoke Tauri, and the Rubrics surface remains a visible retry path. + Isolated frontend and Rust tests pass; installed WebView-to-native + qualification remains an explicit upgrade gate. + The isolated hosted lane passes 81 Swift tests, all nine XCUITests, Debug and + coverage-free Release macOS builds, and unsigned preview packaging without + using the operator's desktop. A read-only + current-package receipt binds the exact + `qualification-5r7JG4` candidate and measures a 62.4% smaller app bundle and + 92.4% smaller host executable than + the retained Tauri Release bundle; it does not refresh the historical launch + or settled-memory comparison. + Release-only fat LTO and one Rust codegen unit reduce the CLI and MCP + companions by 14.6% and 27.5% versus the prior candidate. Native Release + postprocessing removes test coverage instrumentation, strips the shipped host + while preserving its adjacent dSYM, and is enforced by the package gate. + The exact packaged MCP passes its fully sampled 50-start/200-round + qualification with 8.52 ms cold-initialize p95, 30.11 MiB ending RSS, 28 + strict read-only tools, and no TCP listener. + The complete all-feature Rust lane now passes 1,105 tests with 31 intentional + ignores, strict Clippy, and formatting. The retained frontend passes 680 unit + tests with one intentional skip, its separate 20-scenario live warm gate, + package-scoped TypeScript, and a production Vite build. + The Rust-generated capability glossary now has a deterministic native render + proving the external-collector split: CLI execution is available, a native + collector workflow remains planned, and agent authority remains unavailable. + Stale matched-comparison next steps now point to exact-package foreground, + responsiveness, energy, and long-session evidence instead of asking for an + already-completed comparison. + A manifest-locked local gallery exposes all 35 original-pixel dark/light + renders without external assets or network calls; its test fails on missing + or duplicate states. The final audit replaced the active Rubric pack's + washed-out disabled amber action with a high-contrast green `Selected` + receipt in both appearances; only available packs retain the amber action. + This evidence is ready for owner review but does not infer visual acceptance. + +- **2026-09-02 — Native hosted qualification (unreleased source):** GitHub + Actions run 33609288529 passed at exact source commit `824a9e8b`. The existing + Linux product lane remained green, while the isolated arm64 `xcode-27` lane + passed 81 Swift tests, all nine XCUITests, Debug and coverage-free Release + builds, the 33-state owner packet, ad-hoc ZIP/DMG packaging, and read-only + release inspection. Hosted large-receipt render p95s were 96.536 ms Repo + Unpack, 105.896 ms Usage, 121.065 ms Performance, 55.087 ms Testing, and + 64.132 ms Runs, all below the unchanged 150 ms gate. The exact package and + evidence identities are recorded in + `evidence/verification/native-hosted-qualification-2026-09-02.md`. The + hosted images are internally manifest-bound but not byte-identical to the + earlier local packet, so owner visual acceptance remains explicit. The + candidate correctly remains `shipping_ready: false`: production identifier + transfer, Developer ID signing, Library Validation, updater inputs, + notarization/Gatekeeper, installed upgrade/rollback, exact-package runtime + evidence, and the Tauri retirement decision remain open. + +- **2026-09-02 — Native release preflight (unreleased source):** a read-only + `codevetter.native-release-readiness/v1` inspector now binds the exact staged + app to its local qualification, verifies package/signature/runtime/updater + boundaries, and accepts only archive-bound notarization plus + production-identity installed-upgrade proofs. The current preview passes 7 + of 17 checks but correctly remains `shipping_ready: false` on ten production + gates: bundle transfer, Developer ID host/companion signing and one team, + Library Validation, HTTPS appcast, EdDSA key, archive-bound appcast, + Gatekeeper, notarization, and + installed upgrade/relaunch/data/rollback evidence. The installed proof now + binds the exact archive and build and requires a non-empty stable-record + fingerprint across native relaunch and rollback. A dependency-free, + read-only SQLite probe produces that content-free continuity projection and + fails on any missing incumbent identity. Its qualification used isolated + fixtures only. The inspection did not read credentials, sign, notarize, + install, publish, or change the installed Tauri application. + +- **2026-09-01 — Native macOS package candidate (unreleased source):** the + native AppKit/SwiftUI Evidence Workbench now builds as a hardened, + intentionally non-sandboxed Release app with exact Sparkle 2.9.6 wiring that + remains disabled for the preview identifier. A repository-owned qualifier + reuses the existing Rust and ccusage sidecar builders, packages the canonical + `codevetter`, `codevetter-mcp`, `ccusage`, and performance runtime capsule, + preserves Sparkle framework symlinks, smoke-tests every companion, verifies + deep signatures and runpaths, and produces local ZIP/DMG artifacts. The + staged package passed five alternating, surface-confirmed launches on the + populated Performance workspace at 117,424 KiB median process-tree RSS and + has checked true-black visual evidence. A matched five-by-five Release + comparison records startup parity (435.120 ms native versus 419.018 ms Tauri + first-visible-window median), 30.5% lower native settled RSS, and a 51.5% + smaller qualified native bundle. Native Review now enters one + Tauri-independent Rust application service: a bounded request id correlates + `codevetter.verification-command/v1`, ordered `codevetter.progress/v2` + events, request-scoped `codevetter.verification-cancel/v1`, and the distinct + preflight or final canonical receipt. Foreign progress, cancellation, and + terminal receipts fail closed. One shared no-confidence fixture now proves + equivalent request, stage, limitation, verdict, and exit semantics through + Rust, CLI, and native; the repository-scoped MCP + `verification_get_receipt` projection reads that same persisted canonical + receipt without gaining start or cancellation authority. The final packaged + sidecar exposes 28 strict read-only tools with no TCP listener. Native Review, the CLI, and local agent + invocation also share a Rust-owned, explicit-consent isolated-fix contract: + one detached worktree, bounded diff, recorded correctness rerun, + source-qualified re-review, per-finding fixed/reproduced/unchecked status, + retained owner inspection, and separately confirmed discard. There is no + commit, merge, or push action. Repo Unpack can now create a model-free local + snapshot from native UI or `codevetter unpack --operation scan` through one + Rust-owned scan/persistence boundary, then inspect stored Overview, Brief, + Activity, Inventory, and bounded Graph evidence in the native workspace. + The versioned receipt removes the raw file list from the client projection, + reports scan and persistence profiles separately, and continues to label + topology, history, and health as non-executable evidence. The native Graph + desk and `codevetter unpack --operation query` share the versioned + `codevetter.repo-query/v2` projection over the canonical structural and + temporal query services already used by MCP. Search, node explanation, + bounded impact, directed path, and causal trace retain index freshness, + trust, source anchors, and explicit unavailable coverage; Swift performs no + ranking or traversal. A scoped read-only worker prepares one search-only + canonical snapshot in the background, then upgrades it in place with compact + traversal edges only when requested while rechecking snapshot identity and + live-Git freshness. On the qualified 115,884-node graph, warm Release medians + measured 36.07 ms search, 35.15 ms explain, 116.55 ms impact, 68.38 ms path, + 32.46 ms history search, and 32.21 ms causal trace. Search-only RSS measured + 242.6 MiB and rich traversal RSS 307.6 MiB after rejecting a 511.9 MiB full + snapshot prototype. Contract/parser/render tests, 76 Swift tests, inspected + true-black evidence, and the quiet native compile gate pass. Native repository + selection now restores one + security-scoped bookmark across launches, and Usage applies 1w/30d/90d/all + windows consistently to ccusage charts, totals, models, and sessions while + the separate indexed Devin desk follows the same window for sessions, + generated/cache tokens, cost, and model rows. Live quota telemetry remains a + separate credential-sensitive migration. Review receipts now add a + Rust-owned `codevetter.review-intent-diagnostic/v1` projection across native, + CLI JSON, and local-agent output: it preserves the stated goal, deterministic + changed-surface classes, source-review and recorded-QA signals, gaps, and a + human-only closure boundary. Native Review gives this diagnostic a dedicated + true-black evidence desk and can reveal validated recorded QA artifacts in + Finder without promoting legacy QA into revision-exact proof. Its execution + action hands the exact repository and range or pull request to Testing, + clears stale proof and consent, and requires a preview plus fresh explicit + browser-run confirmation there. Native Testing, `codevetter qa`, and the + scoped read-only `qa_workspace_inspect` MCP tool now consume one Rust-owned + `codevetter.qa-workspace/v1` receipt for saved workflows and targets, + repository Playwright spec discovery, and deterministic post-fix rerun + setup. Legacy fields are projected into a separate native preference without + storage-state paths or arbitrary external commands; a selected route and goal + enter the canonical T-REX receipt, while preview consent is always reset. + Review will not regain a second browser execution authority. A real-agent fix + plus real saved-flow rerun smoke remains open. The populated + native app now also consumes `codevetter.onboarding/v1` for first-run state: + it honors the incumbent completion preference, checks executable presence + without inspecting authentication or credentials, transactionally saves + only the declared default adapter plus completion, and renders four + true-black Purpose, Readiness, Agent, and Workbench states. The same receipt + is available through `codevetter onboarding`, while About can reopen the tour + without changing completion state. The populated + native Performance receipt now lazily renders visible evidence rows and + repeated its 100-row gate three times at 41.867, 46.009, and 35.226 ms render + p95. Qualification conservatively uses the 46.009 ms worst run against the + unchanged 150 ms gate. This is not a release: Developer ID signing, + notarization, production appcast and EdDSA inputs, installed update/rollback, + workload/energy/long-session comparison, remaining feature/accessibility + parity, and owner retirement approval are still open. + +- **2026-08-31 — External performance, testing, and MCP evidence adapters + (unreleased source):** verification-receipt ingestion now accepts Playwright + JSON, JUnit XML, LCOV, Cobertura XML, Lighthouse JSON, and Chrome trace JSON + while preserving raw artifact identity, hashing failure text, rejecting XML + entities, and keeping observation-only formats at `no_confidence`. A three-run + Lighthouse CI trial passed the proposed landing-page gates, but the dependency + was removed after the high-severity audit exposed an unpatched transitive + archive traversal. Upstream Size Limit now follows rather than replaces the + desktop's Tauri-aware bundle budgets. `pnpm verification:dogfood` now runs the + active Playwright and c8 suites, emits Playwright JSON, JUnit, LCOV, and + Cobertura through their built-in reporters, and successfully ingests all four + while retaining `no_confidence` outside their proof. The run also removed the + retired Work/Board E2E inventory and exposed then fixed a duplicate React key + in Testing. The maintainer Codex client has pinned + isolated Chrome DevTools and Playwright MCPs plus named CodeVetter runtime and + receipt MCPs. GitHub's checksum-verified official v1.11.0 local server replaces + the PAT-dependent remote registration, with read-only relevant toolsets and + narrow in-memory browser OAuth. Live repository, pull-request, Actions, issue, + and CodeQL reads verify the connection without exposing a write tool. The + packaged CodeVetter graph/history MCP is also enabled for the canonical + checkout from the app-generated opaque configuration. Its history and + tree-sitter structural indexes are current at the checked-out HEAD, and direct + protocol smoke tests returned bounded `history_search` and `graph_query` + results with local access-audit rows. No product release or deployment is + claimed; work is tracked in issue #200. + +- **2026-08-31 — Bounded external collector foundation (unreleased source):** + Added `codevetter collect` and a shared Rust `codevetter.tool-collection/v1` + receipt for one exact clean checked-out Git range. The first adapter executes + an exact Gitleaks 8.30.1 bundle sibling or explicit debug/test override without a + shell, with a minimal environment, timeout, bounded output, binary/config + identity, repository-relative normalization, and raw secret fields removed + before serialization. cargo-audit and cargo-llvm-cov currently return + explicit `unavailable` evidence after optional exact-version preflight; they + do not install tools, fetch advisory data, or claim coverage. No sidecar is + yet bundled and no release claim is made; packaging and the remaining + collectors stay tracked in issue #198. + +- **2026-08-31 — Apple Container trial qualified with a containment caveat:** + Installed the signed/notarized 1.3.1 CLI after owner authorization and + exercised a 1-CPU/256-MB, internal-network, no-DNS, read-only-root sandbox on + the supported arm64 macOS 27 host. The cached no-op run took 0.61 seconds and + teardown left zero containers. A controlled traversal fixture proved the CLI + does not enforce CodeVetter's workspace-root boundary, so any adapter must + canonicalize and reject out-of-root mounts itself. This is external-prerequisite + qualification, not a bundled dependency or architecture approval; issue #197 + remains open. + - **2026-08-24 — Unified local change check (unreleased source):** the packaged `codevetter` CLI now accepts one clean checked-out PR head or Git range plus task intent and emits `codevetter.local-check/v1`. The runner resolves exact @@ -397,7 +653,7 @@ Internal (fleet): - **2026-07-03 — Surface consolidation + finishes (multi-agent pass):** removed redundant standalone pages QaReplay (`/qa-replay`) and IntentDebugger (`/intent-debugger`) — their functionality lives in Review. Finished Rubrics (review↔pack linkage via `local_reviews.standards_pack`, exact prompt preview, per-pack usage stats, pack cloning), T-Rex (per-watcher error recovery + retry, run drill-down dialog with persisted findings/log excerpt, pre-flight gh/token validation, per-PR base-branch inference), and AgentMemories (copy-as-markdown export, substring//regex/ line filter, git-diff-vs-HEAD view with secret redaction). Refactored QuickReview.tsx 6,264→3,050 lines into 12 components + 4 lib modules (behavior-preserving, 15 commits). Raw-Claude baseline scored on the 27 public benchmark cases (catch 0.931 / precision 0.397 / F1 0.557); CodeVetter's own comparator slot still needs generation before head-to-head claims. - **2026-07-03 (shipped in v1.2.8) — By-model cost attribution fix:** session-level `model_used` is last-model-wins, so multi-model Claude sessions booked ALL tokens/cost to the final model (a 211MB session with 17k opus-4-7 messages + 1.6k fable-5 messages billed $3.6k entirely to fable). Fix: per-message `session_model_usage` table populated by the indexer + one-time streaming backfill over existing Claude JSONL; by-model panel and per-session costs now sum per-model parts. Also added Fable/Mythos 5 pricing ($10/$50; was falling to sonnet default), folded `` into "unknown", and removed the Top-projects cost panel from Home (with its query/command/IPC). Verified by replaying the fix over the live DB: opus-4-7 $21,986→$29,473 (was under-credited), fable-5 correctly repriced. Guarded by `multi_model_claude_session_splits_usage_per_model`. - **2026-07-03:** Removed legacy Next.js landing page (`apps/landing-page`) — fully superseded by Astro site; `next-env.d.ts` git-removed, stale doc references cleaned up. -- **2026-07-03:** Published 27 hand-labeled public benchmark cases (`benchmark/cases/`) covering 7 languages (TypeScript, Python, Go, Rust, JavaScript, Java) and 15+ vulnerability types (SQL injection, XSS, hardcoded secrets, race conditions, path traversal, SSRF, prototype pollution, regex DoS, zip bombs, etc.). Scorer script (`scripts/run-public-benchmark.mjs`) validates labels and computes catch-rate/precision/F1 per reviewer. `pnpm bench:public`. Enterprise claims now backed by external, repeatable proof. +- **2026-07-03:** Published 27 hand-labeled public benchmark cases (`benchmarks/public-catch-rate/cases/`) covering 7 languages (TypeScript, Python, Go, Rust, JavaScript, Java) and 15+ vulnerability types (SQL injection, XSS, hardcoded secrets, race conditions, path traversal, SSRF, prototype pollution, regex DoS, zip bombs, etc.). Scorer script (`scripts/run-public-benchmark.mjs`) validates labels and computes catch-rate/precision/F1 per reviewer. `pnpm bench:public`. Enterprise claims now backed by external, repeatable proof. - **2026-07-02/03:** Streamlined telemetry + fleet navigation, guarded manual deploy command in CI, polished repo intelligence evidence surfaces. - **2026-06-28:** Devin agent indexing, agent hide/show filter, Grok parser improvements; PROJECT_STATUS audited as source of truth. - **2026-06-21 (v1.1.99) — Codex cost over-count fix:** Codex reports session-CUMULATIVE token totals; the incremental indexer was ADDING that running total every pass, inflating one session to 61.5B tokens / $35k (true: 391M / ~$220) and making "today" read ~$12.9k. Fix: `tokens_absolute` flag so cumulative tokens are SET not added, plus a one-time `fix_codex_token_totals` repair re-reading each Codex file. Verified on a live-DB copy: today $12,896→$377, year $82k→$38k (Claude cache-read costs, which are real, dominate the remainder). Guarded by `eval_append_delta_sets_cumulative_tokens_but_adds_per_message`. @@ -471,7 +727,7 @@ Internal (fleet): The corpus passes its contract-readiness publication gates; it does not by itself establish agent quality, provider superiority, or product value. - `--evidence-comparison=with:without` mode compares stored outputs with and without deterministic evidence search. -- 27 hand-labeled public benchmark cases (`benchmark/cases/`) covering 7 languages and 15+ vulnerability types; `pnpm bench:public` scores catch-rate/precision/F1. +- 27 hand-labeled public benchmark cases (`benchmarks/public-catch-rate/cases/`) covering 7 languages and 15+ vulnerability types; `pnpm bench:public` scores catch-rate/precision/F1. ### Evidence Pattern Search - Deterministic risk candidate packets from changed files, sensitive paths, optional `ast-grep` structural matches, blast/history context, and verification signals; top candidates and procedure gates injected into review prompts. @@ -499,7 +755,7 @@ Internal (fleet): ### Queryable codebase history - Repo Unpacked persists a backward-compatible schema-v2 history graph connecting bounded commit files, decisions, verification hints, and co-change leads with citations and trust labels. - Local queries prefer exact file/ID/label matches, rank broader terms, expand one hop, and state confidence, no-match, and truncation explicitly without mutating snapshots or creating findings. -- Settings can expose one explicitly enabled indexed repository through the packaged read-only `codevetter-mcp` stdio sidecar. Twenty-four strict tools cover task-level review preparation, graph queries, releases, search, as-of state, lineage, explanations, causal traces, comparisons, review manifests, business-rule archaeology, and evidence hydration; opaque versioned resources provide paginated discovery without absolute paths or credentials. +- Settings can expose one explicitly enabled indexed repository through the packaged read-only `codevetter-mcp` stdio sidecar. Twenty-six strict tools cover capability and evidence-scope discovery, task-level review preparation, graph queries, releases, search, as-of state, lineage, explanations, causal traces, comparisons, review manifests, business-rule archaeology, and evidence hydration; opaque versioned resources provide paginated discovery without absolute paths or credentials. ### App shell and UX - Home opens to usage dashboard (Today / Week / Month / Year counters); Repo holds repository context and Activity; Settings holds operational tools and preferences. diff --git a/apps/landing-page-astro/src/pages/benchmark.astro b/apps/landing-page-astro/src/pages/benchmark.astro index f5b3b17d..5e50cbc6 100644 --- a/apps/landing-page-astro/src/pages/benchmark.astro +++ b/apps/landing-page-astro/src/pages/benchmark.astro @@ -5,7 +5,7 @@ import Footer from '@/components/Footer.astro'; import dataset from '@/../public/benchmark/codevetter-benchmark-v1.json'; import results from '@/data/benchmark-results.json'; -// Real numbers, sourced from benchmark/cases + benchmark/reviews scored by +// Real numbers, sourced from benchmarks/public-catch-rate/cases + .../reviews scored by // scripts/run-public-benchmark.mjs. Regenerate with `node /tmp/gen-benchmark.mjs` // (or the committed generator) after any case/review change. const cv = results.codevetter; @@ -185,7 +185,7 @@ const datasetJsonLd = {

Methodology

    -
  1. Each case lives in benchmark/cases/<id>/ with a source.<ext> and a hand-written label.json ground truth (type, severity, line range, description).
    2. +
  2. Each case lives in benchmarks/public-catch-rate/cases/<id>/ with a source.<ext> and a hand-written label.json ground truth (type, severity, line range, description).
  3. A reviewer's output is normalized into reviews/<case-id>.<reviewer>.json with a matched_ground_truth array per finding.
  4. The scorer (scripts/run-public-benchmark.mjs) computes catch rate, precision, F1, false positives, and redundant matches.
  5. Catch rate = matched ground-truth issues ÷ total expected. Precision = matched ÷ (matched + false positives + redundant). F1 = harmonic mean.
    6. diff --git a/docs/architecture/data-model.md b/docs/architecture/data-model.md index ff203628..f02c74af 100644 --- a/docs/architecture/data-model.md +++ b/docs/architecture/data-model.md @@ -65,8 +65,10 @@ repairs run as idempotent migrations guarded by feature flags. The groups: ## What is not persisted -- **LLM API keys** — stored in user settings via Tauri preferences, not in - SQLite review tables. +- **Direct LLM API keys** — not persisted by the active review/standards +configuration. Legacy provider fields are allowlist-scrubbed from localStorage +on the next read; installed agent CLI credentials remain external to +CodeVetter. - **Raw CLI agent transcripts** — read from disk on demand; only parsed summaries land in SQLite. - **Structural graph for unopened repos** — built on demand and persisted per diff --git a/docs/architecture/mcp-sidecar.md b/docs/architecture/mcp-sidecar.md index 8934dfab..334e6ae5 100644 --- a/docs/architecture/mcp-sidecar.md +++ b/docs/architecture/mcp-sidecar.md @@ -67,6 +67,8 @@ limits, stable links, and structured data. | Tool | Purpose | |---|---| | `prepare_review` | Compose a bounded `codevetter.review-packet/v1` for one exact change from existing graph, history, prior-review, and verification-candidate evidence. It does not run a reviewer or execute a check. | +| `resolve_evidence_scope` | Resolve a bounded flow, exact change, or codebase portfolio into canonical testing or performance candidates. It never executes the candidates and therefore returns planning evidence, not runtime proof. | +| `verification_get_receipt` | Read one persisted canonical local-check receipt by bounded run ID inside the authorized repository scope. It cannot start, cancel, or mutate verification. | | `graph_query` | Search the structural graph or return a compact overview. | | `graph_get_node` | Explain one stable node and its source-backed relationships. | | `graph_get_neighbors` | Read bounded incoming, outgoing, or bidirectional neighbors. | @@ -96,6 +98,43 @@ or the business-rule catalog. Follow stable IDs into explanation, lineage, trace, or hydration calls, and request only citations the agent actually needs. Normal execution never makes a model or provider call. +### Resolve verification evidence + +`resolve_evidence_scope` is the direct agent projection of the same Rust planner +used by native Testing/Performance and `codevetter scope`. Choose `testing` or +`performance`, then provide a `flow`, `change`, or `codebase` scope. Flow and +change scopes require `scope_value`; codebase scope omits it. + +```json +{ + "consumer": "performance", + "scope_kind": "change", + "scope_value": "main...feature" +} +``` + +The response preserves candidate confidence, source leads, uncovered paths, +dirty state, and limitations. MCP remains read-only: use the UI or CLI to admit +and execute a selected workload. + +### Read a verification receipt + +`verification_get_receipt` is the read-only agent projection of a local check +that already ran through native Review, the CLI, or another explicit local CLI +consent boundary. It requires the exact persisted `run_id`, verifies that the +receipt belongs to the MCP server's authorized repository, decodes the +canonical `codevetter.local-check/v1` contract, and applies the normal MCP +redaction and response-size policy. + +```json +{ + "run_id": "local-check-01234567-89ab-cdef-0123-456789abcdef" +} +``` + +The tool never executes verification. Agents that need execution must use the +separate local CLI consent boundary; enabling MCP does not grant that authority. + ### Prepare a review `prepare_review` is the task-level entry point for a review agent. Use the diff --git a/docs/architecture/native-migration-matrix.md b/docs/architecture/native-migration-matrix.md new file mode 100644 index 00000000..b15e3be6 --- /dev/null +++ b/docs/architecture/native-migration-matrix.md @@ -0,0 +1,98 @@ +--- +title: Native macOS migration matrix +description: The no-regression contract for transferring every current CodeVetter surface to the native Evidence Workbench. +--- + +# Native macOS migration matrix + +The native macOS client is a full product migration, not a reduced viewer and +not a new five-tab product. The current Tauri application remains the shipped +authority until every retained surface below passes output, interaction, +performance, accessibility, and owner visual-acceptance gates. + +This matrix tracks product capability rather than React component count. Native +views may combine repeated setup or evidence patterns when that makes the +workflow clearer, but they must preserve every input, state, action, receipt, +limitation, historical record, and machine-readable boundary that remains +reachable in the current product. + +## Primary surfaces + +| Current surface | Current retained capability | Native destination | Current migration state | Replacement gate | +| --- | --- | --- | --- | --- | +| Usage | Local Claude, Codex, Grok, and bundled `ccusage` history; separate Devin and provider quota telemetry; agent/model filters, trends, and adapter health | **Usage** workspace | The canonical Rust `LocalUsageReport` now powers `codevetter usage` and the native local-usage desk. Day/week/month granularity; 1w/30d/90d/all-time windows shared by the ccusage chart, totals, models, and sessions; Claude/Codex/Grok filters; generated/cache/cost/session totals; model mix; adapter health; refresh; stale/unavailable validation; dark/light evidence; XCUITest; and a 365-period/100-session render gate are implemented. The same receipt projects indexed Devin sessions, generated/cache tokens, cost, and model rows from CodeVetter SQLite for 1w/30d/90d/all-time. The native Devin desk follows the selected window as a visibly separate source that is never folded into ccusage totals or described as live quota; `codevetter usage` exposes the same window receipt. Native Usage settings and `codevetter history-roots` now share `codevetter.history-roots/v1` for bounded additional Codex homes: Rust canonicalizes selected session folders, rejects unrelated paths, reports missing roots, and never reads or deletes transcripts during configuration. Reconciliation remains a separate explicit action. Credential-safe live provider telemetry remains pending. The previously exported Tauri `SessionScorecardPanel` has no mounted caller and is not treated as a retained visible surface. | Same providers, time ranges, filters, totals, quota boundaries, empty/error states, and refresh behavior; bounded chart rendering and no implied verification claim | +| Repo Unpack | Repository selection and snapshots; Overview, Handoff, Rules, Analysis, Activity, Inventory, Graph, and Delta; exports and comparisons | **Repo Unpack** workspace with an AppKit source outline and native inspectors | Native and CLI can now create the same persisted, model-free snapshot through the shared Rust scanner and `codevetter.unpack-scan/v1`, with supervised cancellation in the native client. The client receipt strips the raw file list and exposes separate full-scan and SQLite-persistence profiles. The native workspace lists and inspects Rust-owned snapshots through Overview, stored Brief, Activity, Inventory, bounded Graph, deterministic Analysis, observed Rules, source-qualified Handoff, and commit-range Delta desks while preserving repository/commit identity, history limits, outline bounds, and explicit topology/health non-proof labels. Handoff falls back to inventory entrypoints and test leads when no model-labelled report exists. Delta reuses the Rust Git range parser with a 24-commit bound. Native and CLI export the same Rust-rendered Markdown, offline HTML, graph JSON, agent-context Markdown, and repository-memory Markdown through `codevetter.unpack-export/v1`; the user chooses the native destination and Swift does not reinterpret content. The native Graph desk and `codevetter unpack --operation query` now consume one `codevetter.repo-query/v2` read-only receipt over the same structural-graph and history services used by MCP. Structure search, node explanation, impact, path, History search, and causal trace preserve repository identity, index freshness, trust, source anchors, a 100-result protocol bound, and explicit unavailable-index states; Swift performs no ranking or traversal. Exact Rust/parser/runner contracts cover both domains and the fail-closed unindexed boundary; 70 Swift tests, the quiet native compile, and inspected true-black query-desk evidence pass. Model synthesis execution, cleanup, source lineage, and final owner acceptance remain in Tauri or MCP. | Same persisted identities and exports; stable graph/history queries; large-repository bounds; no topology-as-runtime-proof regression | +| Review | Exact PR/range and task intent; source-qualified findings, coverage gaps, blast radius, memory graph, deterministic manifests, executable evidence links, fix handoff, X-Ray/export, synthetic QA and intent diagnostics | **Review** verification workspace | Exact-change plan/execute/cancel/receipt runs through the Tauri-independent `codevetter.verification-command/v1` Rust application service and supervised CLI worker. One bounded request id now correlates ordered `codevetter.progress/v2` events, process-level cancellation, and the distinct preflight/final canonical receipt; native ignores foreign progress and rejects mismatched terminal receipts. Optional repository-contained Markdown specs are hashed and parsed by the Rust core; native preflight exposes their explicit requirements and requires a fresh exact selection binding before execution. Final receipts render requirement-level review/executable/verified coverage and exact evidence references. Native Review and `codevetter check --agent cross` can request independent sequential Claude and Codex passes over the same immutable target and original context. Rust never sends first-review output to the second reviewer, reconciles only exact source-qualified identities, preserves unique and conflicting findings, and fails closed without a composite claim on missing executors, target drift, incomplete coverage, or interruption. Reviewer manifests, qualified candidates, readiness, duration, and availability limits persist in `codevetter.cross-review/v1`; read-only MCP projects the same receipt without execution authority. Cross-review remains optional until provider-backed caught-bug recall, false-positive, latency, and usage benchmarking passes. The native receipt promotes Rust-qualified findings above raw JSON, preserves severity/confidence/source line/suggestion, opens only repository-contained recorded paths, and exports the unmodified canonical JSON. A separate proof map projects the canonical readiness, deterministic manifest and qualification counts, bounded memory/blast-radius context, revision-qualified structural graph, recorded QA, evidence candidates, and procedure gates while explicitly separating execution proof from navigation leads. The same Rust result now carries `codevetter.review-intent-diagnostic/v1`: native renders the captured goal, deterministic changed-surface classes, source-review and QA signals, gaps, evidence chronology, and a fail-closed closure state that always requires human disposition. Recorded QA artifacts can be explicitly revealed through validated repository-relative or existing absolute paths; a legacy QA pass is never promoted to revision-exact proof. Review now hands the exact repository, range or pull request, and discovery scope to Testing while clearing stale proof and prior execution consent. This enforces the canonical boundary that Testing owns browser execution and Review consumes its evidence instead of recreating two runner authorities. Persisted reviews can build and atomically save the same fail-closed JSON, Markdown, or offline HTML Agent PR X-Ray through native UI and `codevetter xray`; public-source confirmation, current-preview identity, the Rust sanitizer, and optional per-finding suggestion approval remain explicit gates. Selected persisted findings can also become a bounded `codevetter.agent-fix-packet/v1` through native UI and `codevetter fix-packet`, carrying exact task/acceptance context, qualified source locations, recorded evidence to preserve, route advice, limitations, and copyable Markdown without claiming the fix is correct. Native UI, CLI, and local agent invocation now share an explicit-consent `codevetter.fix-attempt/v1`: Rust materializes the recorded head as a detached app-data worktree, supervises one selected agent, bounds the diff, runs `git diff --check`, reruns the recorded correctness target, source-qualifies a `WORKTREE` review, and classifies each selected finding as fixed, reproduced, or unchecked. The worktree remains uncommitted and retained until separately confirmed discard; there is no merge or push action. Typed decoding, exact CLI arguments, focused rendering, and true-black visual evidence are recorded. Native Plan, Execute, and Cancel expose keyboard and accessibility contracts; a Rust `no_confidence` preflight now fails closed, disables execution, names the missing binding, and cannot render an empty finding list as a green pass. A real-agent isolated-fix plus saved-flow post-fix rerun smoke remains pending; saved QA targets, repository spec discovery, and rerun preparation now live in Testing through the shared Rust QA-workspace receipt. | Equivalent Rust receipt semantics, every review state and export, source navigation, keyboard operation, cancellation, and no-confidence behavior | +| Testing | Direct preview verification, saved QA workflows and targets, Playwright spec discovery, post-fix rerun setup, changed-capability checks, scenarios, portfolio planning, PR watchers, confirmation, execution progress, failures, receipts, and limitations | **Testing** verification workspace | Direct PR/range preview verification runs through `codevetter trex` with explicit network confirmation, supervised cancellation, schema and verdict/exit validation, preview identity, derived routes, an optional selected saved route and goal, journey/artifact/console evidence, raw JSON, limitations, and a Runs handoff. `codevetter.qa-workspace/v1` is shared by native Testing and `codevetter qa`; the scoped `qa_workspace_inspect` MCP tool exposes the same read-only setup. Rust projects safe legacy fields into a separate native preference, never projects credential-bearing storage-state paths, refuses arbitrary external-command execution, discovers bounded repository Playwright specs without executing them, and prepares the same-flow post-fix rerun without restoring network consent or starting the browser. The shared Rust `evidence-scope` planner is projected through `codevetter scope --consumer testing` and native Testing for deterministic flow, exact-change, and bounded codebase target portfolios. Warm changed proof runs through `codevetter warm`; differential verification runs through `codevetter differential` with a separate preparation gate and the rule that comparison evidence never creates pass evidence. Scenario authoring runs through `codevetter scenario` and the native Scenario Foundry over the incumbent Rust bridge: free/local generation creates expiring candidates, validation and dry-run remain non-persistent, acceptance rechecks the candidate hash, selected destinations, and replacement approval, and rejection writes no project files. Incoming PR watcher configuration, bounded run history, foreground polling, and exact-head recovery now share `codevetter watcher`: native scheduling lasts only for the open app session; consent is not persisted; each automatic poll looks across open PRs, executes only a newly arrived PR or an existing PR with a new head SHA, may contact GitHub, execute isolated project code and the configured agent, post commit statuses, and stays supervised until receipts persist. The Rust boundary validates and fetches the exact immutable `refs/pull//head` object without changing the user's branch, index, worktree, `FETCH_HEAD`, or durable refs; Node projects install with their declared pnpm/npm/Yarn/Bun lockfile contract; and status authentication can reuse existing `gh` authority in memory without persisting or logging the token. Automatic polls skip unchanged PR heads, while a separately confirmed CLI/native Retry action reruns one exact currently open PR and persists a replacement attempt. MCP discovery never starts these runtimes. Exact Rust/CLI/native arguments and schemas, a local bare-remote materialization test, safe non-network configuration smokes, real-repository warm/differential failure-boundary smokes, XCUITest reachability for every Testing workspace, Swift watcher rendering and supervised consent/execution contracts, dark warm/differential/scenario/watcher visual evidence, and bounded direct-preview render gates are recorded. The [shared surface-parity receipt](../../evidence/verification/native-surface-parity-2026-09-01.md) proves that one repository-owned fixture preserves the same Rust, CLI, native, and read-only MCP evidence-scope semantics. The [live watcher qualification](../../evidence/verification/native-watcher-live-qualification-2026-09-01.md) proves one explicitly approved PR head through exact fetch, pnpm install, repository lint, conservative verdict, persistence, GitHub status posting, and a second same-head explicit recovery attempt. Final owner acceptance remains pending | Same `evidence-scope/v1` resolution and canonical receipts, bounded coverage, confirmation gate, cancellation, watcher lifecycle, and browser evidence | +| Performance | Intent/scope resolution, workload admission, zero-egress evidence, samples/warmups/timeouts, diagnosis, paired optimization, campaign handoff, cleanup, and receipts | **Performance** verification workspace | Exact-workload admission, capture, and paired verification run through `codevetter performance`. The same Rust `evidence-scope` planner now powers native flow/change/codebase discovery, `codevetter scope --consumer performance`, the read-only MCP `resolve_evidence_scope` tool, and `prepare_review` target suggestions; MCP never executes the workload. Selecting a closed native candidate fills the exact adapter, target, and optional workload name before admission. The diagnosis-to-campaign handoff preserves baseline/candidate/promotion states, requires an exact baseline checkout, and keeps promotion blocked until the paired Rust receipt confirms it. Native controls preserve all six Rust/Tauri adapters, bounded samples/warmups/timeouts, contained target identity, stale-plan invalidation, cancellation, exit/state validation, observed/inferred/unverified separation, cleanup, limitations, and canonical JSON. Recorded-run inspection invokes the existing digest-validating Rust operation and renders stored identity, lifecycle, policy, child outcome, bounded capture sizes, diagnosis, and limitations. Every supervised plan/capture/paired/inspection receipt also records 75 ms owned-process-tree samples with peak RSS, peak process count, sampler identity, count, and the explicit between-sample limitation. Exact CLI/native argument and schema tests, Rust sampler tests, MCP schema/validation/protocol coverage, a real-repository blocked-plan/resource smoke, dark rendered evidence, XCUITest coverage, and 100-row receipt rendering are recorded. The [shared surface-parity receipt](../../evidence/verification/native-surface-parity-2026-09-01.md) closes the same-fixture Rust/CLI/native/MCP discovery gate while preserving MCP's non-executing authority. The [runtime qualification](../../evidence/performance/native-runtime-qualification.json) passes explicit release launch, steady RSS, bridge latency, progress throughput, cancellation, crash recovery, and large-receipt gates. The [matched Tauri comparison](../../evidence/performance/native-tauri-comparison.json) confirms startup parity, 30.5% lower native settled Performance-workspace RSS, and a 51.5% smaller qualified native bundle across five alternating, surface-confirmed Release launches per app. Workload-execution, energy, and long-session comparisons remain unclaimed. | Same accepted adapters and workload identity; equivalent statistics, inferred/observed labels, cleanup, limitations, and paired-proof rules | +| Runs | Local-check, preview, T-Rex PR, synthetic QA, warm, differential, and audience histories currently embedded in their originating Tauri workflows | Native **Runs** evidence ledger | One bounded `codevetter.run-history/v1` projection and native inspector cover every incumbent listable verification history, including audience responses and synthetic-QA artifacts. Repository filtering, canonical JSON export, bounded arrow-key traversal, a 700-run Rust benchmark, and a 100-run/100-response Swift host-render gate are implemented. Foreground XCUITest and final owner interaction acceptance remain | Every retained receipt family, canonical JSON, verdict and limitation fidelity, repository filtering, keyboard access, bounded rendering, and export | +| Settings | General, Appearance, Integrations, Agents, Agent Island, Agent MCP, Notifications, Usage, Rubrics, Ops, Memories, and About | Native **Settings** window with the same sections | All 12 sections now have a native information architecture. `codevetter.native-settings/v1` and `codevetter settings` provide 28 validated non-secret General, Appearance, Agents, Notifications, and Agent Island values; one value is saved per receipt, unknown keys/options are rejected, and `github_token` plus undeclared credentials are excluded. The 12 Agent Island rows preserve the incumbent opt-in, speech, quiet-hour, and voice keys/defaults/options across Rust, CLI, native UI, and the retained helper. Native labels configuration as live and runtime transfer as pending: it does not launch the helper, consume live session content, speak, or action provider requests. First-run state now shares `codevetter.onboarding/v1` across native and `codevetter onboarding`: it honors the incumbent `onboarding_complete` preference, checks only executable presence, never inspects authentication or credentials, and transactionally saves only the selected declared adapter plus completion. The four-step true-black native flow explains the evidence contract, reports bounded tool readiness, selects Codex or Claude Code, and maps app, CLI, and agent surfaces; About can reopen it without resetting state. Agent MCP preserves repository selection, indexed/stale/enabled state, tools/resources, readiness, copyable client config and invocation, bounded audit, explicit enable/disable, and confirmed audit clearing through `codevetter.mcp-settings/v1` and `codevetter mcp`. Usage recovery and archive maintenance share `codevetter.history-roots/v1` and `codevetter.session-retention/v1`: bounded additional Codex homes, dry-run candidates and protected reasons, fail-closed apply, checkpoint, and separately confirmed VACUUM are native; provider transcripts and source sessions remain out of scope. Memories now share the read-only `codevetter.memories/v1` receipt across native and `codevetter memories`: Rust discovers bounded known locations, returns only existing sources through opaque identities and non-absolute display paths, caps source/read/output volume, and applies heuristic secret-line redaction to content and Git diff. Editing, agent, and MCP authority remain unavailable. Rubrics now share `codevetter.rubric-settings/v1`: built-in and custom packs, active selection, usage attribution, exact prompt preview, duplicate/create flows, and `codevetter check` consumption are aligned across native UI, CLI, incumbent Tauri, and agent invocation. Every incumbent Tauri startup now attempts the sanitized WebView-local migration until Rust owns the canonical preference; existing canonical state wins, invalid legacy state writes nothing, and opening Rubrics remains a visible retry path. Ops now shares the read-only `codevetter.ops-status/v1` receipt across native and `codevetter ops`: fixed 7, 30, and 90 day windows expose only configuration presence, normalized webhook flavor, and bounded aggregate run evidence. Credentials, endpoint values, provider refresh, webhook sends, configuration writes, and agent/MCP authority remain excluded. About reports the native bundle/version, and Sparkle 2.9.6 is embedded behind a fail-closed preview policy. The local package gate proves the three companions, runtime capsule, framework symlinks, hardened launch, ZIP, and DMG without touching the installed app. Exact argument/schema tests, Rust round trips, dark/light evidence, and XCUITest cover the transferred slices. Integration credential controls, live Ops provider/webhook operations, the Agent Island live helper runtime, Developer ID/notarization, production appcast/EdDSA inputs, installed update/rollback, and installed qualification of the WebView-local rubric bridge remain pending. | No preference loss, secret leakage, authority expansion, or silent default change; destructive archive maintenance stays outside agent authority and rubric migration must preserve incumbent custom packs | + +The Repo Unpack query desk now uses a supervised read-only Rust worker with one +search snapshot that upgrades in place with compact traversal edges. It +rechecks live Git freshness and the latest stored snapshot identity on every +request, drops the process on cancellation, and keeps the exact one-shot CLI +query as compatibility fallback. On the qualified 115,884-node graph, warm +Release medians measured 36.07 ms search, 35.15 ms explain, 116.55 ms impact, +68.38 ms path, 32.46 ms history search, and 32.21 ms causal trace. Search-only +RSS was 242.6 MiB and rich traversal RSS 307.6 MiB after a 511.9 MiB prototype +was rejected. This closes the richer-query performance gate while keeping the +memory and cold-upgrade tradeoffs explicit. +The latest background-native gate passes 80 Swift package tests and the macOS +Debug build; this supersedes earlier per-row test counts in the matrix. The +current 33-state packet preserves a true-black canvas with 1--4% near-black +working-plane separation, explicit search-only and rich query states, and +light-appearance counterparts for Review, Testing, Performance, Runs, bounded +history recovery, the memory inspector, Agent Island configuration, and +read-only Ops status. +Appearance-aware evidence foregrounds meet a checked 4.5:1 contrast floor +against dark canvas, warm light canvas, and white evidence planes. The +repository-query worker also closes stdin, grants a bounded 200 ms termination +grace, and uses a final kill only for its exclusively owned read-only child; +the cancellation gate requires settlement within one second. + +The [shared surface-parity receipt](../../evidence/verification/native-surface-parity-2026-09-01.md) +now binds the same canonical no-confidence local-check fixture to Rust, CLI, +native, and the repository-scoped read-only MCP `verification_get_receipt` +projection. MCP retains no verification start or cancellation authority. + +## Shared application behavior + +| Current behavior | Native requirement | +| --- | --- | +| Persistent routes preserve in-progress page state | AppKit window controllers and shared feature models preserve in-progress runs and forms across navigation and window changes. | +| Command palette and ordinary accessible navigation | Native menus, commands, toolbar items, search, keyboard shortcuts, VoiceOver labels, and focus restoration cover the same reachable actions. | +| Onboarding and update state | The shared Rust onboarding receipt, incumbent completion-state compatibility, default-adapter persistence, four native states, CLI projection, secret boundary, and deterministic rendering are qualified. A read-only `codevetter.native-release-readiness/v1` receipt now machine-checks the package, signing, updater, notarization, Gatekeeper, and installed-upgrade boundaries without exercising release authority. The preview passes 7 of 16 checks; production migration still requires the nine reported release gates, and Tauri remains authoritative beforehand. | +| Local SQLite records | Rust remains the only persistence authority. Swift receives versioned projections and never opens or reinterprets SQLite directly. | +| Error boundaries and explicit loading/empty states | Every migrated surface has native loading, empty, limited, failed, cancelled, stale, and unavailable states with written meaning. | + +## Machine surfaces + +The visual migration cannot fork product semantics. The Rust capability +registry, command contracts, and receipt schemas are projected through: + +- native UI commands and inspectors; +- `codevetter check`, `fix-packet`, `fix`, `xray`, `scope`, `trex`, `warm`, `differential`, `scenario`, `performance`, `usage`, `history-roots`, `memories`, `ops`, `unpack --operation scan|list|inspect|compare|export|query`, `settings`, `mcp`, `collect`, and `capabilities`, plus the structural + graph driver; +- repository-scoped MCP graph, history, review-preparation, archaeology, and + capability-catalog tools; +- versioned JSON receipts and exports. + +A native row may be marked complete only when its fixtures prove the same +identity, authority, verdict, limitations, and qualification state as the CLI +or MCP projection that shares the capability. + +The measured Rust/Swift ownership split is defined in +[Native Rust boundary](native-rust-boundary.md). Read-only projections may earn +an in-process path; verification execution remains supervised. + +## Retirement rule + +Tauri retirement is a separate owner decision after every row is complete. +Visual polish, a successful native build, or one working verification slice is +not sufficient evidence. Any missing feature or weaker interaction keeps the +corresponding Tauri surface operational. + +The local package qualification is evidence for the packaging mechanism, not +permission to transfer `com.codevetter.desktop`, publish an update, retire the +Tauri updater, or close the migration issue. diff --git a/docs/architecture/native-rust-boundary.md b/docs/architecture/native-rust-boundary.md new file mode 100644 index 00000000..dc28f005 --- /dev/null +++ b/docs/architecture/native-rust-boundary.md @@ -0,0 +1,93 @@ +--- +title: Native Rust boundary +description: Evidence-backed ownership split between the macOS process and supervised Rust workers. +--- + +# Native Rust boundary + +The native macOS app uses a hybrid Rust boundary. Bounded, read-only projections +may move in-process after their schema, memory, crash, and packaging gates pass. +Verification and other risky or long-running work remains in supervised Rust +workers. Swift never owns verification policy, receipt meaning, SQLite handles, +or Rust object lifetimes. + +This is an ownership decision, not permission to expose the entire Rust crate +through FFI. + +## Evidence + +The release benchmark moved the exact 40,047-byte generated capability registry +through two paths: + +| Path | Median | p95 | Samples | +| --- | ---: | ---: | ---: | +| Rust FFI byte copy | 1.083 µs | 1.208 µs | 10,000 | +| Rust FFI copy plus Swift JSON decode | 106.708 µs | 171.791 µs | 1,000 | +| Release `codevetter capabilities --json` worker round trip | 82.255 ms | 84.644 ms | 20 | + +Both paths produced semantically identical `codevetter.capabilities.v1` JSON. +The complete machine-readable receipt is in +`evidence/performance/native-bridge-benchmark.json`; the reproducible probe is +under `benchmarks/native-bridge`. + +The approximately 82 ms worker startup cost is material for frequent lightweight +reads but negligible beside repository checks, model review, browser journeys, +or performance sampling. Those operations benefit more from cancellation, +process supervision, bounded authority, and crash isolation than from removing +startup latency. + +## Selected split + +| Operation | Boundary | Reason | +| --- | --- | --- | +| Capability registry and other immutable generated projections | Candidate in-process read | Frequent, bounded, deterministic, no ambient authority | +| Small read-only SQLite projections | Benchmark before admission | Latency may matter, but SQLite handles remain Rust-owned | +| Verification planning | Supervised worker initially | Shares source validation and execution identity with the run | +| Correctness, review, and performance execution | Supervised worker | Long-running, cancellable, provider and project-tool authority | +| Browser journeys and collectors | Supervised child workers | Highest crash, timeout, resource, and authority risk | +| PR watcher timing | Native app-lifetime task; each poll is a supervised Rust worker | Swift may wait and request a confirmed poll, but Rust owns PR discovery, sandbox execution, agent use, GitHub status writes, and persistence | +| Receipt persistence and verdict construction | Rust worker/service | One semantic authority across UI, CLI, and MCP | + +## In-process admission gate + +An in-process projection must satisfy all of these before production use: + +1. The payload has a versioned Rust-owned schema and a deterministic fixture. +2. Swift receives owned bytes or values, never Rust pointers with shared lifetime. +3. Invalid, future, and oversized payload fixtures fail closed. +4. The call is read-only, bounded, non-blocking, and cannot invoke a provider, + project command, browser, or network operation. +5. Crash containment and release packaging are qualified for the real library, + not inferred from the 16 KB probe. +6. CLI and MCP projections remain semantically equivalent. + +If any gate fails, the operation stays on the worker boundary. + +## Worker contract + +The native Review slice now enters one Tauri-independent Rust application +service through `codevetter.verification-command/v1`. The caller supplies or +receives one bounded request id; `codevetter.progress/v2` JSON lines on stderr +carry that identity plus a monotonic sequence, and the distinct preflight or +final canonical receipt on stdout carries the same identity. Native rejects +foreign progress and a terminal receipt whose identity does not match the +supervised command. + +`codevetter.verification-cancel/v1` identifies the same request for supervised +cancellation. Native refuses a cancellation aimed at another active request; +an accepted cancellation terminates the request's supervised process group and +records no success receipt. It remains a transport-level terminal action rather +than a persisted canonical engine event, so concurrent daemon-style +cancellation is not implied by this service contract. + +`codevetter watcher --operation poll` additionally requires `--confirm-run`. +The CLI is deliberately not a daemon: it remains alive until all newly observed +head-SHA runs have persisted receipts. Native macOS owns only a cancellable timer +for the open app session, and does not persist execution consent. + +## Claim boundary + +This decision completes the boundary prototype and selection. It does not prove +production FFI packaging, active-run memory, high-volume progress throughput, +large-receipt rendering, or crash recovery. Those remain qualification gates; +Tauri stays operational until the full migration matrix closes. diff --git a/docs/architecture/review-pipeline.md b/docs/architecture/review-pipeline.md index 664938a5..ffb94518 100644 --- a/docs/architecture/review-pipeline.md +++ b/docs/architecture/review-pipeline.md @@ -97,6 +97,18 @@ removed without discarding otherwise valid evidence. Qualification diagnostics and rejected/stale/unresolved counts stay in the manifest so the UI cannot turn partial evidence into full confidence. +## External collector boundary + +The unreleased `codevetter collect` path resolves the same exact clean Git +change before invoking an optional local sidecar. `tool_collectors.rs` owns +binary/config identity, no-shell supervision, output limits, normalization, and +limitations; the external tool does not own the CodeVetter verdict. The first +implemented adapter is Gitleaks 8.30.1 and drops raw match/secret fields before +serialization. cargo-audit and cargo-llvm-cov remain claim-closed preflights +until their offline database and LLVM prerequisites are packaged. The +[qualification receipt](https://github.com/Codevetter/codevetter/blob/main/evidence/verification/tool-collector-foundation-2026-08-31.md) +records the proven slice; issue #198 owns packaging and remaining execution. + ## Manifest and interruption behavior SQLite stores additive run, unit, attempt, qualification, and checkpoint state. @@ -117,9 +129,36 @@ roots, prompts, or raw provider output. 1. User selects findings (dismissed findings are excluded from bulk selection). 2. `agent-fix-packet` is built from selected findings: goal, acceptance criteria, non-goals, browser/QA evidence refs, usage-routing advice. -3. Fix attempts run in **isolated git worktrees** (Rust `sandbox.rs`). -4. Re-review runs the same pipeline against the fix diff. -5. Per-finding re-check status: `fixed` / `reproduced` / `unchecked`. +3. After a separate explicit confirmation, `codevetter fix --operation execute` + materializes the recorded head as a detached worktree under CodeVetter app + data and runs exactly one selected coding-agent CLI there. +4. Rust bounds the changed-file list and diff, runs `git diff --check`, reruns + the correctness target from the source receipt, and source-qualifies a + `WORKTREE` re-review. +5. Per-finding re-check status is `fixed`, `reproduced`, or `unchecked`. + `verified_fixed` requires a clean diff, a passing executable target, a + completed re-review, and no reproduced or unchecked selected finding. +6. The worktree remains uncommitted and owner-inspectable. CodeVetter never + merges or pushes it. `codevetter fix --operation discard` requires a second + explicit confirmation before removing it. + +Codex uses its ephemeral workspace-write sandbox. Claude uses non-persistent +`acceptEdits` mode with an empty strict MCP configuration, and Gemini uses its +sandbox with `auto_edit` plus extensions disabled. Git credential prompting and +plain implicit pushes are disabled for the child process. Rust also verifies +that detached `HEAD` still equals the recorded source SHA; any agent-created +commit or branch movement fails closed and blocks all recheck claims. These +controls bound CodeVetter's invocation, but an externally configured CLI or +provider remains a separate local trust dependency. + +Completed local-check receipts additionally support a narrower deterministic +handoff through native Review and `codevetter fix-packet`. Rust reloads the +persisted receipt, rejects unknown or unqualified selected finding identities, +and binds the exact task, attached acceptance requirements, source locations, +runtime/procedure evidence, route advice, and limitations into +`codevetter.agent-fix-packet/v1`. The native Review sheet, CLI, and local agent +invocation then share `codevetter.fix-attempt/v1`; read-only MCP tools do not +gain execution authority. ## Verification proof @@ -152,19 +191,31 @@ previewed in a sandboxed iframe. The checked-in landing gallery is a local build artifact until its examples are manually adjudicated and deployment is separately authorized. +The Tauri panel, native Review sheet, and `codevetter xray` use the same Rust +builder, sanitizer, and atomic-save implementation. Native preview summarizes +eligibility, omissions, stages, and public findings without introducing a +WebView; the selected JSON, Markdown, or HTML artifact is still rendered and +written by Rust. + ## Standards packs -`StandardsPack` (`review-service.ts`) groups checks by focus -(`product-safety`, `security-boundary`, …). The active pack is persisted in -user settings (`codevetter_review_config` localStorage key, mirrored to Tauri -preferences) and linked to reviews via `local_reviews.standards_pack`. The -Rubrics page (`/rubrics`) handles pack authoring, exact prompt preview, -per-pack usage stats, and cloning. +`RubricPackInput` (`commands/rubric_settings.rs`) groups checks by focus +(`product-safety`, `security-boundary`, …). The Rust core owns built-ins, +validation, the active selection, custom packs, exact prompt rendering, and +the `codevetter.rubric-settings/v1` receipt. Completed reviews link the selected +id through `local_reviews.standards_pack`. The incumbent Rubrics page imports +the previous allowlisted `codevetter_review_config` localStorage record once, +then mirrors the canonical Rust receipt back for compatibility with older +frontend code. Native Settings and `codevetter rubrics` use the same receipt; +`codevetter check` consumes its active prompt context directly. ## Key files -- `apps/desktop/src/lib/review-service.ts` — config and standards packs. +- `apps/desktop/src-tauri/src/commands/rubric_settings.rs` — canonical rubric config and receipts. +- `apps/desktop/src/lib/review-service.ts` — incumbent compatibility mirror and prompt fallback. - `apps/desktop/src/lib/agent-fix-packet.ts` — fix packet construction. +- `apps/desktop/src-tauri/src/commands/fix_packet.rs` — receipt-bound native/CLI fix handoff. +- `apps/desktop/src-tauri/src/commands/fix_attempt.rs` — confirmed detached-worktree execution, bounded diff, executable recheck, re-review, and discard receipt. - `apps/desktop/src/lib/review-proof.ts` — verification handoff. - `apps/desktop/src/lib/quick-review-*.ts{x}` — QuickReview state, code, format, procedure. - `apps/desktop/src/components/quick-review/` — 13 panels (setup, editor, findings, fix diff, verification summary, audience, synthetic QA, history context, review memory graph, evidence insights, create preview, agent status timeline). diff --git a/docs/development/benchmark.md b/docs/development/benchmark.md index 146c1a26..2d65abf2 100644 --- a/docs/development/benchmark.md +++ b/docs/development/benchmark.md @@ -25,6 +25,10 @@ npm run bench:curation -- benchmarks/agent-prs/cases # after adding at least one benchmarks/agent-prs/cases/*.json file npm run bench:catch-rate -- benchmarks/agent-prs/cases --reviewer=codevetter npm run test:benchmark +# Provider-backed independent Claude/Codex comparison (writes ignored artifacts) +pnpm bench:cross-review +# Resume or deterministically rescore a checkpoint without repeating completed calls +pnpm bench:cross-review -- --resume artifacts/cross-review-benchmark/ --rescore ``` Fixture contract: @@ -95,7 +99,7 @@ avoid overfitting 27 cases. Protocol: every case ran through the REAL production pipeline (risk tiers, specialists, coordinator, dedup) via the ignored Rust harness `diag_benchmark_generate_codevetter_reviews`; raw outputs live in -`benchmark/reviews-raw/`. Ground-truth mapping was proposed mechanically +`benchmarks/public-catch-rate/reviews-raw/`. Ground-truth mapping was proposed mechanically (`scripts/map-benchmark-reviews.mjs`) and then hand-judged per finding under one rule: a finding matches only if its CORE CLAIM identifies the defect; process findings (no-tests, shipped-with-comment) never match. @@ -111,3 +115,21 @@ defect-only ground truth. With redundants collapsed, precision would be These are synthetic single-file cases — the planned real agent-PR case curation still stands before external head-to-head claims. + +## Results — independent cross-review (2026-09-02) + +The real `codevetter check --agent cross` path ran all 27 public cases with one +generic task and fresh isolated repositories. Human review corrected one narrow +keyword-mapper miss; the exact summary and limitations are recorded in the +[cross-review evidence](../../evidence/verification/native-cross-review-2026-09-02.md). + +| Strategy | Catch rate | Findings | Strict precision | F1 | Mean review time | +| --- | ---: | ---: | ---: | ---: | ---: | +| Claude | 29/29 | 134 | 0.216 | 0.356 | 88.4 s | +| Codex | 28/29 | 46 | 0.609 | 0.747 | 99.1 s | +| Claude + Codex | 29/29 | 99 | 0.293 | 0.453 | 187.5 s | + +The union recovered one low-severity dead-code label over Codex but more than +doubled its finding burden. Cross-review therefore remains an explicit +high-recall option, not the default. Usage/cost was unavailable from both local +executors, and synthetic single-file results do not establish real-PR quality. diff --git a/docs/development/configuration.md b/docs/development/configuration.md index 0d818d1c..334bef02 100644 --- a/docs/development/configuration.md +++ b/docs/development/configuration.md @@ -24,7 +24,8 @@ the Settings tab and persisted via Tauri preferences. | `DEBUG_TAURI_DRIVER` | `.env` (optional) | Debug flag for the (removed) tauri-driver path; kept for compatibility. | | LLM provider keys | Settings tab → Tauri preferences | Anthropic / OpenAI / OpenRouter. Never written to SQLite review tables. | | `gatewayBaseUrl`, `gatewayApiKey`, `gatewayModel` | `codevetter_review_config` (localStorage) mirrored to Tauri preferences | `ReviewConfig` in `apps/desktop/src/lib/review-service.ts`. | -| `reviewTone`, `customRules`, `activeStandardsPack`, `standardsPacks` | same | Standards packs authored in `/rubrics`. | +| `reviewTone` | same | Incumbent review presentation default. | +| `customRules`, `activeStandardsPack`, `standardsPacks` | Rust `review_rubric_config_v1` preference; allowlisted localStorage compatibility mirror | Standards packs authored in Tauri or native Settings and available through `codevetter rubrics`; the first incumbent open migrates older local-only packs. | | Auto-updater pubkey + endpoint | `apps/desktop/src-tauri/tauri.conf.json` | `@tauri-apps/plugin-updater` consumes `latest.json` from GitHub Releases. | ## CSP diff --git a/docs/development/docs.md b/docs/development/docs.md index 6fae096e..72081661 100644 --- a/docs/development/docs.md +++ b/docs/development/docs.md @@ -72,7 +72,7 @@ run `node_modules/.bin/blume validate`. ## Render with Blume -Blume reads `blume.config.ts` at the repo root and renders `docs/` as a +Blume reads `docs-site/blume.config.ts` and renders `docs/` as a static site. It is **not** the source of truth — it only presents the Markdown. diff --git a/docs/development/native-macos.md b/docs/development/native-macos.md new file mode 100644 index 00000000..995cec11 --- /dev/null +++ b/docs/development/native-macos.md @@ -0,0 +1,394 @@ +--- +title: Native macOS development +description: Reproducible tooling and ownership boundaries for the AppKit and SwiftUI Evidence Workbench. +--- + +# Native macOS development + +The native client lives in `apps/macos`. It is a macOS-only projection of the +Rust verification engine: AppKit owns application lifecycle, windows, menus, +split views, and dense desktop behavior; SwiftUI composes bounded feature and +evidence views. Verification policy, execution, verdicts, and receipt identity +remain Rust-owned. + +The existing Tauri application remains operational while the native client is +qualified. Native scaffolding or feature presence is not replacement proof. +The candidate therefore uses `com.codevetter.desktop.native-preview`, separate +from the shipped Tauri identifier. Transfer `com.codevetter.desktop` only after +every migration row passes and the owner makes the retirement decision. + +## Pinned baseline + +| Layer | Repository contract | +| --- | --- | +| Xcode project automation | `xcodebuildmcp@2.7.0`, invoked through `npx -y` and `.xcodebuildmcp/config.yaml` | +| Xcode project | `apps/macos/CodeVetter.xcworkspace`, shared `CodeVetter` scheme | +| Swift | Apple Swift toolchain selected by Xcode; package manifest requires Swift 6.1 or newer | +| Swift formatting | `swift format` from the selected Apple toolchain | +| Swift tests | Swift Testing for package behavior; XCTest/XCUITest for application behavior | +| Rust | Repository Cargo lockfile plus the selected Rust toolchain; `rustfmt`, Clippy, and Cargo tests are the minimum gates | +| Updater | Exact Sparkle 2.9.6 package on the app target; fail-closed preview configuration | +| Dependency policy | Existing `cargo-deny` policy; no new runtime or bridge dependency without qualification | + +The bounded seven-family Runs projection and native host rendering have checked +benchmarks at `evidence/performance/native-run-history-benchmark.json`. The Rust +projection covers 700 stored runs plus 100 audience responses and returns the +newest 100 in 1.077 ms p95. Swift decodes 100 rows in 2.741 ms p95 and creates, +lays out, and displays the 1280x800 ledger with 100 selected response rows in +26.826 ms p95. The claims exclude CLI startup, window-server frame pacing, and +interactive scrolling. + +The native direct-preview Testing slice has a separate checked gate at +`evidence/performance/native-testing-benchmark.json`. A canonical fixture with +100 browser journeys and 100 changed paths decodes in 0.510 ms p95 and creates, +lays out, and displays the 980x640 receipt desk in 19.271 ms p95. This measures +receipt decoding and native host rendering, not Rust execution, preview network +latency, browser runtime, window-server frame pacing, or interactive scrolling. + +The native exact-workload Performance slice is checked at +`evidence/performance/native-performance-benchmark.json`. A canonical diagnosis +with 100 observed evidence rows was repeated through three independent focused +gates. Decode p95s were 1.502, 1.792, and 1.551 ms; render p95s were 41.867, +46.009, and 35.226 ms. Qualification conservatively uses the 1.792 ms decode +and 46.009 ms render worst runs, both well inside the unchanged 25 ms/150 ms +gates. This measures receipt decoding and native host rendering, not CLI +startup, Node or workload execution, window-server frame pacing, or +interactive scrolling. + +The native local-usage slice is checked at +`evidence/performance/native-usage-benchmark.json`. A canonical fixture with +365 daily periods, 52 weekly periods, 12 monthly periods, and 100 sessions +decodes in 7.727 ms p95 and creates, lays out, and displays the 980x640 Usage +workspace in 18.346 ms p95. The Swift view bounds day/week/month chart rows and +visible model/session rows. This measures canonical JSON decoding and native +host rendering, not ccusage process startup, filesystem scanning, +window-server frame pacing, or interactive scrolling. + +Additional Codex history recovery is checked at +`evidence/verification/native-history-roots-2026-09-02.md`. Native Usage +settings and `codevetter history-roots` share one Rust-owned bounded receipt. +Selected `sessions` or `archived_sessions` directories normalize to their +canonical Codex home; unrelated directories, malformed paths, duplicates, and +more than 16 roots fail closed. Configuration reports availability but never +reads or deletes transcript content, and reconciliation remains a separate +explicit Usage action. + +Native memory inspection is checked at +`evidence/verification/native-memories-2026-09-02.md`. Native Settings and +`codevetter memories` share `codevetter.memories/v1` for bounded list, read, +and Git-diff operations. Rust discovers known locations but emits only existing +sources, addresses them with opaque SHA-256 identities, replaces absolute paths +with display paths, caps source and content volume, and applies heuristic +secret-line redaction. The contract is intentionally read-only and private: +Swift does not open arbitrary paths, and agent/MCP projections cannot read or +modify memory content. + +Read-only Ops status is checked at +`evidence/verification/native-ops-status-2026-09-02.md`. Native Settings and +`codevetter ops` share `codevetter.ops-status/v1` for fixed 7, 30, and 90 +day windows. Rust exposes only configuration-presence booleans, a normalized +webhook flavor, and bounded aggregate observability rows. Credentials, webhook +URLs, provider responses, absolute paths, live billing refreshes, webhook +sends, configuration writes, and agent/MCP authority remain outside this +contract. + +The first native Repo Unpack slice is checked at +`evidence/performance/native-unpack-benchmark.json`. A canonical fixture with +100 stored snapshots, 700 graph nodes, and 1,000 root tree rows decodes in +5.179 ms p95 and creates, lays out, and displays the 1280x800 workspace in +22.921 ms p95. The client receives a Rust-bounded projection and never opens +SQLite or receives the raw file inventory. This measures canonical JSON +decoding and native host rendering, not repository scanning, graph queries, +window-server frame pacing, or interactive scrolling. + +The aggregate runtime qualification is checked at +`evidence/performance/native-runtime-qualification.json`. The release client +reaches a responsive first frame in 0.580 seconds on average and settles at +117,424 KiB median RSS on the populated Performance workspace. The supervised worker +delivers 1,000 typed progress events in 342.278 ms, settles cancellation in +0.326 ms, rejects a deliberately crashed worker, and accepts a fresh worker in +3.739 ms. All five retained large-fixture decode/render gates remain below +25 ms/150 ms p95. + +The historical matched Release launch and settled-memory comparison is checked at +`evidence/performance/native-tauri-comparison.json`. Five alternating launches +per application reached an accessibility-confirmed Performance workspace. The +native and Tauri first-visible-window medians were 435.120 ms and 419.018 ms, +which qualifies startup parity rather than a speed claim. Native settled at +117,424 KiB process-tree RSS versus 169,024 KiB for Tauri, a 30.5% reduction, +for that qualified build. The current read-only package receipt at +`evidence/performance/native-current-package-footprint.json` binds the exact +`qualification-5r7JG4` candidate without launching either app: its 62,060 KiB +bundle is 62.4% smaller and its host executable is 92.4% smaller than the +retained 165,144 KiB Tauri Release bundle. These claims exclude +responsive-frame timing, current-tree startup or RSS, scrolling, workload +execution, energy, and long-session behavior. + +XcodeBuildMCP is deliberately a development tool rather than an application +dependency. Its project-local configuration exposes only the macOS, package, +coverage, discovery, scaffolding, diagnostic, and cleanup workflows required by +this migration. Sparkle 2.9.6 is the one exact third-party Swift package on the +native app target. The independently tested feature package remains free of the +binary framework so policy and receipt tests do not depend on app embedding. + +The Review handoff can execute one explicitly confirmed isolated fix through +the bundled Rust CLI. The selected receipt head is materialized under app data, +the selected coding agent can edit only that detached worktree, and Rust owns +the bounded diff, correctness rerun, source-qualified re-review, and +`codevetter.fix-attempt/v1` result. The native UI can reveal the retained +worktree or separately confirm its discard; it cannot commit, merge, or push. +Read-only MCP retains no execution authority. + +## Local package qualification + +`pnpm native:package:qualify` consumes an XcodeBuildMCP-produced Release app; +it does not invoke Xcode itself. The qualifier reuses the existing release +sidecar builders, stages a new bundle under `artifacts/native-package`, embeds +`codevetter`, `codevetter-mcp`, `ccusage`, the runtime performance capsule, and +Sparkle, then verifies hardened signatures and emits ZIP/DMG archives plus a +machine-readable receipt. The visible app remains `CodeVetter.app`, while its +host executable is `CodeVetterNative` so the lowercase `codevetter` CLI remains +distinct on the default case-insensitive filesystem. + +The current checked local run is recorded in +[Native package qualification](../../evidence/verification/native-package-qualification-2026-09-01.md). +The current package-only candidate is 62,060 KiB with a 17,427,235-byte ZIP +and 20,034,510-byte DMG. It embeds Sparkle 2.9.6, passes three executable +companion smokes, preserves framework symlinks, requires rich repository-query +CLI parity, and passes a signed packaged-sidecar smoke with 28 strict read-only +MCP tools and no listener. It was deliberately not launched on the operator's +active desktop. The earlier 80,116 KiB candidate remains the five-launch +Performance-workspace evidence at 117,424 KiB median settled RSS; the current +package-only check does not silently inherit a new launch or memory claim. + +The first complete isolated hosted run is recorded in +[Native hosted qualification](../../evidence/verification/native-hosted-qualification-2026-09-02.md). +At commit `824a9e8b`, it passed 81 Swift tests, all nine XCUITests, Debug and +coverage-free Release builds, the 33-state packet, package qualification, and +the read-only readiness inspector on GitHub's arm64 runner. Its ZIP, DMG, app, +companions, dSYM, metrics, and hashes are bound in that receipt. It does not +inherit the older package's launch or settled-memory result. + +### Release optimization boundary + +The exact current package uses Rust fat LTO with one codegen unit while +retaining unwind panic semantics. The native Release host disables coverage +instrumentation, enables dead-code stripping and deployment postprocessing, +and preserves external debug symbols in an adjacent dSYM. The package +qualifier fails closed if the source host contains LLVM coverage or profile +sections. Relative to the immediately prior package, these changes reduce the +bundle by 31.6%, host by 77.0%, CLI by 14.6%, MCP by 27.5%, ZIP by 18.8%, and +DMG by 21.9%. The exact package and fully sampled MCP receipt is +`evidence/performance/native-release-optimization.json`. + +The tradeoff is Release build throughput: the observed native clean Release +build took 34.0 seconds, and the latest two optimized Rust sidecar links took +200 and 159 seconds. Debug and test profiles are unaffected. This receipt does not +claim a new foreground launch, current-package RSS, scrolling, energy, or +long-session result, and production must archive the matching dSYM separately. + +This is deliberately not a +shipping claim: the preview bundle has no update feed or key and is ad-hoc +signed. Because ad-hoc components have no shared Team ID, local staged and +Debug previews disable Library Validation; the checked-in Release entitlement +stays empty and the production Developer ID build must prove Library Validation. + +## Release-readiness inspection + +`pnpm native:release:inspect` is the read-only gate between local packaging and +release operations. It binds the exact inspected app to its package receipt, +checks bundle/version/companion identity, signatures, Hardened Runtime, +Library Validation, execution authority, Sparkle configuration, Gatekeeper, +and optional notarization and installed-upgrade proofs. It emits +`codevetter.native-release-readiness/v1` and fails closed by setting +`shipping_ready` to `false`; it never signs, notarizes, installs, publishes, +enumerates identities, or reads credentials. + +The current preview result is recorded in +[Native release-readiness inspection](../../evidence/verification/native-release-readiness-2026-09-02.md). +Seven of 17 local checks pass and ten production gates remain blocked. The +additional gate requires an offline-verified Sparkle appcast bound to the exact +production feed, public key, version, build, and qualified archive. +The exact current-source candidate is +`artifacts/native-package/qualification-5r7JG4/CodeVetter.app`; its package +receipt, archive hashes, and exact bundled-MCP smoke are recorded in +[Native macOS package qualification](../../evidence/verification/native-package-qualification-2026-09-01.md). + +Installed migration evidence uses `pnpm native:data-continuity`. With every +CodeVetter process fully quit, `capture` reads only durable record identities +from the resolved `com.codevetter.desktop/codevetter.db`, hashes them with a +per-run nonce, and never reads messages or preference values. Capture the +baseline before installation, then capture against that same baseline after +the native relaunch and again after rollback. `compare` refuses empty evidence, +any missing incumbent identity, the wrong Application Support root, database +integrity failure, or a changed baseline fingerprint; legitimate new rows do +not fail continuity. The resulting `codevetter.native-data-continuity/v1` +projection is nested in the separately provenance-qualified installed-upgrade +proof. Running the probe does not authorize installation, launch, rollback, or +production-identity transfer. + +## Commands + +Run these from the repository root: + +```bash +npx -y xcodebuildmcp@2.7.0 project-discovery discover-projects --scan-path apps/macos +pnpm test:native +pnpm test:native:ui -- --foreground --desktop-idle +pnpm test:native:full -- --foreground --desktop-idle +pnpm native:build:release +pnpm native:package:qualify +pnpm native:package:finalize -- --qualification +pnpm test:native-package +pnpm native:appcast:inspect -- --app --appcast --qualification --out +pnpm native:notarization:prove -- --app --archive --qualification --submission --out +pnpm native:installed-upgrade:qualify -- --incumbent-app --native-app --qualification --run-root --out --foreground --hosted-ephemeral +pnpm test:native-review-gallery +pnpm native:data-continuity -- capture --database /codevetter.db --phase before --out +pnpm native:data-continuity -- capture --database /codevetter.db --phase after_upgrade --baseline --out +pnpm native:data-continuity -- capture --database /codevetter.db --phase after_rollback --baseline --out +pnpm native:data-continuity -- compare --before --after-upgrade --after-rollback --out +pnpm test:native-data-continuity +pnpm native:release:inspect -- --app --qualification --out +pnpm test:native-release +pnpm native:runtime:compare -- --native-app --tauri-app --runs 5 --settle-ms 5000 --out --foreground +pnpm test:native-runtime-compare +swift format lint --recursive apps/macos/CodeVetter apps/macos/CodeVetterPackage/Sources apps/macos/CodeVetterPackage/Tests apps/macos/CodeVetterUITests +cargo fmt --manifest-path apps/desktop/src-tauri/Cargo.toml --check +cargo clippy --manifest-path apps/desktop/src-tauri/Cargo.toml --all-targets -- -D warnings +cargo test --manifest-path apps/desktop/src-tauri/Cargo.toml +``` + +`pnpm test:native` is the default automation lane. It lowers scheduling +priority, runs all Swift package behavior and offscreen render gates, and +compiles the macOS application without launching CodeVetter or taking focus. +`pnpm test:native:background` is an explicit alias for the same lane. + +`pnpm test:native:ui` is deliberately foreground-only. It runs only the nine +XCUITest interaction cases and can activate CodeVetter, move focus, or open +menus while it is running. `pnpm test:native:full` runs the quiet lane first and +then the foreground lane. Neither package script grants foreground access. +Both fail closed unless the operator adds the just-in-time +`--foreground --desktop-idle` flags for that invocation. Run interaction +automation only while the local Mac is idle or on a dedicated graphical +session. A Space on the same active login is not an isolation boundary because +XCUITest can switch focus or Spaces. + +For zero-disruption interaction qualification, use a dedicated remote Mac (or +a separate graphical login on that Mac) and keep the session unlocked for +XCUITest. Offscreen host-render tests remain the local visual regression gate; +they do not need a visible application window. + +The manual-only `native-qualification.yml` workflow is the repository-owned +hosted path. It runs on GitHub's arm64 `xcode-27` image, grants no release or +signing authority, and uploads only the unsigned preview package, dSYM, local +qualification, current-tree 35-state owner-review packet, and release-readiness +evidence for seven days. Interaction tests remain opt-in through the dispatch +input; ordinary pushes and pull requests do not start this workflow. This is +the preferred way to qualify XCUITest without borrowing the operator's active +desktop. + +Before the workflow exists on the default branch, dispatch the branch through +the already-registered `ci.yml` workflow with `native_qualification=true` and +set `native_interaction=true` when the isolated runner should execute XCUITest. +Both inputs default to false, so ordinary CI dispatches remain native-quiet. + +The first complete hosted receipt is +[run 33609288529](https://github.com/Codevetter/codevetter/actions/runs/33609288529) +at source commit `824a9e8b`. It passed the quiet lane, all nine opt-in +XCUITests, the Release build, unsigned preview packaging, and the fail-closed +readiness inspection without using the operator's desktop. The artifact is +retained for seven days; durable results and hashes live in +[the checked evidence](../../evidence/verification/native-hosted-qualification-2026-09-02.md). + +`native-production-qualification.yml` is the separately protected, manual-only +production-candidate lane. It has read-only repository permission, performs +credential use in an ephemeral keychain, and may sign, notarize, create a +Sparkle appcast, and exercise an isolated installed migration. It cannot +publish a release. Run +[33628919883](https://github.com/Codevetter/codevetter/actions/runs/33628919883) +stopped at its protected-input preflight because all eight required repository +secrets were absent; no signing, notarization, installation, or release action +ran. + +The XcodeBuildMCP CLI and MCP server use the same tool implementations and the +same project-local defaults. If a current Codex session started before the MCP +registration, the pinned CLI is the supported in-session path; later sessions +receive the scoped MCP tools automatically. + +The native executable accepts `--appearance light` or `--appearance dark` for +repeatable visual qualification. This affects only the launched process; it +does not write a system or application preference. + +## Qualification boundary + +- Development builds use a supervised local Rust process until the bridge + admits a specific read-only projection. The selected hybrid ownership rule is + documented in [Native Rust boundary](../architecture/native-rust-boundary.md). +- Review checks enter `codevetter.verification-command/v1` with a bounded + `--request-id`. They keep the correlated canonical receipt alone on stdout + and opt into ordered `codevetter.progress/v2` JSON lines on stderr with + `--json --progress-json`. Preflight receipts remain distinct from final + receipts; native ignores progress for another request, rejects a mismatched + receipt, and scopes `codevetter.verification-cancel/v1` to the active request. + Cancellation cannot produce or preserve a success claim. +- Direct preview verification invokes the existing `codevetter trex` contract. + Swift performs form admission and receipt rendering only; Rust still owns Git + identity, preview validation, route derivation, browser execution, persistence, + verdicts, and limitations. Failed and no-confidence exit codes remain valid + inspectable receipts when they agree with the canonical verdict. +- Exact-workload performance verification invokes `codevetter performance`. + Planning is read-only and fingerprints the exact local workload before + execution; Swift cannot enable capture after the scope changes. Rust and the + existing local performance capsule retain authority over zero-egress + admission, execution, diagnosis, paired comparison, cleanup, and receipt + semantics. Exit states 0, 1, and 2 remain inspectable only when the outer + receipt state agrees. +- Isolated Review fixes invoke `codevetter fix`. Swift only supplies the exact + persisted run/finding identities, selected agent, and explicit confirmation, + then validates the canonical receipt against CLI exit state. Rust creates and + retains the detached worktree, supervises the agent, reruns the recorded + correctness target, re-reviews `WORKTREE`, and requires a separate confirmed + discard. No native or CLI merge path exists. +- Local usage invokes `codevetter usage`, which reuses the Tauri Rust service + and opens the existing SQLite database read-only only when it is present. + Ready, stale, and unavailable reports remain inspectable only when exit 0, 1, + or 2 agrees. ccusage accounts for Claude, Codex, and Grok; Devin and live + provider quotas remain explicitly separate and are not inferred by Swift. +- Repo Unpack history invokes `codevetter unpack`, which opens the existing + SQLite database read-only and projects stored snapshot identities plus a + Rust-trimmed inventory. Swift does not query SQLite, recompute graph/history + semantics, or treat deterministic topology and health leads as runtime proof. +- Native Settings invokes `codevetter settings`. Rust owns an explicit + non-secret key allowlist, value validation, and SQLite persistence; Swift + renders only `codevetter.native-settings/v1`. Unknown keys and options are + rejected, only one declared value is saved per receipt, and credential keys + such as `github_token` never enter the native projection. +- Agent Island configuration is one bounded slice of that receipt. The 12 + opt-in, speech, quiet-hour, and voice preferences use the same keys, defaults, + and options as the retained supervised helper and are editable from native UI + or `codevetter settings`. The Evidence Workbench does not yet launch the + helper, read live sessions, speak updates, or action provider requests; the + preview is non-activating and agent/MCP authority remains unavailable. +- Native memory inspection invokes `codevetter memories`. Rust owns bounded + source discovery, opaque source identity, canonical path admission, output + limits, heuristic redaction, and Git-diff extraction. Swift receives only the + versioned read-only receipt; there is no edit, delete, agent, or MCP authority. +- Agent MCP invokes `codevetter mcp` and renders + `codevetter.mcp-settings/v1`. Rust retains repository canonicalization, + indexed/stale state, enablement, tool/resource catalogs, redaction limits, + client configuration, and bounded access metadata. Native audit clearing is + explicitly confirmed; its rows never include arguments, prompts, query text, + credentials, or evidence content. +- Debug and Release builds are intentionally outside App Sandbox because the + product must execute user-selected repository tools and supervised helpers. + Release retains Hardened Runtime and Library Validation; notarized production + signing must prove the same authority without adding ambient credentials. +- The app may read and execute within repositories selected by the user; it + receives no ambient credential authority. Each Rust contract remains + responsible for repository containment, egress, and subprocess limits. +- Shipping requires contract parity across native, CLI, and MCP plus measured + launch, memory, cancellation, large-receipt, accessibility, and visual gates. +- Local packaging and disabled-preview updater wiring are qualified. Developer + ID signing, notarization, production appcast/EdDSA inputs, installed upgrade, + rollback, identifier transfer, and Tauri retirement remain separate gates. diff --git a/docs/development/performance.md b/docs/development/performance.md index 350219ad..4cd0788c 100644 --- a/docs/development/performance.md +++ b/docs/development/performance.md @@ -150,21 +150,33 @@ static import closure plus the default Home route—not the sum of every lazy ro wrap the per-file diff in `React.memo`. Deferred — speculative without a profile, and risky in a 6k-line file. -### Bundle budget guard (`bench:bundle`) +### Bundle budget guards (`bench:bundle`) Reads Vite's manifest to compute the actual entry + Home static closure and fails if that exceeds **550 KB raw**, if any individual chunk exceeds **500 KB raw**, or -if the complete lazy distribution exceeds **1,800 KB raw**. This catches startup +if the complete lazy distribution exceeds **1,800 KB raw**. Size Limit then +independently checks the emitted `out/assets/*.js` distribution against the same +1.8 MB ceiling. The upstream tool is deliberately additive: it cannot express +the Tauri-specific entry + Home closure or individual lazy-chunk limits. This catches startup regressions without treating intentionally deferred code as startup work: | chunk / closure | raw KB | gzip KB | note | |-----------------|-------:|--------:|------| -| initial + Home | 452.8 | — | startup parse boundary | -| `AgentPanel-*` | 457.0 | 114.8 | largest lazy feature chunk | -| `index-*` | 396.7 | 127.4 | entry/vendor | -| `RepoPage-*` | 239.4 | 58.2 | lazy route | -| `QuickReview-*` | 200.8 | 52.6 | lazy route | -| **all lazy routes** | **1,601.3** | **444.7** | distribution guard | +| initial + Home | 472.5 | — | startup parse boundary | +| `index-*` | 411.5 | 132.5 | entry/vendor | +| `RepoPage-*` | 304.4 | 75.0 | lazy route | +| `QuickReview-*` | 254.2 | 67.9 | lazy route | +| **all lazy routes** | **1,232.3** | **353.6** | distribution guard; Size Limit reports 1.26 MB decimal | + +### Browser performance evidence + +Chrome DevTools MCP is the maintained browser-performance path for local traces, +network diagnostics, and Core Web Vitals investigation. Lighthouse JSON from an +external producer can be ingested by the verification-receipt CLI, but remains +observational evidence rather than a correctness or release verdict. Lighthouse +CI 0.15.1 was exercised for three successful local landing-page runs, then +rejected as a repository dependency because its current graph fails the +high-severity dependency gate, including an unpatched archive traversal. ## 4. Release-history graph — backfill, time travel, and scrubbing @@ -780,9 +792,9 @@ It deterministically generates 20,000, 200,000, and 800,000 the official sorted min/mean/max output contract with UTF-8 station names, and emits the existing `size=ms/op` contract. Dataset construction is outside the timed region. The benchmark's -[README](../../benchmarks/runtime-challenges/temperature-aggregation/README.md) +[README](https://github.com/Codevetter/codevetter/blob/main/benchmarks/runtime-challenges/temperature-aggregation/README.md) contains the exact diagnosis command, while its -[artifact record](../../benchmarks/runtime-challenges/temperature-aggregation/ARTIFACT.md) +[artifact record](https://github.com/Codevetter/codevetter/blob/main/benchmarks/runtime-challenges/temperature-aggregation/ARTIFACT.md) records attribution and the differences from the official Java challenge. The initial qualification used CodeVetter to select the parser at 77.64% CPU @@ -913,7 +925,7 @@ pnpm --silent runtime:campaign -- promote \ The same six campaign operations are available from `runtime:mcp`. Start that server with `--repo /path/to/candidate --incumbent-repo /path/to/incumbent` so the promotion checkout is fixed outside tool arguments. The checked-in -[agent program](../../scripts/runtime-failure-capsule/AUTONOMOUS_OPTIMIZATION_PROGRAM.md) +[agent program](https://github.com/Codevetter/codevetter/blob/main/scripts/runtime-failure-capsule/AUTONOMOUS_OPTIMIZATION_PROGRAM.md) defines the loop and its authority boundary. Campaign artifacts are local JSON under the declared directory: one manifest, diff --git a/docs/development/testing.md b/docs/development/testing.md index fb135a26..766b6e93 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -88,6 +88,34 @@ pnpm bench:public # 27 public cases, catch-rate/precision/F1 See [benchmark.md](./benchmark.md). +## External producer compatibility + +CodeVetter does not require other projects to replace their test stack. The +verification-receipt loader normalizes Playwright JSON and JUnit XML, plus LCOV +and Cobertura coverage, while preserving the raw artifact hash and missing +evidence as explicit limitations. Run its focused contracts with: + +```bash +pnpm test:verification-receipts +``` + +The repository also emits and consumes those upstream formats itself: + +```bash +pnpm verification:dogfood +``` + +Playwright writes its built-in JSON and JUnit reports alongside the list and +HTML reporters. c8 writes LCOV and Cobertura coverage in addition to the console +summary. Generated reports stay ignored under `apps/desktop/test-results/`, +`apps/desktop/coverage/`, and `artifacts/`; they are reproducible evidence, not +committed snapshots. Tests for the retired Work and Board surfaces are no longer +part of the active E2E inventory. + +See [verification-receipts.md](./verification-receipts.md) for supported +formats and the distinction between producer observations and a qualified +CodeVetter verdict. + ## Native Agent Island On macOS, the Apple-framework-only helper has a framework-independent protocol @@ -133,7 +161,8 @@ release-mode stdio lifecycle. A failure stops the pipeline. owned by [GitHub issue #116](https://github.com/Codevetter/codevetter/issues/116) for review by 2026-09-11 because its fix requires the Astro 7 major upgrade. - **Clippy zero-warning** in release qualification. -- **Bundle budgets** via `apps/desktop/scripts/bundle-budget.mjs`. +- **Bundle budgets** via `apps/desktop/scripts/bundle-budget.mjs`, followed by + the additive upstream Size Limit distribution cap. - **GitHub-Issue specs** for non-trivial features; no repo-local spec tree is part of the quality gate. - **Pre-commit** (`.husky/pre-commit`): `lint-staged` runs `biome check --write` on staged `apps/desktop/src/**/*.{ts,tsx}`. diff --git a/docs/development/verification-receipts.md b/docs/development/verification-receipts.md index 9f3a1ab5..ec906021 100644 --- a/docs/development/verification-receipts.md +++ b/docs/development/verification-receipts.md @@ -2,7 +2,7 @@ title: Verification receipts description: Machine-first ingestion and comparison of project-owned test and performance evidence. sidebar: - order: 4 + order: 5 --- # Verification receipts @@ -32,7 +32,7 @@ The canonical producer schema is - limitations that bound the producer's claims. The complete executable example is -[`codevetter-local-fast-2026-08-07.json`](../../scripts/verification-receipts/fixtures/codevetter-local-fast-2026-08-07.json). +[`codevetter-local-fast-2026-08-07.json`](https://github.com/Codevetter/codevetter/blob/main/scripts/verification-receipts/fixtures/codevetter-local-fast-2026-08-07.json). That projection intentionally remains `no_confidence` for inventory because its source receipt contains aggregate outcomes instead of individual test and selection identities. Its command-level RSS is retained as partial evidence, @@ -46,6 +46,34 @@ keeping the exact raw-file SHA-256 as the source identity. Missing inventory, attempts, resource measurements, budgets, or network evidence stay missing; pre-test runner failures become operational `no_confidence` evidence. +The same loader accepts these established producer formats: + +- Playwright JSON, preserving terminal attempts, retries, durations, and hashed + failure signatures without copying raw errors; +- JUnit XML, including skipped tests and failed testcase error outcomes; +- LCOV and Cobertura XML coverage summaries; +- Lighthouse JSON category and navigation metrics; and +- Chrome trace JSON event count and trace duration. + +Test reports can support a bounded correctness projection. Coverage, +Lighthouse, and trace artifacts are observations only: without revision-bound +inventory, selection, resource, safety, and repeated comparison evidence they +remain `no_confidence` and never become a shipping verdict. + +CodeVetter dogfoods the installed Playwright reporters and c8 coverage +reporters end to end: + +```bash +pnpm verification:dogfood +``` + +The command runs the active desktop browser suite, writes Playwright JSON and +JUnit XML, runs the frontend unit suite with LCOV and Cobertura output, ingests +all four artifacts, and writes the ignored integration summary to +`artifacts/verification-dogfood/summary.json`. This is an integration check, +not release qualification: it fails on a producer or ingestion error, while +preserving `no_confidence` for verdict dimensions the reports cannot prove. + The adapter takes the repository identity from the scoped project's `package.json`. Unsupported producer formats and producer-native receipts without a stable repository identity fail before analysis. Raw executable @@ -62,7 +90,7 @@ qualified bundle is emitted. Ingest one repository-relative receipt: ```bash -pnpm verification:ingest -- \ +pnpm verification:ingest \ --repo /path/to/project \ --receipt artifacts/verification.json ``` @@ -70,7 +98,7 @@ pnpm verification:ingest -- \ Compare two receipts: ```bash -pnpm verification:compare -- \ +pnpm verification:compare \ --repo /path/to/project \ --baseline artifacts/baseline.json \ --current artifacts/current.json @@ -90,7 +118,7 @@ limitation and never claim a controlled speedup. Start the separate repository-scoped, read-only stdio process: ```bash -pnpm verification:mcp -- --repo /path/to/project +pnpm verification:mcp --repo /path/to/project ``` It exposes two tools: @@ -131,5 +159,7 @@ pnpm test:verification-receipts The suite covers deterministic ingestion, independent budgets, transient rechecks, same- and cross-commit comparison, incompatible identities, unsafe selector narrowing, privacy rejection, filesystem containment, CLI/MCP parity, -the real CodeVetter local-runner projection, and complete plus pre-test-failure -Vault E2E receipt adaptation. +the real CodeVetter local-runner projection, complete plus pre-test-failure +Vault E2E receipt adaptation, Playwright/JUnit attempt normalization, +LCOV/Cobertura observations, Lighthouse/Chrome trace observations, and hostile +XML entity rejection. diff --git a/docs/index.md b/docs/index.md index a4e75995..ce02abe2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -38,6 +38,10 @@ is only the presentation and search layer. - [history-evidence-import.md](./architecture/history-evidence-import.md) — importing provider-side outcomes. - [native-agent-island.md](./architecture/native-agent-island.md) — supervised Swift status, speech, and provider-action boundary. - [verification-workbench.md](./architecture/verification-workbench.md) — additive evidence identities, retention, managed-run, intent, and performance records. +- [native-migration-matrix.md](./architecture/native-migration-matrix.md) — + no-regression map from every retained Tauri surface to the native macOS app. +- [native-rust-boundary.md](./architecture/native-rust-boundary.md) — measured + hybrid ownership split between in-process reads and supervised execution. ### Decisions @@ -50,6 +54,8 @@ is only the presentation and search layer. - [setup.md](./development/setup.md) — prerequisites, install, run. - [testing.md](./development/testing.md) — the four test surfaces. - [performance.md](./development/performance.md) — perf harness and baselines. +- [native-macos.md](./development/native-macos.md) — native Evidence Workbench + toolchain, ownership, and qualification boundaries. - [benchmark.md](./development/benchmark.md) — catch-rate benchmark. - [agent-task-corpus.md](./development/agent-task-corpus.md) — immutable coding-agent task packages and fail-closed readiness. diff --git a/docs/knowledge/codebase-context-tools-landscape.md b/docs/knowledge/codebase-context-tools-landscape.md index f606029c..488b3574 100644 --- a/docs/knowledge/codebase-context-tools-landscape.md +++ b/docs/knowledge/codebase-context-tools-landscape.md @@ -1,6 +1,8 @@ --- title: Codebase context tools landscape -description: Retained April 2026 research survey of context, indexing, documentation, and review tools. +description: Survey of codebase indexing, context-provider, and documentation tools evaluated for CodeVetter (research snapshot, April 2026). +sidebar: + order: 90 --- # Codebase Context Tools for AI Agents & AI Code Review diff --git a/docs/knowledge/performance-lab-publication.md b/docs/knowledge/performance-lab-publication.md index b8a35f89..d68f54e5 100644 --- a/docs/knowledge/performance-lab-publication.md +++ b/docs/knowledge/performance-lab-publication.md @@ -55,7 +55,7 @@ product's end game. Merge status does not upgrade evidence strength: the directional, unverified, guardrail, and synthetic-stress entries are not independent claims of proven customer impact. The full inactive-project disposition is retained in -[`artifacts/performance/old-local-projects-results-2026-08-09.md`](../../artifacts/performance/old-local-projects-results-2026-08-09.md). +[`evidence/performance/old-local-projects-results-2026-08-09.md`](https://github.com/Codevetter/codevetter/blob/main/evidence/performance/old-local-projects-results-2026-08-09.md). ## External open-source work diff --git a/docs/knowledge/tooling-decisions.md b/docs/knowledge/tooling-decisions.md index 61057b85..7c46fa52 100644 --- a/docs/knowledge/tooling-decisions.md +++ b/docs/knowledge/tooling-decisions.md @@ -26,7 +26,20 @@ decay — re-check before acting on anything here. Items the research could not confirm are marked UNVERIFIED on the detail pages and should be treated as open questions, not findings. -## Adopt +## Status vocabulary + +- **Approved** — qualified for a bounded lane, but not executable there yet. +- **Trialled** — exercised against this repository with measured output, but not + part of a maintained path. +- **Wired** — invoked by a tracked local command, hook, workflow, or product + adapter. +- **Configured** — registered with a maintainer client, but not yet callable + because authentication, product authorization, or a client restart is pending. +- **Rejected** — disqualified for the named lane. Rejection in the private + customer-code product lane does not automatically reject a public-repository + maintainer aid. + +## Product adoption | Tool | License | Surface | Category detail | |---|---|---|---| @@ -38,10 +51,67 @@ questions, not findings. | Terminal-Bench 4.0 (Harbor) | Apache-2.0 | Corpus under `benchmarks/` | [agent benchmarks](./tooling-agent-benchmarks.md) | | `libkrun` | Apache-2.0 | In-process VMM in `src-tauri/` | [sandboxing](./tooling-sandboxing.md) | -`cargo-audit` is the highest-leverage item: the `rustsec` crate runs inside the -existing Rust backend with no new process boundary, no sidecar to codesign, and -no subprocess. Its SARIF 2.1.0 output is real but undocumented in the README and -absent from the changelog — pin `>= 0.22.0` and trust the source, not the docs. +These are product-lane decisions, not implementation receipts. In particular, +Gitleaks is now **wired** for this repository's hooks and CI, while bundling it +as a signed desktop sidecar remains **approved** work. + +## Repository and maintainer tooling + +| Tool | Status | Bounded use | +|---|---|---| +| DeepWiki MCP | Wired | Maintainer questions about this public repository; a new Codex session is required after MCP configuration changes | +| GitHub CodeQL default setup | Wired | GitHub-hosted scanning of this public repository only | +| Biome SARIF | Wired | Local artifact generation plus code-scanning upload | +| Gitleaks 8.30.1 | Wired | Staged-change hook, full-history local check, and checksum-pinned CI binary | +| zizmor 1.29 / action 0.6.2 | Wired | Offline local workflow audit plus GitHub code-scanning upload | +| actionlint 1.7.12 + ShellCheck 0.11.0 | Wired | Workflow syntax/semantics and embedded-shell validation; checksum-pinned CI binaries and local `pnpm quality:workflows` command | +| cargo-deny 0.20.2 | Wired | Offline Rust license, source, wildcard-requirement, and duplicate-version policy; native SARIF for actionable policy violations | +| ast-grep 0.45.2 | Trialled, not wired | Structural locations were correct, but native `--format sarif` emits an invalid root format version; no missing-rule case justifies a converter yet | +| Trivy 0.74.0 config scan | Trialled, not wired | Embedded checks found no supported first-party IaC surface; the unbounded scan targeted a dependency Dockerfile and the bounded scan misclassified a JSON fixture | +| OSV-Scanner 2.5.1 | Repository runner wired | `pnpm quality:vulnerabilities` produces offline SARIF plus a database-identity receipt; the 35-package baseline and remediation are tracked in issue #195 | +| StrykerJS 10.0.0 | Bounded local command wired | Accounting oracle: 218 mutants, 185 killed, 33 survived, 84.86% score; `pnpm quality:mutation:accounting`, tracked in issue #196 | +| Schemathesis 4.25.2 | Rejected for current surface | CLI availability verified, but CodeVetter has no OpenAPI/Swagger contract or HTTP server to exercise | +| Apple `container` CLI | Approved for measured trial | Host qualifies (arm64, macOS 27), but the signed admin-installed system service is absent; tracked in issue #197 | +| Lighthouse CI 0.15.1 | Trialled, rejected as a repo dependency | Three local landing-page runs passed the proposed category/Core Web Vitals gates, but the package introduced three high advisories including unpatched `extract-zip` traversal; raw Lighthouse JSON ingestion remains supported | +| Size Limit 13.0.3 | Wired, additive | Caps the complete emitted desktop JS distribution after the existing Tauri-aware entry/Home and per-chunk budget gate; it does not replace those product-specific calculations | +| `fast-xml-parser` 5.11.1 | Wired | Closed JUnit and Cobertura XML ingestion with DTD/entity rejection before parsing | +| `@friedemannsommer/lcov-parser` 8.0.0 | Wired | Local LCOV ingestion without bundling the GPL Perl `lcov` tools | +| Playwright built-in JSON/JUnit reporters + c8 LCOV/Cobertura reporters | Wired and dogfooded | `pnpm verification:dogfood` runs the active suites, emits all four upstream formats, ingests them, and keeps unproven verdict dimensions at `no_confidence` | +| Chrome DevTools MCP 1.8.0 | Wired | Pinned maintainer MCP for traces and browser diagnostics; isolated/headless profile, telemetry and CrUX lookup disabled, response headers redacted | +| Playwright MCP 0.0.79 | Wired | Pinned isolated/headless maintainer browser with service workers blocked and a bounded local/codevetter.com origin allowlist | +| GitHub MCP Server 1.11.0 | Wired and verified | Checksum-verified official Darwin arm64 binary registered as local stdio with `repos`, `issues`, `pull_requests`, `actions`, and `code_security`; strict read-only mode and narrow browser OAuth scopes (`read:org`, `security_events`) replace the earlier PAT-dependent remote registration, with live repository, PR, Actions, issue, and CodeQL reads verified | +| CodeVetter packaged graph/history MCP | Wired and verified | Enabled for the canonical checkout through the product-generated opaque repository scope; current history and tree-sitter structural indexes returned bounded `history_search` and `graph_query` results, with operational access metadata recorded locally | + +These tools subsidize discovery and evidence collection. CodeVetter still owns +receipt qualification, taxonomy, budgets, and the final measurable verdict. +The maintainer client also registers CodeVetter's repository-scoped runtime MCP +as `codevetter-runtime-exec` and its non-executing receipt MCP as +`codevetter-verification`; the names preserve the execution boundary. The +product-generated packaged sidecar is registered separately as +`codevetter-history`. Newly added registrations appear as callable client tools +in the next Codex session, so the packaged sidecar was also protocol-smoked +directly in this session. GitHub's browser OAuth is complete, and its token +remains in the server process rather than Codex configuration. Future repository +scopes must still be enabled in CodeVetter and copied from its opaque generated +configuration rather than synthesizing database arguments. + +The first CodeQL run also exposed a cleartext localStorage API-key field. Source +tracing showed that the old browser gateway execution path had already been +removed and only its non-functional Settings panel remained. The remediation +removes that panel and allowlist-migrates the shared record to rubric fields +instead of adding a credential dependency to preserve dead behavior; issue +#194 remains open until pushed CodeQL evidence confirms the alert is closed. + +The OSV baseline changes the order of operations for repository dependency work: +fix or classify the measured lockfile baseline before adding another scanner. +The tracked [baseline evidence](https://github.com/Codevetter/codevetter/blob/main/evidence/security/osv-baseline-2026-08-31.md) +records 52 advisory/package matches and the exact database hashes; generated +SARIF and receipts remain ignored scratch. +`cargo-audit` remains the highest-leverage product embedding candidate because +the `rustsec` crate runs inside the existing Rust backend with no process +boundary or sidecar to codesign. Its SARIF 2.1.0 output is real but undocumented +in the README and absent from the changelog — pin `>= 0.22.0` and trust the +source, not the docs. ## Reject @@ -54,13 +124,14 @@ absent from the changelog — pin `>= 0.22.0` and trust the source, not the docs | npm / pnpm audit | No offline mode; npm's fallback path uploads the full dependency tree plus machine metadata | | `lcov` / `genhtml` Perl tooling | GPL-2.0 — do not bundle (the *format* is unencumbered) | | Meta OpenApps | CC-BY-NC-4.0, commercial use prohibited | -| DeepWiki | Hosted; private repos need a paid Devin account — see [documentation tooling](./tooling-documentation.md) | -| CodeQL | License forbids use on non-open-source codebases without paid GHAS — see [sandboxing](./tooling-sandboxing.md) | +| DeepWiki as product/docs authority | Hosted; private repos need a paid Devin account — see [documentation tooling](./tooling-documentation.md) | +| CodeQL on customer repositories | License forbids use on non-open-source codebases without paid GHAS — see [sandboxing](./tooling-sandboxing.md) | | Firecracker, gVisor | Linux-kernel only; no macOS host mode exists | | Docker Desktop | Proprietary GUI app, license-gated at 250 employees / $10M revenue | | WorkArena, WebVoyager | Require live third-party websites or hosted SaaS | | detect-secrets | Dormant since 2024; no SARIF | | Nosey Parker | Archived 2026-04-24, superseded by Titus | +| Lighthouse CI as a repository dependency | Current 0.15.1 dependency graph fails the repository's high-severity audit, including unpatched `extract-zip` traversal; do not hide the same package behind an untracked `dlx` invocation | ## Adopt only with explicit configuration diff --git a/docs/knowledge/tooling-documentation.md b/docs/knowledge/tooling-documentation.md index 7e014b72..930e3101 100644 --- a/docs/knowledge/tooling-documentation.md +++ b/docs/knowledge/tooling-documentation.md @@ -1,18 +1,23 @@ --- -title: Documentation tooling — the DeepWiki decision -description: Why DeepWiki is not adopted for CodeVetter's documentation, and what would fit if auto-generated architecture docs are wanted. +title: Documentation tooling — the DeepWiki boundary +description: Where DeepWiki helps CodeVetter maintainers, where it is rejected, and what would fit for generated architecture docs. sidebar: order: 14 --- -# Documentation tooling — the DeepWiki decision +# Documentation tooling — the DeepWiki boundary Verified **2026-08-30**. See [tooling-decisions.md](./tooling-decisions.md) for the cross-category summary. -**Verdict: reject** DeepWiki as a documentation pipeline dependency. The only -form that composes with the existing stack is a passive outbound link (a README -or landing badge to `deepwiki.com/Codevetter/codevetter`), not an integration. +**Verdict: wired as a maintainer query aid; rejected as a documentation pipeline +or product dependency.** The public DeepWiki MCP endpoint is configured in the +owner's Codex environment for questions about this public repository. It is +not tracked repository configuration, does not ingest customer repositories, +and does not replace committed documentation or executable proof. + +Codex discovers MCP servers at session startup. After adding or changing the +DeepWiki entry, start a new session before claiming the tools are callable. ## Why @@ -41,8 +46,8 @@ not the hosted product. ### It fails on process grounds independently of privacy -Even setting privacy aside, DeepWiki conflicts with this repo's own documented -rules: +Promoting its generated pages to product documentation would still conflict +with this repo's own documented rules: - **"Markdown under `docs/` is the source of truth."** A hosted generated wiki is a second, non-authoritative home for the same facts. @@ -57,8 +62,8 @@ assessed DeepWiki in **April 2026** and reached the same conclusion. That document sat in a top-level `research/` directory that nothing linked to, so the finding was effectively lost and the question got asked again. -The August 2026 re-check confirmed the April verdict still holds, and that the -three self-hosted alternatives it named remain active: +The August 2026 re-check confirmed the product/docs boundary still holds, and +that the three self-hosted alternatives it named remain active: | Project | Stars | License | Last push | |---|---|---|---| diff --git a/docs/knowledge/tooling-sandboxing.md b/docs/knowledge/tooling-sandboxing.md index 597ce8f1..beee82d2 100644 --- a/docs/knowledge/tooling-sandboxing.md +++ b/docs/knowledge/tooling-sandboxing.md @@ -7,7 +7,7 @@ sidebar: # Sandboxed execution and static analysis -Verified **2026-08-30**. See [tooling-decisions.md](./tooling-decisions.md) for +Verified **2026-08-31**. See [tooling-decisions.md](./tooling-decisions.md) for the cross-category summary. Sandboxed execution is the mechanism CodeVetter's verdicts rest on: if the @@ -15,7 +15,27 @@ isolation is not reproducible, the evidence is not either. This page covers isolation on Apple Silicon, the determinism controls that make runs comparable, and static analyzers that emit SARIF. -## Recommended: `libkrun`, with a VZ upgrade path +## Recommended sequence: prove the contract with Apple `container`, then embed + +On a supported host, qualify the official Apple `container` CLI first. It gives +the team a maintained, OCI-compatible lightweight-VM implementation with which +to measure containment, mount policy, networking, resource bounds, startup, +and teardown before accepting an in-process VMM dependency. The signed 1.3.1 +package is now qualified on the Apple Silicon macOS 27 trial host. A cached +no-op container started in 0.61 seconds; read-only root/workspace, bounded CPU +and memory, an internal no-DNS network, dropped capabilities, host-environment +absence, and teardown all behaved as expected. The first image/init-image run +took 20.56 seconds and the cached images occupied 1.45 GB. + +One contract failed: the CLI accepted a controlled bind source containing `..` +when it resolved outside the intended fixture root. CodeVetter must canonicalize +and enforce workspace containment itself; Apple Container's mount validation is +not that policy. The [qualification receipt](https://github.com/Codevetter/codevetter/blob/main/evidence/verification/apple-container-qualification-2026-08-31.md) +records identities, measurements, teardown, and remaining gates. Issue #197 +keeps the architecture decision open. + +If the measured contract is sound, choose between consuming Apple's +Containerization Swift package through a sidecar and embedding `libkrun`. **`libkrun`** (Apache-2.0, `containers/libkrun`, 2,643★) is a small VMM **library** written in Rust and built on Apple's `Hypervisor.framework`. It is @@ -24,10 +44,11 @@ backend via its C API, boots a minimal Linux microVM per verification run, and tears it down after. That matches the local-first, no-server constraint without requiring the user to install and license a GUI application. -**`apple/containerization`** (Apache-2.0, Swift, 8.9k★) is the follow-on once a -Swift sidecar is acceptable. It claims **sub-second** boot and is the only -option here with **native VM state save/restore** — the actual warm-start -mechanism. It requires **macOS 26** and Apple Silicon. +**`apple/containerization`** (Apache-2.0, Swift, 8.9k★) is the library beneath +that first-party CLI and is the follow-on once a Swift sidecar is acceptable. +It claims **sub-second** boot and is the only option here with **native VM state +save/restore** — the actual warm-start mechanism. It requires **macOS 26** and +Apple Silicon. **Colima + Lima** (MIT / Apache-2.0) is the pragmatic fallback for pre-macOS-26 or Intel machines: a one-time Homebrew install rather than a bundled component. @@ -37,7 +58,8 @@ or Intel machines: a one-time Homebrew install rather than a bundled component. | Option | License | Daemon? | Bundleable? | macOS ARM? | |---|---|---|---|---| | **libkrun** | Apache-2.0 | No — in-process C API | Yes, as a Rust dependency | Yes, explicit HVF backend | -| **apple/containerization** | Apache-2.0 | No (the `container` CLI wrapping it does) | Via a Swift sidecar | Apple Silicon, **macOS 26 only** | +| Apple `container` CLI | Apache-2.0 | Yes, system service | External prerequisite | Apple Silicon, macOS 26+ | +| **apple/containerization** | Apache-2.0 | No at library level | Via a Swift sidecar | Apple Silicon, macOS 26+ | | **Colima + Lima** | MIT / Apache-2.0 | CLI-managed VM, no GUI daemon | Prerequisite only | Yes (`vz` driver, default since Lima v1.0) | | **Podman** | Apache-2.0 | `podman machine` VM | Prerequisite only | Yes | | **Docker Desktop** | Engine Apache-2.0; **Desktop app proprietary** | Yes, GUI app + background VM | No | Yes, but license-gated | @@ -99,13 +121,13 @@ give you this. | Tool | License | Offline | Native SARIF | Verdict | |---|---|---|---|---| -| **Biome** | Apache-2.0 | Yes | Present, fidelity **UNVERIFIED** | **Check first** — already the repo's linter | -| **ast-grep** | MIT | Yes | Present, fidelity **UNVERIFIED** | Rust-native, good stack fit | +| **Biome** | Apache-2.0 | Yes | Verified SARIF 2.1.0 | **Wired** for repository evidence | +| **ast-grep** | MIT | Yes | Envelope invalid in 0.45.2 | Structural matching is sound, but do not upload its native SARIF | | **Ruff** | MIT | Yes | Yes (`--output-format=sarif`) | Python scope only | | **Semgrep** | CLI is LGPL-2.1 | Yes | Yes | Engine fine; **rules are the problem** | | **Clippy** | Apache-2.0 | Yes | **No** | Needs `clippy-sarif` converter | | **ESLint** | MIT | Yes | No | Third-party formatter | -| **CodeQL** | Custom | Yes | Yes | 🚫 **Legally disqualified** | +| **CodeQL** | Custom | Yes | Yes | Repository-only; 🚫 customer-code product use | ### 🚫 CodeQL is a hard blocker for this product @@ -130,9 +152,12 @@ Running Semgrep against a user's own code is internal use and fine. area** — that needs counsel before shipping, and is not cleared by the engine's LGPL alone. -**Lowest-friction path: verify Biome's own SARIF fidelity first.** The repo -already runs Biome for `pnpm lint`, so if its SARIF output is adequate, no -second linter is needed at all. +Biome's SARIF path is now verified and wired for the public repository, so no +second generic linter is justified without a missing-rule case. A focused +ast-grep 0.45.2 trial found correct structural locations but an invalid SARIF +root: `version` contains the tool version rather than `2.1.0`, and `$schema` is +absent. The [tracked receipt](https://github.com/Codevetter/codevetter/blob/main/evidence/security/ast-grep-sarif-qualification-2026-08-31.md) +keeps this a measured rejection instead of an assumed capability. ## Observability — do not add a dependency @@ -162,7 +187,5 @@ Flagged UNVERIFIED and worth closing before committing engineering time: before relying on it. - **`libkrun` cold-boot time on Apple Silicon.** "Smallest possible boot time" is a stated design goal; no published benchmark was found. -- **Biome and ast-grep SARIF fidelity.** Both have SARIF code in-repo; neither - was validated against real output. - **Podman's default macOS backend** (applehv vs libkrun vs QEMU) and its exact network/CPU/memory flags. diff --git a/docs/knowledge/tooling-secrets-and-supply-chain.md b/docs/knowledge/tooling-secrets-and-supply-chain.md index ebfbb8c4..a93b285e 100644 --- a/docs/knowledge/tooling-secrets-and-supply-chain.md +++ b/docs/knowledge/tooling-secrets-and-supply-chain.md @@ -36,10 +36,25 @@ the underlying `rustsec` crate is a **library**: it runs in-process in [RustSec/advisory-db](https://github.com/RustSec/advisory-db) at `~/.cargo/advisory-db` — trivially vendorable and shippable. - Pair with **cargo-deny** (`--offline`) for SPDX license-policy enforcement, - which nothing else here does for Rust. Note its output is structured JSON log - lines, **not SARIF** — the mapping would be yours to write. + which nothing else here does for Rust. Version 0.20.2 emits native SARIF + 2.1.0; the repository now verifies that output instead of relying on the + earlier, stale JSON-only assessment. -### `gitleaks` — bundle as a sidecar +### `cargo-deny` — wired repository policy + +The tracked `apps/desktop/src-tauri/deny.toml` evaluates the shipped Apple +Silicon target, permits only the license families present in the qualified +graph, denies wildcard requirements, denies unknown registries and all Git +dependencies, and reports duplicate versions without failing on Tauri-owned +convergence. `pnpm quality:rust-policy` runs the locked offline check locally. + +`repository-security.yml` downloads the exact 0.20.2 Linux musl binary, +verifies its publisher digest, uploads native license/source SARIF, and enforces +the bans lane separately. The qualified baseline has clean licenses and sources +plus 14 non-blocking duplicate-version warnings. Exact artifact identities and +limitations are in the [tracked evidence](https://github.com/Codevetter/codevetter/blob/main/evidence/security/cargo-deny-baseline-2026-08-31.md). + +### `gitleaks` — repository gate plus an unreleased product collector foundation **MIT**, and the core scanner's license is unchanged. Fully offline: no DB, no network, rules embedded or from `.gitleaks.toml`. Emits SARIF. Static Go binary, @@ -52,6 +67,60 @@ the commercial relicensing applies **only to `gitleaks/gitleaks-action` v2.0.0+* Caution: v8.30.1 shipped 2026-03-21 and recent commits are largely Dependabot. Not stale, but feature velocity has slowed. +The repository integration invokes the binary directly rather than the +commercially relicensed action: + +- `.husky/pre-commit` scans staged changes when Gitleaks is installed; +- `.husky/pre-push` scans complete Git history, with the previous pattern scan + retained only as a limited fallback; +- `pnpm quality:secrets` is the reproducible local command; +- `repository-security.yml` downloads v8.30.1, verifies its embedded SHA-256, + emits redacted SARIF, uploads the result, and fails on a finding. + +`.gitleaksignore` contains four exact historical fingerprints. It does not +allowlist whole paths or rules, so later findings in those files remain visible. +The unreleased Rust backend and `codevetter collect` CLI now add a bounded, +versioned `codevetter.tool-collection/v1` foundation. It resolves one clean +checked-out Git range, accepts only an exact 8.30.1 bundle sibling or explicit +debug/test override, records the binary SHA-256 and configuration identity, +invokes without a shell under time/output/environment bounds, and normalizes +only rule and repository-relative location metadata. Raw `Secret` and `Match` +fields are dropped before serialization, and the JSON report remains in a +bounded process pipe instead of a temporary file. Missing cargo-audit and cargo-llvm-cov +prerequisites remain explicit `unavailable` evidence. + +This source wiring is not evidence that any sidecar has shipped. No Gitleaks +binary is yet declared in Tauri resources or qualified inside a signed app +bundle, and cargo-audit/coverage execution remains unimplemented under issue +#198. + +### `osv-scanner` — offline repository runner wired, remediation required + +`pnpm quality:vulnerabilities` invokes the repository-owned 2.5.1 runner with +`--offline --offline-vulnerabilities`. It hashes the preseeded ecosystem +databases and writes SARIF plus a versioned receipt under ignored `artifacts/`. +Database refresh remains an explicit, separate network operation. The qualified +warm scan took 8.942 seconds and found 35 affected locked package versions and +52 advisory/package matches: 18 Rust packages and 17 docs-site npm packages. +The remainder is dominated by unmaintained GTK3 and Unicode Rust crates that +need platform/reachability classification rather than a blind allowlist. See +the [tracked baseline evidence](https://github.com/Codevetter/codevetter/blob/main/evidence/security/osv-baseline-2026-08-31.md). + +This explains why the root `pnpm audit` result was insufficient: it did not +cover the independent docs-site lockfile or RustSec. The trial is not yet a +gate because the current baseline would fail and each finding still needs +scope/reachability review. Integration and remediation are tracked in issue +#195. Database refresh must stay separate from the offline scan so a product +run cannot turn a transient network path into an implicit manifest upload. + +The approved Blume 1.5.3 and event-listener 5.4.2 maintenance reduced the same +offline result from 52 matches to 39 result instances (36 normalized rules) at +revision `855202998b56c1658b9decda22298a1b63fb5caf`. It did not make the graph +clean: the docs-site production audit still reports 12 high, 7 moderate, and 1 +low advisory in current transitive build/documentation paths. The +[baseline receipt](https://github.com/Codevetter/codevetter/blob/main/evidence/security/osv-baseline-2026-08-31.md) +keeps both the improvement and remaining exposure explicit. + ## Rejected ### TruffleHog — two independent blockers @@ -137,6 +206,16 @@ corrections to common belief: Useful detail: the misconfiguration checks bundle is **embedded in the binary at build time**, so that scanning survives with zero network. +A bounded 0.74.0 repository trial confirmed that +`--disable-telemetry --skip-version-check --skip-check-update` suppresses the +notification request and uses the embedded checks. It also showed why this is +not yet a useful maintained lane: the unbounded scan reported against a +dependency-owned Dockerfile, while the dependency-excluded scan misclassified a +warm-verification JSON fixture as CloudFormation and found no supported +first-party IaC surface. Trivy config scanning is therefore **trialled, not +wired** until such a surface exists. See the +[tracked receipt](https://github.com/Codevetter/codevetter/blob/main/evidence/security/trivy-config-qualification-2026-08-31.md). + ## SBOM formats Target **CycloneDX 1.7** (ECMA-424 2nd Edition, patch 1.7.1 2026-06-02) and diff --git a/docs/operations/automation-contract.md b/docs/operations/automation-contract.md index 29aad18a..5f234e4e 100644 --- a/docs/operations/automation-contract.md +++ b/docs/operations/automation-contract.md @@ -34,7 +34,7 @@ Do not duplicate this matrix elsewhere — link here. | Desktop Rust backend | `apps/desktop/src-tauri/src/` | `ci.yml` MCP tests; `release.yml` Tauri build | Sarthak | | Local SQLite | `rusqlite` in Rust backend (no server) | Local only; `observability.rs` aggregates locally | Sarthak | | MCP sidecar | `apps/desktop/src-tauri/src/bin/codevetter-mcp.rs` | `ci.yml` MCP protocol + stdio lifecycle tests; `mcp/sanitize.rs` redaction | Sarthak | -| Benchmark | `benchmark/` + `scripts/run-catch-rate-benchmark.mjs` | `pnpm test:benchmark`; public cases committed | Sarthak | +| Benchmark | `benchmarks/public-catch-rate/` + `scripts/run-catch-rate-benchmark.mjs` | `pnpm test:benchmark`; public cases committed | Sarthak | | Release pipeline | `auto-release.yml` → `release.yml` → GitHub Releases | Release assets + `latest.json` manifest | Sarthak | | Auto-updater | `@tauri-apps/plugin-updater` consuming `latest.json` | `scripts/verify-release-manifest.mjs` validates linkage | Sarthak | | Docs | `docs/` + `docs-site/` (Blume) | `docs.yml` link + structure validation | Sarthak | diff --git a/docs/operations/ci.md b/docs/operations/ci.md index 9daad516..9abc42a0 100644 --- a/docs/operations/ci.md +++ b/docs/operations/ci.md @@ -7,15 +7,15 @@ sidebar: # CI -`.github/workflows/ci.yml` runs on every push and pull request. +`.github/workflows/ci.yml` runs on every push to `main` and every pull request. ## Job: `lint-and-typecheck` (ubuntu-latest) Steps, in order (a failure stops the job): -1. `actions/checkout@v6` -2. `pnpm/action-setup@v4` + `actions/setup-node@v6` (Node 22, pnpm cache) -3. `dtolnay/rust-toolchain@stable` +1. SHA-pinned `actions/checkout` with persisted credentials disabled +2. SHA-pinned `pnpm/action-setup` + `actions/setup-node` (Node 22, pnpm cache) +3. SHA-pinned `dtolnay/rust-toolchain` 4. Install Tauri Linux deps (`libwebkit2gtk-4.1-dev`, `libayatana-appindicator3-dev`, `librsvg2-dev`, `libxdo-dev`) 5. `pnpm install --frozen-lockfile` 6. **Lint** — `pnpm run lint` in `apps/desktop` (Biome) @@ -43,11 +43,45 @@ Steps, in order (a failure stops the job): |---|---|---|---| | `auto-release.yml` | push to `main` on `tauri.conf.json` version bump | cut `v` release + dispatch `release.yml` | [release-pipeline.md](./release-pipeline.md) | | `release.yml` | `release.created` or `workflow_dispatch` | build/sign/upload Tauri binaries + `latest.json` | [release-pipeline.md](./release-pipeline.md) | +| `native-qualification.yml` | `workflow_dispatch` (manual) | qualify the unsigned native macOS preview and optionally run XCUITest on an isolated hosted desktop | [../development/native-macos.md](../development/native-macos.md) | | `deploy-landing.yml` | `workflow_dispatch` (manual) | deploy Astro site to Cloudflare Pages | [landing-deploy.md](./landing-deploy.md) | | `weekly.yml` | cron `0 9 * * 1` (Mon 09:00 UTC) + `workflow_dispatch` | lockfile-agnostic quality check (lint/typecheck/test/build if defined) | [jobs/weekly-quality.md](./jobs/weekly-quality.md) | | `docs.yml` | push/PR | doc link + structure validation | [../development/docs.md](../development/docs.md) | +| `repository-security.yml` | push to `main`/PR/manual | actionlint + ShellCheck, Biome SARIF, cargo-deny Rust policy, full-history Gitleaks, and zizmor code-scanning uploads | [../knowledge/tooling-decisions.md](../knowledge/tooling-decisions.md) | +| `osv-offline.yml` | manual | explicit OSV database refresh followed by a separate offline lockfile scan and evidence upload | [../knowledge/tooling-secrets-and-supply-chain.md](../knowledge/tooling-secrets-and-supply-chain.md) | + +The first complete hosted native qualification passed at commit `824a9e8b`: +81 Swift tests, nine XCUITests, Debug and coverage-free Release builds, the +33-state owner packet, unsigned ZIP/DMG packaging, and read-only readiness +inspection. The exact run, hashes, metrics, and remaining production gates are +recorded in [the hosted qualification receipt](../../evidence/verification/native-hosted-qualification-2026-09-02.md). + +`repository-security.yml` uses no application dependency. Gitleaks, actionlint, +ShellCheck, and cargo-deny are checksum-pinned binaries; every third-party +action is pinned to a commit; and job permissions are declared narrowly. +Publish jobs do not use dependency or toolchain caches, so an offline pedantic +zizmor review has no unsuppressed security findings. Its two remaining +informational suggestions prefer runner shell commands over the pinned Rust +toolchain setup action; the maintained, commit-pinned action is retained +deliberately. + +Run `pnpm quality:workflows` when actionlint is installed. It parses every +workflow, validates GitHub expression and event semantics, and delegates shell +fragments to ShellCheck. The first qualified audit and exact artifact identities +are recorded in [the tracked evidence](https://github.com/Codevetter/codevetter/blob/main/evidence/security/actionlint-baseline-2026-08-31.md). ## Local pre-commit / pre-push -- **pre-commit** (`.husky/pre-commit`): `lint-staged` → `biome check --write` on staged `apps/desktop/src/**/*.{ts,tsx}`. -- **pre-push** (`.husky/pre-push`): `npm run lint` (root `lint` script, which runs Biome) + secret-pattern scan over tracked files. Exclusions are anchored to known dirs (`benchmark/`, `apps/landing-page-astro/public/benchmark/`, fixtures, `secret_policy.rs`). +- **pre-commit** (`.husky/pre-commit`): `lint-staged` → `biome check --write` on staged `apps/desktop/src/**/*.{ts,tsx}`, then Gitleaks staged-diff scanning when the binary is installed. +- **pre-push** (`.husky/pre-push`): `pnpm run lint`, then full-history Gitleaks. A limited tracked-file pattern scan remains as a fallback for contributors without the binary; CI always uses Gitleaks. + +Generate a local Biome SARIF 2.1.0 artifact at +`artifacts/tooling/biome.sarif` with `pnpm quality:sarif`. Generated artifacts +remain ignored scratch; the workflow uploads them to GitHub code scanning. + +Run `pnpm quality:vulnerabilities` after explicitly refreshing the OSV databases. +The scan itself is offline and produces `artifacts/tooling/osv/results.sarif` +plus a receipt containing scanner, source, and database identities. Exit `1` +means findings; exit `2` means the scanner or evidence path failed. This is not +yet a push/PR gate because the measured baseline must be remediated rather than +silently accepted. The manual workflow remains red while findings exist. diff --git a/docs/operations/release-pipeline.md b/docs/operations/release-pipeline.md index f50e366b..d296f7c9 100644 --- a/docs/operations/release-pipeline.md +++ b/docs/operations/release-pipeline.md @@ -63,11 +63,22 @@ See [runbooks/cut-a-release.md](./runbooks/cut-a-release.md). ## Signing -`release.yml` uses Apple signing/notarization secrets (`APPLE_*`) stored as -GitHub Actions secrets. Signed release publication is the last gate; the +`release.yml` signs the Tauri updater archive with +`TAURI_SIGNING_PRIVATE_KEY`; it does not currently Developer ID-sign or +notarize the macOS application bundle (`signingIdentity` is null). Updater +signature verification and Apple Gatekeeper trust are separate claims. The graph + MCP budget qualification runs before the build (see [development/performance.md](../development/performance.md)). +The unreleased native client has a separate protected +`native-production-qualification.yml` workflow. It requires Apple Developer ID +and notarization inputs plus a Sparkle EdDSA key pair, performs all credential +work only on an ephemeral hosted runner, verifies the exact appcast/archive, +and qualifies isolated installed upgrade/data/rollback before it may report +`shipping_ready: true`. It does not publish or retire Tauri. The observed +preflight run failed closed because all protected inputs were absent; no secret +value was inspected or logged. + The optional [Native Agent Island](../architecture/native-agent-island.md) is bundled as a nested universal sidecar. Release verification checks arm64 and x86_64 slices plus its nested code signature before assets are uploaded. diff --git a/docs/operations/signing-and-updates.md b/docs/operations/signing-and-updates.md index 14d24caa..3dc3677d 100644 --- a/docs/operations/signing-and-updates.md +++ b/docs/operations/signing-and-updates.md @@ -62,3 +62,23 @@ signing-key setup and the auto-update mechanism. - The `pubkey` in `tauri.conf.json` must match the key pair used to sign builds. Without it, update verification will fail. - The `TAURI_SIGNING_PRIVATE_KEY` env var is only needed at build time, never at runtime. - Auto-update checks fail silently if the endpoint is unreachable or no update is available. + +## Native macOS production candidate + +The Swift/AppKit migration uses Sparkle rather than the Tauri updater and has a +separate, non-publishing qualification workflow: +`.github/workflows/native-production-qualification.yml`. Its protected input +names are `APPLE_CERTIFICATE`, `APPLE_CERTIFICATE_PASSWORD`, +`APPLE_SIGNING_IDENTITY`, `APPLE_ID`, `APPLE_PASSWORD`, `APPLE_TEAM_ID`, +`SPARKLE_EDDSA_PRIVATE_KEY`, and `SPARKLE_EDDSA_PUBLIC_KEY`. Never place their +values in repository files, commands, receipts, or logs. + +The workflow imports the certificate into an ephemeral keychain, builds the +production identifier, signs every nested executable, notarizes and staples +the exact archive, generates and cryptographically inspects the Sparkle +appcast, and runs an isolated incumbent-to-native-to-rollback migration proof. +It uploads evidence for inspection but has no release publication permission. +Run 33628919883 proved the preflight is fail-closed: all eight protected inputs +were absent, so signing and every downstream production operation were +skipped. Provisioning those repository secrets and rerunning the workflow is a +release-owner gate, not application implementation work. diff --git a/docs/product/overview.md b/docs/product/overview.md index 0def6952..6181e561 100644 --- a/docs/product/overview.md +++ b/docs/product/overview.md @@ -56,7 +56,7 @@ work, and did the affected audience succeed with it?* ## Benchmark evidence -27 hand-labeled public benchmark cases (`benchmark/cases/`) covering 7 +27 hand-labeled public benchmark cases (`benchmarks/public-catch-rate/cases/`) covering 7 languages and 15+ vulnerability types. The coordinator dedup fix (2026-07-11) flipped the head-to-head vs raw Claude: catch 1.000 vs 0.931, precision 0.433 vs 0.397, F1 0.604 vs 0.557. Real agent-PR case curation diff --git a/docs/product/surfaces.md b/docs/product/surfaces.md index 7149e4d1..31f54b6e 100644 --- a/docs/product/surfaces.md +++ b/docs/product/surfaces.md @@ -40,7 +40,7 @@ Source: `navItems` in `apps/desktop/src/components/sidebar.tsx`. | Usage | `/` | `apps/desktop/src/pages/Home.tsx` | Local spend and token trends from bundled `ccusage` for Claude/Codex/Grok plus CodeVetter's separately queried Devin tracker; provider remaining-usage and quota telemetry stays separate and unchanged. | | Repo Unpack | `/unpack` | `apps/desktop/src/pages/RepoPage.tsx` | Whole-repo evidence-backed system brief. Tab `match`es `/unpack` and `/intel`. Scanner in `src-tauri/src/commands/unpack*.rs`; persisted to `repo_unpacked_reports`. See [architecture/repo-unpacked.md](../architecture/repo-unpacked.md). | | Review | `/review` | `apps/desktop/src/pages/QuickReview.tsx` | First change-checking stage: select an exact change, inspect source-qualified findings and coverage gaps, attach evidence, and export a local Agent PR X-Ray. The setup panel reports Agent MCP readiness and exposes the exact local `codevetter check` command for the selected range. Findings remain leads until executable evidence supports the decision. | -| Testing | `/trex` | `apps/desktop/src/pages/TRex.tsx` | Runtime-evidence stage: resolve a human-described flow, exact PR/change, or bounded codebase portfolio into runnable tests; confirm the plan; then capture receipts. Direct preview, changed-capability verification, scenarios, and PR watchers remain available. See [trex-change-preview.md](./trex-change-preview.md). | +| Testing | `/trex` | `apps/desktop/src/pages/TRex.tsx` | Runtime-evidence stage: resolve a human-described flow, exact PR/change, or bounded codebase portfolio into runnable tests; confirm the plan; then capture receipts. Direct preview, changed-capability verification, scenarios, and PR watchers share Rust-owned CLI/native contracts. Native watcher timing is app-lifetime only and every execution session requires explicit consent. See [trex-change-preview.md](./trex-change-preview.md). | | Performance | `/performance` | `apps/desktop/src/pages/Performance.tsx` | Uses the same intent resolver, then admits one exact Node test/script, Vitest, Playwright, or Go benchmark workload. Shows zero-egress admission, observed versus inferred evidence, limitations, one next action, cleanup state, and machine-readable receipts. | Settings (`/settings`) is a labelled utility separated at the bottom of the @@ -58,6 +58,52 @@ codevetter check --range main...HEAD \ --json ``` +After a completed persisted review, `codevetter xray --review-id ` and the +native Review sheet share the existing Rust public-export contract. Preview is +allowed to return explicit publication blockers; saving additionally requires +an exact destination and an eligible freshly rebuilt packet. JSON, Markdown, +and self-contained HTML remain deterministic and provider-free. + +`codevetter fix-packet --run-id ` and the native Review handoff sheet also +share one Rust projection. A user selects persisted, source-qualified findings; +the packet binds the exact task, explicit acceptance requirements, evidence to +preserve, route advice, and limitations into copyable Markdown. It remains a +fix instruction packet, never evidence that the fix succeeded. + +After that handoff, `codevetter fix --operation execute` and the native Review +sheet share one explicit-consent `codevetter.fix-attempt/v1` contract. One +configured agent may edit only a detached worktree materialized from the +recorded head. Rust bounds the diff, reruns the recorded correctness target, +and source-qualifies a re-review before any selected finding can be labelled +fixed. The worktree is retained for inspection; CodeVetter has no commit, +merge, or push action, and discard requires separate confirmation. Agents can +invoke the same local CLI contract, while MCP remains read-only. + +`codevetter warm --operation status|start|stop|run|cancel|cleanup|current` and +the native Testing warm-proof workspace share the incumbent Rust bridge to the +repository-owned verifier. Native changed proof keeps daemon ownership, +worktree identity, deterministic scenario selection, observations, redacted +artifacts, limitations, persistence, and outcome semantics in Rust. Cleanup is +fail-closed in the CLI: it requires exactly one of `--dry-run` or +`--apply-cleanup`. The agent/MCP target-discovery surface stays read-only and +cannot start the verifier. + +`codevetter differential --operation prepare|run|cancel|cleanup` and the native +paired-evidence workspace share the same exact reference/candidate selection, +materialization, parity policies, bounded delta previews, cleanup, persistence, +classification, and zero-model-call contract. Preparation is a separate gate; +commit/range candidates require an exact revision, and differential output is +explicitly additive evidence that can never manufacture a passing verdict. + +`codevetter scenario --operation inspect|generate|validate|dry-run|accept|reject|cleanup` +and the native Scenario Foundry preserve the incumbent compiler pipeline. +Generation is restricted to the declared free/local provider and produces an +expiring candidate only. Validation and dry-run remain separate and cannot +persist evidence or update baselines. Acceptance rechecks the exact candidate +hash, writes only selected repository-contained destinations, and requires a +separate replacement approval when a selected file already exists. +Candidate cleanup is CLI-only and requires explicit `--apply-cleanup`. + `check` requires a clean local checkout at the resolved change head. It composes the existing review, correctness, and performance engines into one versioned receipt. It may discover the highest-confidence closed target or @@ -70,15 +116,38 @@ The Repo surface (`apps/desktop/src/pages/RepoPage.tsx`) consolidates Unpack, Activity, Graph, Inventory, Analysis, Handoff, and past snapshots. `RepoUnpacked.tsx` is a child view within it, not the `/unpack` page itself. +The unreleased native migration reuses the same Rust scanner and SQLite +persistence boundary through `codevetter unpack --operation scan`. Its current +Repo Unpack workspace can create model-free snapshots and inspect stored +Overview, Brief, Activity, Inventory, bounded Graph, and commit-range Delta +evidence. Deterministic Analysis, observed Rules, and source-qualified Handoff +desks keep health and historical leads distinct from executable proof. It also +saves the incumbent Rust-rendered Markdown, offline HTML, +graph JSON, agent-context, and repository-memory exports. Model synthesis +execution and cleanup remain with the shipped Tauri authority until their +replacement gates pass. The native Graph desk and +`codevetter unpack --operation query` now share one read-only Rust receipt for +bounded structural search, node explanation, impact, directed path, temporal +search, and causal trace. It exposes index freshness, trust, sources, evidenced +versus qualified-lead history links, and unavailable coverage without +duplicating ranking or traversal in Swift. Native repeated queries use one +supervised read-only worker with a single search projection that upgrades in +place with compact traversal edges only when required. Bounded result edges +then regain their full evidence and source anchors from SQLite. Every request +rechecks live freshness and snapshot identity; framing, cancellation by process +termination, and the exact one-shot CLI compatibility fallback remain intact. + The fixed rail and command search intentionally omit per-destination mnemonic codes. The former custom `g` navigation chords were removed; destinations use ordinary accessible links and command search instead. -Testing and Performance share `evidence-scope/v1`. Human phrases are bounded -local path/content searches and are never executed. PR/change plans are pinned -to exact Git identities. Codebase plans are capped portfolios with explicit -uncovered paths; they do not claim complete behavioral coverage. A resolved -plan must be confirmed before any test or profile starts. +Testing and Performance share `evidence-scope/v1` across native UI, +`codevetter scope`, MCP `resolve_evidence_scope`, and the target suggestions in +`prepare_review`. Human phrases are bounded local path/content searches and are +never executed. PR/change plans are pinned to exact Git identities. Codebase +plans are capped portfolios with explicit uncovered paths; they do not claim +complete behavioral coverage. MCP remains read-only; a resolved UI/CLI plan +must be confirmed before any test or profile starts. ## URL-only surfaces From 342bbf8962c98edbd250abf02cc941f21fbdc26a Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Thu, 3 Sep 2026 13:17:30 +0530 Subject: [PATCH 2/3] fix(native): drain process output before completion --- .../VerificationRunner.swift | 104 ++++++++---------- .../CodeVetterFeatureTests.swift | 44 ++++---- 2 files changed, 68 insertions(+), 80 deletions(-) diff --git a/apps/macos/CodeVetterPackage/Sources/CodeVetterFeature/VerificationRunner.swift b/apps/macos/CodeVetterPackage/Sources/CodeVetterFeature/VerificationRunner.swift index ec6c295c..90c5228d 100644 --- a/apps/macos/CodeVetterPackage/Sources/CodeVetterFeature/VerificationRunner.swift +++ b/apps/macos/CodeVetterPackage/Sources/CodeVetterFeature/VerificationRunner.swift @@ -3511,12 +3511,12 @@ public final class CodeVetterProcessRunner: @unchecked Sendable { private func runReadOnly(executable: URL, arguments: [String]) async throws -> Data { try await withCheckedThrowingContinuation { continuation in let process = Process() - let stdout = LockedData() - let stderr = LockedData() + let stdout = LockedPipeCapture() + let stderr = LockedPipeCapture() let stdoutPipe = Pipe() let stderrPipe = Pipe() - stdoutPipe.fileHandleForReading.readabilityHandler = { stdout.append($0.availableData) } - stderrPipe.fileHandleForReading.readabilityHandler = { stderr.append($0.availableData) } + stdoutPipe.fileHandleForReading.readabilityHandler = { stdout.consume(from: $0) } + stderrPipe.fileHandleForReading.readabilityHandler = { stderr.consume(from: $0) } process.executableURL = executable process.arguments = arguments process.standardOutput = stdoutPipe @@ -3524,15 +3524,16 @@ public final class CodeVetterProcessRunner: @unchecked Sendable { process.terminationHandler = { completed in stdoutPipe.fileHandleForReading.readabilityHandler = nil stderrPipe.fileHandleForReading.readabilityHandler = nil - stdout.append(stdoutPipe.fileHandleForReading.readDataToEndOfFile()) - stderr.append(stderrPipe.fileHandleForReading.readDataToEndOfFile()) + let output = stdout.finish(from: stdoutPipe.fileHandleForReading) + let errors = stderr.finish(from: stderrPipe.fileHandleForReading) guard completed.terminationStatus == 0 else { continuation.resume( throwing: VerificationRunnerError.launchFailed( - stderr.string.trimmingCharacters(in: .whitespacesAndNewlines))) + String(decoding: errors, as: UTF8.self) + .trimmingCharacters(in: .whitespacesAndNewlines))) return } - continuation.resume(returning: stdout.value) + continuation.resume(returning: output) } do { try process.run() @@ -3549,18 +3550,13 @@ public final class CodeVetterProcessRunner: @unchecked Sendable { requestID: String? = nil, onStderrLine: (@Sendable (String) -> Void)? = nil ) async throws -> TrackedProcessOutput { - let stdout = LockedData() - let stderr = LockedData() + let stdout = LockedPipeCapture() + let stderr = LockedPipeCapture() let stdoutPipe = Pipe() let stderrPipe = Pipe() - let stderrLines = LockedLineBuffer() - stdoutPipe.fileHandleForReading.readabilityHandler = { stdout.append($0.availableData) } + stdoutPipe.fileHandleForReading.readabilityHandler = { stdout.consume(from: $0) } stderrPipe.fileHandleForReading.readabilityHandler = { handle in - let data = handle.availableData - stderr.append(data) - for line in stderrLines.append(data) { - onStderrLine?(line) - } + stderr.consume(from: handle, onLine: onStderrLine) } let process = Process() @@ -3574,20 +3570,19 @@ public final class CodeVetterProcessRunner: @unchecked Sendable { process.terminationHandler = { [weak self] completed in stdoutPipe.fileHandleForReading.readabilityHandler = nil stderrPipe.fileHandleForReading.readabilityHandler = nil - stdout.append(stdoutPipe.fileHandleForReading.readDataToEndOfFile()) - let trailingErrors = stderrPipe.fileHandleForReading.readDataToEndOfFile() - stderr.append(trailingErrors) - for line in stderrLines.append(trailingErrors) + stderrLines.finish() { - onStderrLine?(line) - } + let output = stdout.finish(from: stdoutPipe.fileHandleForReading) + let errors = stderr.finish( + from: stderrPipe.fileHandleForReading, + onLine: onStderrLine + ) let wasCancelled = self?.clear(completed) ?? false if wasCancelled { continuation.resume(throwing: CancellationError()) } else { continuation.resume( returning: TrackedProcessOutput( - stdout: stdout.value, - stderr: stderr.value, + stdout: output, + stderr: errors, status: completed.terminationStatus )) } @@ -3661,60 +3656,51 @@ private func decodeProgress( onProgress(progress) } -private final class LockedData: @unchecked Sendable { +private final class LockedPipeCapture: @unchecked Sendable { private let lock = NSLock() private var data = Data() + private var pending = Data() + private var finished = false - var string: String { + func consume( + from handle: FileHandle, + onLine: (@Sendable (String) -> Void)? = nil + ) { lock.lock() defer { lock.unlock() } - return String(data: data, encoding: .utf8) ?? "" + guard !finished else { return } + append(handle.availableData, onLine: onLine) } - var value: Data { + func finish( + from handle: FileHandle, + onLine: (@Sendable (String) -> Void)? = nil + ) -> Data { lock.lock() defer { lock.unlock() } + guard !finished else { return data } + append(handle.readDataToEndOfFile(), onLine: onLine) + if let onLine, !pending.isEmpty, + let line = String(data: pending, encoding: .utf8) + { + onLine(line) + } + pending.removeAll() + finished = true return data } - func append(_ chunk: Data) { + private func append(_ chunk: Data, onLine: (@Sendable (String) -> Void)?) { guard !chunk.isEmpty else { return } - lock.lock() data.append(chunk) - lock.unlock() - } -} - -private final class LockedLineBuffer: @unchecked Sendable { - private let lock = NSLock() - private var pending = Data() - - func append(_ chunk: Data) -> [String] { - guard !chunk.isEmpty else { return [] } - lock.lock() - defer { lock.unlock() } + guard let onLine else { return } pending.append(chunk) - return drainCompleteLines() - } - - func finish() -> [String] { - lock.lock() - defer { lock.unlock() } - guard !pending.isEmpty else { return [] } - defer { pending.removeAll() } - guard let line = String(data: pending, encoding: .utf8) else { return [] } - return [line] - } - - private func drainCompleteLines() -> [String] { - var lines: [String] = [] while let newline = pending.firstIndex(of: 0x0A) { let line = pending[.. Date: Thu, 3 Sep 2026 13:37:46 +0530 Subject: [PATCH 3/3] fix(native): serialize hosted performance gates --- scripts/run-native-checks.mjs | 9 +++++---- scripts/run-native-checks.test.mjs | 5 ++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/run-native-checks.mjs b/scripts/run-native-checks.mjs index 10b7ec28..2ebb2c14 100644 --- a/scripts/run-native-checks.mjs +++ b/scripts/run-native-checks.mjs @@ -81,10 +81,11 @@ export function nativeCheckCommands({ mode }, environment = process.env) { arguments: [ 'swift-package', 'test', - '--package-path', - 'apps/macos/CodeVetterPackage', - '--parallel', - 'false', + '--json', + JSON.stringify({ + packagePath: 'apps/macos/CodeVetterPackage', + parallel: false, + }), ], }, { diff --git a/scripts/run-native-checks.test.mjs b/scripts/run-native-checks.test.mjs index 20adce10..48cd7351 100644 --- a/scripts/run-native-checks.test.mjs +++ b/scripts/run-native-checks.test.mjs @@ -25,7 +25,10 @@ test('native automation defaults to the non-activating background lane', () => { (command) => !command.arguments.includes('test') || command.arguments[0] === 'swift-package' ) ); - assert.deepEqual(commands[0].arguments.slice(-2), ['--parallel', 'false']); + assert.deepEqual(JSON.parse(commands[0].arguments[3]), { + packagePath: 'apps/macos/CodeVetterPackage', + parallel: false, + }); }); test('UI automation fails closed without explicit foreground approval', () => {