diff --git a/evidence/design/reviews/sidebar.json b/evidence/design/reviews/sidebar.json new file mode 100644 index 00000000..216a8e7c --- /dev/null +++ b/evidence/design/reviews/sidebar.json @@ -0,0 +1,59 @@ +{ + "$schema": "fleet.design-review.v1", + "version": 1, + "project": "codevetter", + "target": "desktop primary navigation sidebar", + "mode": "preserve", + "register": "product", + "context": { + "product": "PRODUCT.md", + "design": "DESIGN.md" + }, + "direction": { + "references": [], + "probes": [], + "selected": "existing-design", + "approval": "not-required", + "before": "artifacts/design/trex-before-1440.png" + }, + "evidence": { + "screenshots": [ + { + "width": 390, + "path": "artifacts/design/sidebar-after-390.png" + }, + { + "width": 768, + "path": "artifacts/design/sidebar-after-768.png" + }, + { + "width": 1440, + "path": "artifacts/design/sidebar-after-1440.png" + } + ], + "projectCheck": { + "command": "biome check touched files; tsc --noEmit; targeted Playwright visual-system and smoke tests; git diff --check", + "status": "pass" + }, + "critique": { + "score": 33, + "maximum": 40 + }, + "audit": { + "score": 18, + "maximum": 20 + }, + "unresolved": { + "p0": 0, + "p1": 0 + }, + "detector": { + "posture": "advisory", + "findings": [] + } + }, + "ownerFeedback": { + "decision": "keep", + "note": "Owner confirmed the implemented sidebar is better and chose to stay with this direction." + } +} diff --git a/evidence/design/reviews/trex-change-and-preview-verification.json b/evidence/design/reviews/trex-change-and-preview-verification.json new file mode 100644 index 00000000..c81244de --- /dev/null +++ b/evidence/design/reviews/trex-change-and-preview-verification.json @@ -0,0 +1,59 @@ +{ + "$schema": "fleet.design-review.v1", + "version": 1, + "project": "codevetter", + "target": "T-Rex change and preview verification", + "mode": "preserve", + "register": "product", + "context": { + "product": "PRODUCT.md", + "design": "DESIGN.md" + }, + "direction": { + "references": [], + "probes": [], + "selected": "existing-design", + "approval": "not-required", + "before": "artifacts/design/trex-before-1440.png" + }, + "evidence": { + "screenshots": [ + { + "width": 390, + "path": "artifacts/design/trex-after-390.png" + }, + { + "width": 768, + "path": "artifacts/design/trex-after-768.png" + }, + { + "width": 1440, + "path": "artifacts/design/trex-after-1440.png" + } + ], + "projectCheck": { + "command": "cd apps/desktop && pnpm exec tsc --noEmit", + "status": "pass" + }, + "critique": { + "score": 37, + "maximum": 40 + }, + "audit": { + "score": 19, + "maximum": 20 + }, + "unresolved": { + "p0": 0, + "p1": 0 + }, + "detector": { + "posture": "advisory", + "findings": [] + } + }, + "ownerFeedback": { + "decision": "delegated", + "note": "Owner authorized the bounded preserve-lane implementation with 'go for it'; the incumbent CodeVetter workbench language remains authoritative." + } +} diff --git a/evidence/design/sidebar-after-1440.png b/evidence/design/sidebar-after-1440.png new file mode 100644 index 00000000..a1d13d79 Binary files /dev/null and b/evidence/design/sidebar-after-1440.png differ diff --git a/evidence/design/sidebar-after-390.png b/evidence/design/sidebar-after-390.png new file mode 100644 index 00000000..7781868a Binary files /dev/null and b/evidence/design/sidebar-after-390.png differ diff --git a/evidence/design/sidebar-after-768.png b/evidence/design/sidebar-after-768.png new file mode 100644 index 00000000..40c01f56 Binary files /dev/null and b/evidence/design/sidebar-after-768.png differ diff --git a/evidence/design/structural-context-evaluation.html b/evidence/design/structural-context-evaluation.html new file mode 100644 index 00000000..ade3647c --- /dev/null +++ b/evidence/design/structural-context-evaluation.html @@ -0,0 +1,123 @@ + + + + + + + + +Synthetic structural-context agent evaluation + + + +
+
CodeVetter · structural context evaluationsynthetic-structural-context-contract
+
+

Does the graph help the agent ship better?

+

The same tasks and agent identity, paired with and without CodeVetter structural context. Executable hidden checks decide success; activity metrics only explain the path.

+
+ unqualified +

Synthetic contract fixture only. No real structural-context value claim is authorized.

+
+
+ +
+

Paired outcome

2 complete A/B pairs across 2 tasks.

Success means every required hidden check passed with no recorded regression.

+
+
without structural context50%1/2 successful runs
+
VS
+
with structural context100%2/2 successful runs
+
+
+50% success delta1 treatment wins0 control wins1 ties0 regression delta
+

unqualified · synthetic receipts · Synthetic contract fixture only. No real structural-context value claim is authorized.

+
+ +
+

What changed task by task

Open a decision trace to see which graph question oriented the treatment arm and which source paths it returned.

+
+
+
trial 1

Preserve authenticated route access

+ tie pass +
+
+
without structural contextsuccess2/2 checks
+ +
with structural contextsuccess2/2 checks
+
+
+

Changed checks

No check outcome changed between arms.

+
Agent decision trace
  1. graph_querycallers:requireAuthenticatedUser

    Confirmed the guard is shared by both protected route branches.

    src/auth.ts · src/routes.ts
+
+
+
+
trial 1

Invalidate stale checkout cache entries

+ treatment win +
+
+
without structural contextcheck failure1/2 checks
+ +
with structural contextsuccess2/2 checks
+
+
+

Changed checks

  • Stale checkout cache is clearedcontrol: failtreatment: pass
+
Agent decision trace
  1. graph_querysymbol:invalidateCheckoutCache

    Located the cache invalidator and its checkout caller.

    src/cache.ts · src/checkout.ts
  2. graph_pathcheckout route to cache invalidator

    Confirmed the missing invalidation edge on the write path.

    src/routes.ts · src/checkout.ts · src/cache.ts
+
+
+
+ +
+

Can this result be trusted?

Qualification is policy-driven. Favorable synthetic numbers still fail the real-evidence gate.

+
  • FAILreal evidenceSynthetic fixtures cannot establish product value.
  • PASScomplete pairs2/2 complete A/B pairs
  • PASSdistinct tasks2/2 distinct tasks
  • PASSaa pairs2/2 complete A/A pairs
  • PASSaa noise0% discordance; maximum 0%
  • PASSpair integrity0 invalid or contaminated pairs
  • PASSsuccess improvement+50%; minimum +20%
  • PASSregression control0 regressions; maximum 0
+ +
+ +
+

Activity diagnostics

Secondary signals describe cost and search behavior. They never substitute for task success, and missing values remain missing.

+
Means use only pairs where both arms captured the metric. A negative delta means less activity, not necessarily a better result.
MetricControlTreatmentDeltaCoverage
Verification selections1102/2
Files inspected2.52-0.52/2
Files modified1102/2
Tool calls22.5+0.52/2
Input tokens19501700-2502/2
Output tokens520450-702/2
Elapsed time (ms)6450055500-90002/2
Cost (USD)0.0180.016-0.0022/2
+
+ +
+

Boundaries

What this artifact does not prove.

+

Declared limitations

  • Synthetic receipts prove the scorer contract only.
  • No coding agent or hidden acceptance runner was invoked.
  • Activity diagnostic fields are illustrative and not product measurements.

Invalid or contaminated pairs

None detected.

+
Authorized claim

Synthetic contract fixture only. No real structural-context value claim is authorized.

+
+ + +
+ + + \ No newline at end of file diff --git a/evidence/design/trex-after-1440.png b/evidence/design/trex-after-1440.png new file mode 100644 index 00000000..acbdf8cb Binary files /dev/null and b/evidence/design/trex-after-1440.png differ diff --git a/evidence/design/trex-after-390.png b/evidence/design/trex-after-390.png new file mode 100644 index 00000000..9bf63da6 Binary files /dev/null and b/evidence/design/trex-after-390.png differ diff --git a/evidence/design/trex-after-768.png b/evidence/design/trex-after-768.png new file mode 100644 index 00000000..62836436 Binary files /dev/null and b/evidence/design/trex-after-768.png differ diff --git a/evidence/design/trex-before-1440.png b/evidence/design/trex-before-1440.png new file mode 100644 index 00000000..7fc7c4ba Binary files /dev/null and b/evidence/design/trex-before-1440.png differ diff --git a/evidence/design/trex-result-1440.png b/evidence/design/trex-result-1440.png new file mode 100644 index 00000000..3a842f26 Binary files /dev/null and b/evidence/design/trex-result-1440.png differ diff --git a/evidence/design/usage-telemetry-after-1440.png b/evidence/design/usage-telemetry-after-1440.png new file mode 100644 index 00000000..dddec2ae Binary files /dev/null and b/evidence/design/usage-telemetry-after-1440.png differ diff --git a/evidence/design/usage-telemetry-after-390.png b/evidence/design/usage-telemetry-after-390.png new file mode 100644 index 00000000..77a40854 Binary files /dev/null and b/evidence/design/usage-telemetry-after-390.png differ diff --git a/evidence/design/usage-telemetry-after-768.png b/evidence/design/usage-telemetry-after-768.png new file mode 100644 index 00000000..abcfa19a Binary files /dev/null and b/evidence/design/usage-telemetry-after-768.png differ diff --git a/evidence/design/usage-telemetry-before-1440.png b/evidence/design/usage-telemetry-before-1440.png new file mode 100644 index 00000000..4a6aee59 Binary files /dev/null and b/evidence/design/usage-telemetry-before-1440.png differ diff --git a/evidence/performance/calorie-exercise-guidance-2026-08-10.md b/evidence/performance/calorie-exercise-guidance-2026-08-10.md new file mode 100644 index 00000000..f4ecc36c --- /dev/null +++ b/evidence/performance/calorie-exercise-guidance-2026-08-10.md @@ -0,0 +1,132 @@ +# Calorie exercise-guidance performance trial — 2026-08-10 + +This records a local proof run, not a customer-impact case study. The benchmark +used no production database, hosted endpoint, deployment, or paid model. The +publication ledger should label any later review URL as a synthetic stress +result unless representative product-scale evidence is added. + +## Scope + +- Project: `/Users/sarthak/Desktop/fleet/calorie` +- Starting revision: `4db2075170a27c5147ad88fe3622dd69760d90ec` +- Runtime: Node `v24.19.0` on Darwin arm64 +- Exact flow: `exercise guidance performance scales across representative food histories` +- Benchmark: `src/lib/recommendations.performance.test.ts` +- Candidate: `calculateGymGuidance` in `src/lib/recommendations.ts` +- Inputs: 1,000, 10,000, and 35,000 food-history entries; 100 in-process + iterations per input +- Correctness oracle: the selected exercise window must belong to the most + recent eligible meal, independent of input order; future and expired meals do + not qualify + +The benchmark is intentionally a scale/stress workload. A 35,000-entry local +history is useful for distinguishing algorithmic behavior, but this artifact +does not establish that typical users have that history size or that the +operation is currently a visible UX bottleneck. + +## Why CodeVetter selected it + +`runtime:plan-flow-campaign` discovered the timed Vitest flow without a manual +target hint. The independent source-profile passes inside a five-sample run +repeated `calculateGymGuidance` as the repository-owned CPU candidate. The +captured supported-scale cost after the change was approximately `0.17 ms/op` +at 35,000 entries. + +The planner classified the candidate as actionable but used neutral frequency +and user-impact weights because Calorie has no project-owned priority manifest. +Therefore production frequency and customer impact remain unverified. + +## Change + +The baseline implementation cloned the history, filtered it, fully sorted the +eligible entries, mapped every entry into a window object, and then selected the +first still-open window. The candidate performs one pass and retains only the +most recent eligible open window. + +The change preserves the baseline's stable-tie behavior: when two entries have +the same timestamp, the first input entry remains selected. + +## Paired result + +CodeVetter alternated baseline and candidate executions for ten measurement +pairs after one warmup per side. Both checkouts used the same exact workload +digest: + +`ebf9f02b4b14632ea6f245be39693a196b5276ea42bc6cedd46000819e80c0ec` + +| History entries | Baseline median | Candidate median | Change | +| ---: | ---: | ---: | ---: | +| 1,000 | 0.034 ms/op | 0.016 ms/op | -52.941% | +| 10,000 | 0.253 ms/op | 0.046 ms/op | -81.818% | +| 35,000 | 1.203 ms/op | 0.162 ms/op | -86.534% | + +The endpoint scale exponent moved from `1.003` to `0.651`. CodeVetter returned: + +- verdict: `confirmed` +- mechanically confirmed: `true` +- materially useful under the recorded benchmark policy: `true` +- shipping recommended at the ten-sample floor: `true` +- verifier limitations: none + +Interpretation: the algorithmic improvement is strongly supported. The absolute +benefit is only about `1.041 ms/op` at the largest stress input, so this is not +yet evidence of customer-visible latency. A future UI must show absolute and +relative movement together. + +## Correctness and repository checks + +- Focused Vitest run: 2 files, 17 tests passed +- Full Calorie test run before the final iteration-count increase: 28 files, + 120 tests passed +- TypeScript: `pnpm typecheck` passed +- Biome on the three touched files passed +- `git diff --check` passed + +## CodeVetter improvements caused by this trial + +The first paired Vitest attempt compared runner startup time and ignored the +benchmark's console `ms/op` series. That produced an inconclusive result even +though both sides emitted comparable domain metrics. CodeVetter now retains the +median console metrics from repeated paired Vitest measurements, matching the +existing Node-test and Node-script behavior. A regression test covers the +failure mode. + +The later campaign run also exposed a source-coordinate mismatch: the raw V8 +profile reported generated line 178 while bounded source inspection uniquely +anchored `calculateGymGuidance` to original TypeScript line 254. The campaign +planner now emits the source-anchored line and preserves the raw profiler line +as `reported_line`. This prevents an agent or UI from silently presenting the +generated coordinate as the original source location. + +Relevant CodeVetter validation after both fixes: 34 focused runtime-performance +tests passed, including paired execution, Vitest metric retention, source-line +alignment, Node profiling, and Go benchmark coverage. + +## Other candidate screening in this pass + +| Project | Evidence found | Decision | +| --- | --- | --- | +| India Standards | Existing benchmark depends on a local DuckDB/data path and requires explicit arguments. | Not executed in the default local campaign; the workload needs a declared fixture boundary first. | +| Karte | Generic tests but no direct timing evidence; important paths are database/network-shaped. | No synthetic optimization forced. | +| SWE Interview Prep | Generic deterministic tests over small fixed catalogs, with no representative timed flow. | Kept as a correctness surface; no scale claim manufactured. | +| ChatGPT Memory Insights | Local export normalization and semantic-analysis code were inspected. The dominant semantic flow depends on model inference, while the only current performance test checks timer bookkeeping. | No candidate changed. A representative staged analysis fixture is needed before optimizing source. | + +These negative decisions are part of the proof: discovery breadth is not the +same as evidence quality, and a campaign should be allowed to return “needs a +better workload.” + +## UI implications earned by this evidence + +Per application, the eventual surface should show: + +1. flow inventory: tested, excluded, and missing-workload flows; +2. exact revision, workload identity, samples, and correctness gate; +3. observed absolute and relative measurements; +4. source-anchored candidate plus raw profiler coordinate when they differ; +5. confirmed, rejected, guardrail, or needs-better-workload status; +6. inferred product impact separately from measured runtime impact; +7. the concrete tool improvement or product patch produced by the trial. + +This trial argues against building a generic profiler dashboard first. The +useful UI object is an evidence-backed application-flow review with explicit +coverage and missing-evidence states. diff --git a/evidence/performance/native-bridge-benchmark.json b/evidence/performance/native-bridge-benchmark.json new file mode 100644 index 00000000..99683131 --- /dev/null +++ b/evidence/performance/native-bridge-benchmark.json @@ -0,0 +1,48 @@ +{ + "schema_version": "codevetter.native-bridge-benchmark/v1", + "measured_at": "2026-09-01T23:00:30Z", + "source": { + "base_sha": "b34ce70edbcfab8cd9463be87b7453de7e623ed3", + "state": "working tree includes the uncommitted native migration", + "platform": "arm64-apple-macosx27.0.0", + "rust": "rustc 1.98.0", + "swift": "Apple Swift 6.4", + "xcodebuildmcp": "2.7.0" + }, + "payload_bytes": 40047, + "ffi_transfer": { + "iterations": 10000, + "minimum_us": 0.875, + "median_us": 1.083, + "p95_us": 1.208, + "average_us": 1.0816948999999492 + }, + "ffi_transfer_and_decode": { + "iterations": 1000, + "minimum_us": 90.917, + "median_us": 106.708, + "p95_us": 171.791, + "average_us": 116.45407299999995 + }, + "worker_round_trip": { + "iterations": 20, + "minimum_us": 76987.042, + "median_us": 82255.209, + "p95_us": 84643.667, + "average_us": 81877.04169999999 + }, + "artifact_bytes": { + "ffi_probe_dylib": 49888, + "release_worker_cli": 44109408, + "swift_benchmark": 133208 + }, + "semantic_parity": true, + "selected_boundary": "hybrid: in-process for bounded read-only projections; supervised worker for verification execution", + "limitations": [ + "The FFI probe measures a deterministic read-only projection, not long-running verification.", + "Debug and release application integration overhead is not included.", + "Process startup is intentionally included in the worker round trip.", + "The probe dylib size is not a production core-library size forecast.", + "The pinned XcodeBuildMCP package is resolved through a benchmark-local npm cache so repeated runs do not depend on the user's shared npx cache." + ] +} diff --git a/evidence/performance/native-current-package-footprint.json b/evidence/performance/native-current-package-footprint.json new file mode 100644 index 00000000..d0f1b28a --- /dev/null +++ b/evidence/performance/native-current-package-footprint.json @@ -0,0 +1,64 @@ +{ + "schema_version": "codevetter.native-package-footprint/v1", + "recorded_at": "2026-09-02T06:20:30Z", + "status": "qualified_with_bounded_claims", + "source": { + "base_sha": "b34ce70edbcfab8cd9463be87b7453de7e623ed3", + "branch": "feat/native-macos-evidence-workbench", + "working_tree": "includes the uncommitted native migration", + "platform": "darwin", + "architecture": "arm64", + "macos": "27.0" + }, + "method": { + "measurement": "Read-only du -sk bundle measurement and stat executable measurement on exact local Release app bundles.", + "foreground_app_launches": 0, + "installed_application_touched": false + }, + "applications": { + "native": { + "bundle_identifier": "com.codevetter.desktop.native-preview", + "version": "1.11.0", + "build": "11100", + "bundle_path": "artifacts/native-package/qualification-5r7JG4/CodeVetter.app", + "package_receipt": "artifacts/native-package/qualification-5r7JG4/qualification.json", + "bundle_kib": 62060, + "host_executable": "CodeVetterNative", + "host_executable_bytes": 5667872, + "host_executable_sha256": "95c81746c4ceac17b731867ab8f02c057875cc8ef9f098904ee7cfd689f56c8c" + }, + "tauri": { + "bundle_identifier": "com.codevetter.desktop", + "version": "1.11.0", + "build": "1.11.0", + "bundle_path": "apps/desktop/src-tauri/target/release/bundle/macos/CodeVetter.app", + "bundle_kib": 165144, + "host_executable": "codevetter-desktop", + "host_executable_bytes": 74639312, + "host_executable_sha256": "6e010b2b1b759a892c8c34ae04a6a2ea243b9b6455ec290ba570e88508bdccff" + } + }, + "metrics": { + "bundle_kib": { + "native": 62060, + "tauri": 165144, + "native_to_tauri_ratio": 0.376, + "native_reduction_percent": 62.421, + "classification": "improved" + }, + "host_executable_bytes": { + "native": 5667872, + "tauri": 74639312, + "native_to_tauri_ratio": 0.076, + "native_reduction_percent": 92.406, + "classification": "improved" + } + }, + "claim": "The exact current native Release candidate has a 62.4% smaller app bundle and a 92.4% smaller host executable than the retained Tauri Release bundle.", + "limitations": [ + "This receipt measures package and host-executable footprint only.", + "It does not refresh startup, responsiveness, settled RSS, workload execution, scrolling, energy, or long-session behavior.", + "The existing native-tauri launch and settled-RSS comparison remains historical evidence for its recorded qualified build.", + "Refreshing launch or settled-RSS evidence requires foreground application automation and explicit owner approval." + ] +} diff --git a/evidence/performance/native-macos-baseline.md b/evidence/performance/native-macos-baseline.md new file mode 100644 index 00000000..a6106964 --- /dev/null +++ b/evidence/performance/native-macos-baseline.md @@ -0,0 +1,51 @@ +# Native macOS performance baseline + +Date: 2026-08-31 +Target: `CodeVetter` Debug, Apple silicon, macOS +Harness: XcodeBuildMCP 2.7.0 and `XCTApplicationLaunchMetric` + +## Responsive first frame + +`XCTApplicationLaunchMetric(waitUntilResponsive: true)` measured five launches: + +| Sample | Seconds | +| --- | ---: | +| 1 | 0.396736 | +| 2 | 0.390666 | +| 3 | 0.392087 | +| 4 | 0.498724 | +| 5 | 0.434117 | + +Average: **0.422 seconds** +Relative standard deviation: **9.787%** + +The test passed with Xcode's default 10% relative-deviation tolerance. The +metric waits for the first frame to become responsive; it is not a shell-process +startup approximation. + +## Idle footprint + +Ten seconds after a dark-appearance launch: + +- resident memory: **105,952 KiB**; +- Debug application bundle: **3,056 KiB**. + +## Claim boundary + +These measurements established the initial Debug baseline. They remain useful +as historical XCTest responsive-frame evidence but do not carry the later +Release comparison claim by themselves. + +The historical same-machine Release launch and settled-memory evidence lives in +`evidence/performance/native-tauri-comparison.json`. Five alternating, +surface-confirmed runs classify first-visible-window startup as parity and +measure 117,424 KiB native versus 169,024 KiB Tauri median settled +Performance-workspace process-tree RSS, a 30.5% native reduction. Those runtime +claims remain bound to that receipt's qualified build. + +The read-only current-tree package measurement lives in +`evidence/performance/native-current-package-footprint.json`. The exact +`qualification-5r7JG4` native candidate is 62.4% smaller as an app bundle and +92.4% smaller as a host executable than the retained Tauri Release bundle. It +does not refresh launch, RSS, workload, energy, scrolling, or long-session +claims. diff --git a/evidence/performance/native-performance-benchmark.json b/evidence/performance/native-performance-benchmark.json new file mode 100644 index 00000000..7996c488 --- /dev/null +++ b/evidence/performance/native-performance-benchmark.json @@ -0,0 +1,40 @@ +{ + "schema_version": "codevetter.native-performance-benchmark/v2", + "recorded_at": "2026-09-02", + "target": "aarch64-apple-darwin", + "command": "env npm_config_cache=artifacts/native-checks/xcodebuildmcp-npm-cache npx -y xcodebuildmcp@2.7.0 swift-package test --package-path apps/macos/CodeVetterPackage --filter hundredRowPerformanceReceiptDecodesAndRendersWithinTheNativeGate --parallel false", + "dataset": { + "observed_rows": 100, + "inferred_rows": 1, + "unverified_rows": 1, + "viewport": "980x640" + }, + "measurement": { + "independent_gate_runs": 3, + "swift_decode": { + "warmups_per_run": 10, + "samples_per_run": 100, + "p95_us_runs": [1502, 1792, 1551], + "median_p95_us": 1551, + "worst_p95_us": 1792, + "gate_us": 25000 + }, + "swift_host_render": { + "warmups_per_run": 3, + "samples_per_run": 20, + "p95_us_runs": [41867, 46009, 35226], + "median_p95_us": 41867, + "worst_p95_us": 46009, + "gate_us": 150000 + } + }, + "qualification": { + "swift_focused_gate": "three focused runs passed, 0 failed, 0 skipped", + "historical_macos_ui_test": "8 passed, 0 failed, 0 skipped; not rerun during the 2026-09-02 background refresh", + "historical_dark_setup": "artifacts/design/native-performance-dark.jpeg", + "historical_light_setup": "artifacts/design/native-performance-light.jpeg", + "historical_dark_receipt": "artifacts/design/native-performance-receipt-dark.png", + "headless_plan": "Both admitted exit 0 and blocked exit 2 branches observed" + }, + "claim_boundary": "Measures three independent repetitions of canonical performance JSON decoding and fresh NSHostingView creation, lazy visible-row layout, and display for one 100-row diagnosis. Conservative qualification uses the worst run-level p95. It excludes CLI startup, Node and repository workload execution, active-run CPU and memory, window-server frame pacing, and interactive scrolling." +} diff --git a/evidence/performance/native-release-optimization.json b/evidence/performance/native-release-optimization.json new file mode 100644 index 00000000..4b7be66b --- /dev/null +++ b/evidence/performance/native-release-optimization.json @@ -0,0 +1,88 @@ +{ + "schema_version": "codevetter.native-release-optimization/v1", + "recorded_at": "2026-09-02T06:20:11.066Z", + "status": "qualified_with_bounded_claims", + "source": { + "base_sha": "b34ce70edbcfab8cd9463be87b7453de7e623ed3", + "branch": "feat/native-macos-evidence-workbench", + "working_tree": "includes the uncommitted native migration", + "platform": "arm64-apple-macosx27.0.0" + }, + "change": { + "rust_release": { + "lto": "fat", + "codegen_units": 1, + "panic": "unwind" + }, + "native_release": { + "coverage_instrumentation": false, + "dead_code_stripping": true, + "deployment_postprocessing": true, + "debug_symbols": "adjacent dSYM" + } + }, + "baseline": { + "candidate": "qualification-60JfB0", + "bundle_kib": 90732, + "host_bytes": 24691040, + "cli_bytes": 51339024, + "mcp_bytes": 10356416, + "zip_bytes": 21468174, + "dmg_bytes": 25649028 + }, + "optimized": { + "candidate": "qualification-5r7JG4", + "bundle_kib": 62060, + "host_bytes": 5667872, + "host_sha256": "95c81746c4ceac17b731867ab8f02c057875cc8ef9f098904ee7cfd689f56c8c", + "cli_bytes": 43853088, + "cli_sha256": "8aa8b3670fddfb2ca91b2c8669b7468187becdbad8fe310b7f1b95f6b4434879", + "mcp_bytes": 7506752, + "mcp_sha256": "0f000679f481ae313f77fdead93ae79cb00e43f75389f5f221b92c4ca517aee1", + "zip_bytes": 17427235, + "zip_sha256": "d5dc651c85ac064b932e233bb96d11e18e3f332a8c30693cae1866e7548f98ea", + "dmg_bytes": 20034510, + "dmg_sha256": "63f5ad84ce3239b4b9c45ac67ceef6a6c677c498bcbf0c7223bf07f79584fb8a", + "coverage_segments": 0, + "adjacent_dsym_kib": 34144 + }, + "improvement_percent": { + "bundle": 31.601, + "host": 77.045, + "cli": 14.581, + "mcp": 27.516, + "zip": 18.823, + "dmg": 21.890 + }, + "retained_tauri_comparison_percent": { + "bundle_reduction": 62.421, + "host_reduction": 92.406 + }, + "packaged_mcp_qualification": { + "protocol": "2025-11-25", + "startup_recorded": 50, + "workload_rounds": 200, + "cold_initialize_p95_ms": 8.524, + "mixed_concurrency_4_p95_ms": 24.828, + "idle_rss_mib": 30.109, + "second_half_rss_growth_mib": 2.641, + "tool_count": 28, + "strict_schemas": true, + "read_only_annotations": true, + "tcp_listeners": 0, + "budgets_applied": true, + "passed": true + }, + "tradeoffs": { + "native_clean_release_build_seconds": 34.0, + "rust_cli_release_link_seconds": 200, + "rust_mcp_release_link_seconds": 159, + "decision": "Keep the optimization because shipping footprint and fully sampled MCP qualification improve materially while runtime budgets and panic semantics remain intact." + }, + "limitations": [ + "The optimization receipt measures package footprint and the packaged MCP workload only.", + "It does not refresh foreground native launch, current-package settled RSS, scrolling, energy, or long-session behavior.", + "Fat LTO materially increases Release link time but does not affect Debug or test profiles.", + "Production must archive the matching dSYM separately from end-user ZIP and DMG artifacts." + ] +} diff --git a/evidence/performance/native-repository-query-benchmark.json b/evidence/performance/native-repository-query-benchmark.json new file mode 100644 index 00000000..824f6f9a --- /dev/null +++ b/evidence/performance/native-repository-query-benchmark.json @@ -0,0 +1,35 @@ +{ + "schema_version": "codevetter.native-repository-query-benchmark/v1", + "recorded_at": "2026-09-01", + "binary_profile": "release", + "surface": "native-supervised-cli-boundary", + "repository": { + "label": "indexed incumbent CodeVetter checkout", + "graph_indexed": true, + "history_indexed": true, + "stale_relative_to_live_checkout": true + }, + "query": "verification", + "limit": 12, + "measurements": { + "graph": { + "wall_ms": [3160, 2210, 2150, 2170, 2170], + "median_ms": 2170, + "result_count": 12 + }, + "history": { + "wall_ms": [820, 820, 810, 820, 810], + "median_ms": 820, + "result_count": 1 + } + }, + "method": "/usr/bin/time -p over five independent CLI processes; values are rounded to the reported 10 ms wall-clock precision", + "interpretation": "History is sub-second. Structural search is usable for explicit submit but not instant because each native query starts a fresh sidecar and reconstructs the canonical snapshot query index.", + "next_step": "Completed by native-repository-query-worker-benchmark.json: the scoped worker now has measured parity, lifecycle, cancellation, memory, stale-index behavior, and supervised fallback.", + "limitations": [ + "This is one local Mac and one indexed repository, not a cross-machine distribution.", + "Independent processes intentionally exclude MCP in-process cache reuse.", + "The indexed snapshots were stale and the receipt disclosed that state.", + "Debug timings are not release qualification; observed Debug medians were approximately 7.39 s for graph and 1.94 s for history." + ] +} diff --git a/evidence/performance/native-repository-query-rich-worker-benchmark.json b/evidence/performance/native-repository-query-rich-worker-benchmark.json new file mode 100644 index 00000000..d656b5f8 --- /dev/null +++ b/evidence/performance/native-repository-query-rich-worker-benchmark.json @@ -0,0 +1,74 @@ +{ + "schema_version": "codevetter.native-repository-query-rich-worker-benchmark/v1", + "recorded_at": "2026-09-01T18:15:44.601Z", + "binary_profile": "release", + "surface": "native-persistent-read-only-query-worker", + "repository": { + "label": "indexed incumbent CodeVetter checkout", + "graph_indexed": true, + "history_indexed": true, + "graph_nodes": 115884, + "graph_edges": 193399, + "stale_relative_to_live_checkout": true + }, + "measurements": { + "process_start_and_search_prepare_ms": 2454.8, + "first_traversal_upgrade_ms": 987.44, + "graph_search": { + "wall_ms": [36.99, 36.25, 36.0, 34.81, 36.07], + "median_ms": 36.07 + }, + "graph_explain": { + "wall_ms": [35.43, 35.5, 35.15, 34.05, 34.41], + "median_ms": 35.15 + }, + "graph_impact": { + "wall_ms": [115.49, 116.54, 116.55, 119.41, 117.56], + "median_ms": 116.55 + }, + "graph_path": { + "wall_ms": [70.21, 68.08, 68.04, 69.54, 68.38], + "median_ms": 68.38 + }, + "history_search": { + "wall_ms": [36.96, 32.46, 32.37, 31.55, 32.72], + "median_ms": 32.46 + }, + "history_trace": { + "wall_ms": [33.76, 32.21, 31.54, 32.35, 31.68], + "median_ms": 32.21 + }, + "search_only_worker_rss_kib": 248416, + "search_only_worker_rss_mib": 242.6, + "rich_worker_rss_kib": 315008, + "rich_worker_rss_mib": 307.6 + }, + "qualified_result": { + "impact_affected_nodes": 1, + "path_hops": 1, + "history_trace_episodes": 0 + }, + "comparison": { + "independent_process_graph_search_median_ms": 2170, + "independent_process_history_search_median_ms": 820, + "warm_graph_search_latency_reduction_percent": 98.3, + "warm_history_search_latency_reduction_percent": 96.0, + "rejected_full_interactive_snapshot_rss_mib": 511.9, + "final_rich_worker_rss_reduction_percent": 39.9 + }, + "method": "One exact-tree Release worker over bounded JSON lines: prepare the search projection once; issue five sequential samples per graph search, explain, outgoing impact, directed path, history search, and causal trace; read RSS after search preparation and again after compact traversal upgrade. The selected graph result contained one outgoing affected node and a one-hop path. RSS uses ps; wall time uses Node performance.now.", + "interpretation": "The native app retains a search-only canonical node projection for routine retrieval. The first rich graph operation upgrades that same snapshot in place with compact traversal edges; only bounded result edges are then hydrated with full evidence and source anchors. Ranking and traversal remain Rust-owned and shared with MCP.", + "lifecycle": { + "maximum_cached_graph_snapshots": 1, + "cancellation": "Task cancellation terminates the scoped worker and rejects partial receipts; the next request starts a clean worker.", + "fallback": "A CLI without persistent transport uses the exact supervised one-shot query command; canonical errors never trigger a weaker fallback.", + "shutdown": "Closing stdin or terminating the native parent stops the worker." + }, + "limitations": [ + "This is one local Apple-silicon Mac and one large indexed repository, not a cross-machine distribution.", + "The 2.45 s search preparation and 0.99 s first traversal upgrade remain observable if background preparation has not completed.", + "The rich worker retained 307.6 MiB after traversal; this measured speed-memory tradeoff must remain visible in runtime qualification.", + "The causal selector was valid but returned no indexed episodes in this repository snapshot, so latency and empty-state decoding are proven but a populated live trace is not.", + "The indexed snapshots were stale and every receipt disclosed that state." + ] +} diff --git a/evidence/performance/native-repository-query-worker-benchmark.json b/evidence/performance/native-repository-query-worker-benchmark.json new file mode 100644 index 00000000..4f7833a5 --- /dev/null +++ b/evidence/performance/native-repository-query-worker-benchmark.json @@ -0,0 +1,48 @@ +{ + "schema_version": "codevetter.native-repository-query-worker-benchmark/v1", + "recorded_at": "2026-09-01T17:30:11Z", + "binary_profile": "release", + "surface": "native-persistent-read-only-query-worker", + "repository": { + "label": "indexed incumbent CodeVetter checkout", + "graph_indexed": true, + "history_indexed": true, + "graph_nodes": 115884, + "graph_edges": 193399, + "stale_relative_to_live_checkout": true + }, + "measurements": { + "process_start_and_graph_prepare_ms": 2490.4, + "first_graph_after_prepare_ms": 51.63, + "graph": { + "wall_ms": [50.27, 48.98, 48.09, 50.5, 51.18, 52.26, 50.94], + "median_ms": 50.5 + }, + "history": { + "wall_ms": [41.14, 36.76, 36.55, 35.4, 33.77], + "median_ms": 36.55 + }, + "worker_rss_kib": 248192, + "worker_rss_mib": 242.4 + }, + "comparison": { + "independent_process_graph_median_ms": 2170, + "independent_process_history_median_ms": 820, + "graph_latency_reduction_percent": 97.7, + "history_latency_reduction_percent": 95.5 + }, + "method": "One Release worker over bounded JSON lines: prepare graph once, issue one first graph query, then seven graph and five history queries sequentially. Wall time uses Node performance.now; RSS uses ps after both domains. Process startup is included only in preparation.", + "interpretation": "The native app prepares the one-snapshot search projection quietly when a retained Repo Unpack snapshot opens. Intentional queries then reuse the canonical Rust snapshot and index while rechecking live Git freshness and the latest stored snapshot identity.", + "lifecycle": { + "maximum_cached_graph_snapshots": 1, + "cancellation": "Task cancellation terminates the scoped worker and rejects partial receipts; the next request starts a clean worker.", + "fallback": "An older or test-only CLI that lacks the worker uses the exact supervised one-shot query command.", + "shutdown": "Closing stdin or terminating the native parent stops the worker." + }, + "limitations": [ + "This is one local Apple-silicon Mac and one large indexed repository, not a cross-machine distribution.", + "The 2.49 s graph preparation remains observable if the user submits before background preparation completes.", + "The retained 242.4 MiB worker RSS is a measured speed-memory tradeoff and must remain visible in future runtime qualification.", + "The indexed snapshots were stale and every receipt disclosed that state." + ] +} diff --git a/evidence/performance/native-run-history-benchmark.json b/evidence/performance/native-run-history-benchmark.json new file mode 100644 index 00000000..c2cb54e5 --- /dev/null +++ b/evidence/performance/native-run-history-benchmark.json @@ -0,0 +1,41 @@ +{ + "schema_version": "codevetter.native-run-history-benchmark/v2", + "recorded_at": "2026-08-31", + "target": "aarch64-apple-darwin", + "commands": { + "rust": "cargo test --release --manifest-path apps/desktop/src-tauri/Cargo.toml benchmark_seven_family_projection_over_seven_hundred_rows --lib -- --ignored --nocapture --test-threads=1", + "swift": "swift test --filter hundredRunLedgerDecodesAndRendersWithinTheNativeGate" + }, + "dataset": { + "stored_run_rows": 700, + "stored_audience_response_rows": 100, + "families": 7, + "returned_rows": 100, + "swift_selected_response_rows": 100 + }, + "measurement": { + "rust_projection": { + "warmups": 20, + "samples": 250, + "median_us": 892, + "p95_us": 1077 + }, + "swift_decode": { + "warmups": 10, + "samples": 100, + "p95_us": 2741, + "gate_us": 25000 + }, + "swift_host_render": { + "warmups": 3, + "samples": 20, + "viewport": "1280x800", + "p95_us": 26826, + "gate_us": 150000 + } + }, + "claim_boundary": { + "rust": "Measures in-process SQLite projection, one batched audience-response query, metadata assembly, global sort, and truncation. It excludes CLI process startup.", + "swift": "Measures canonical JSON decoding and fresh NSHostingView creation, layout, and display for 100 runs with 100 selected audience responses. It excludes CLI startup, window-server frame pacing, and interactive scrolling." + } +} diff --git a/evidence/performance/native-runtime-gate-audit-2026-09-01.md b/evidence/performance/native-runtime-gate-audit-2026-09-01.md new file mode 100644 index 00000000..7f382bbf --- /dev/null +++ b/evidence/performance/native-runtime-gate-audit-2026-09-01.md @@ -0,0 +1,62 @@ +# Native runtime gate audit — 2026-09-01 + +This audit closes the measurement task for the native migration without +claiming that a historical foreground run was repeated after every source +change. The recorded Release runtime qualification remains the authority for +launch, memory, cancellation, crash recovery, progress throughput, and large +receipt rendering. The Rust/Swift bridge was remeasured against the current +working tree because that boundary changed materially. + +## Explicit gates + +| Gate | Limit | Observed | Result | +| --- | ---: | ---: | --- | +| Responsive Release launch average | 1.0 s | 0.58 s | Pass | +| Settled native Performance-workspace RSS | 204,800 KiB | 117,424 KiB | Pass | +| FFI transfer and decode p95 | 250 us | 170.583 us | Pass | +| Supervised worker round trip p95 | 150,000 us | 80,312.500 us | Pass | +| 1,000 progress events | 2,000 ms | 342.278 ms | Pass | +| Cancellation acknowledgement | 500 ms | 0.326 ms | Pass | +| Fresh worker after crash | 1,000 ms | 3.739 ms | Pass | +| Large receipt render p95 | 150,000 us | 30,561–77,597 us | Pass | + +The matched Release comparison also classified startup as parity with the +Tauri application, measured 30.529% lower settled RSS, and measured a 51.486% +smaller qualified bundle at the time of that comparison. + +## Current-tree refresh + +`pnpm bench:native-bridge` rebuilt the Rust probe, Release worker CLI, and +Release Swift benchmark through the pinned XcodeBuildMCP 2.7.0 workflow. The +40,047-byte fixture retained semantic parity across the in-process and +supervised-worker boundaries. The exact receipt is +`evidence/performance/native-bridge-benchmark.json`. + +The benchmark and background native-check runner now use persistent, +gitignored npm caches under `artifacts/native-bridge/` and +`artifacts/native-checks/`. This avoids depending on or repeatedly repopulating +the user's shared `npx` cache while preserving the pinned tool version. + +The current background-safe regression evidence is 80 Swift package tests, +1,105 executed Rust tests with 31 intentional ignores, a successful macOS Debug +build, and a successful Release host build. The Swift package lane now +explicitly uses XcodeBuildMCP's +`--parallel false`: AppKit render and supervised-process latency gates measure +one shared Mac, so concurrent test workers were competing for the exact +resources being measured and could manufacture threshold failures. In the +serialized full gate, 1,000 progress events completed in 67.939 ms and the five +large-surface conservative render p95 values range from 30.561 ms to 55.588 ms. +The current 100-row Performance surface was repeated three times and uses its +46.009 ms worst run-level p95 rather than the 41.867 ms median. No +foreground XCUITest or matched-app launch was run during this refresh because +those workflows activate application windows and would interrupt the +operator's desktop. + +## Release boundary + +All explicit runtime gates have passing evidence. Before an authorized release, +repeat the foreground matched-app launch/RSS comparison against the exact +signed candidate and record the result separately. That refresh is a release +qualification step, not a reason to disturb an active desktop session now. +The comparator now fails closed unless its invocation includes `--foreground`, +matching the native UI-test runner's explicit desktop-control acknowledgement. diff --git a/evidence/performance/native-runtime-qualification.json b/evidence/performance/native-runtime-qualification.json new file mode 100644 index 00000000..41708edb --- /dev/null +++ b/evidence/performance/native-runtime-qualification.json @@ -0,0 +1,133 @@ +{ + "schema_version": "codevetter.native-runtime-qualification/v1", + "recorded_at": "2026-09-01", + "source": { + "base_sha": "b34ce70edbcfab8cd9463be87b7453de7e623ed3", + "state": "working tree includes the uncommitted native migration", + "platform": "arm64-apple-macosx27.0", + "rust": "rustc 1.98.0", + "swift": "Apple Swift 6.4", + "xcodebuildmcp": "2.7.0" + }, + "gates": { + "responsive_release_launch": { + "metric": "XCTApplicationLaunchMetric.ApplicationFirstFramePresentationResponsive", + "samples_seconds": [0.556924, 0.627633, 0.583197, 0.575797, 0.558257], + "average_seconds": 0.58, + "relative_standard_deviation_percent": 4.428, + "maximum_average_seconds": 1.0, + "pass": true + }, + "release_workspace_memory": { + "surface": "Performance with the repository selected", + "rss_samples_kib": [ + 117440, + 117424, + 117120, + 117920, + 116928 + ], + "steady_rss_kib": 117424, + "maximum_steady_rss_kib": 204800, + "release_app_bundle_kib_after_worker_packaging": 80116, + "source_receipt": "evidence/performance/native-tauri-comparison.json", + "pass": true + }, + "matched_tauri_comparison": { + "source_receipt": "evidence/performance/native-tauri-comparison.json", + "samples_per_application": 5, + "surface_confirmed_samples": 10, + "first_visible_window_native_median_ms": 435.12, + "first_visible_window_tauri_median_ms": 419.018, + "first_visible_window_classification": "parity", + "settled_process_tree_rss_native_median_kib": 117424, + "settled_process_tree_rss_tauri_median_kib": 169024, + "settled_process_tree_rss_native_reduction_percent": 30.529, + "bundle_native_reduction_percent": 51.486, + "pass": true + }, + "rust_swift_bridge": { + "fixture_bytes": 13248, + "ffi_transfer_and_decode_p95_us": 48.292, + "maximum_ffi_transfer_and_decode_p95_us": 250, + "supervised_worker_round_trip_p95_us": 73745.625, + "maximum_supervised_worker_round_trip_p95_us": 150000, + "semantic_parity": true, + "pass": true + }, + "progress_throughput": { + "events": 1000, + "elapsed_ms": 342.278, + "maximum_elapsed_ms": 2000, + "events_per_second": 2921.6, + "pass": true + }, + "cancellation": { + "elapsed_ms": 0.326, + "maximum_elapsed_ms": 500, + "successful_receipt_accepted": false, + "pass": true + }, + "worker_crash_recovery": { + "crashed_receipt_accepted": false, + "fresh_worker_elapsed_ms": 3.739, + "maximum_fresh_worker_elapsed_ms": 1000, + "pass": true + }, + "large_receipt_decode_and_render": { + "maximum_decode_p95_us": 25000, + "maximum_render_p95_us": 150000, + "surfaces": { + "repo_unpack": { + "dataset": "100 snapshots, 700 graph nodes, 1000 tree rows", + "decode_p95_us": 7587, + "render_p95_us": 55588, + "pass": true + }, + "usage": { + "dataset": "365 daily periods and 100 sessions", + "decode_p95_us": 8838, + "render_p95_us": 30561, + "pass": true + }, + "performance": { + "dataset": "100 observed evidence rows", + "source_receipt": "evidence/performance/native-performance-benchmark.json", + "independent_gate_runs": 3, + "decode_p95_us": 1792, + "render_p95_us": 46009, + "pass": true + }, + "testing": { + "dataset": "100 journeys and 100 changed paths", + "decode_p95_us": 575, + "render_p95_us": 33341, + "pass": true + }, + "runs": { + "dataset": "100 runs and 100 selected responses", + "decode_p95_us": 2826, + "render_p95_us": 34535, + "pass": true + } + }, + "pass": true + } + }, + "qualification": { + "all_gates_pass": true, + "swift_runtime_gate_test": "1 passed, 0 failed", + "release_launch_test": "historical 1 passed, 0 failed; current exact package not foreground-launched", + "swift_package_suite": "79 passed, 0 failed, 0 skipped", + "macos_suite": "historical 64 passed, 0 failed, including 9 XCUITests", + "release_build": "succeeded" + }, + "selected_boundary": "Hybrid: bounded immutable read projections may use an in-process bridge; verification and other risky or long-running work stays in supervised workers.", + "limitations": [ + "The release launch metric used ENABLE_TESTABILITY=YES for the qualification build; the separate release application build succeeded without changing shipping settings.", + "RSS samples cover the native application process on the selected Performance workspace; supervised worker process-tree memory is recorded by Rust execution receipts rather than folded into the app RSS.", + "Host-render measurements exclude window-server frame pacing and interactive scrolling.", + "The matched comparison qualifies lower settled Performance-workspace RSS and smaller package footprint, not workload-execution, energy, or long-session performance.", + "Xcode emitted one internal QoS priority-inversion advisory during the command-palette XCUITest; all 64 tests passed and no product failure accompanied it." + ] +} diff --git a/evidence/performance/native-tauri-comparison.json b/evidence/performance/native-tauri-comparison.json new file mode 100644 index 00000000..929f079f --- /dev/null +++ b/evidence/performance/native-tauri-comparison.json @@ -0,0 +1,142 @@ +{ + "schema_version": "codevetter.native-tauri-comparison/v1", + "recorded_at": "2026-09-01T10:29:32.088Z", + "status": "qualified_with_bounded_claims", + "source": { + "base_sha": "b34ce70edbcfab8cd9463be87b7453de7e623ed3", + "branch": "feat/native-macos-evidence-workbench", + "working_tree": "includes the uncommitted native migration", + "node": "v26.7.0" + }, + "machine": { + "platform": "darwin", + "architecture": "arm64", + "model": "Mac17,8", + "logical_cpu": 18, + "memory_bytes": 51539607552, + "macos": "27.0" + }, + "method": { + "runs_per_application": 5, + "settle_ms": 5000, + "surface": "Performance", + "launch_order": [ + ["native", "tauri"], + ["tauri", "native"], + ["native", "tauri"], + ["tauri", "native"], + ["native", "tauri"] + ], + "launch": "Exact bundle executable; LaunchServices and the installed app were not used.", + "state_isolation": "Each sample received a unique CODEVETTER_APP_DATA_DIR.", + "surface_confirmation": { + "native": "Measure what changed.", + "tauri": "Choose a workload to measure", + "confirmed_samples": 10 + }, + "resource_scope": "One settled recursively owned process-tree observation after the surface marker appeared." + }, + "applications": { + "native": { + "bundle_identifier": "com.codevetter.desktop.native-preview", + "version": "1.11.0", + "build": "11100", + "executable": "CodeVetterNative", + "executable_bytes": 17198368, + "bundle_kib": 80116, + "package_receipt": "artifacts/native-package/qualification-jUbwnc/qualification.json" + }, + "tauri": { + "bundle_identifier": "com.codevetter.desktop", + "version": "1.11.0", + "build": "1.11.0", + "executable": "codevetter-desktop", + "executable_bytes": 74639312, + "bundle_kib": 165144, + "package_source": "apps/desktop/src-tauri/target/release/bundle/macos/CodeVetter.app" + } + }, + "metrics": { + "first_visible_window_ms": { + "native": { + "samples": [524.982, 436.857, 342.756, 435.12, 317.447], + "minimum": 317.447, + "median": 435.12, + "average": 411.432, + "p95": 524.982, + "maximum": 524.982 + }, + "tauri": { + "samples": [426.733, 419.018, 388.931, 588.98, 373.399], + "minimum": 373.399, + "median": 419.018, + "average": 439.412, + "p95": 588.98, + "maximum": 588.98 + }, + "median_delta_ms": 16.102, + "native_to_tauri_median_ratio": 1.038, + "classification": "parity", + "claim": "No material first-visible-window improvement is claimed." + }, + "settled_process_tree_rss_kib": { + "native": { + "samples": [117440, 117424, 117120, 117920, 116928], + "minimum": 116928, + "median": 117424, + "average": 117366.4, + "p95": 117920, + "maximum": 117920 + }, + "tauri": { + "samples": [169024, 159568, 136288, 180864, 183168], + "minimum": 136288, + "median": 169024, + "average": 165782.4, + "p95": 183168, + "maximum": 183168 + }, + "median_delta_kib": -51600, + "native_to_tauri_median_ratio": 0.695, + "native_reduction_percent": 30.529, + "classification": "improved", + "claim": "The native Performance workspace used 30.5% less settled process-tree RSS at the median." + }, + "bundle_kib": { + "native": 80116, + "tauri": 165144, + "native_to_tauri_ratio": 0.485, + "native_reduction_percent": 51.486, + "classification": "improved" + }, + "host_executable_bytes": { + "native": 17198368, + "tauri": 74639312, + "native_to_tauri_ratio": 0.23, + "native_reduction_percent": 76.958, + "classification": "improved" + } + }, + "qualification": { + "all_samples_reached_expected_surface": true, + "all_samples_terminated_cleanly": true, + "installed_application_touched": false, + "startup_non_regression_gate": true, + "settled_rss_improvement_gate": true, + "bundle_size_improvement_gate": true, + "native_performance_claim": "Qualified only for lower settled Performance-workspace RSS and smaller package/executable footprint; startup remains parity." + }, + "reproduction": { + "script": "scripts/compare-desktop-runtime.mjs", + "test": "pnpm test:native-runtime-compare", + "command": "pnpm native:runtime:compare -- --native-app --tauri-app --runs 5 --settle-ms 5000 --out ", + "raw_artifact": "artifacts/performance/native-tauri-comparison-current.json" + }, + "limitations": [ + "First visible window is observed through macOS Accessibility and is not XCTest ApplicationFirstFramePresentationResponsive.", + "The Tauri Performance route is selected through its shipped Command-K palette; native uses its repository-owned qualification launch argument.", + "RSS is one settled observation and may miss short-lived descendants between launch and the five-second sample.", + "CODEVETTER_APP_DATA_DIR isolates SQLite and generated artifacts. The incumbent Tauri release still uses its normal WebKit data store and normal release background behavior.", + "The comparison does not measure scrolling, workload execution, energy, long-session behavior, signing/notarization, installed updates, or rollback." + ] +} diff --git a/evidence/performance/native-testing-benchmark.json b/evidence/performance/native-testing-benchmark.json new file mode 100644 index 00000000..32e85f6c --- /dev/null +++ b/evidence/performance/native-testing-benchmark.json @@ -0,0 +1,35 @@ +{ + "schema_version": "codevetter.native-testing-benchmark/v1", + "recorded_at": "2026-08-31", + "target": "aarch64-apple-darwin", + "command": "CODEVETTER_TESTING_SCREENSHOT_PATH=artifacts/design/native-testing-receipt-dark.png npx -y xcodebuildmcp@2.7.0 swift-package test --package-path apps/macos/CodeVetterPackage --filter hundredJourneyTestingReceiptDecodesAndRendersWithinTheNativeGate", + "dataset": { + "journeys": 100, + "changed_paths": 100, + "derived_routes": 6, + "limitations": 1, + "viewport": "980x640" + }, + "measurement": { + "swift_decode": { + "warmups": 10, + "samples": 100, + "p95_us": 510, + "gate_us": 25000 + }, + "swift_host_render": { + "warmups": 3, + "samples": 20, + "p95_us": 19271, + "gate_us": 150000 + } + }, + "qualification": { + "swift_package_test": "passed", + "macos_test": "16 passed, 0 failed, 0 skipped", + "dark_setup": "artifacts/design/native-testing-dark.png", + "light_setup": "artifacts/design/native-testing-light.png", + "dark_receipt": "artifacts/design/native-testing-receipt-dark.png" + }, + "claim_boundary": "Measures canonical T-Rex JSON decoding and fresh NSHostingView creation, layout, and display for one 100-journey, 100-path receipt. It excludes CLI startup, Rust execution, Git resolution, preview network latency, browser runtime, window-server frame pacing, and interactive scrolling." +} diff --git a/evidence/performance/native-unpack-benchmark.json b/evidence/performance/native-unpack-benchmark.json new file mode 100644 index 00000000..4f098856 --- /dev/null +++ b/evidence/performance/native-unpack-benchmark.json @@ -0,0 +1,36 @@ +{ + "schema_version": "codevetter.native-unpack-benchmark/v1", + "recorded_at": "2026-08-31", + "target": "aarch64-apple-darwin", + "command": "CODEVETTER_UNPACK_SCREENSHOT_PATH= CODEVETTER_UNPACK_LIGHT_SCREENSHOT_PATH= npx -y xcodebuildmcp@2.7.0 swift-package test --package-path apps/macos/CodeVetterPackage --filter largeUnpackProjectionDecodesAndRendersWithinTheNativeGate", + "dataset": { + "snapshots": 100, + "graph_nodes": 700, + "graph_edges": 699, + "tree_rows": 1000, + "viewport": "1280x800" + }, + "measurement": { + "swift_decode": { + "warmups": 10, + "samples": 100, + "p95_us": 5179, + "gate_us": 40000 + }, + "swift_host_render": { + "warmups": 3, + "samples": 20, + "p95_us": 22921, + "gate_us": 150000 + } + }, + "qualification": { + "swift_package_test": "23 passed, 0 failed", + "macos_test": "29 passed, 0 failed, 0 skipped", + "repo_unpack_xcuitest": "read-only Rust authority and refresh controls passed", + "dark_render": "artifacts/design/native-unpack-dark.png", + "light_render": "artifacts/design/native-unpack-light.png", + "live_read_only_smoke": "Stored CodeVetter snapshot listed and inspected without exposing raw all_files" + }, + "claim_boundary": "Measures UnpackHistoryReceipt and UnpackSnapshotRecord decoding, nested inventory decoding, and fresh NSHostingView creation, layout, and display for a bounded fixture. It excludes CLI startup, SQLite query latency, repository scanning, analysis, graph queries, window-server frame pacing, and interactive scrolling." +} diff --git a/evidence/performance/native-usage-benchmark.json b/evidence/performance/native-usage-benchmark.json new file mode 100644 index 00000000..a072f433 --- /dev/null +++ b/evidence/performance/native-usage-benchmark.json @@ -0,0 +1,37 @@ +{ + "schema_version": "codevetter.native-usage-benchmark/v1", + "recorded_at": "2026-08-31", + "target": "aarch64-apple-darwin", + "command": "CODEVETTER_USAGE_SCREENSHOT_PATH= CODEVETTER_USAGE_LIGHT_SCREENSHOT_PATH= npx -y xcodebuildmcp@2.7.0 swift-package test --package-path apps/macos/CodeVetterPackage --filter largeUsageReportDecodesAndRendersWithinTheNativeGate", + "dataset": { + "daily_periods": 365, + "weekly_periods": 52, + "monthly_periods": 12, + "sessions": 100, + "agents": 3, + "viewport": "980x640" + }, + "measurement": { + "swift_decode": { + "warmups": 10, + "samples": 100, + "p95_us": 7727, + "gate_us": 25000 + }, + "swift_host_render": { + "warmups": 3, + "samples": 20, + "p95_us": 18346, + "gate_us": 150000 + } + }, + "qualification": { + "swift_package_test": "20 passed, 0 failed", + "macos_test": "25 passed, 0 failed, 0 skipped", + "usage_xcuitest": "provider boundaries passed independently of filesystem scan duration", + "dark_render": "artifacts/design/native-usage-dark.jpeg", + "light_render": "artifacts/design/native-usage-light.png", + "headless_usage": "Real local ready report observed with exit 0" + }, + "claim_boundary": "Measures canonical LocalUsageReport JSON decoding and fresh NSHostingView creation, layout, and display for a bounded 365-period and 100-session fixture. It excludes CLI and ccusage startup, filesystem scanning, window-server frame pacing, interactive scrolling, Devin activity, and live provider telemetry." +} diff --git a/evidence/performance/old-local-portfolio-manifest.json b/evidence/performance/old-local-portfolio-manifest.json new file mode 100644 index 00000000..36770752 --- /dev/null +++ b/evidence/performance/old-local-portfolio-manifest.json @@ -0,0 +1,17 @@ +{ + "schema_version": "runtime-qualification-portfolio-manifest/v1", + "repositories": [ + { "id": "aliveville", "path": "../../../../fleet-inactive-projects/aliveville" }, + { "id": "companion-robot", "path": "../../../../fleet-inactive-projects/companion-robot" }, + { "id": "elves-hq", "path": "../../../../fleet-inactive-projects/elves-hq" }, + { "id": "everythingrated", "path": "../../../../fleet-inactive-projects/everythingrated" }, + { "id": "forecast-lab", "path": "../../../../fleet-inactive-projects/forecast-lab" }, + { "id": "materia", "path": "../../../../fleet-inactive-projects/materia" }, + { "id": "open-historia", "path": "../../../../fleet-inactive-projects/open-historia" }, + { "id": "protein-index", "path": "../../../../fleet-inactive-projects/protein-index" }, + { "id": "saas-ideas", "path": "../../../../fleet-inactive-projects/saas-ideas" }, + { "id": "today-little-log", "path": "../../../../fleet-inactive-projects/today-little-log" }, + { "id": "truehire", "path": "../../../../fleet-inactive-projects/truehire" }, + { "id": "web-playables", "path": "../../../../fleet-inactive-projects/web-playables" } + ] +} diff --git a/evidence/performance/old-local-projects-results-2026-08-09.md b/evidence/performance/old-local-projects-results-2026-08-09.md new file mode 100644 index 00000000..e7bb1d21 --- /dev/null +++ b/evidence/performance/old-local-projects-results-2026-08-09.md @@ -0,0 +1,125 @@ +# Old local project performance pass — 2026-08-09 + +## Scope + +This pass inspected every locally checked-out Fleet project whose canonical +lifecycle is `past`. It deliberately used local execution only: no production +database, external API, Cloud Player, deployment, or cloud benchmark was run. +The historical `mashup` project was excluded because it has been incorporated +into Foundry and has no independent checkout. + +The qualification manifest is +`artifacts/performance/old-local-portfolio-manifest.json`. + +## Portfolio result + +Before adding a benchmark, CodeVetter classified the 12 checked-out projects as: + +- `ready`: 0 +- `needs_selection`: 8 +- `no_representative_workload`: 4 + +After the Web Playables benchmark was added, CodeVetter independently selected +that exact test and classified the repository as `ready`: + +- adapter: `vitest` +- target: `games/idle-startup/test/performance.test.ts` +- test: `scales deterministic company ticks` +- qualification score: 70 + +This is a useful product result: qualification can rediscover a good workload, +but it cannot manufacture representative workloads from generic correctness +tests. The portfolio initially had zero workloads with direct timing evidence. + +## Verified optimization + +### Web Playables — idle startup tick + +The benchmark advances deterministic game state for 1,000, 10,000, and 50,000 +ticks. The production tick function calculated gross revenue and burn, then +called `netPerSec()` for milestone checks, redundantly calculating both again. +The change reuses `gross - burn` within the same tick. + +Observed CodeVetter comparison: + +| Scale | Baseline | Optimized | Change | +| ---: | ---: | ---: | ---: | +| 1,000 ticks | 3.613 ms | 2.807 ms | -22.31% | +| 10,000 ticks | 21.666 ms | 12.338 ms | -43.05% | +| 50,000 ticks | 104.190 ms | 57.964 ms | -44.37% | + +The enclosing Vitest wall time improved from 831 ms to 655 ms (-21.18%). +CodeVetter returned `confirmed`, `mechanically_confirmed: true`, and +`materially_useful: true`. It correctly withheld `shipping_recommended` +because the baseline had only three samples and independent Vitest profiles did +not all contain application source frames. + +Validation after the change: + +- 34 tests passed (27 simulation, 6 prestige, 1 performance) +- TypeScript `tsc --noEmit` passed +- Biome passed for the changed production and benchmark files +- `git diff --check` passed + +The checkout was clean before the experiment. It now contains only the scoped +simulation change, the new benchmark, and the local `.codevetter` evidence. +The lockfile-existing dependencies were installed locally with scripts disabled; +69 packages came from cache and four were downloaded. No dependency was added. + +## Project-by-project disposition + +| Project | Qualification / inspection result | Action | +| --- | --- | --- | +| Aliveville | Generic Vitest tests suggested simulation work, but no timed workload existed and the Web3D dependency tree was not installed. | Inspected the world tick and catch-up paths; no evidence-backed change made. Best next workload is a deterministic multi-NPC catch-up benchmark. | +| Companion Robot | No representative code/test workload; effectively a project shell. | No optimization to make. | +| Elves HQ | Build/dev workspace with no benchmark or test workload. | No evidence-backed optimization to make. | +| EverythingRated | Generic tests only; checkout already had 109 dirty entries. | Preserved existing work and made no speculative edit. | +| Forecast Lab | No representative code/test workload; effectively a project shell. | No optimization to make. | +| Materia | Generic content tests only; checkout already had 18 dirty entries. | Preserved existing work and made no speculative edit. | +| Open Historia | Correctness tests, no timed workload. Storage restore already uses keyed lookup rather than a quadratic scan. | No material local hotspot found; no edit made. | +| Protein Index | Highest-signal work depends on database/network behavior. | Excluded from this backend-independent pass. | +| SaaS Ideas | No representative workload; checkout already had three dirty entries. | Preserved existing work and made no speculative edit. | +| Today Little Log | Browser/auth-oriented tests and lifecycle is deleted. | Excluded from local CPU optimization. | +| TrueHire | Generic tests only; likely runtime cost is external repository fetching and inputs are small. | A synthetic scoring microbenchmark would not be representative, so no edit was made. | +| Web Playables | Newly qualified deterministic tick benchmark. | Verified 44.37% improvement at 50,000 ticks. | + +## CodeVetter gaps exposed + +1. **Representative workload discovery is the limiting step.** File and test + names are not enough; direct timing evidence was absent in all 12 projects. +2. **Vitest source profiling is incomplete.** V8 CPU profiling captured the + Vitest/Vite runner process, not useful application frames from the fork that + executed the test. The measurement and before/after verdict worked, but the + tool did not locate the redundant calculation by itself. +3. **Dirty-snapshot qualification is coarse.** The repository is marked dirty, + but untracked benchmark/evidence identity needs to remain explicit in every + receipt. +4. **The shipping gate is usefully conservative.** It did not turn a three-sample + baseline into a shipping claim even after a large observed improvement. + +## One Billion Row Challenge status + +The current CodeVetter artifact is a bounded Node parser experiment, not an +official 1BRC implementation: + +- largest measured input: 800,000 rows +- latest parser-only time: 34.325 ms (the retained supervised run was 32.465 ms) +- latest throughput: approximately 23.31 million rows/second +- naive linear parser-only projection to one billion rows: 42.91 seconds +- official 8-core winner: 1.535 seconds +- optimistic gap: 27.95 times slower, or 3.58% of the winner's throughput + +That gap is intentionally labelled optimistic. The current metric excludes file +I/O and startup, uses an in-memory string and a small generated station set, and +the file entry point reads the entire input with `readFile(..., "utf8")`. It is +therefore not yet capable of a credible official 12 GB run. Cross-machine +wall-clock results are not directly comparable either. + +The next honest milestone is not a smaller microbenchmark number. It is a +streaming or chunked parser that can process the official file with bounded +memory, followed by an end-to-end same-machine benchmark against a baseline. + +## Change policy + +No commit, push, deployment, production configuration, or cloud resource was +created by this pass. diff --git a/evidence/reviews/pr157-codevetter-review.png b/evidence/reviews/pr157-codevetter-review.png new file mode 100644 index 00000000..876be9ea Binary files /dev/null and b/evidence/reviews/pr157-codevetter-review.png differ diff --git a/evidence/reviews/pr173-codevetter-review.png b/evidence/reviews/pr173-codevetter-review.png new file mode 100644 index 00000000..31321657 Binary files /dev/null and b/evidence/reviews/pr173-codevetter-review.png differ diff --git a/evidence/security/actionlint-baseline-2026-08-31.md b/evidence/security/actionlint-baseline-2026-08-31.md new file mode 100644 index 00000000..3b446dff --- /dev/null +++ b/evidence/security/actionlint-baseline-2026-08-31.md @@ -0,0 +1,45 @@ +# Actionlint and ShellCheck qualification — 2026-08-31 + +This receipt qualifies repository workflow validation; it is not product +runtime evidence. + +## Tool identity + +| Tool | Version | License | Qualified artifact | +|---|---|---|---| +| actionlint | 1.7.12 | MIT | Darwin arm64 archive SHA-256 `aba9ced2dee8d27fecca3dc7feb1a7f9a52caefa1eb46f3271ea66b6e0e6953f` | +| ShellCheck | 0.11.0 | GPL-3.0 | Linux x86-64 archive SHA-256 `b7af85e41cc99489dcc21d66c6d5f3685138f06d34651e6d34b42ec6d54fe6f6` | + +The CI lane uses actionlint's publisher-provided Linux x86-64 checksum +`8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8`. +ShellCheck's release does not publish a checksum file; its hardcoded digest was +qualified from the GitHub release asset fetched through the authenticated +GitHub API. + +## First audit + +The initial full-workflow audit found three findings: + +- `ci.yml`: SC2046 identified an unquoted computed ccusage binary path. The + target is now captured separately and the complete executable path is quoted. +- `weekly.yml`: two SC2129 style findings identified repeated redirects to + `GITHUB_OUTPUT` and `GITHUB_STEP_SUMMARY`. Each set now uses one bounded group + redirect. + +The complete rerun exits zero with no ignored rules. The permanent +`repository-security.yml` job downloads both exact binaries, verifies their +hashes, and runs actionlint with ShellCheck discovery enabled. No application +dependency or product-runtime invocation was added. + +## Companion pedantic zizmor audit + +Running zizmor 1.29.0 with `--offline --pedantic` after actionlint exposed one +remaining shell-template expansion, workflow-level write permissions, missing +concurrency controls, undocumented permission purposes, and unnamed jobs. The +workflows now pass the structural findings. The remaining output is the four +low-confidence cache-poisoning heuristics on trusted deploy/release workflows +plus two informational suggestions to replace the pinned Rust toolchain action +with runner shell commands. The publish jobs now disable or omit dependency and +toolchain caches, eliminating all four cache findings. The informational +suggestions were not applied: retaining one maintained, commit-pinned setup +action is clearer than duplicating toolchain setup shell logic. diff --git a/evidence/security/ast-grep-sarif-qualification-2026-08-31.md b/evidence/security/ast-grep-sarif-qualification-2026-08-31.md new file mode 100644 index 00000000..b3ef72e6 --- /dev/null +++ b/evidence/security/ast-grep-sarif-qualification-2026-08-31.md @@ -0,0 +1,40 @@ +# ast-grep SARIF qualification — 2026-08-31 + +## Scope + +This trial evaluates ast-grep's native SARIF output before considering it for +repository code-scanning upload. It does not evaluate ast-grep's matching +engine generally and does not add a rule pack. + +## Tool and fixture + +- Tool: ast-grep 0.45.2 +- License: MIT +- Rule: an inline TypeScript pattern matching + `localStorage.setItem($KEY, $VALUE)` +- Target: `apps/desktop/src/lib/review-service.ts` + +The matcher found both expected calls with exact file, line, column, byte, and +snippet locations. That establishes useful structural-match fidelity for this +fixture. + +## SARIF result + +The `--format sarif` envelope is not valid SARIF 2.1.0: + +- the root `version` is `0.45.2`, which is the ast-grep tool version rather + than the required SARIF format version `2.1.0`; +- the root `$schema` declaration is absent; +- the tool driver omits its own version even though that information was + written into the incompatible root field. + +Locations, rule IDs, severity, and messages are otherwise present. The output +should not be uploaded to GitHub code scanning or treated as interoperable +SARIF without a repository-owned repair step and schema validation. + +## Decision + +Trialled, not wired. Biome and CodeQL already cover the current repository +lanes. Add ast-grep only when a concrete structural rule closes a demonstrated +gap, and either upstream emits a conforming envelope or a bounded converter is +qualified with fixtures. diff --git a/evidence/security/cargo-deny-baseline-2026-08-31.md b/evidence/security/cargo-deny-baseline-2026-08-31.md new file mode 100644 index 00000000..204ab567 --- /dev/null +++ b/evidence/security/cargo-deny-baseline-2026-08-31.md @@ -0,0 +1,54 @@ +# cargo-deny policy qualification — 2026-08-31 + +## Scope + +This receipt qualifies cargo-deny as a repository policy gate for the Rust +desktop dependency graph. It does not add cargo-deny to the shipped desktop +binary and does not fetch an advisory database during the check. + +## Tool identity + +| Tool | Version | License | Qualified artifacts | +|---|---:|---|---| +| cargo-deny | 0.20.2 | MIT OR Apache-2.0 | macOS arm64 SHA-256 `fe67d82a10d8597a3549364cb733a3f9cc1bfff9031b7ae46384a9f2a72090c3`; CI Linux x86-64 musl SHA-256 `9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f` | + +Both digests match the release assets published for cargo-deny 0.20.2. The +macOS archive digest was reproduced after download before executing the binary. + +## Policy + +The tracked `apps/desktop/src-tauri/deny.toml` policy: + +- evaluates the shipped `aarch64-apple-darwin` graph; +- allows the 15 permissive or file-level-copyleft SPDX families currently + present in the graph; +- ignores unpublished workspace packages and marks the desktop crate + `publish = false` so it cannot be published accidentally; +- denies wildcard dependency requirements; +- denies unknown registries and all Git dependencies; +- reports duplicate transitive versions as warnings rather than pretending + Tauri-owned convergence is immediately actionable. + +## Baseline result + +The offline, locked policy run exits zero: + +```text +bans ok, licenses ok, sources ok +``` + +The graph contains 14 duplicate-version warnings. They are visible in command +output but are not uploaded as code-scanning alerts. The SARIF lane covers +license and source-policy violations, while the bans lane remains an enforced +human-readable check. cargo-deny 0.20.2 emits native SARIF 2.1.0; this corrects +the earlier documentation claim that only JSON output was available. + +## Maintained path + +- `pnpm quality:rust-policy` runs the full offline policy when the pinned tool + is installed locally. +- `repository-security.yml` downloads the exact Linux archive, verifies its + SHA-256, uploads license/source SARIF, and separately enforces bans. +- RustSec advisory scanning remains in the OSV lane until the existing + vulnerability baseline is remediated; this policy gate does not duplicate + or suppress those findings.