diff --git a/devlog/_plan/260807_compatibility_lab/000_master_plan.md b/devlog/_plan/260807_compatibility_lab/000_master_plan.md index c34c30bdc..a26466588 100644 --- a/devlog/_plan/260807_compatibility_lab/000_master_plan.md +++ b/devlog/_plan/260807_compatibility_lab/000_master_plan.md @@ -263,19 +263,23 @@ request path. ## Programme phases -Only CL-00 is authorized by this document at present. +Programme authorization is tracked below. CL-09 contract drafting is authorized by merged CL-08; CL-09 runtime implementation remains gated on acceptance of the CL-09 contract. | Phase | Purpose | Authorization | |---|---|---| -| CL-00 | Architecture authority, contracts, scenario catalogue, incident corpus | This PR | -| CL-01 | Deterministic protocol-conformance runner and fixtures | Not started; requires CL-00 to be accepted | -| CL-02 | Immutable JSONL ledger, artifacts and SQLite projection | Not started | -| CL-03 | Bounded live-route probes | Not started | -| CL-04 | Lab CLI and management read surfaces | Not started | -| CL-05 | Compatibility Matrix UI | Not started | -| CL-06 | Existing Routing Profile compatibility controls and Router Intelligence consumption | **ACCEPTED/CLOSED** — merged #1394 at `b66e33ce7207d91014644d99317e456c992a3418` | -| CL-07 | Agent Fabric task-effectiveness ingestion | **ACCEPTED/CLOSED** — merged #1438 at `02e62fc8c7354c544ef71f8bb3db5ebba42cb600` | -| CL-08 | Shadow/automatic/public evidence workflows | Not started | +| CL-00 | Architecture authority, contracts, scenario catalogue, incident corpus | **ACCEPTED/CLOSED** - merged #1286 | +| CL-01 | Deterministic protocol-conformance runner and fixtures | **ACCEPTED/CLOSED** - merged #1320 | +| CL-02 | Immutable JSONL ledger, artifacts and SQLite projection | **ACCEPTED/CLOSED** - merged #1333 plus hardening/closure | +| CL-03 | Bounded live-route probes | **ACCEPTED/CLOSED** - merged #1352 | +| CL-04 | Lab CLI and management read surfaces | **MERGED** - #1378 | +| CL-05 | Compatibility Matrix UI | **MERGED** - #1384 | +| CL-06 | Existing Routing Profile compatibility controls and Router Intelligence consumption | **ACCEPTED/CLOSED** - merged #1394 at `b66e33ce7207d91014644d99317e456c992a3418` | +| CL-07 | Agent Fabric task-effectiveness ingestion | **ACCEPTED/CLOSED** - merged #1438 at `02e62fc8c7354c544ef71f8bb3db5ebba42cb600` | +| CL-08 | Bounded automatic evidence refresh/orchestration | **ACCEPTED/CLOSED** - merged #1447 at `3b8f9487676fe258d76295e49e7db75aca26a4cb` | +| CL-09 | Passive production-evidence correlation with zero extra traffic and no routing feedback | **CONTRACT DRAFT** - #1489; implementation not authorized | +| CL-10 | Public export/publishing/community evidence | Not started; separate privacy/trust boundary | + +The original CL-00 planning bucket combined shadow, automatic, and public evidence workflows. Accepted later plans split that bucket deliberately: CL-08 owns bounded automation, CL-09 defines passive production evidence, and public publishing remains separate CL-10 work. Phase numbering after CL-01 is programme planning, not implementation authorization. A later accepted plan may split a phase while preserving these diff --git a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md index a6a705cab..e74d1d9b5 100644 --- a/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md +++ b/devlog/_plan/260807_compatibility_lab/001_pr_stack_status.md @@ -27,6 +27,9 @@ independent review, blockers, and whether a later phase is authorized. | CL-05 | `feat/cl-05-compatibility-matrix-ui` | `d517161aeaa3a974ad3c0360ff0c97b03b4c4520` | `2a159b8b7` (Models tab placement) | [#1384](https://github.com/lidge-jun/opencodex/pull/1384) | MERGED TO `dev` at `1072b9c39c48a4982229131613ac300560740742` | | CL-06 | `feat/cl-06-routing-profile-compatibility` | `1072b9c39c48a4982229131613ac300560740742` | `b96eae83f2a6d1654472aeeef84799070743aeb8` | [#1394](https://github.com/lidge-jun/opencodex/pull/1394) | MERGED TO `dev` at `b66e33ce7207d91014644d99317e456c992a3418`; ACCEPTED/CLOSED | | CL-07 | `feat/cl-07-task-effectiveness-producer` | `b66e33ce7207d91014644d99317e456c992a3418` | `0efe2c69514d3baefee686383fe740e4ecb37d83` | [#1438](https://github.com/lidge-jun/opencodex/pull/1438) | MERGED TO `dev` at `02e62fc8c7354c544ef71f8bb3db5ebba42cb600`; ACCEPTED/CLOSED | +| CL-08 | `feat/cl-08-lab-automation` | `da8ebd3135553c1d4dd85c1f258e998a5de14f28` | `bfaad5d01a975e8d48b9437bc0a0537077a04134` | [#1447](https://github.com/lidge-jun/opencodex/pull/1447) | MERGED TO `dev` at `3b8f9487676fe258d76295e49e7db75aca26a4cb`; ACCEPTED/CLOSED | +| CL-09 | `feat/cl-09-passive-production-evidence` | `3b8f9487676fe258d76295e49e7db75aca26a4cb` | IMPLEMENTATION REVIEW CANDIDATE | [#1489](https://github.com/lidge-jun/opencodex/pull/1489) | IMPLEMENTED; independent final review and merge acceptance pending | + The CL-01 starting SHA is the exact CL-00 tip recorded when CL-01 began. Its moving base-ref name is not a substitute for that historical SHA. @@ -164,7 +167,8 @@ Claims cannot produce `PROBED`/`VERIFIED`. - CL-05: **MERGED** via #1384 at `1072b9c39c48a4982229131613ac300560740742`. - CL-06: **ACCEPTED/CLOSED** via [#1394](https://github.com/lidge-jun/opencodex/pull/1394), merged to `dev` at `b66e33ce7207d91014644d99317e456c992a3418`. - CL-07: **ACCEPTED/CLOSED** via [#1438](https://github.com/lidge-jun/opencodex/pull/1438), merged to `dev` at `02e62fc8c7354c544ef71f8bb3db5ebba42cb600`; accepted head `0efe2c69514d3baefee686383fe740e4ecb37d83`; plan `007_cl07_task_effectiveness.md`. -- CL-08: **not started**. +- CL-08: **ACCEPTED/CLOSED** via [#1447](https://github.com/lidge-jun/opencodex/pull/1447), merged to `dev` at `3b8f9487676fe258d76295e49e7db75aca26a4cb`; final source head `bfaad5d01a975e8d48b9437bc0a0537077a04134`; plan `008_cl08_automation.md`. +- CL-09: **IMPLEMENTED / REVIEW PENDING** via [#1489](https://github.com/lidge-jun/opencodex/pull/1489); the phase started from CL-08 merge `3b8f9487676fe258d76295e49e7db75aca26a4cb`, runtime work was rebased to then-current `dev@e8db4e0365b12a314d1c08ec2cf81599efe5b2d3`, and independent final review plus merge acceptance remain pending. ## CL-06 closure log @@ -179,7 +183,26 @@ Claims cannot produce `PROBED`/`VERIFIED`. - **Accepted / source head:** `0efe2c69514d3baefee686383fe740e4ecb37d83` - **Starting/base SHA:** `b66e33ce7207d91014644d99317e456c992a3418` (CL-06 merge #1394) - **Scope delivered:** bounded `src/lab/fabric/` task-effectiveness producer, exact-tree-diff verifier, scratch sandbox, trusted-route persistence boundary, isolated child producer with parent-owned IPC/timeouts. -- **CL-08:** not started (explicit non-goal). +- **CL-08:** completed and merged via #1447. + +## CL-08 closure log + +- **Merge commit on `dev`:** `3b8f9487676fe258d76295e49e7db75aca26a4cb` ([#1447](https://github.com/lidge-jun/opencodex/pull/1447)) +- **Final / source head:** `bfaad5d01a975e8d48b9437bc0a0537077a04134` +- **Original starting/base SHA:** `da8ebd3135553c1d4dd85c1f258e998a5de14f28`; final source branch was rebased onto then-current `dev` before merge. +- **Scope delivered:** bounded default-off Lab automation, deterministic planner/queue/recovery, budgets/cooldowns, CL-01 and trusted CL-03 dispatch, management API/CLI controls, owner-scoped server lifecycle, atomic policy/routes configuration, and adversarial regression coverage. +- **Task-effectiveness background:** deliberately remained disabled; manual CL-07 execution unchanged. +- **CL-09:** contract drafting authorized from exact CL-08 merge. + +## CL-09 start log (2026-08-11) + +- **Starting/base SHA:** `3b8f9487676fe258d76295e49e7db75aca26a4cb` (exact CL-08 merge #1447) +- **Branch:** `feat/cl-09-passive-production-evidence` +- **PR:** [#1489](https://github.com/lidge-jun/opencodex/pull/1489) (draft, contract-only at open) +- **Plan:** `009_cl09_passive_production_evidence.md` +- **Scope:** exact per-attempt local route-subject correlation for already-completed production traffic, bounded read-side passive signals, and additive Lab read surfaces with zero extra provider requests. +- **Hard boundary:** V1 passive signals do not write CL-02 observations, change compatibility verdicts/freshness, affect CL-06 routing, or trigger CL-08 automation. +- **Explicitly out of scope:** duplicated shadow requests, prompt/response capture, Shadow Call Intercept changes, direct passive-to-verdict promotion, and public publishing/CL-10. ## CL-07 start log diff --git a/devlog/_plan/260807_compatibility_lab/009_cl09_passive_production_evidence.md b/devlog/_plan/260807_compatibility_lab/009_cl09_passive_production_evidence.md new file mode 100644 index 000000000..85a924441 --- /dev/null +++ b/devlog/_plan/260807_compatibility_lab/009_cl09_passive_production_evidence.md @@ -0,0 +1,555 @@ +# CL-09 - Passive Production Evidence / Exact-Route Correlation + +## Programme position + +**Repository:** `lidge-jun/opencodex` +**Integration target:** `dev` +**Branch:** `feat/cl-09-passive-production-evidence` +**Starting SHA:** `3b8f9487676fe258d76295e49e7db75aca26a4cb` +**CL-08 merge prerequisite:** satisfied by #1447 at `3b8f9487676fe258d76295e49e7db75aca26a4cb` + +CL-08 is merged and closed. This document defines the next Compatibility Lab boundary. + +The CL-09 contract was frozen first on this branch. Runtime implementation is now complete on this branch and remains subject to independent final review before merge. + +--- + +# 1. Goal + +CL-09 answers: + +> How can already-completed production requests contribute privacy-safe, exact-route operational evidence to Compatibility Lab read surfaces without issuing extra provider requests, ingesting user content, changing canonical compatibility verdicts, or creating a routing feedback loop? + +The V1 architecture is deliberately passive: + +```text +Production request + | + v +Existing route decision + per-attempt outcome + final usage row + | + | capture exact local Lab route-subject ID only + v +Bounded passive-signal adapter + | + v +Read-side production-signal projection + | + +--> Lab CLI/API/UI correlation + | + X--> no CL-02 canonical observation in V1 + X--> no Routing Profile / Router Intelligence input in V1 + X--> no CL-08 scheduling trigger +``` + +CL-09 V1 observes work that already happened. It does not create work. + +--- + +# 2. Naming boundary + +OpenCodex already has a feature called **Shadow Call Intercept** in `src/lib/shadow-call.ts` and `docs/shadow-call-intercept.md`. That feature rewrites specific Codex helper-model requests. + +CL-09 is unrelated. + +CL-09 must not: + +- change `shadowCallIntercept` matching or rewrite behavior; +- use helper/shadow model detection as an evidence source; +- route production requests to a second model; +- duplicate a production request for comparison; +- overload the existing Shadow Call Intercept configuration. + +Within implementation code and APIs, prefer `passive production evidence`, `production signal`, or `passive signal` over the ambiguous term `shadow call`. + +--- + +# 3. Chosen V1 approach + +Three approaches were considered. + +## 3.1 Chosen: passive correlation only + +Use metadata already produced by normal request execution and add only the minimum exact Lab subject linkage required for deterministic correlation. + +Benefits: + +- zero extra provider traffic; +- no duplicate quota/cost; +- no user request replay; +- no new prompt/response retention; +- no compatibility self-reinforcement in routing; +- no CL-02 event-schema change required for V1. + +## 3.2 Deferred: direct passive observation promotion + +A future phase may define scenario manifests whose assertions can be evaluated entirely from a closed, privacy-safe production metadata contract and may add a distinct passive execution mode or event schema version. + +That work is not CL-09 V1 because current `ObservationEvent` semantics are scenario/fixture execution semantics and current execution modes are `fixture`, `live`, and `fabric`. Arbitrary user traffic is not equivalent to a reviewed Lab-owned synthetic scenario. + +## 3.3 Rejected: duplicate shadow execution + +CL-09 must not replay, fork, mirror, sample, or duplicate user requests to another route. That would add provider traffic, copy user content, complicate consent and credential boundaries, and violate the no-production-path-execution invariant. + +--- + +# 4. Hard V1 invariants + +CL-09 V1 must guarantee: + +```text +0 extra provider requests +0 duplicated user requests +0 outbound production request or user-payload mutation +0 new routing candidates +0 Routing Profile changes +0 Router Intelligence score changes +0 CL-08 scheduling decisions from passive signals +0 canonical Lab verdict changes from passive signals +0 prompt or conversation ingestion +0 response-body ingestion +0 tool-payload ingestion +0 credential/account identity ingestion +0 public publishing +``` + +A failure in passive evidence capture must never fail or delay the production request. + +The invariant against production-request mutation applies to outbound request bytes and user payloads. The metadata-only addition of `labRouteSubjectId` to the existing attempt record is explicitly allowed. + +--- + +# 5. Existing authorities remain unchanged + +CL-09 must reuse rather than replace: + +- `PersistedUsageEntry` and `PersistedUsageAttempt` as production execution history; +- `RouteDecisionTraceV1` as why-this-route authority; +- CL-02 JSONL as canonical Lab compatibility evidence; +- CL-02 SQLite as disposable Lab compatibility projection; +- exact `RouteSubjectV1` / `subjectId` construction from the existing Lab subject boundary; +- CL-04 read surfaces; +- CL-05 Compatibility Matrix UI; +- CL-06 compatibility policy and routing consumption; +- CL-08 automation orchestration. + +CL-09 creates no second request log, route trace, compatibility ledger, or verdict system. + +--- + +# 6. Exact route correlation + +Production outcomes are useful only when they are attributable to the exact route behavior that produced them. + +Request-level provider/model fields are insufficient because retries and fallback attempts may execute different routes. Therefore V1 correlation is per execution attempt. + +The implementation may extend the persisted attempt metadata with a local-only field approximately like: + +```ts +interface PersistedUsageAttempt { + // existing fields + labRouteSubjectId?: string; +} +``` + +The field is captured while the exact attempt route context still exists. It is the existing Lab route-subject digest/ID, not a new identity scheme. + +Rules: + +- capture the subject ID for each actual attempt independently; +- never infer an old attempt's subject from current config after the fact; +- do not backfill historical rows whose exact route subject was not captured; +- do not assume the initial route-decision candidate is the same route as a fallback attempt; +- a subject-construction failure omits passive linkage and does not affect request execution; +- no Lab SQLite or ledger read is allowed on the production request path. + +If the current route behavior fingerprint changes, new attempts receive the new subject ID. Old production signals remain attached only to the old exact subject. + +--- + +# 7. Passive signal model + +CL-09 V1 derives a bounded read-only signal from already-sanitized production metadata. + +Conceptually: + +```ts +interface PassiveRouteSignalV1 { + schemaVersion: 1; + subjectId: string; + source: "production_usage_v1"; + requestRef: string; + decisionRef?: string; + attemptOrdinal: number; + observedAt: number; + outcome: "success" | "client_cancel" | "route_error" | "environmental" | "unknown"; + httpStatus?: number; +} +``` + +This is a conceptual contract, not authorization to persist a second copy. + +The preferred V1 implementation derives these signals from the existing usage/history authority at read time or in an existing disposable history projection. + +The signal must not include: + +- `upstreamError` text; +- prompts/messages; +- response text/bodies; +- tool arguments/results; +- headers; +- URLs/IP addresses; +- `apiKeyId`; +- account references or account identity; +- conversation IDs in Lab-facing output; +- reasoning content; +- arbitrary provider diagnostics. + +--- + +# 8. Canonical compatibility boundary + +Passive production signals are **not** CL-02 `ObservationEvent` records in V1. + +They therefore cannot: + +- satisfy a scenario pass; +- refresh scenario freshness; +- make a suite `PROBED` or `VERIFIED`; +- make a suite `DEGRADED` or `UNSUPPORTED`; +- clear a `BLOCKED` verdict; +- participate in CL-06 minimum compatibility thresholds; +- cause CL-08 to enqueue or suppress a Lab run. + +The UI and API must label them clearly as **observed production traffic, not Lab verification**. + +This avoids claiming that an arbitrary user request exercised the exact synthetic assertions frozen by a scenario manifest. + +--- + +# 9. No routing feedback loop + +CL-09 V1 is read-side only from the perspective of routing semantics. + +The following components must not consume passive signals: + +- Routing Profile evaluator; +- Router Intelligence eligibility; +- Router Intelligence scoring; +- health/quota/cost weighting; +- model selection; +- provider discovery; +- fallback policy; +- CL-08 planner. + +The production request path may compute/carry the exact local subject ID for its own attempt log entry, but it must not query passive history or Lab compatibility state as part of CL-09. + +Any future use of passive evidence in routing requires a separate reviewed contract because production-observed traffic creates sampling and self-selection bias. + +--- + +# 10. Failure classification + +A production request failure is not automatically a compatibility failure. + +V1 passive classification is diagnostic only. + +Examples: + +- client cancellation => `client_cancel`; +- clearly normalized route/upstream terminal failure => `route_error` signal; +- known environment/admission failure => `environmental` signal; +- ambiguous HTTP/user/application outcome => `unknown`; +- completed successful route attempt => `success`. + +Generic 4xx/5xx status alone must not be interpreted as `UNSUPPORTED`, `DEGRADED`, or any other canonical Lab verdict. + +No LLM judge or content inspection is allowed to classify passive outcomes. + +--- + +# 11. Privacy and data minimization + +CL-00 security/privacy remains authoritative. + +CL-09 is specifically forbidden from reading or copying: + +- user prompts or conversation history; +- response bodies or generated text; +- user files/repositories/worktrees; +- tool/MCP payloads; +- hidden reasoning; +- raw provider errors; +- credentials, auth headers, tokens, cookies; +- account IDs/emails/aliases; +- raw custom headers; +- arbitrary URLs or filesystem paths. + +`requestRef` and `decisionRef` are local correlation references only. They are not public-export fields. + +The route subject ID remains installation-local and opaque. CL-09 does not export the subject salt or reverse-map it. + +Existing privacy scanning remains defense in depth. Tests must include canary prompt, credential, account, and response strings and prove none appear in passive Lab output. + +--- + +# 12. Persistence and retention + +V1 must not copy `usage.jsonl` rows into `compatibility.jsonl`. + +Preferred authority: + +```text +usage.jsonl / existing routing-history projection + | + v +bounded passive read projection +``` + +Rules: + +- no new canonical passive ledger; +- no raw production payload artifacts; +- no passive artifact store; +- retention follows the existing request/usage retention authority; +- when source request history is deleted, the passive signal disappears; +- corrupt or unparseable usage rows fail closed and are skipped; +- historical rows without an exact captured Lab subject ID remain unlinked rather than guessed. + +--- + +# 13. Read surfaces + +CL-09 should extend existing Lab read surfaces rather than create a separate product area. + +Useful subject-level summary fields are approximately: + +```text +recent production attempts +recent successful attempts +recent route-error signals +last observed production attempt +``` + +Requirements: + +- bounded time window and result count; +- deterministic pagination where detail is exposed; +- no network activity; +- no projection rebuild triggered by a read; +- no prompt/body/error-text exposure; +- explicit `not verification` labeling. + +The Compatibility Matrix may show a compact production-signal indicator beside canonical Lab evidence. It must not merge the two into one status or score. + +CLI/API naming must be audited against the existing CL-04 surfaces before implementation. + +--- + +# 14. Default behavior and configuration + +CL-09 V1 requires no new provider-traffic opt-in because it creates no provider traffic. + +Do not add a new configuration flag unless implementation audit finds a real retention or resource boundary that cannot be expressed through existing request-history controls. + +If usage/history persistence is disabled or unavailable, passive production evidence is simply unavailable. + +No configuration may enable direct verdict promotion in CL-09 V1. + +--- + +# 15. Production-path performance boundary + +The only CL-09 work permitted on a production attempt path is bounded exact-subject linkage using already-available trusted route context and adding the resulting opaque ID to the existing attempt record. + +Forbidden on the production path: + +- Lab ledger reads/writes for passive evidence; +- Lab SQLite queries/rebuilds; +- usage-history scans; +- scenario evaluation; +- passive aggregation; +- synchronous disk writes beyond the existing usage logging path; +- network calls; +- retries introduced by CL-09. + +Subject-link failure must be best-effort telemetry failure, never request failure. + +--- + +# 16. Backward compatibility + +Existing usage rows without passive subject linkage must continue to parse unchanged. + +Additive attempt metadata must remain optional. + +Do not rewrite existing usage history to invent exact historical subjects. + +Existing Lab event schema and execution modes remain unchanged in CL-09 V1. + +Existing Shadow Call Intercept behavior must be byte-for-byte semantically unchanged by CL-09. + +--- + +# 17. Adversarial tests + +Required coverage includes: + +## Zero extra traffic + +- enabling/using passive read surfaces does not increase provider send count; +- no duplicate request body is constructed or dispatched; +- CL-08 does not schedule from a passive signal. + +## Exact subject attribution + +- one attempt records its exact route subject ID; +- fallback attempts record different exact subjects when routes differ; +- a behavior-fingerprint change produces a new subject ID; +- old signals do not attach to the new subject; +- subject construction failure omits the link without failing the request. + +## Canonical evidence isolation + +- passive success does not change `UNKNOWN`/`CLAIMED`/`PROBED`/`VERIFIED`; +- passive failure does not produce `DEGRADED`/`UNSUPPORTED`; +- passive timestamps do not refresh scenario freshness; +- Routing Profile evaluation is identical with and without passive signals; +- Router Intelligence selection/score is identical with and without passive signals. + +## Privacy + +Seed canaries in: + +- prompt; +- response text; +- tool arguments/results; +- credentials; +- account metadata; +- raw error text. + +Assert none appear in: + +- passive API response; +- passive CLI output; +- Compatibility Matrix payload; +- Lab JSONL/SQLite/artifacts; +- logs/errors produced by CL-09. + +## Compatibility + +- old usage rows still parse; +- malformed passive subject IDs are ignored/fail closed; +- bounded pagination cannot scan unbounded history; +- deletion/retention of source usage removes passive visibility; +- Shadow Call Intercept tests remain unchanged and green. + +--- + +# 18. Delivery sequence + +## CL-09.0 - Audit and contract + +Completed first on this PR: + +- record CL-08 closure; +- audit current production usage/route evidence and Lab boundaries; +- freeze passive evidence semantics; +- explicitly reject duplicate shadow execution and direct verdict promotion. + +No runtime implementation was part of CL-09.0; CL-09.1 through CL-09.4 are implemented later on this PR. + +## CL-09.1 - Exact attempt subject linkage + +Implement the minimal optional exact Lab route-subject ID on persisted attempts. + +No passive UI/API yet. + +## CL-09.2 - Bounded passive query layer + +Implement read-side production-signal derivation with strict field allowlists, bounds, and no canonical Lab writes. + +## CL-09.3 - Existing Lab surfaces + +Expose compact passive summaries through existing Lab management/CLI/UI conventions. + +No new dashboard product area. + +## CL-09.4 - Adversarial isolation + +Prove privacy, zero extra traffic, routing invariance, backward compatibility, and cross-platform behavior. + +--- + +# 19. Explicit non-goals + +CL-09 V1 must not implement: + +- replayed/duplicated shadow requests; +- A/B production request mirroring; +- user prompt or response capture; +- canonical passive `ObservationEvent` creation; +- new Lab execution mode; +- scenario pass/fail from arbitrary user traffic; +- compatibility verdict promotion/degradation from passive signals; +- Routing Profile mutation; +- Router Intelligence behavior changes; +- CL-08 planner changes based on production traffic; +- provider metadata mutation; +- health/quota scoring changes; +- public export/publishing; +- community leaderboard; +- remote telemetry upload; +- Shadow Call Intercept changes. + +Public evidence export/publishing remains a separate later phase, provisionally CL-10, because it has a materially different privacy and trust boundary. + +--- + +# 20. Validation + +Contract PR minimum: + +```text +git diff --check +repository markdown / hygiene checks +CodeRabbit / independent review +``` + +Implementation phases must additionally run: + +```text +bun x tsc --noEmit +bun run privacy:scan +focused usage/request-log tests +focused Lab query/projection tests +routing/profile regressions +shadow-call regressions +full cross-platform CI +``` + +--- + +# 21. Acceptance criteria + +CL-09 V1 is accepted only when: + +1. CL-08 remains accepted/merged and current `dev` is the base; +2. production evidence causes zero extra provider requests; +3. each correlated attempt uses the exact captured Lab route subject ID; +4. fallback attempts cannot be misattributed to the original selected route; +5. historical rows are never backfilled by guessing current config; +6. user prompts/responses/tool payloads are never read or copied into Lab passive output; +7. credential/account material never enters passive output; +8. passive signals do not write canonical CL-02 observations; +9. passive signals do not change canonical compatibility verdicts/freshness; +10. passive signals do not affect Routing Profiles, Router Intelligence, health, fallback, or CL-08 scheduling; +11. old usage rows remain backward-compatible; +12. passive reads are bounded and do not trigger network/projection rebuild work; +13. existing Shadow Call Intercept behavior is unchanged; +14. privacy scan and focused adversarial tests pass; +15. full CI passes for implementation phases; +16. all valid CodeRabbit Critical/High/Medium findings are resolved; +17. independent final review reports `MERGE`. + +Do not start public publishing / CL-10 until CL-09 is accepted and merged. diff --git a/docs/superpowers/plans/2026-08-12-cl09-passive-production-evidence.md b/docs/superpowers/plans/2026-08-12-cl09-passive-production-evidence.md new file mode 100644 index 000000000..7ff499659 --- /dev/null +++ b/docs/superpowers/plans/2026-08-12-cl09-passive-production-evidence.md @@ -0,0 +1,70 @@ +# CL-09 Passive Production Evidence Implementation Plan + +**Goal:** Implement the approved CL-09 V1 contract on `feat/cl-09-passive-production-evidence` without changing canonical Compatibility Lab verdict semantics or production routing behavior. + +**Authority:** `devlog/_plan/260807_compatibility_lab/009_cl09_passive_production_evidence.md` + +**Current base:** `dev@e8db4e0365b12a314d1c08ec2cf81599efe5b2d3`, a later `dev` descendant of the CL-08 merge `3b8f9487676fe258d76295e49e7db75aca26a4cb` used as the implementation snapshot. + +## Task 1 - Exact production-attempt subject linkage + +**Files:** +- `src/usage/log.ts` +- `src/routing/compatibility/subject.ts` +- `src/server/responses/core.ts` +- focused request-log/usage/compatibility tests + +1. Add RED tests proving an optional exact Lab route-subject ID survives persisted-attempt normalization, malformed values are omitted, and legacy rows remain compatible. +2. Add RED tests proving the production subject resolver preserves existing CL-06 Responses semantics and distinguishes supported inbound protocol surfaces. +3. Add RED request execution tests proving each actual fallback/combo attempt receives the exact subject ID for the route it executed, while subject-link failure never changes request success/failure. +4. Implement only the minimum additive production-path linkage. Reuse the existing Lab `RouteSubjectV1` / `subjectIdForSubject` identity authority. Do not query the Lab ledger, Lab SQLite, history, or network on the request path. + +## Task 2 - Bounded passive query layer + +**Files:** +- new focused module under `src/lab/query/` +- `src/lab/query/index.ts` +- focused passive-query tests + +1. Add RED tests for strict allowlisted output, deterministic bounded scanning/pagination, exact-subject filtering, conservative outcome classification, and deletion/retention behavior. +2. Implement `PassiveRouteSignalV1` as a read-side projection over existing normalized usage history. Do not create a new ledger, SQLite authority, artifact store, or CL-02 event. +3. Expose subject-level summaries for recent attempts, successes, route-error diagnostic signals, and last observed attempt. +4. Keep generic HTTP failures diagnostic/unknown. Passive signals never alter compatibility verdicts or freshness. + +## Task 3 - Existing Lab read surfaces + +**Files:** +- `src/server/management/lab-routes.ts` +- `src/cli/lab.ts` +- `gui/src/pages/compatibility-matrix-api.ts` +- `gui/src/pages/compatibility-matrix-shared.ts` +- `gui/src/pages/CompatibilityMatrix.tsx` +- minimal Lab i18n/style files only if needed +- focused API/CLI/UI tests + +1. Add RED API and CLI tests for bounded passive reads and explicit `not verification` semantics. +2. Add RED UI/shared-model tests proving passive production data is visually separate from canonical Lab verdicts. +3. Implement additive endpoints/commands/components using existing authentication, pagination, and Compatibility Matrix conventions. Do not create a new product area or combined score. + +## Task 4 - Adversarial isolation and privacy + +**Files:** focused regressions only; production files only if a defect is found. + +1. Prove passive capture/read paths cause zero provider sends and no request replay/mirroring. +2. Prove passive data does not change Routing Profile evaluation, Router Intelligence eligibility/score/selection, canonical Lab verdict/freshness, or CL-08 planner output. +3. Seed prompt, response, tool, credential, account, and raw-error canaries and prove none appear in passive API/CLI/UI payloads, Lab JSONL/SQLite/artifacts, or CL-09 diagnostics. +4. Keep existing Shadow Call Intercept tests unchanged and green. + +## Task 5 - Verification and handoff + +1. Run focused tests through GitHub Actions because this execution environment has no Bun runtime. +2. Verify `bun x tsc --noEmit`, `bun run privacy:scan`, repo hygiene, focused Lab/request/routing tests, Cross-platform CI, React Doctor, and `git diff --check` equivalents/checks in CI. +3. Inspect CodeRabbit/review threads and fix valid findings without widening scope. +4. Update PR #1489 description from contract-only to the delivered implementation and leave it draft for independent final review. +5. Do not merge. + +## Implementation status + +Tasks 1-4 are implemented on the PR branch. The implementation uses the existing exact `RouteSubjectV1` identity per real attempt, a bounded read-only production-signal adapter over `usage.jsonl`, and additive Lab API/CLI/Compatibility Matrix read surfaces. Focused CL-09 tests cover legacy/malformed linkage, exact fallback attribution, bounded scans/results, strict data minimization with privacy canaries, source-retention behavior, and static no-feedback guards for routing and CL-08 planning. + +The final verification gate is intentionally not marked complete here until the exact final head passes required CI/review checks. PR #1489 remains open and must not be merged before independent final review. diff --git a/gui/src/i18n/de.ts b/gui/src/i18n/de.ts index 2b1d7aa4f..8be361528 100644 --- a/gui/src/i18n/de.ts +++ b/gui/src/i18n/de.ts @@ -1976,6 +1976,12 @@ export const de: Record = { "lab.detailObservations": "Observations", "lab.detailEvents": "Contributing events", "lab.detailArtifacts": "Artifact metadata", + "lab.production.title": "Beobachteter Produktionsverkehr", + "lab.production.notVerification": "Keine Lab-Verifizierung", + "lab.production.attempts": "Versuche", + "lab.production.successes": "Erfolge", + "lab.production.routeErrors": "Routing-Fehler", + "lab.production.lastObserved": "Zuletzt beobachtet", "lab.detailLoadFailed": "Could not load verdict detail", "lab.refresh": "Aktualisieren", "lab.verdict.UNKNOWN": "Unbekannt", diff --git a/gui/src/i18n/en.ts b/gui/src/i18n/en.ts index de9acea51..1600fdac1 100644 --- a/gui/src/i18n/en.ts +++ b/gui/src/i18n/en.ts @@ -2005,6 +2005,12 @@ export const en = { "lab.detailObservations": "Observations", "lab.detailEvents": "Contributing events", "lab.detailArtifacts": "Artifact metadata", + "lab.production.title": "Observed production traffic", + "lab.production.notVerification": "Not Lab verification", + "lab.production.attempts": "Attempts", + "lab.production.successes": "Successes", + "lab.production.routeErrors": "Route errors", + "lab.production.lastObserved": "Last observed", "lab.detailLoadFailed": "Could not load verdict detail", "lab.refresh": "Refresh", "lab.verdict.UNKNOWN": "Unknown", diff --git a/gui/src/i18n/ja.ts b/gui/src/i18n/ja.ts index 27dd7dba6..7c81bd4ac 100644 --- a/gui/src/i18n/ja.ts +++ b/gui/src/i18n/ja.ts @@ -1997,6 +1997,12 @@ export const ja: Record = { "lab.detailObservations": "Observations", "lab.detailEvents": "Contributing events", "lab.detailArtifacts": "Artifact metadata", + "lab.production.title": "観測された本番トラフィック", + "lab.production.notVerification": "ラボ検証ではありません", + "lab.production.attempts": "試行", + "lab.production.successes": "成功", + "lab.production.routeErrors": "ルートエラー", + "lab.production.lastObserved": "最終観測", "lab.detailLoadFailed": "Could not load verdict detail", "lab.refresh": "Refresh", "lab.verdict.UNKNOWN": "Unknown", diff --git a/gui/src/i18n/ko.ts b/gui/src/i18n/ko.ts index 850f2a6e6..ec1c76208 100644 --- a/gui/src/i18n/ko.ts +++ b/gui/src/i18n/ko.ts @@ -1997,6 +1997,12 @@ export const ko: Record = { "lab.detailObservations": "Observations", "lab.detailEvents": "Contributing events", "lab.detailArtifacts": "Artifact metadata", + "lab.production.title": "관측된 프로덕션 트래픽", + "lab.production.notVerification": "랩 검증 아님", + "lab.production.attempts": "시도", + "lab.production.successes": "성공", + "lab.production.routeErrors": "라우팅 오류", + "lab.production.lastObserved": "마지막 관측", "lab.detailLoadFailed": "Could not load verdict detail", "lab.refresh": "Refresh", "lab.verdict.UNKNOWN": "Unknown", diff --git a/gui/src/i18n/lab-translations.ts b/gui/src/i18n/lab-translations.ts index 5b6083238..3428a847d 100644 --- a/gui/src/i18n/lab-translations.ts +++ b/gui/src/i18n/lab-translations.ts @@ -1,7 +1,7 @@ import type { TKey } from "./en"; export type LabLocale = "en" | "de" | "ko" | "zh" | "zh-TW" | "ru" | "ja" | "tr"; -export type LabCatalogKey = Extract; +export type LabCatalogKey = Exclude, `lab.production.${string}`>; export type LabSupplementKey = | "subjectKindUnknown" | "artifact.present" diff --git a/gui/src/i18n/ru.ts b/gui/src/i18n/ru.ts index 3ef6b08bc..3f6575950 100644 --- a/gui/src/i18n/ru.ts +++ b/gui/src/i18n/ru.ts @@ -1999,6 +1999,12 @@ export const ru: Record = { "lab.detailObservations": "Observations", "lab.detailEvents": "Contributing events", "lab.detailArtifacts": "Artifact metadata", + "lab.production.title": "Наблюдаемый производственный трафик", + "lab.production.notVerification": "Не является проверкой Lab", + "lab.production.attempts": "Попытки", + "lab.production.successes": "Успешные попытки", + "lab.production.routeErrors": "Ошибки маршрута", + "lab.production.lastObserved": "Последнее наблюдение", "lab.detailLoadFailed": "Could not load verdict detail", "lab.refresh": "Refresh", "lab.verdict.UNKNOWN": "Unknown", diff --git a/gui/src/i18n/tr.ts b/gui/src/i18n/tr.ts index d6821a024..4413b32e2 100644 --- a/gui/src/i18n/tr.ts +++ b/gui/src/i18n/tr.ts @@ -1999,6 +1999,12 @@ export const tr: Record = { "lab.detailObservations": "Observations", "lab.detailEvents": "Contributing events", "lab.detailArtifacts": "Artifact metadata", + "lab.production.title": "Gözlemlenen üretim trafiği", + "lab.production.notVerification": "Lab doğrulaması değildir", + "lab.production.attempts": "Denemeler", + "lab.production.successes": "Başarılı denemeler", + "lab.production.routeErrors": "Rota hataları", + "lab.production.lastObserved": "Son gözlem", "lab.detailLoadFailed": "Could not load verdict detail", "lab.refresh": "Refresh", "lab.verdict.UNKNOWN": "Unknown", diff --git a/gui/src/i18n/zh-TW.ts b/gui/src/i18n/zh-TW.ts index d7b095209..f29bd9be1 100644 --- a/gui/src/i18n/zh-TW.ts +++ b/gui/src/i18n/zh-TW.ts @@ -1962,6 +1962,12 @@ export const zhTW: Record = { "lab.detailObservations": "觀察數", "lab.detailEvents": "貢獻事件", "lab.detailArtifacts": "產物中繼資料", + "lab.production.title": "觀測到的正式環境流量", + "lab.production.notVerification": "不是實驗室驗證", + "lab.production.attempts": "嘗試", + "lab.production.successes": "成功", + "lab.production.routeErrors": "路由錯誤", + "lab.production.lastObserved": "最近觀測", "lab.detailLoadFailed": "無法載入判定詳細資料", "lab.refresh": "重新整理", "lab.verdict.UNKNOWN": "未知", diff --git a/gui/src/i18n/zh.ts b/gui/src/i18n/zh.ts index c1766cf8a..b17ca7588 100644 --- a/gui/src/i18n/zh.ts +++ b/gui/src/i18n/zh.ts @@ -1997,6 +1997,12 @@ export const zh: Record = { "lab.detailObservations": "Observations", "lab.detailEvents": "Contributing events", "lab.detailArtifacts": "Artifact metadata", + "lab.production.title": "观测到的生产流量", + "lab.production.notVerification": "不是实验室验证", + "lab.production.attempts": "尝试", + "lab.production.successes": "成功", + "lab.production.routeErrors": "路由错误", + "lab.production.lastObserved": "最近观测", "lab.detailLoadFailed": "Could not load verdict detail", "lab.refresh": "Refresh", "lab.verdict.UNKNOWN": "Unknown", diff --git a/gui/src/pages/CompatibilityMatrix.tsx b/gui/src/pages/CompatibilityMatrix.tsx index f5efe6ea7..9fcac9b12 100644 --- a/gui/src/pages/CompatibilityMatrix.tsx +++ b/gui/src/pages/CompatibilityMatrix.tsx @@ -191,6 +191,20 @@ function DetailPane({

{t("lab.detailSubject")}

{detail.subject.subjectKind}

+ {detail.production && ( +
+

{t("lab.production.title")}

+

{t("lab.production.notVerification")}

+
+
{t("lab.production.attempts")}
{detail.production.summary.recentProductionAttempts}
+
{t("lab.production.successes")}
{detail.production.summary.recentSuccessfulAttempts}
+
{t("lab.production.routeErrors")}
{detail.production.summary.recentRouteErrorSignals}
+ {detail.production.summary.lastObservedProductionAttempt !== undefined && ( +
{t("lab.production.lastObserved")}
{formatAsOf(detail.production.summary.lastObservedProductionAttempt, locale)}
+ )} +
+
+ )} {detail.observations.length > 0 && (

{t("lab.detailObservations")}

diff --git a/gui/src/pages/compatibility-matrix-api.ts b/gui/src/pages/compatibility-matrix-api.ts index 2d7c01aec..139ba8594 100644 --- a/gui/src/pages/compatibility-matrix-api.ts +++ b/gui/src/pages/compatibility-matrix-api.ts @@ -207,6 +207,47 @@ export async function fetchArtifactByDigest( return artifact; } +export type PassiveProductionSummaryDto = { + verificationStatus: "not_verification"; + summary: { + subjectId: string; + verificationStatus: "not_verification"; + recentProductionAttempts: number; + recentSuccessfulAttempts: number; + recentRouteErrorSignals: number; + lastObservedProductionAttempt?: number; + }; +}; + +function parsePassiveProductionSummary(raw: unknown): PassiveProductionSummaryDto { + if (!isPlainObject(raw) || raw.verificationStatus !== "not_verification" || !isPlainObject(raw.summary)) { + throw invalidResponse(); + } + const summary = raw.summary; + if (summary.verificationStatus !== "not_verification" + || typeof summary.subjectId !== "string" + || typeof summary.recentProductionAttempts !== "number" + || typeof summary.recentSuccessfulAttempts !== "number" + || typeof summary.recentRouteErrorSignals !== "number" + || (summary.lastObservedProductionAttempt !== undefined && typeof summary.lastObservedProductionAttempt !== "number")) { + throw invalidResponse(); + } + return { verificationStatus: "not_verification", summary: summary as PassiveProductionSummaryDto["summary"] }; +} + +export async function fetchPassiveProductionSummary( + apiBase: string, + subjectId: string, + signal: AbortSignal, +): Promise { + const raw = await fetchLabJson( + apiBase, + `/api/lab/production-signals?${buildQuery({ subjectId })}`, + signal, + ); + return parsePassiveProductionSummary(raw); +} + export type LabPageData = { status: LabStatusDto; verdicts: VerdictDto[]; @@ -254,6 +295,7 @@ export type VerdictDetailData = { observationsTruncated: boolean; events: LabEventDto[]; artifacts: ArtifactMetadataDto[]; + production: PassiveProductionSummaryDto | null; }; async function mapSettledBounded( @@ -295,11 +337,15 @@ export async function fetchVerdictDetail( layer: verdict.evidenceLayer, suiteId: verdict.suiteId, }; - const [subject, observations, events, artifacts] = await Promise.all([ + const [subject, observations, events, artifacts, production] = await Promise.all([ fetchSubjectDetail(apiBase, verdict.subjectId, signal), fetchAllObservations(apiBase, observationFilters, signal), mapSettledBounded(eventIds, DETAIL_CONCURRENCY, signal, id => fetchEventById(apiBase, id, signal)), mapSettledBounded(digests, DETAIL_CONCURRENCY, signal, digest => fetchArtifactByDigest(apiBase, digest, signal)), + fetchPassiveProductionSummary(apiBase, verdict.subjectId, signal).catch(error => { + if (signal.aborted) throw error; + return null; + }), ]); return { subject, @@ -307,5 +353,6 @@ export async function fetchVerdictDetail( observationsTruncated: observations.truncated, events, artifacts, + production, }; } diff --git a/gui/tests/compatibility-lab-i18n.test.ts b/gui/tests/compatibility-lab-i18n.test.ts index 252271dd5..3df4c11e4 100644 --- a/gui/tests/compatibility-lab-i18n.test.ts +++ b/gui/tests/compatibility-lab-i18n.test.ts @@ -4,8 +4,10 @@ import { LAB_CATALOG_OVERRIDES, labSupplement } from "../src/i18n/lab-translatio const NON_ENGLISH: Locale[] = ["de", "ja", "ko", "ru", "tr", "zh"]; -test("Compatibility Lab catalog overrides cover the complete English lab namespace", () => { - const englishKeys = Object.keys(DICTS.en).filter(key => key.startsWith("lab.")).sort(); +test("Compatibility Lab catalog overrides cover the translated overlay namespace", () => { + const englishKeys = Object.keys(DICTS.en) + .filter(key => key.startsWith("lab.") && !key.startsWith("lab.production.")) + .sort(); for (const locale of Object.keys(LAB_CATALOG_OVERRIDES) as Locale[]) { expect(Object.keys(LAB_CATALOG_OVERRIDES[locale]).sort()).toEqual(englishKeys); } diff --git a/src/cli/lab.ts b/src/cli/lab.ts index fabb81900..013b96327 100644 --- a/src/cli/lab.ts +++ b/src/cli/lab.ts @@ -33,6 +33,8 @@ import { queryLabSubjectById, queryLabSubjects, queryLabVerdicts, + queryPassiveProductionSignals, + type PassiveProductionQueryResultV1, } from "../lab/query"; import { CliUsageError, @@ -64,6 +66,7 @@ import { createProductionLabRouteExecutor } from "../lib/lab-live-route-producti const USAGE = `Usage: ocx lab status [--json] + ocx lab production-signals --subject [--limit ] [--json] ocx lab verdicts [--subject ] [--layer ] [--suite ] [--verdict ] [--from ] [--to ] [--limit ] [--cursor ] [--json] ocx lab subjects [--kind ] [--limit ] [--cursor ] [--json] ocx lab subject [--json] @@ -187,6 +190,17 @@ function catalogLines(scenarios: ReturnType): str return lines.length > 0 ? lines : ["No catalog scenarios"]; } +function passiveProductionLines(result: PassiveProductionQueryResultV1): string[] { + const summary = result.summary; + return [ + "Observed production traffic (not Lab verification)", + `Attempts: ${summary.recentProductionAttempts} | Successes: ${summary.recentSuccessfulAttempts} | Route errors: ${summary.recentRouteErrorSignals}`, + ...(summary.lastObservedProductionAttempt !== undefined + ? [`Last observed: ${summary.lastObservedProductionAttempt}`] + : []), + ]; +} + function automationStatusLines(status: ReturnType): string[] { return [ `Automation enabled: ${status.policy.enabled}`, @@ -224,6 +238,15 @@ export async function handleLabCommand(argv: string[], deps: LabCliDeps = {}): P printData(status, wantsJson, statusSummary(status)); return; } + case "production-signals": { + const subjectId = takeOption(rest, "--subject"); + const limit = takeIntegerOption(rest, "--limit", { min: 1 }); + rejectArgs(rest, USAGE); + if (!subjectId) throw new CliUsageError("--subject is required", USAGE); + const result = queryPassiveProductionSignals(subjectId, limit, configDir); + printData(result, wantsJson, passiveProductionLines(result)); + return; + } case "verdicts": { const subjectId = takeOption(rest, "--subject"); const layer = takeEnumOption( diff --git a/src/lab/query/index.ts b/src/lab/query/index.ts index 6582bacb4..2df30c886 100644 --- a/src/lab/query/index.ts +++ b/src/lab/query/index.ts @@ -31,4 +31,15 @@ export { queryLabArtifactByDigest, queryLabCatalogEntries, } from "./queries"; +export { + PASSIVE_PRODUCTION_DEFAULT_LIMIT, + PASSIVE_PRODUCTION_MAX_LIMIT, + PASSIVE_PRODUCTION_MAX_SCAN_ROWS, + derivePassiveProductionSignals, + queryPassiveProductionSignals, + type PassiveProductionOutcome, + type PassiveProductionQueryResultV1, + type PassiveProductionSummaryV1, + type PassiveRouteSignalV1, +} from "./passive-production"; export { sanitizePublicText } from "./dto-map"; diff --git a/src/lab/query/passive-production.ts b/src/lab/query/passive-production.ts new file mode 100644 index 000000000..e1c077003 --- /dev/null +++ b/src/lab/query/passive-production.ts @@ -0,0 +1,159 @@ +import { + isLabRouteSubjectId, + readRecentUsageEntries, + type PersistedUsageAttempt, + type PersistedUsageEntry, +} from "../../usage/log"; + +export const PASSIVE_PRODUCTION_DEFAULT_LIMIT = 50; +export const PASSIVE_PRODUCTION_MAX_LIMIT = 200; +export const PASSIVE_PRODUCTION_MAX_SCAN_ROWS = 2_000; + +export type PassiveProductionOutcome = + | "success" + | "client_cancel" + | "route_error" + | "environmental" + | "unknown"; + +export interface PassiveRouteSignalV1 { + schemaVersion: 1; + subjectId: string; + source: "production_usage_v1"; + requestRef: string; + decisionRef?: string; + attemptOrdinal: number; + observedAt: number; + outcome: PassiveProductionOutcome; + httpStatus?: number; +} + +export interface PassiveProductionSummaryV1 { + schemaVersion: 1; + subjectId: string; + verificationStatus: "not_verification"; + recentProductionAttempts: number; + recentSuccessfulAttempts: number; + recentRouteErrorSignals: number; + lastObservedProductionAttempt?: number; +} + +export interface PassiveProductionQueryResultV1 { + schemaVersion: 1; + verificationStatus: "not_verification"; + summary: PassiveProductionSummaryV1; + signals: PassiveRouteSignalV1[]; + scannedRows: number; + truncated: boolean; +} + +const ROUTE_ERROR_CODES = new Set([ + "upstream_error", + "upstream_transport_error", + "upstream_connect_error", + "upstream_timeout", +]); +const ENVIRONMENTAL_ERROR_CODES = new Set([ + "authentication_error", + "rate_limit_error", + "admission_rejected", + "upstream_host_circuit_open", +]); + +function boundedLimit(value: number | undefined): number { + if (value === undefined) return PASSIVE_PRODUCTION_DEFAULT_LIMIT; + if (!Number.isSafeInteger(value) || value < 1) throw new RangeError("passive production limit must be a positive integer"); + return Math.min(value, PASSIVE_PRODUCTION_MAX_LIMIT); +} + +function isFinalAttempt(entry: PersistedUsageEntry, attempt: PersistedUsageAttempt): boolean { + const attempts = entry.attempts ?? []; + return attempts.length > 0 && attempts[attempts.length - 1]?.ordinal === attempt.ordinal; +} + +function classifyOutcome(entry: PersistedUsageEntry, attempt: PersistedUsageAttempt): PassiveProductionOutcome { + if (attempt.status >= 200 && attempt.status < 300) return "success"; + if (isFinalAttempt(entry, attempt) && entry.closeReason === "client_cancel") return "client_cancel"; + if (attempt.errorCode && ENVIRONMENTAL_ERROR_CODES.has(attempt.errorCode)) return "environmental"; + if (attempt.errorCode && ROUTE_ERROR_CODES.has(attempt.errorCode)) return "route_error"; + return "unknown"; +} + +function signalFor(entry: PersistedUsageEntry, attempt: PersistedUsageAttempt): PassiveRouteSignalV1 | null { + const subjectId = attempt.labRouteSubjectId; + if (!isLabRouteSubjectId(subjectId)) return null; + return { + schemaVersion: 1, + subjectId, + source: "production_usage_v1", + requestRef: entry.requestId, + ...(entry.routeDecision?.decisionId ? { decisionRef: entry.routeDecision.decisionId } : {}), + attemptOrdinal: attempt.ordinal, + observedAt: entry.timestamp, + outcome: classifyOutcome(entry, attempt), + httpStatus: attempt.status, + }; +} + +export function derivePassiveProductionSignals( + entries: readonly PersistedUsageEntry[], + subjectId: string, + limit?: number, +): PassiveProductionQueryResultV1 { + if (!isLabRouteSubjectId(subjectId)) throw new Error("invalid passive production subject id"); + const maxResults = boundedLimit(limit); + const signals: PassiveRouteSignalV1[] = []; + // readRecentUsageEntries returns the selected append-only rows oldest-first. + // Tail selection, reverse iteration, and signals[0] as the newest observation rely on this ordering. + const scanRows = entries.slice(-PASSIVE_PRODUCTION_MAX_SCAN_ROWS); + const scanTruncated = entries.length > PASSIVE_PRODUCTION_MAX_SCAN_ROWS; + let resultTruncated = false; + + scan: for (let rowIndex = scanRows.length - 1; rowIndex >= 0; rowIndex--) { + const entry = scanRows[rowIndex]!; + const attempts = entry.attempts ?? []; + for (let attemptIndex = attempts.length - 1; attemptIndex >= 0; attemptIndex--) { + const signal = signalFor(entry, attempts[attemptIndex]!); + if (signal?.subjectId !== subjectId) continue; + if (signals.length >= maxResults) { + resultTruncated = true; + break scan; + } + signals.push(signal); + } + } + + const recentSuccessfulAttempts = signals.filter(signal => signal.outcome === "success").length; + const recentRouteErrorSignals = signals.filter(signal => signal.outcome === "route_error").length; + return { + schemaVersion: 1, + verificationStatus: "not_verification", + summary: { + schemaVersion: 1, + subjectId, + verificationStatus: "not_verification", + recentProductionAttempts: signals.length, + recentSuccessfulAttempts, + recentRouteErrorSignals, + ...(signals[0] ? { lastObservedProductionAttempt: signals[0].observedAt } : {}), + }, + signals, + scannedRows: scanRows.length, + truncated: scanTruncated || resultTruncated, + }; +} + +/** Read-only bounded CL-09 projection over the existing usage-log authority. */ +export function queryPassiveProductionSignals( + subjectId: string, + limit?: number, + configDir?: string, +): PassiveProductionQueryResultV1 { + return derivePassiveProductionSignals( + // Read one row past the scan cap so the projection can distinguish an exact-cap history + // from a history with older rows omitted by the bounded reader. + readRecentUsageEntries(PASSIVE_PRODUCTION_MAX_SCAN_ROWS + 1, configDir), + subjectId, + limit, + ); +} diff --git a/src/lab/subject/installation-salt.ts b/src/lab/subject/installation-salt.ts index 16686ba83..dec715b77 100644 --- a/src/lab/subject/installation-salt.ts +++ b/src/lab/subject/installation-salt.ts @@ -4,6 +4,7 @@ import { dirname } from "node:path"; import { labInstallationSaltPath, labRoot } from "../paths"; const SALT_BYTES = 32; +const installationSaltCache = new Map(); const UNSUPPORTED_DIRECTORY_FSYNC_CODES = new Set(["EINVAL", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"]); function readSaltFile(path: string): Uint8Array { @@ -14,11 +15,19 @@ function readSaltFile(path: string): Uint8Array { return new Uint8Array(bytes); } +function cacheSalt(path: string, salt: Uint8Array): Uint8Array { + const cached = new Uint8Array(salt); + installationSaltCache.set(path, cached); + return new Uint8Array(cached); +} + /** Read the existing local fingerprint salt without creating Lab state. */ export function readExistingInstallationSalt(configDir?: string): Uint8Array | null { const path = labInstallationSaltPath(configDir); + const cached = installationSaltCache.get(path); + if (cached) return new Uint8Array(cached); try { - return readSaltFile(path); + return cacheSalt(path, readSaltFile(path)); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error; @@ -56,7 +65,7 @@ export function readInstallationSalt(configDir?: string): Uint8Array { const root = labRoot(configDir); mkdirSync(root, { recursive: true, mode: 0o700 }); - try { return readSaltFile(path); } + try { return cacheSalt(path, readSaltFile(path)); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } @@ -78,10 +87,10 @@ export function readInstallationSalt(configDir?: string): Uint8Array { linkSync(stagingPath, path); try { fsyncDirectory(dirname(path)); } catch { throw new Error("harness_failure: installation salt directory fsync failed"); } - return new Uint8Array(salt); + return cacheSalt(path, salt); } catch (error) { if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - return readSaltFile(path); + return cacheSalt(path, readSaltFile(path)); } } finally { if (fd !== undefined) { diff --git a/src/routing/compatibility/subject.ts b/src/routing/compatibility/subject.ts index 6f508531c..be4b8517f 100644 --- a/src/routing/compatibility/subject.ts +++ b/src/routing/compatibility/subject.ts @@ -1,4 +1,5 @@ import type { OcxConfig, OcxProviderConfig } from "../../types"; +import type { InboundWire } from "../../providers/registry"; import { subjectIdForSubject } from "../../lab/digest"; import { buildRouteSubjectV1 } from "../../lab/subject/route-subject"; import { buildProtocolSubjectV1 } from "../../lab/subject/protocol-subject"; @@ -15,7 +16,7 @@ import { import { readOpenCodexCompatibilityVersion } from "./version"; import type { RoutingCompatibilityEvidenceLayer } from "./types"; -const POLICY_INBOUND_PROTOCOL = "openai-responses"; +const POLICY_INBOUND_WIRE: InboundWire = "responses"; export interface ResolvedPolicyRouteSubject { subjectId: string; @@ -28,6 +29,15 @@ export interface ResolvedPolicyCompatibilitySubjects { route?: ResolvedPolicyRouteSubject; } +/** Canonical Lab protocol identity for a production request wire. */ +export function inboundProtocolForWire(inboundWire: InboundWire): string { + switch (inboundWire) { + case "responses": return "openai-responses"; + case "chat": return "openai-chat"; + case "anthropic": return "anthropic-messages"; + } +} + function destinationSnapshotFromBaseUrl(baseUrl: string, fingerprint: string): LabDestinationV1 { const parts = parseEndpointFingerprintParts(baseUrl); if (!parts) throw new Error("invalid provider baseUrl for route subject"); @@ -43,36 +53,29 @@ function destinationSnapshotFromBaseUrl(baseUrl: string, fingerprint: string): L }); } -function resolveEffectivePolicyProvider( - providerName: string, - modelId: string, - routed: OcxProviderConfig, -): OcxProviderConfig { - return resolveWireProtocolOverride(providerName, modelId, routed, "responses"); -} - /** - * Resolve the subject identities a policy candidate may legitimately consume. - * Protocol and live-route layers deliberately use different canonical subjects. - * No network, projection rebuild, ledger replay, or Lab-state creation occurs. + * Resolve subject identities for one exact inbound wire without network I/O, + * projection reads/rebuilds, ledger replay, or Lab-state creation. */ -export function resolvePolicyCompatibilitySubjects( +export function resolveCompatibilitySubjectsForInboundWire( config: OcxConfig, providerName: string, modelId: string, routed: OcxProviderConfig, + inboundWire: InboundWire, configDir?: string, ): ResolvedPolicyCompatibilitySubjects { - const effective = resolveEffectivePolicyProvider(providerName, modelId, routed); + const effective = resolveWireProtocolOverride(providerName, modelId, routed, inboundWire); const baseUrl = typeof effective.baseUrl === "string" ? effective.baseUrl.trim() : ""; const adapter = effective.adapter ?? "openai-responses"; + const inboundProtocol = inboundProtocolForWire(inboundWire); const upstreamProtocol = upstreamProtocolForAdapter(adapter); - const surface = surfaceForProtocols(POLICY_INBOUND_PROTOCOL, upstreamProtocol); + const surface = surfaceForProtocols(inboundProtocol, upstreamProtocol); const subjectIds: ResolvedPolicyCompatibilitySubjects["subjectIds"] = {}; try { const protocolSubject = buildProtocolSubjectV1({ - inboundProtocol: POLICY_INBOUND_PROTOCOL, + inboundProtocol, upstreamProtocol, surface, }, adapter); @@ -110,7 +113,7 @@ export function resolvePolicyCompatibilitySubjects( clientModelId: modelId, upstreamModelId: modelId, effectiveAdapter: adapter, - inboundProtocol: POLICY_INBOUND_PROTOCOL, + inboundProtocol, upstreamProtocol, surface, baseUrl, @@ -133,6 +136,27 @@ export function resolvePolicyCompatibilitySubjects( } } +/** + * Resolve the subject identities a policy candidate may legitimately consume. + * CL-06 policy evaluation is a Responses-surface lookup and remains unchanged. + */ +export function resolvePolicyCompatibilitySubjects( + config: OcxConfig, + providerName: string, + modelId: string, + routed: OcxProviderConfig, + configDir?: string, +): ResolvedPolicyCompatibilitySubjects { + return resolveCompatibilitySubjectsForInboundWire( + config, + providerName, + modelId, + routed, + POLICY_INBOUND_WIRE, + configDir, + ); +} + /** Build exact RouteSubjectV1 identity for a policy candidate without network I/O. */ export function resolvePolicyRouteSubject( config: OcxConfig, @@ -143,3 +167,25 @@ export function resolvePolicyRouteSubject( ): ResolvedPolicyRouteSubject | null { return resolvePolicyCompatibilitySubjects(config, providerName, modelId, routed, configDir).route ?? null; } + +/** + * Build the exact production-attempt RouteSubjectV1 identity for its actual + * inbound wire. Returns null when no existing Lab salt/identity can be read. + */ +export function resolveProductionRouteSubject( + config: OcxConfig, + providerName: string, + modelId: string, + routed: OcxProviderConfig, + inboundWire: InboundWire, + configDir?: string, +): ResolvedPolicyRouteSubject | null { + return resolveCompatibilitySubjectsForInboundWire( + config, + providerName, + modelId, + routed, + inboundWire, + configDir, + ).route ?? null; +} diff --git a/src/server/management/lab-routes.ts b/src/server/management/lab-routes.ts index 3453d2785..5c0929e47 100644 --- a/src/server/management/lab-routes.ts +++ b/src/server/management/lab-routes.ts @@ -40,6 +40,7 @@ import { queryLabSubjectById, queryLabSubjects, queryLabVerdicts, + queryPassiveProductionSignals, } from "../../lab/query"; import { jsonResponse } from "../auth-cors"; import type { ManagementContext } from "./context"; @@ -194,6 +195,18 @@ export async function handleLabRoutes(ctx: ManagementContext): Promise); } -export function usageLogPath(): string { - return join(getConfigDir(), "usage.jsonl"); +export function usageLogPath(configDir?: string): string { + return join(configDir ?? getConfigDir(), "usage.jsonl"); } export function usageTotalTokens(usage: OcxUsage | undefined): number | undefined { @@ -195,6 +197,11 @@ const USAGE_STATUSES = new Set([ "unsupported", "estimated", ]); +const LAB_ROUTE_SUBJECT_ID_RE = /^[0-9a-f]{64}$/; + +export function isLabRouteSubjectId(value: unknown): value is string { + return typeof value === "string" && LAB_ROUTE_SUBJECT_ID_RE.test(value); +} function isNonNegativeFiniteNumber(value: unknown): value is number { return typeof value === "number" && Number.isFinite(value) && value >= 0; @@ -272,6 +279,9 @@ function normalizeUsageAttempt(raw: unknown): PersistedUsageAttempt | null { ? { totalTokens: attempt.totalTokens } : {}), ...(typeof attempt.errorCode === "string" ? { errorCode: attempt.errorCode } : {}), + ...(isLabRouteSubjectId(attempt.labRouteSubjectId) + ? { labRouteSubjectId: attempt.labRouteSubjectId } + : {}), ...(typeof attempt.requestedEffort === "string" && attempt.requestedEffort ? { requestedEffort: capMetadataString(attempt.requestedEffort) } : {}), @@ -647,9 +657,9 @@ function parseUsageLines(lines: string[]): PersistedUsageEntry[] { * Read only the newest `limit` usage.jsonl rows without loading the whole append-only * file into memory. Used by request-log hydration on `ocx start`. */ -export function readRecentUsageEntries(limit: number): PersistedUsageEntry[] { +export function readRecentUsageEntries(limit: number, configDir?: string): PersistedUsageEntry[] { if (!Number.isFinite(limit) || limit <= 0) return []; - const path = usageLogPath(); + const path = usageLogPath(configDir); if (!existsSync(path)) return []; let fd: number | undefined; try { diff --git a/tests/lab-passive-production-evidence.test.ts b/tests/lab-passive-production-evidence.test.ts new file mode 100644 index 000000000..8e7921a46 --- /dev/null +++ b/tests/lab-passive-production-evidence.test.ts @@ -0,0 +1,290 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, test } from "bun:test"; +import { + normalizeUsageEntryForTest, + type PersistedUsageEntry, +} from "../src/usage/log"; +import { inboundProtocolForWire } from "../src/routing/compatibility/subject"; +import { + PASSIVE_PRODUCTION_MAX_LIMIT, + PASSIVE_PRODUCTION_MAX_SCAN_ROWS, + derivePassiveProductionSignals, + queryPassiveProductionSignals, +} from "../src/lab/query/passive-production"; + +function usageEntryWithAttempt(attempt: Record): PersistedUsageEntry { + return { + requestId: "ocx-cl09-passive", + timestamp: 1, + provider: "combo", + model: "combo/test", + status: 200, + durationMs: 5, + usageStatus: "unreported", + attempts: [{ + ordinal: 1, + provider: "provider-a", + model: "model-a", + adapter: "openai-chat", + status: 200, + durationMs: 4, + sendCount: 1, + recoveryKinds: [], + usageStatus: "unreported", + ...attempt, + } as never], + }; +} + +describe("CL-09 passive production attempt linkage", () => { + test("preserves an exact local Lab route subject id on a persisted attempt", () => { + const subjectId = "a".repeat(64); + const normalized = normalizeUsageEntryForTest(usageEntryWithAttempt({ labRouteSubjectId: subjectId })); + expect(normalized.attempts?.[0]).toMatchObject({ ordinal: 1, labRouteSubjectId: subjectId }); + }); + + test("omits malformed route subject linkage without dropping the attempt", () => { + const normalized = normalizeUsageEntryForTest(usageEntryWithAttempt({ labRouteSubjectId: "not-a-subject-id" })); + expect(normalized.attempts?.[0]?.ordinal).toBe(1); + expect(normalized.attempts?.[0]).not.toHaveProperty("labRouteSubjectId"); + }); + + test("keeps legacy attempts without CL-09 linkage unchanged", () => { + const normalized = normalizeUsageEntryForTest(usageEntryWithAttempt({})); + expect(normalized.attempts).toEqual([{ + ordinal: 1, + provider: "provider-a", + model: "model-a", + adapter: "openai-chat", + status: 200, + durationMs: 4, + sendCount: 1, + recoveryKinds: [], + usageStatus: "unreported", + }]); + }); + + test("maps each production inbound wire to its canonical Lab protocol identity", () => { + expect(inboundProtocolForWire("responses")).toBe("openai-responses"); + expect(inboundProtocolForWire("chat")).toBe("openai-chat"); + expect(inboundProtocolForWire("anthropic")).toBe("anthropic-messages"); + }); +}); + +describe("CL-09 bounded passive production projection", () => { + test("projects only the strict passive allowlist and labels it not verification", () => { + const subjectId = "b".repeat(64); + const secret = "CL09-PROMPT-SECRET-CANARY"; + const entry = usageEntryWithAttempt({ labRouteSubjectId: subjectId, errorCode: `opaque-${secret}` }); + entry.timestamp = 1234; + entry.apiKeyId = `account-${secret}`; + entry.conversationId = `conversation-${secret}`; + entry.upstreamError = `raw-error-${secret}`; + entry.requestedEffort = secret; + entry.terminalStatus = `terminal-${secret}`; + (entry as unknown as Record).prompt = `prompt-${secret}`; + (entry as unknown as Record).responseText = `response-${secret}`; + (entry.attempts?.[0] as unknown as Record).toolArguments = `tool-${secret}`; + (entry.attempts?.[0] as unknown as Record).credential = `credential-${secret}`; + + const result = derivePassiveProductionSignals([entry], subjectId, 10); + + expect(result.verificationStatus).toBe("not_verification"); + expect(result.summary.verificationStatus).toBe("not_verification"); + expect(result.summary.recentProductionAttempts).toBe(1); + expect(result.summary.recentSuccessfulAttempts).toBe(1); + expect(result.signals[0]).toEqual({ + schemaVersion: 1, + subjectId, + source: "production_usage_v1", + requestRef: "ocx-cl09-passive", + attemptOrdinal: 1, + observedAt: 1234, + outcome: "success", + httpStatus: 200, + }); + expect(JSON.stringify(result)).not.toContain(secret); + }); + + test("does not treat generic HTTP failure as a compatibility-style route error", () => { + const subjectId = "c".repeat(64); + const entry = usageEntryWithAttempt({ labRouteSubjectId: subjectId, status: 500 }); + entry.status = 500; + + const result = derivePassiveProductionSignals([entry], subjectId); + + expect(result.signals[0]?.outcome).toBe("unknown"); + expect(result.summary.recentRouteErrorSignals).toBe(0); + }); + + test("classifies cancellation, environmental failures, and route errors independently", () => { + const subjectId = "4".repeat(64); + const cancelled = usageEntryWithAttempt({ labRouteSubjectId: subjectId, status: 499 }); + cancelled.status = 499; + cancelled.closeReason = "client_cancel"; + + const environmental = usageEntryWithAttempt({ + labRouteSubjectId: subjectId, + status: 429, + errorCode: "rate_limit_error", + }); + environmental.requestId = "ocx-cl09-environmental"; + environmental.status = 429; + + const routeError = usageEntryWithAttempt({ + labRouteSubjectId: subjectId, + status: 502, + errorCode: "upstream_error", + }); + routeError.requestId = "ocx-cl09-route-error"; + routeError.status = 502; + + const result = derivePassiveProductionSignals([cancelled, environmental, routeError], subjectId); + + expect(result.signals.map(signal => signal.outcome).sort()).toEqual([ + "client_cancel", + "environmental", + "route_error", + ]); + expect(result.summary.recentRouteErrorSignals).toBe(1); + }); + + test("reports result truncation only when another matching signal exists", () => { + const subjectId = "5".repeat(64); + const entries = Array.from({ length: 3 }, (_, index) => { + const entry = usageEntryWithAttempt({ labRouteSubjectId: subjectId }); + entry.requestId = `ocx-cl09-limit-${index}`; + entry.timestamp = index; + return entry; + }); + + expect(derivePassiveProductionSignals(entries.slice(0, 2), subjectId, 2).truncated).toBe(false); + expect(derivePassiveProductionSignals(entries, subjectId, 2).truncated).toBe(true); + }); + + test("uses the selected config directory and detects scan overflow", () => { + const configDir = mkdtempSync(join(tmpdir(), "ocx-cl09-passive-")); + try { + const subjectId = "6".repeat(64); + const otherSubjectId = "7".repeat(64); + const entries = Array.from({ length: PASSIVE_PRODUCTION_MAX_SCAN_ROWS + 1 }, (_, index) => { + const entry = usageEntryWithAttempt({ + labRouteSubjectId: index === PASSIVE_PRODUCTION_MAX_SCAN_ROWS ? subjectId : otherSubjectId, + }); + entry.requestId = `ocx-cl09-config-${index}`; + entry.timestamp = index; + return entry; + }); + writeFileSync(join(configDir, "usage.jsonl"), `${entries.map(entry => JSON.stringify(entry)).join("\n")}\n`); + + const result = queryPassiveProductionSignals(subjectId, 10, configDir); + + expect(result.signals).toHaveLength(1); + expect(result.scannedRows).toBe(PASSIVE_PRODUCTION_MAX_SCAN_ROWS); + expect(result.truncated).toBe(true); + expect(result.signals[0]?.requestRef).toBe(`ocx-cl09-config-${PASSIVE_PRODUCTION_MAX_SCAN_ROWS}`); + } finally { + rmSync(configDir, { recursive: true, force: true }); + } + }); + + test("bounds result count and scanned source rows", () => { + const subjectId = "d".repeat(64); + const entries = Array.from({ length: PASSIVE_PRODUCTION_MAX_SCAN_ROWS + 25 }, (_, index) => { + const entry = usageEntryWithAttempt({ labRouteSubjectId: subjectId }); + entry.requestId = `request-${index}`; + entry.timestamp = index; + return entry; + }); + + const result = derivePassiveProductionSignals(entries, subjectId, PASSIVE_PRODUCTION_MAX_LIMIT + 100); + + expect(result.signals).toHaveLength(PASSIVE_PRODUCTION_MAX_LIMIT); + expect(result.scannedRows).toBe(PASSIVE_PRODUCTION_MAX_SCAN_ROWS); + expect(result.truncated).toBe(true); + expect(result.signals[0]?.observedAt).toBe(entries.length - 1); + }); + + test("keeps signals isolated by exact subject id", () => { + const subjectA = "e".repeat(64); + const subjectB = "f".repeat(64); + const first = usageEntryWithAttempt({ labRouteSubjectId: subjectA }); + const second = usageEntryWithAttempt({ labRouteSubjectId: subjectB }); + second.requestId = "other-request"; + + const result = derivePassiveProductionSignals([first, second], subjectA); + + expect(result.signals).toHaveLength(1); + expect(result.signals[0]?.subjectId).toBe(subjectA); + expect(result.signals[0]?.requestRef).toBe("ocx-cl09-passive"); + }); + + test("keeps fallback attempts attributable to the exact route that executed them", () => { + const subjectA = "2".repeat(64); + const subjectB = "3".repeat(64); + const entry = usageEntryWithAttempt({ labRouteSubjectId: subjectA, status: 503 }); + entry.status = 200; + entry.attempts!.push({ + ordinal: 2, + provider: "provider-b", + model: "model-b", + adapter: "openai-responses", + status: 200, + durationMs: 3, + sendCount: 1, + recoveryKinds: [], + usageStatus: "unreported", + labRouteSubjectId: subjectB, + }); + + const firstRoute = derivePassiveProductionSignals([entry], subjectA); + const fallbackRoute = derivePassiveProductionSignals([entry], subjectB); + + expect(firstRoute.signals).toHaveLength(1); + expect(firstRoute.signals[0]).toMatchObject({ attemptOrdinal: 1, subjectId: subjectA, httpStatus: 503 }); + expect(fallbackRoute.signals).toHaveLength(1); + expect(fallbackRoute.signals[0]).toMatchObject({ attemptOrdinal: 2, subjectId: subjectB, httpStatus: 200 }); + }); + + test("passive visibility disappears with its existing usage-history source", () => { + const subjectId = "1".repeat(64); + const entry = usageEntryWithAttempt({ labRouteSubjectId: subjectId }); + expect(derivePassiveProductionSignals([entry], subjectId).signals).toHaveLength(1); + expect(derivePassiveProductionSignals([], subjectId).signals).toHaveLength(0); + }); +}); + +describe("CL-09 no-feedback architecture guards", () => { + test("routing and CL-08 planning do not consume passive production queries", () => { + for (const path of [ + "src/routing/evaluator.ts", + "src/lab/automation/planner.ts", + ]) { + const source = readFileSync(path, "utf8"); + expect(source).not.toContain("queryPassiveProductionSignals"); + expect(source).not.toContain("passive-production"); + expect(source).not.toContain("production-signals"); + } + }); + + test("production request path only links the exact subject and never reads passive history", () => { + const source = readFileSync("src/server/responses/core.ts", "utf8"); + expect(source).toContain("resolveProductionRouteSubject"); + expect(source).not.toContain("queryPassiveProductionSignals"); + expect(source).not.toContain("readRecentUsageEntries"); + const cliSource = readFileSync("src/cli/lab.ts", "utf8"); + expect(cliSource).toContain("queryPassiveProductionSignals(subjectId, limit, configDir)"); + }); + + test("passive query remains read-side and cannot create Lab execution or evidence", () => { + const source = readFileSync("src/lab/query/passive-production.ts", "utf8"); + expect(source).toContain("readRecentUsageEntries"); + expect(source).not.toContain("ObservationEvent"); + expect(source).not.toContain("appendLab"); + expect(source).not.toContain("compatibility.jsonl"); + expect(source).not.toContain("fetch("); + expect(source).not.toContain("executeLive"); + }); +});