From ee58332fb901d2786f3859ebedde81679faa2497 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:20:33 -0400 Subject: [PATCH 01/25] feat: deepen Bombadil exploration diagnostics --- README.md | 20 +- docs/verification.md | 143 +++- src/exports.test.ts | 4 + src/tooling/bombadil-campaign.test.ts | 101 ++- src/tooling/bombadil-campaign.ts | 120 ++++ src/tooling/bombadil-runner.test.ts | 326 ++++++++- src/tooling/bombadil-runner.ts | 937 +++++++++++++++++++++++++- src/tooling/bombadil.ts | 21 + 8 files changed, 1652 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index e31c3e6..ee5cecd 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ A quiet probe means the declared deterministic work settled. It does not prove t | `@hraness/direct/web` | Atomic browser installation, with low-level bridge and firewall escape hatches | Browser only | | `@hraness/direct/tooling/browser-verification` | Protocol-bound bridge reads, bounded agent-browser commands, local server leases, and artifact writes | Bun 1.3.14 with Node APIs | | `@hraness/direct/tooling/bombadil-campaign` | Direct property and conservative action factories for a Bombadil specification | Bombadil 0.7.2 specification compiler | -| `@hraness/direct/tooling/bombadil` | Local server ownership, native Bombadil lifecycle, trace attestation, replay, and diagnostic artifacts | Bun 1.3.14 with Node APIs | +| `@hraness/direct/tooling/bombadil` | Local server ownership, native Bombadil lifecycle, serial campaign matrices, trace attestation and summaries, replay, and diagnostic artifacts | Bun 1.3.14 with Node APIs | | `@hraness/direct/tooling/bundle-boundary` | Deterministic emitted-file scans and exact versioned-wire evidence | Bun 1.3.14 with Node APIs | The tooling subpaths are development-only. They are built separately from the @@ -272,7 +272,23 @@ path, and any additional safe actions. Call `runDirectBombadilFuzz` from `@hraness/direct/tooling/bombadil` in a small Bun wrapper. The runner accepts only an explicit local HTTP origin, starts an argv-only server command, invokes the exact native 0.7.2 binary, attests the bounded trace with Direct's canonical -parsers, writes pass or failure artifacts, and releases its owned processes. +parsers, writes pass or failure artifacts plus a compact exploration summary, +and releases its owned processes. Use `runDirectBombadilFuzzMatrix` when a +product owns several scenarios; it runs them serially and requires one exact +campaign selector for replay. + +Keep liveness formulas time-bounded. Prefer guarded product actions with +explicit weights over unrestricted browser actions, and name small JSON +snapshots that expose semantic state without retaining page content. Run short +12–30 second campaigns while editing and longer 60–300 second matrices in a +scheduled diagnostic lane. Inspect and replay a retained failing trace, then +promote the smallest readable failure to a deterministic product regression. +When a campaign must exercise an interaction, require a named product value to +change after a non-Wait action so bootstrap and idle transitions do not satisfy +the exploration policy. +The raw trace remains authoritative and may contain screenshots, URLs, typed +text, accessible labels, and local paths; treat it as potentially sensitive. +Summary counts and hashes help triage exploration but are not Direct coverage. See [Verification](./docs/verification.md#run-a-bounded-bombadil-campaign) for the complete configuration and proof limits. diff --git a/docs/verification.md b/docs/verification.md index 52c3e58..c4ae45a 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -237,6 +237,11 @@ deterministic checks. This path does not replace product-owned semantic or accessibility assertions. It also does not prove a substituted service, adapter, browser host, operating system, or device. +Bombadil is an experimental 0.x tool. Review the official +[manual](https://antithesishq.github.io/bombadil/) when changing its pinned +version, specification, actions, properties, or CLI invocation; minor releases +may change interfaces or trace details. + Install the one supported release directly in the consumer. Direct declares it as an exact optional peer so products that do not use fuzzing do not install a browser tool: @@ -253,20 +258,36 @@ TypeScript source. Import it only from a Bombadil specification; use the built Create a specification such as `direct/bombadil-campaign.ts`: ```ts +import { always, eventually } from "@antithesishq/bombadil"; import { createDirectBombadilActions, + createDirectBombadilNamedSnapshot, createDirectBombadilProperties, + createDirectBombadilResourceLeakProperty, } from "@hraness/direct/tooling/bombadil-campaign"; export * from "@antithesishq/bombadil/browser/defaults/properties"; const direct = createDirectBombadilProperties(); +const phase = createDirectBombadilNamedSnapshot({ + fallback: "unavailable", + name: "todos.phase", + read: ({ window }) => Reflect.get(window, "__todosPhase"), +}); export const direct_safe_actions = createDirectBombadilActions(); export const direct_exact_contract = direct.exactContract; export const direct_stable_catalog = direct.stableCatalog; export const direct_no_declared_violations = direct.noDeclaredViolations; export const direct_eventual_quiescence = direct.eventualQuiescence; +export const todos_phase_is_known = always( + eventually(() => phase.current !== "unavailable").within(5, "seconds"), +); +export const no_dom_node_leak = createDirectBombadilResourceLeakProperty({ + metric: "dom_nodes", + growthLimit: 500, + windowMillis: 10_000, +}); ``` The Direct action generator deliberately excludes reload, history traversal, @@ -275,6 +296,31 @@ Enter key. It retains ordinary buttons, text input, scrolling, and an always-eligible low-weight wait. This preserves the post-handshake contract within one document. Add product actions only when their navigation and form effects are understood, and keep product-specific assertions in the campaign. +Guard domain actions on the state that makes them valid, then weight valuable +state-changing actions above Wait. Do not increase throughput by admitting +reload, navigation, submission, destructive controls, or arbitrary generated +input. A generated action sequence is useful only while it preserves the +scenario boundary and exercises behavior the product can interpret. + +`createDirectBombadilNamedSnapshot` gives product properties and post-run +diagnostics a small semantic signal. It requires a safe bounded name, clones no +more than two million JSON characters, and fails closed to the explicit +fallback when a page getter throws or returns unsuitable data. Extract state, +not page content or credentials. Named values are represented only by canonical +SHA-256 hashes in the exploration summary; the authoritative raw trace still +contains the original values. + +Bombadil's 0.7.2 manual documents a sliding-window resource property at +`@antithesishq/bombadil/browser/extras/resources`, but the published npm +package omits that subpath from its `exports` map. Use Direct's +`createDirectBombadilResourceLeakProperty` implementation until a reviewed +Bombadil release exports the official helper. Add a tuned property when +repeated product actions allocate DOM nodes, listeners, layout objects, or +heap. Prefer DOM-node or listener thresholds when they express the defect +because heap samples move with garbage collection. Measure a normal run in +Inspect before setting a growth limit, and keep the window longer than ordinary +rendering bursts. The summary's resource high-water marks help choose and +review those thresholds; a maximum alone does not prove or disprove a leak. Create a Bun wrapper such as `direct/fuzz-browser.ts`: @@ -297,6 +343,14 @@ await runDirectBombadilFuzz({ repositoryRoot, scenario: "todos.populated", specificationPath: resolve(directRoot, "bombadil-campaign.ts"), + viewport: { width: 1_024, height: 768, deviceScaleFactor: 2 }, + explorationPolicy: { + minNonWaitActions: 1, + requiredNamedSnapshots: ["direct", "todos.phase"], + minDistinctNamedSnapshotValues: { "todos.phase": 2 }, + minNamedSnapshotChangesAfterNonWait: { "todos.phase": 1 }, + requireStableTargetUrl: true, + }, server: { command: [ process.execPath, @@ -327,6 +381,26 @@ the runner rejects Direct's reserved scenario and fixture keys there because it binds the requested scenario itself. The repository root, campaign path, server working directory, and replay trace are resolved canonically before use; a symlink that escapes the configured repository is rejected. +`viewport` is optional and defaults to Bombadil 0.7.2's 1024×768 viewport at a +device scale factor of 2. The runner always passes the validated exact values +to random and replay invocations and records them in `run.json`. +Assign one reviewed viewport to each existing scenario campaign. Alternate a +stable wide and narrow viewport across the matrix instead of duplicating every +scenario at every size; add a second size only when the scenario owns a +responsive behavior that needs separate exploration. + +`explorationPolicy` is optional. When present, it can require a minimum count +of non-Wait actions, particular action kinds and named snapshots, minimum +distinct hashed values for named snapshots, named-value changes observed after +a non-Wait action, and an exact stable target URL. The change requirement keeps +bootstrap or Wait-only transitions from satisfying a product-interaction +threshold. The trace records the last action with the resulting state, so this +count identifies a named value transition associated with an active product +action. It does not prove causality. The policy detects a campaign that passed +its properties without doing the intended exploration. It is a diagnostic +sufficiency check, not a coverage claim. Start with observed stable behavior +and raise thresholds only when the action generator makes them reliably +reachable. Run random exploration for 12 to 300 seconds. The default is 20 seconds: @@ -334,19 +408,54 @@ Run random exploration for 12 to 300 seconds. The default is 20 seconds: bun direct/fuzz-browser.ts --time-limit 20s ``` +Use 12–30 seconds in the edit loop. A scheduled diagnostic lane can run each +campaign for 60–300 seconds, serially, with an outer job timeout and retained +failure artifacts. Random exploration should supplement the deterministic +required gate rather than make an otherwise healthy pull request depend on one +particular random path. + +For multiple product scenarios, pass a bounded matrix to the shared runner: + +```ts +import { runDirectBombadilFuzzMatrix } from "@hraness/direct/tooling/bombadil"; + +await runDirectBombadilFuzzMatrix([ + { id: "populated", config: populatedCampaign }, + { id: "empty", config: emptyCampaign }, +], process.argv.slice(2)); +``` + +Without `--campaign`, the matrix runs every unique campaign serially. Select +one for focused work with `--campaign empty`. Replay is intentionally rejected +without that selector so a trace cannot be applied to the wrong scenario. + Use `--base-url` to select another local root origin. Use `--replay` with a repository-local `.jsonl` trace instead of `--time-limit` to reproduce a prior run. The native runner supports Bombadil 0.7.2 on Apple silicon macOS and x64 or arm64 Linux. Unsupported platform and architecture pairs fail before a server starts. +Bombadil's browser Inspect UI is the fastest way to examine the actions, +screenshots, resource timeline, snapshots, and violations in a retained run: + +```sh +bunx bombadil browser inspect artifacts/direct-bombadil/todos//bombadil +bun direct/fuzz-browser.ts --replay artifacts/direct-bombadil/todos//bombadil/trace.jsonl +``` + +Use the same campaign, viewport, specification, scenario, application tree, +and server configuration for replay. Bombadil rejects a replay that diverges, +so reproduction is strong debugging evidence but not guaranteed after the +product changes. + The browser formulas require an exact scenario contract, a nonempty catalog, zero declared violations, and quiescence to recur within ten seconds from every -observed state. Bombadil's temporal engine cannot decide an unbounded `always` -inside a bounded `eventually`, so continuous identity and catalog binding stay -in the host attestation below. A formula result and Bombadil exit status are not -sufficient evidence because a short or incomplete trace could otherwise pass -vacuously. +observed state. Keep every liveness obligation bounded to a product latency +budget. A finite run cannot decide an unbounded future obligation, and nesting +unbounded `always` inside bounded `eventually` does not make it decidable. +Continuous identity and catalog binding therefore stay in the host attestation +below. A formula result and Bombadil exit status are not sufficient evidence +because a short or incomplete trace could otherwise pass vacuously. After every random run or replay, the host runner streams the bounded 0.7.2 JSONL trace from foreign input and requires one named `direct` observation per @@ -369,12 +478,30 @@ configured local server is always stopped through the shared browser-verification lease helpers, and its output drain remains bounded even when cleanup itself fails. -Each attempt writes `run.json`, `bombadil.log`, and `server.log` below +Each attempt writes `run.json`, `exploration-summary.json`, `bombadil.log`, and `server.log` below `artifacts/direct-bombadil///`, including failures. The rolling `manifest.json` points to the latest record. `rawTracePath` reports a regular nonempty trace even if attestation fails; `tracePath` is present only -after exact attestation. Keep these diagnostic artifacts out of source control -unless the consumer explicitly reviews them as fixtures. +after exact attestation. The summary strictly parses the 0.7.2 envelopes and +records the raw trace SHA-256, action-kind and safe target-tag counts, non-Wait +count and longest Wait streak, origin-relative URL fingerprints, non-null +transition-hash cardinality, canonical named-snapshot value hashes, property +violation names and counts, named-value changes after non-Wait actions, and +browser resource high-water marks. It excludes +typed text, accessible names, snapshot values, URLs, screenshots, and absolute +paths. These diagnostics describe what Bombadil happened to explore. They do +not measure code, state, interaction, or Direct catalog coverage, and the raw +trace remains authoritative. + +Keep all generated artifacts out of source control by default. Upload a failed +scheduled run to access-controlled CI storage with a bounded retention period; +the raw trace can contain screenshots, query values, typed text, accessibility +labels, extracted values, and local paths. Preserve it long enough to inspect +and replay. Once the defect is understood, add the smallest deterministic +regression at the owning parser, reducer, port, component, semantic browser, or +Direct scenario boundary. Verify that regression fails before the fix and +passes after it. Retain a reviewed trace fixture only when replay itself adds +durable value; otherwise remove the sensitive trace after promotion. ## Report coverage without promotion diff --git a/src/exports.test.ts b/src/exports.test.ts index 5d42c43..6672cd7 100644 --- a/src/exports.test.ts +++ b/src/exports.test.ts @@ -61,12 +61,16 @@ describe("public package exports", () => { expect(Object.keys(bombadil).toSorted()).toEqual([ "attestDirectBombadilTrace", "runDirectBombadilFuzz", + "runDirectBombadilFuzzMatrix", + "summarizeDirectBombadilTrace", ]); expect(typeof browserVerification.createAgentBrowser).toBe("function"); expect(typeof browserVerification.createDirectBrowserContractReader).toBe("function"); expect(typeof browserVerification.readDirectBrowserContract).toBe("function"); expect(typeof bombadil.runDirectBombadilFuzz).toBe("function"); expect(typeof bombadil.attestDirectBombadilTrace).toBe("function"); + expect(typeof bombadil.runDirectBombadilFuzzMatrix).toBe("function"); + expect(typeof bombadil.summarizeDirectBombadilTrace).toBe("function"); expect(typeof bundleBoundary.checkBundleBoundary).toBe("function"); expect(typeof bundleBoundary.findForbiddenMarkers).toBe("function"); expect("createAgentBrowser" in root).toBeFalse(); diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 22db343..63ac1cb 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -13,7 +13,10 @@ interface FakeCell { current: unknown; name: string | null; readonly named: (name: string) => FakeCell; - readonly read: (state: { readonly window: unknown }) => unknown; + readonly read: (state: { + readonly resources?: Readonly>; + readonly window: unknown; + }) => unknown; } interface FakeActionGenerator { @@ -68,7 +71,9 @@ void mock.module("@antithesishq/bombadil/browser/defaults/actions", () => ({ const { createDirectBombadilActions, + createDirectBombadilNamedSnapshot, createDirectBombadilProperties, + createDirectBombadilResourceLeakProperty, readDirectBombadilObservation, } = await import("./bombadil-campaign.js"); @@ -229,6 +234,100 @@ describe("Direct Bombadil observation", () => { }); }); +describe("Direct Bombadil named snapshots", () => { + test("names bounded JSON and fails closed around hostile or oversized page values", () => { + const snapshot = createDirectBombadilNamedSnapshot({ + fallback: { status: "unavailable" }, + name: "product.phase", + read: (state) => Reflect.get(state.window, "phase"), + }) as unknown as FakeCell; + expect(snapshot.name).toBe("product.phase"); + expect(snapshot.read({ window: { phase: { status: "ready" } } })).toEqual({ + status: "ready", + }); + + const hostile = {}; + Object.defineProperty(hostile, "phase", { + get: () => { + throw new Error("hostile getter"); + }, + }); + expect(snapshot.read({ window: hostile })).toEqual({ status: "unavailable" }); + expect(snapshot.read({ + window: { phase: "x".repeat(2_000_001) }, + })).toEqual({ status: "unavailable" }); + }); + + test("rejects unsafe names and non-JSON fallbacks before registering an extractor", () => { + expect(() => createDirectBombadilNamedSnapshot({ + fallback: null, + name: "unsafe name", + read: () => null, + })).toThrow("safe 1-128 character identifier"); + expect(() => createDirectBombadilNamedSnapshot({ + fallback: undefined as never, + name: "safe", + read: () => null, + })).toThrow("fallback must be bounded JSON"); + }); +}); + +describe("Direct Bombadil resource properties", () => { + const resources = (timestamp: number, domNodes: number) => ({ + documents: 1, + dom_nodes: domNodes, + js_event_listeners: 2, + js_heap_total: 2_000, + js_heap_used: 1_000, + layout_objects: 10, + script_duration: 0, + task_duration: 0, + thread_time: 0, + timestamp, + }); + + test("detects excessive growth across a bounded sliding resource window", () => { + const property = createDirectBombadilResourceLeakProperty({ + growthLimit: 10, + metric: "dom_nodes", + windowMillis: 1_000, + }) as unknown as FakeFormula; + const cell = cells.at(-1); + if (cell === undefined) throw new Error("resource extractor was not registered"); + + cell.current = cell.read({ resources: resources(1, 100), window: {} }); + expect(evaluate(property.body)).toBeTrue(); + cell.current = cell.read({ resources: resources(1.5, 105), window: {} }); + expect(evaluate(property.body)).toBeTrue(); + cell.current = cell.read({ resources: resources(1.75, 120), window: {} }); + expect(evaluate(property.body)).toBeFalse(); + }); + + test("rejects unknown fields, metrics, and unsafe numeric bounds", () => { + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: 1, + metric: "dom_nodes", + windowMillis: 1_000, + extra: true, + } as never)).toThrow("must contain metric"); + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: 1, + metric: "documents" as never, + windowMillis: 1_000, + })).toThrow("metric is unsupported"); + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: Number.POSITIVE_INFINITY, + metric: "dom_nodes", + windowMillis: 1_000, + })).toThrow("positive finite safe number"); + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: 1, + metric: "dom_nodes", + windowMillis: 300_001, + })).toThrow("between 1 and 300000"); + }); +}); + function clickAction(overrides: Readonly> = {}): unknown { return { Click: { diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index 5a932d4..5650e12 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -9,6 +9,7 @@ import { extract, weighted, type ActionGenerator, + type Cell, type Formula, type JSON as BombadilJson, type Tree, @@ -29,6 +30,16 @@ const DIRECT_PROBE_SCHEMA = "direct.probe/v1"; const MAX_RAW_CONTRACT_CHARACTERS = 2_000_000; const BRIDGE_KEYS = new Set(["manifest", "reset", "schema", "snapshot"]); const UNSAFE_CLICK_INPUT_TYPES = new Set(["image", "reset", "submit"]); +const SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +const RESOURCE_LEAK_OPTION_KEYS = new Set(["growthLimit", "metric", "windowMillis"]); +const RESOURCE_METRICS = [ + "dom_nodes", + "js_event_listeners", + "js_heap_total", + "js_heap_used", + "layout_objects", +] as const; +const RESOURCE_METRIC_SET = new Set(RESOURCE_METRICS); export interface DirectBombadilObservation { readonly [key: string | number | symbol]: BombadilJson; @@ -54,6 +65,14 @@ export interface DirectBombadilProperties { readonly eventualQuiescence: Formula; } +export type DirectBombadilResourceMetric = (typeof RESOURCE_METRICS)[number]; + +export interface DirectBombadilResourceLeakOptions { + readonly growthLimit: number; + readonly metric: DirectBombadilResourceMetric; + readonly windowMillis: number; +} + function safeClickAction(action: ActionTemplate): boolean { if (typeof action !== "object" || action === null) return false; const candidate = "Click" in action @@ -126,6 +145,72 @@ function hasExactKeys( return keys.length === expected.size && keys.every((key) => expected.has(key)); } +/** + * Builds Bombadil's documented sliding-window resource growth invariant from + * the public browser state because 0.7.2 omits its extras module from exports. + */ +export function createDirectBombadilResourceLeakProperty( + options: DirectBombadilResourceLeakOptions, +): Formula { + if (!isRecord(options) || !hasExactKeys(options, RESOURCE_LEAK_OPTION_KEYS)) { + throw new Error("Bombadil resource leak options must contain metric, growthLimit, and windowMillis"); + } + if (typeof options.metric !== "string" || !RESOURCE_METRIC_SET.has(options.metric)) { + throw new Error("Bombadil resource leak metric is unsupported"); + } + if ( + typeof options.growthLimit !== "number" + || !Number.isFinite(options.growthLimit) + || options.growthLimit <= 0 + || options.growthLimit > Number.MAX_SAFE_INTEGER + ) { + throw new Error("Bombadil resource leak growthLimit must be a positive finite safe number"); + } + if ( + typeof options.windowMillis !== "number" + || !Number.isSafeInteger(options.windowMillis) + || options.windowMillis < 1 + || options.windowMillis > 300_000 + ) { + throw new Error("Bombadil resource leak windowMillis must be an integer between 1 and 300000"); + } + const metric = options.metric; + const samples: Array<{ readonly timestamp: number; readonly value: number }> = []; + let previousTimestamp = -1; + const window = extract((state) => { + const timestamp = state.resources.timestamp * 1_000; + const value = state.resources[metric]; + if ( + !Number.isFinite(timestamp) + || timestamp < 0 + || timestamp < previousTimestamp + || !Number.isFinite(value) + || value < 0 + ) { + return { baseline: 0, valid: false, value: 0 }; + } + previousTimestamp = timestamp; + samples.push({ timestamp, value }); + const cutoff = timestamp - options.windowMillis; + while (samples.length > 2 && (samples[1]?.timestamp ?? Number.POSITIVE_INFINITY) <= cutoff) { + samples.shift(); + } + return { + baseline: samples[0]?.value ?? value, + valid: true, + value, + }; + }); + return always(() => + window.current.valid + && window.current.value - window.current.baseline <= options.growthLimit + ); +} + function invalidObservation(bridgePresent = false): DirectBombadilObservation { return { activationHash: "", @@ -150,6 +235,41 @@ function boundedJsonClone(value: unknown): BombadilJson | null { return JSON.parse(source) as BombadilJson; } +function optionalBoundedJsonClone(value: unknown): BombadilJson | undefined { + const source = JSON.stringify(value); + if (source === undefined || source.length > MAX_RAW_CONTRACT_CHARACTERS) return undefined; + return JSON.parse(source) as BombadilJson; +} + +/** + * Creates a named, bounded JSON extractor that fails closed to an explicit + * fallback when a page getter throws or returns non-JSON or oversized data. + */ +export function createDirectBombadilNamedSnapshot(options: { + readonly fallback: T; + readonly name: string; + readonly read: (state: BombadilBrowserState) => unknown; +}): Cell { + if ( + options.name.length === 0 + || options.name.length > 128 + || !SNAPSHOT_NAME_PATTERN.test(options.name) + ) { + throw new Error("Bombadil snapshot name must be a safe 1-128 character identifier"); + } + const fallback = optionalBoundedJsonClone(options.fallback); + if (fallback === undefined) { + throw new Error("Bombadil snapshot fallback must be bounded JSON"); + } + return extract((state) => { + try { + return (optionalBoundedJsonClone(options.read(state)) ?? fallback) as T; + } catch { + return fallback as T; + } + }).named(options.name); +} + function readNonNegativeCounters(value: unknown): { readonly valid: boolean; readonly values: number[]; diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index a3e0709..a164b38 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -11,6 +11,8 @@ import { parseDirectBombadilFuzzArguments, runBombadilNativeProcess, runDirectBombadilFuzz, + runDirectBombadilFuzzMatrix, + summarizeDirectBombadilTrace, validateDirectBombadilFuzzConfig, type DirectBombadilFuzzConfig, type DirectBombadilInvocation, @@ -187,13 +189,49 @@ function absentObservation(): Record { }; } -function traceLine(observation: unknown, timestamp: number): string { +interface TraceLineOptions { + readonly action?: unknown; + readonly namedSnapshots?: readonly { readonly name: string; readonly value: unknown }[]; + readonly url?: string; + readonly violations?: readonly unknown[]; +} + +function traceLine( + observation: unknown, + timestamp: number, + options: TraceLineOptions = {}, +): string { return JSON.stringify({ timestamp, - action: null, - state: {}, - snapshots: [{ index: 0, name: "direct", value: observation, time: timestamp }], - violations: [], + action: options.action ?? null, + state: { + url: options.url ?? "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + hash_previous: timestamp === 1 ? null : timestamp - 1, + hash_current: timestamp, + screenshot: `/tmp/${String(timestamp)}.png`, + resources: { + js_heap_used: timestamp * 10, + js_heap_total: timestamp * 20, + dom_nodes: timestamp, + documents: 1, + js_event_listeners: timestamp * 2, + layout_objects: timestamp * 3, + timestamp, + thread_time: timestamp / 10, + task_duration: timestamp / 20, + script_duration: timestamp / 30, + }, + }, + snapshots: [ + { index: 0, name: "direct", value: observation, time: timestamp }, + ...(options.namedSnapshots ?? []).map((snapshot, index) => ({ + index: index + 1, + name: snapshot.name, + value: snapshot.value, + time: timestamp, + })), + ], + violations: options.violations ?? [], }); } @@ -392,6 +430,11 @@ describe("Direct Bombadil configuration and invocation", () => { expect(validated.entryPath).toBe("/"); expect(validated.targetQuery).toEqual({}); expect(validated.server.readinessPath).toBe("/ready"); + expect(validated.viewport).toEqual({ + deviceScaleFactor: 2, + height: 768, + width: 1_024, + }); expect(validated.bombadilExecutable).toEndWith( `node_modules/@antithesishq/bombadil/binaries/${nativeBinaryName()}`, ); @@ -435,6 +478,26 @@ describe("Direct Bombadil configuration and invocation", () => { ...config, server: { ...config.server, readinessPath: "//external.test" }, })).toThrow("origin-relative"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + viewport: { width: 0 }, + })).toThrow("viewport.width"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + viewport: { deviceScaleFactor: Number.POSITIVE_INFINITY }, + })).toThrow("deviceScaleFactor"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { requiredActionKinds: ["Wait", "Wait"] }, + })).toThrow("duplicate kind"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { minDistinctNamedSnapshotValues: { "unsafe name": 2 } }, + })).toThrow("safe bounded snapshot name"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { minNamedSnapshotChangesAfterNonWait: { phase: 10_001 } }, + })).toThrow("between 1 and 10000"); }); test("rejects specification, server cwd, and replay symlinks that escape the repository", async () => { @@ -493,6 +556,7 @@ describe("Direct Bombadil configuration and invocation", () => { specificationPath: "/repo/spec.ts", targetQuery: { workbench: "frame" }, timeLimitSeconds: 20, + viewport: { deviceScaleFactor: 1.25, height: 720, width: 1_280 }, }); expect(invocation.targetUrl).toBe( "http://127.0.0.1:5184/direct/?__direct_scenario=surface.ready&workbench=frame", @@ -508,6 +572,12 @@ describe("Direct Bombadil configuration and invocation", () => { "/repo/artifacts/run/bombadil", "--headless", "--instrument-javascript=", + "--width", + "1280", + "--height", + "720", + "--device-scale-factor", + "1.25", "--exit-on-violation", "--time-limit", "20s", @@ -526,12 +596,65 @@ describe("Direct Bombadil configuration and invocation", () => { timeLimitSeconds: 20, }); expect(invocation.command).toContain("--reproduce"); + expect(invocation.command).toEqual(expect.arrayContaining([ + "--width", "1024", "--height", "768", "--device-scale-factor", "2", + ])); expect(invocation.command).not.toContain("--time-limit"); expect(invocation.command).not.toContain("--exit-on-violation"); expect(invocation.wallClockTimeoutMs).toBe(330_000); }); }); +describe("Direct Bombadil campaign matrix", () => { + test("runs unique bounded campaigns serially and selects exactly one", async () => { + const { config } = await fixture(); + const campaigns = [{ id: "primary", config }, { + id: "secondary", + config: { ...config, artifactName: "fixture-secondary" }, + }] as const; + const allRuntime = dependencies(); + const all = await runDirectBombadilFuzzMatrix( + campaigns, + ["--time-limit=12s"], + allRuntime.overrides, + ); + expect(all).toMatchObject({ + kind: "matrix", + results: [{ campaignId: "primary" }, { campaignId: "secondary" }], + }); + expect(allRuntime.calls.filter((call) => call === "run-bombadil")).toHaveLength(2); + + const selectedRuntime = dependencies(); + const selected = await runDirectBombadilFuzzMatrix( + campaigns, + ["--campaign=secondary", "--time-limit=12s"], + selectedRuntime.overrides, + ); + expect(selected).toMatchObject({ + kind: "matrix", + results: [{ campaignId: "secondary" }], + }); + expect(selectedRuntime.calls.filter((call) => call === "run-bombadil")).toHaveLength(1); + }); + + test("rejects ambiguous replay, duplicate IDs, and unknown selection", async () => { + const { config } = await fixture(); + const campaigns = [{ id: "primary", config }] as const; + expect((await rejection(runDirectBombadilFuzzMatrix( + campaigns, + ["--replay=artifacts/trace.jsonl"], + ))).message).toContain("requires exactly one --campaign"); + expect((await rejection(runDirectBombadilFuzzMatrix( + campaigns, + ["--campaign=missing"], + ))).message).toContain("Unknown Bombadil campaign"); + expect((await rejection(runDirectBombadilFuzzMatrix([ + { id: "same", config }, + { id: "same", config: { ...config, artifactName: "other" } }, + ], []))).message).toContain("unique lowercase kebab"); + }); +}); + describe("Direct Bombadil trace attestation", () => { async function attest(observations: readonly unknown[]) { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-trace-")); @@ -657,6 +780,165 @@ describe("Direct Bombadil trace attestation", () => { }); }); +describe("Direct Bombadil exploration summary", () => { + async function summaryTrace(lines: readonly string[]): Promise { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-summary-")); + temporaryDirectories.push(directory); + const tracePath = join(directory, "trace.jsonl"); + await writeFile(tracePath, `${lines.join("\n")}\n`, "utf8"); + return tracePath; + } + + test("derives deterministic bounded diagnostics without retaining typed text or labels", async () => { + const observation = directObservation(); + const tracePath = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: { b: 2, a: 1 } }], + }), + traceLine(observation, 2, { + action: { + Click: { + fingerprint: { + accessible_name: "sensitive button label", + tag: "button", + }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: { a: 2 } }], + }), + traceLine(observation, 3, { + action: "Wait", + namedSnapshots: [{ name: "phase", value: { a: 2 } }], + }), + traceLine(observation, 4, { + action: { TypeText: { delay_millis: 0, text: "sensitive typed text" } }, + namedSnapshots: [{ name: "phase", value: { a: 2 } }], + violations: [{ + name: "noConsoleErrors", + violation: { False: { condition: "sensitive formula source" } }, + }], + }), + ]); + const options = { + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + minNonWaitActions: 2, + requireStableTargetUrl: true, + requiredActionKinds: ["Click", "TypeText"] as const, + requiredNamedSnapshots: ["phase"], + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }; + const first = await summarizeDirectBombadilTrace(options); + const second = await summarizeDirectBombadilTrace(options); + expect(second).toEqual(first); + expect(first).toMatchObject({ + schema: "direct.bombadil-exploration-summary/v1", + actions: { + byKind: { Click: 1, TypeText: 1, Wait: 1 }, + maxWaitStreak: 1, + nonWaitCount: 2, + targetTags: { button: 1 }, + total: 3, + }, + urls: { + distinctFingerprintCount: 1, + observationCount: 4, + stableTarget: true, + }, + transitions: { distinctNonNullHashCount: 4, nonNullHashCount: 4 }, + propertyViolations: { byName: { noConsoleErrors: 1 }, total: 1 }, + resourceHighWaterMarks: { + domNodes: 4, + jsHeapUsedBytes: 40, + }, + policy: { configured: true, failures: [], satisfied: true }, + }); + expect(first.namedSnapshots.find((entry) => entry.name === "phase")).toMatchObject({ + changeAfterNonWaitCount: 1, + distinctValueCount: 2, + observationCount: 4, + }); + expect(first.trace.sha256).toHaveLength(64); + const serialized = JSON.stringify(first); + expect(serialized).not.toContain("sensitive button label"); + expect(serialized).not.toContain("sensitive typed text"); + expect(serialized).not.toContain("sensitive formula source"); + expect(serialized).not.toContain("127.0.0.1"); + expect(serialized).not.toContain("/tmp/"); + }); + + test("reports strict policy misses while keeping the diagnostic summary", async () => { + const tracePath = await summaryTrace([traceLine(directObservation(), 1)]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNonWaitActions: 1, + requireStableTargetUrl: true, + requiredActionKinds: ["Click"], + requiredNamedSnapshots: ["phase"], + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.policy.satisfied).toBeFalse(); + expect(summary.policy.failures).toHaveLength(4); + }); + + test("does not credit bootstrap or Wait-only state changes to product actions", async () => { + const observation = directObservation(); + const tracePath = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: "loading" }], + }), + traceLine(observation, 2, { + action: "Wait", + namedSnapshots: [{ name: "phase", value: "ready" }], + }), + ]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.find((entry) => entry.name === "phase")).toMatchObject({ + changeAfterNonWaitCount: 0, + distinctValueCount: 2, + }); + expect(summary.policy).toMatchObject({ + failures: [expect.stringContaining("post-non-Wait change minimum")], + satisfied: false, + }); + }); + + test("rejects hostile envelopes, action targets, and excessively deep snapshot JSON", async () => { + const observation = directObservation(); + const badTarget = await summaryTrace([traceLine(observation, 1, { + action: { Click: { fingerprint: { tag: "unsafe tag" } } }, + })]); + expect((await rejection(summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: badTarget, + }))).message).toContain("invalid action target tag"); + + let deep: unknown = null; + for (let index = 0; index < 70; index += 1) deep = [deep]; + const deepTrace = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: [{ name: "deep", value: deep }], + })]); + expect((await rejection(summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: deepTrace, + }))).message).toContain("exceeds JSON depth"); + }); +}); + describe("Direct Bombadil process lifecycle", () => { test("cleans descendants and inherited pipes after a normal leader exit", async () => { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-normal-exit-")); @@ -790,6 +1072,11 @@ describe("Direct Bombadil run lifecycle", () => { }, initialDirect: { source: "scenario", scenario: "surface.ready", route: "/surface" }, server: { logPresent: true }, + explorationSummary: { + schema: "direct.bombadil-exploration-summary/v1", + policy: { configured: false, satisfied: true }, + }, + viewport: { deviceScaleFactor: 2, height: 768, width: 1_024 }, }); const bombadil = record(manifest.bombadil, "bombadil"); expect(bombadil.version).toBe("0.7.2"); @@ -800,6 +1087,35 @@ describe("Direct Bombadil run lifecycle", () => { expect(await readFile(String(bombadil.logPath), "utf8")).toContain("bombadil stdout"); const server = record(manifest.server, "server"); expect(await readFile(String(server.logPath), "utf8")).toContain("server output"); + const summaryPath = String(manifest.explorationSummaryPath); + expect(JSON.parse(await readFile(summaryPath, "utf8"))).toMatchObject({ + schema: "direct.bombadil-exploration-summary/v1", + trace: { lineCount: 2 }, + }); + }); + + test("fails a configured exploration policy while retaining the derived sidecar", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies(); + expect((await rejection(runDirectBombadilFuzz({ + ...config, + explorationPolicy: { + minNonWaitActions: 1, + requiredActionKinds: ["Click"], + }, + }, [], runtime.overrides))).message).toContain("exploration policy was not satisfied"); + const manifest = JSON.parse(await readFile( + join(repositoryRoot, "artifacts", "direct-bombadil", "fixture-product", "manifest.json"), + "utf8", + )) as Record; + expect(manifest).toMatchObject({ + status: "failed", + explorationSummary: { + policy: { configured: true, satisfied: false }, + }, + }); + expect(await readFile(String(manifest.explorationSummaryPath), "utf8")) + .toContain("direct.bombadil-exploration-summary/v1"); }); test("retains an attested failure artifact and server log for a nonzero exit", async () => { diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index 61cf990..f2a2ef2 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -3,6 +3,7 @@ import { readFile, realpath, stat, writeFile } from "node:fs/promises"; import { isAbsolute, join, relative, resolve } from "node:path"; import process from "node:process"; import { createInterface } from "node:readline"; +import { createHash } from "node:crypto"; import { parseDirectProbeSnapshot, @@ -44,6 +45,12 @@ const TRACE_MAX_BYTES = 64 * 1024 * 1024; const TRACE_MAX_LINE_BYTES = 16 * 1024 * 1024; const TRACE_MAX_LINES = 10_000; const TRACE_MAX_SNAPSHOTS_PER_LINE = 4_096; +const TRACE_MAX_NAMED_SNAPSHOT_NAMES = 128; +const TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME = 1_024; +const TRACE_MAX_DISTINCT_URLS = 1_024; +const TRACE_MAX_PROPERTY_NAMES = 128; +const TRACE_MAX_CANONICAL_SNAPSHOT_BYTES = 2 * 1024 * 1024; +const TRACE_MAX_JSON_DEPTH = 64; const RANDOM_RUN_OVERHEAD_MS = 30_000; const REPLAY_WALL_CLOCK_TIMEOUT_MS = MAX_TIME_LIMIT_SECONDS * 1_000 + RANDOM_RUN_OVERHEAD_MS; const PROCESS_TERMINATION_GRACE_MS = 5_000; @@ -52,6 +59,66 @@ const SERVER_OUTPUT_TIMEOUT_MS = 3_000; const DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; const TRACE_LINE_KEYS = new Set(["action", "snapshots", "state", "timestamp", "violations"]); const TRACE_SNAPSHOT_KEYS = new Set(["index", "name", "time", "value"]); +const TRACE_STATE_KEYS = new Set([ + "hash_current", + "hash_previous", + "resources", + "screenshot", + "url", +]); +const TRACE_RESOURCE_KEYS = new Set([ + "documents", + "dom_nodes", + "js_event_listeners", + "js_heap_total", + "js_heap_used", + "layout_objects", + "script_duration", + "task_duration", + "thread_time", + "timestamp", +]); +const TRACE_VIOLATION_KEYS = new Set(["name", "violation"]); +const VIEWPORT_KEYS = new Set(["deviceScaleFactor", "height", "width"]); +const EXPLORATION_POLICY_KEYS = new Set([ + "minDistinctNamedSnapshotValues", + "minNamedSnapshotChangesAfterNonWait", + "minNonWaitActions", + "requireStableTargetUrl", + "requiredActionKinds", + "requiredNamedSnapshots", +]); +const DEFAULT_VIEWPORT_WIDTH = 1_024; +const DEFAULT_VIEWPORT_HEIGHT = 768; +const DEFAULT_DEVICE_SCALE_FACTOR = 2; +const SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +const TARGET_TAG_PATTERN = /^[a-z][a-z0-9-]*$/u; +const ACTION_KINDS = [ + "Back", + "Click", + "DoubleClick", + "Forward", + "MouseDrag", + "PressKey", + "Reload", + "ScrollDown", + "ScrollUp", + "SetFileInputFiles", + "SetViewport", + "TypeText", + "Wait", +] as const; +const ACTION_KIND_SET = new Set(ACTION_KINDS); +const UNIT_ACTION_KINDS = new Set([ + "Back", + "Forward", + "Reload", + "Wait", +]); +const TARGETED_ACTION_KINDS = new Set([ + "Click", + "DoubleClick", +]); const DIRECT_OBSERVATION_KEYS = new Set([ "activationHash", "activeRoute", @@ -76,6 +143,76 @@ export interface DirectBombadilServerConfig { readonly startupTimeoutMs?: number; } +export type DirectBombadilActionKind = (typeof ACTION_KINDS)[number]; + +export interface DirectBombadilViewportConfig { + readonly deviceScaleFactor?: number; + readonly height?: number; + readonly width?: number; +} + +export interface DirectBombadilExplorationPolicy { + readonly minDistinctNamedSnapshotValues?: Readonly>; + readonly minNamedSnapshotChangesAfterNonWait?: Readonly>; + readonly minNonWaitActions?: number; + readonly requireStableTargetUrl?: boolean; + readonly requiredActionKinds?: readonly DirectBombadilActionKind[]; + readonly requiredNamedSnapshots?: readonly string[]; +} + +export interface DirectBombadilExplorationSummary { + readonly schema: "direct.bombadil-exploration-summary/v1"; + readonly trace: { + readonly bytes: number; + readonly lineCount: number; + readonly sha256: string; + }; + readonly actions: { + readonly byKind: Readonly>>; + readonly maxWaitStreak: number; + readonly nonWaitCount: number; + readonly targetTags: Readonly>; + readonly total: number; + }; + readonly urls: { + readonly distinctFingerprintCount: number; + readonly fingerprintSha256: readonly string[]; + readonly observationCount: number; + readonly stableTarget: boolean; + }; + readonly transitions: { + readonly distinctNonNullHashCount: number; + readonly nonNullHashCount: number; + }; + readonly namedSnapshots: readonly { + readonly changeAfterNonWaitCount: number; + readonly distinctValueCount: number; + readonly distinctValueSha256: readonly string[]; + readonly name: string; + readonly observationCount: number; + }[]; + readonly propertyViolations: { + readonly byName: Readonly>; + readonly total: number; + }; + readonly resourceHighWaterMarks: { + readonly documents: number; + readonly domNodes: number; + readonly jsEventListeners: number; + readonly jsHeapTotalBytes: number; + readonly jsHeapUsedBytes: number; + readonly layoutObjects: number; + readonly scriptDurationSeconds: number; + readonly taskDurationSeconds: number; + readonly threadTimeSeconds: number; + }; + readonly policy: { + readonly configured: boolean; + readonly failures: readonly string[]; + readonly satisfied: boolean; + }; +} + export interface DirectBombadilFuzzConfig { readonly artifactName: string; readonly baseUrl: string; @@ -86,6 +223,8 @@ export interface DirectBombadilFuzzConfig { readonly scenario: string; readonly specificationPath: string; readonly targetQuery?: Readonly>; + readonly explorationPolicy?: DirectBombadilExplorationPolicy; + readonly viewport?: DirectBombadilViewportConfig; readonly server: DirectBombadilServerConfig; } @@ -107,6 +246,21 @@ export type DirectBombadilFuzzResult = readonly status: "passed"; }; +export interface DirectBombadilFuzzCampaign { + readonly config: DirectBombadilFuzzConfig; + readonly id: string; +} + +export type DirectBombadilFuzzMatrixResult = + | { readonly kind: "help" } + | { + readonly kind: "matrix"; + readonly results: readonly { + readonly campaignId: string; + readonly result: Extract; + }[]; + }; + export interface DirectBombadilInvocation { readonly abortSignal?: AbortSignal; readonly command: readonly string[]; @@ -184,17 +338,37 @@ interface ProcessSignalEmitter { ) => unknown; } -interface ValidatedConfig extends DirectBombadilFuzzConfig { +type ValidatedConfig = Omit< + DirectBombadilFuzzConfig, + "explorationPolicy" | "server" | "viewport" +> & { readonly artifactRoot: string; readonly baseUrl: string; readonly bombadilExecutable: string; readonly entryPath: `/${string}`; + readonly explorationPolicy: ValidatedExplorationPolicy | null; readonly port: string; readonly targetQuery: Readonly>; + readonly viewport: ValidatedViewport; readonly server: DirectBombadilServerConfig & { readonly readinessPath: `/${string}`; readonly startupTimeoutMs: number; }; +}; + +interface ValidatedViewport { + readonly deviceScaleFactor: number; + readonly height: number; + readonly width: number; +} + +interface ValidatedExplorationPolicy { + readonly minDistinctNamedSnapshotValues: Readonly>; + readonly minNamedSnapshotChangesAfterNonWait: Readonly>; + readonly minNonWaitActions: number; + readonly requireStableTargetUrl: boolean; + readonly requiredActionKinds: readonly DirectBombadilActionKind[]; + readonly requiredNamedSnapshots: readonly string[]; } function readOptionValue( @@ -427,7 +601,177 @@ function exactTraceDirectObservation( }; } -function parseTraceLine(line: string, lineNumber: number): TraceDirectObservation { +interface ParsedTraceAction { + readonly kind: DirectBombadilActionKind; + readonly targetTag: string | null; +} + +interface ParsedTraceState { + readonly currentHash: number | null; + readonly resources: Readonly>; + readonly url: URL; +} + +interface ParsedTraceLine { + readonly action: ParsedTraceAction | null; + readonly directObservation: TraceDirectObservation; + readonly namedSnapshots: readonly { + readonly name: string; + readonly valueSha256: string; + }[]; + readonly propertyViolationNames: readonly string[]; + readonly state: ParsedTraceState; +} + +const RESOURCE_FIELD_MAP = { + documents: "documents", + dom_nodes: "domNodes", + js_event_listeners: "jsEventListeners", + js_heap_total: "jsHeapTotalBytes", + js_heap_used: "jsHeapUsedBytes", + layout_objects: "layoutObjects", + script_duration: "scriptDurationSeconds", + task_duration: "taskDurationSeconds", + thread_time: "threadTimeSeconds", +} as const; + +function canonicalJson(value: unknown, depth = 0): string { + if (depth > TRACE_MAX_JSON_DEPTH) { + throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(TRACE_MAX_JSON_DEPTH)}`); + } + if (value === null || typeof value === "boolean" || typeof value === "string") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("Bombadil named snapshot has a non-finite number"); + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((entry) => canonicalJson(entry, depth + 1)).join(",")}]`; + } + if (!isRecord(value)) throw new Error("Bombadil named snapshot is not JSON"); + const entries = Object.keys(value).sort().map((key) => + `${JSON.stringify(key)}:${canonicalJson(value[key], depth + 1)}` + ); + return `{${entries.join(",")}}`; +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +function namedSnapshotValueSha256(value: unknown): string { + const canonical = canonicalJson(value); + if (Buffer.byteLength(canonical, "utf8") > TRACE_MAX_CANONICAL_SNAPSHOT_BYTES) { + throw new Error( + `Bombadil named snapshot exceeds ${String(TRACE_MAX_CANONICAL_SNAPSHOT_BYTES)} canonical bytes`, + ); + } + return sha256(canonical); +} + +function parseTraceAction(value: unknown, lineNumber: number): ParsedTraceAction | null { + if (value === null) return null; + if (typeof value === "string") { + if (!ACTION_KIND_SET.has(value) || !UNIT_ACTION_KINDS.has(value as DirectBombadilActionKind)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + } + return { kind: value as DirectBombadilActionKind, targetTag: null }; + } + if (!isRecord(value) || Object.keys(value).length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + } + const kind = Object.keys(value)[0]; + if ( + kind === undefined + || !ACTION_KIND_SET.has(kind) + || UNIT_ACTION_KINDS.has(kind as DirectBombadilActionKind) + || !isRecord(value[kind]) + ) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + } + const actionKind = kind as DirectBombadilActionKind; + let targetTag: string | null = null; + if (TARGETED_ACTION_KINDS.has(actionKind)) { + const fingerprint = value[kind].fingerprint; + if (!isRecord(fingerprint)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + const tag = fingerprint.tag; + if ( + typeof tag !== "string" + || tag.length === 0 + || tag.length > 64 + || !TARGET_TAG_PATTERN.test(tag) + ) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target tag`); + } + targetTag = tag; + } + return { kind: actionKind, targetTag }; +} + +function parseNonNegativeFiniteNumber( + value: unknown, + lineNumber: number, + field: string, +): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + return value; +} + +function parseTraceState(value: unknown, lineNumber: number): ParsedTraceState { + if (!isRecord(value) || !hasExactKeys(value, TRACE_STATE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser state`); + } + if (typeof value.url !== "string" || value.url.length === 0 || value.url.length > 8_192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + let url: URL; + try { + url = new URL(value.url); + } catch { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL protocol`); + } + if (typeof value.screenshot !== "string" || value.screenshot.length > 8_192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid screenshot path`); + } + for (const field of ["hash_previous", "hash_current"] as const) { + const hash = value[field]; + if (hash !== null && ( + typeof hash !== "number" + || !Number.isFinite(hash) + || !Number.isInteger(hash) + || hash < 0 + )) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + } + if (!isRecord(value.resources) || !hasExactKeys(value.resources, TRACE_RESOURCE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid browser resources`); + } + const resources: Record = {}; + for (const field of Object.keys(RESOURCE_FIELD_MAP) as (keyof typeof RESOURCE_FIELD_MAP)[]) { + resources[field] = parseNonNegativeFiniteNumber( + value.resources[field], + lineNumber, + `resources.${field}`, + ); + } + parseNonNegativeFiniteNumber(value.resources.timestamp, lineNumber, "resources.timestamp"); + return { + currentHash: value.hash_current as number | null, + resources: resources as Readonly>, + url, + }; +} + +function parseTraceLine(line: string, lineNumber: number): ParsedTraceLine { let input: unknown; try { input = JSON.parse(line) as unknown; @@ -447,7 +791,38 @@ function parseTraceLine(line: string, lineNumber: number): TraceDirectObservatio ) { throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid state fields`); } + const state = parseTraceState(input.state, lineNumber); + const action = parseTraceAction(input.action, lineNumber); const snapshots = input.snapshots as unknown[]; + const namedSnapshots: Array<{ readonly name: string; readonly valueSha256: string }> = []; + const namedSnapshotNames = new Set(); + for (const snapshotValue of snapshots) { + if ( + !isRecord(snapshotValue) + || !hasExactKeys(snapshotValue, TRACE_SNAPSHOT_KEYS) + || !Number.isSafeInteger(snapshotValue.index) + || !Number.isSafeInteger(snapshotValue.time) + || (snapshotValue.index as number) < 0 + || (snapshotValue.time as number) < 0 + || (snapshotValue.name !== null && typeof snapshotValue.name !== "string") + ) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid snapshot`); + } + if (snapshotValue.name !== null) { + const name = validateSnapshotName( + snapshotValue.name, + `Bombadil trace line ${String(lineNumber)} snapshot name`, + ); + if (namedSnapshotNames.has(name)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} repeats named snapshot ${name}`); + } + namedSnapshotNames.add(name); + namedSnapshots.push({ + name, + valueSha256: namedSnapshotValueSha256(snapshotValue.value), + }); + } + } const directSnapshots = snapshots.filter((snapshot): snapshot is Readonly> => isRecord(snapshot) && snapshot.name === "direct" ); @@ -465,7 +840,26 @@ function parseTraceLine(line: string, lineNumber: number): TraceDirectObservatio ) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); } - return parseTraceDirectObservation(snapshot.value); + const propertyViolationNames: string[] = []; + for (const violation of input.violations) { + if (!isRecord(violation) || !hasExactKeys(violation, TRACE_VIOLATION_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + propertyViolationNames.push(validateSnapshotName( + violation.name, + `Bombadil trace line ${String(lineNumber)} property violation name`, + )); + if (!isRecord(violation.violation) || Object.keys(violation.violation).length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + } + return { + action, + directObservation: parseTraceDirectObservation(snapshot.value), + namedSnapshots, + propertyViolationNames, + state, + }; } /** Exact post-run proof over Bombadil 0.7.2's bounded JSONL trace. */ @@ -499,7 +893,7 @@ export async function attestDirectBombadilTrace(options: { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); } - const observation = parseTraceLine(line, observationCount); + const observation = parseTraceLine(line, observationCount).directObservation; const exact = exactTraceDirectObservation(observation); if (exact === null) { if (initial !== null) { @@ -577,6 +971,248 @@ export async function attestDirectBombadilTrace(options: { }; } +function sortedCountRecord( + values: ReadonlyMap, +): Readonly>> { + return Object.freeze(Object.fromEntries( + [...values.entries()].sort(([left], [right]) => left.localeCompare(right)), + ) as Partial>); +} + +/** + * Derives bounded diagnostic exploration metadata from the raw Bombadil trace. + * The hashes and counts are navigation aids, not Direct coverage evidence. + */ +export async function summarizeDirectBombadilTrace(options: { + readonly explorationPolicy?: DirectBombadilExplorationPolicy; + readonly targetUrl: string; + readonly tracePath: string; +}): Promise { + const metadata = await stat(options.tracePath).catch(() => null); + if (metadata === null || !metadata.isFile() || metadata.size === 0) { + throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + } + if (metadata.size > TRACE_MAX_BYTES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_BYTES)} bytes`); + } + let targetUrl: URL; + try { + targetUrl = new URL(options.targetUrl); + } catch { + throw new Error("targetUrl must be an absolute URL"); + } + const policy = validateExplorationPolicy(options.explorationPolicy); + const actionCounts = new Map(); + const targetTags = new Map(); + const urlFingerprints = new Set(); + const transitionHashes = new Set(); + const snapshots = new Map; + }>(); + const propertyViolations = new Map(); + const resources = { + documents: 0, + domNodes: 0, + jsEventListeners: 0, + jsHeapTotalBytes: 0, + jsHeapUsedBytes: 0, + layoutObjects: 0, + scriptDurationSeconds: 0, + taskDurationSeconds: 0, + threadTimeSeconds: 0, + }; + let lineCount = 0; + let totalActions = 0; + let nonWaitCount = 0; + let waitStreak = 0; + let maxWaitStreak = 0; + let nonNullHashCount = 0; + let stableTarget = true; + const stream = createReadStream(options.tracePath, { encoding: "utf8" }); + const lines = createInterface({ input: stream, crlfDelay: Infinity }); + try { + for await (const line of lines) { + lineCount += 1; + if (lineCount > TRACE_MAX_LINES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); + } + if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { + throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); + } + const parsed = parseTraceLine(line, lineCount); + if (parsed.action !== null) { + totalActions += 1; + actionCounts.set(parsed.action.kind, (actionCounts.get(parsed.action.kind) ?? 0) + 1); + if (parsed.action.kind === "Wait") { + waitStreak += 1; + maxWaitStreak = Math.max(maxWaitStreak, waitStreak); + } else { + nonWaitCount += 1; + waitStreak = 0; + } + if (parsed.action.targetTag !== null) { + if (!targetTags.has(parsed.action.targetTag) && targetTags.size >= 128) { + throw new Error("Bombadil trace exceeds 128 distinct action target tags"); + } + targetTags.set( + parsed.action.targetTag, + (targetTags.get(parsed.action.targetTag) ?? 0) + 1, + ); + } + } else { + waitStreak = 0; + } + + const relativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + urlFingerprints.add(sha256(relativeUrl)); + if (urlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct URL fingerprints`, + ); + } + stableTarget &&= parsed.state.url.href === targetUrl.href; + if (parsed.state.currentHash !== null) { + nonNullHashCount += 1; + transitionHashes.add(String(parsed.state.currentHash)); + } + + for (const snapshot of parsed.namedSnapshots) { + let entry = snapshots.get(snapshot.name); + if (entry === undefined) { + if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`, + ); + } + entry = { + changeAfterNonWaitCount: 0, + lastValueSha256: null, + observationCount: 0, + values: new Set(), + }; + snapshots.set(snapshot.name, entry); + } + if ( + parsed.action !== null + && parsed.action.kind !== "Wait" + && entry.lastValueSha256 !== null + && entry.lastValueSha256 !== snapshot.valueSha256 + ) { + entry.changeAfterNonWaitCount += 1; + } + entry.lastValueSha256 = snapshot.valueSha256; + entry.observationCount += 1; + entry.values.add(snapshot.valueSha256); + if (entry.values.size > TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { + throw new Error( + `Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`, + ); + } + } + for (const name of parsed.propertyViolationNames) { + if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`, + ); + } + propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); + } + for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { + resources[outputName] = Math.max( + resources[outputName], + parsed.state.resources[sourceName as keyof typeof RESOURCE_FIELD_MAP], + ); + } + } + } finally { + lines.close(); + stream.destroy(); + } + if (lineCount === 0) throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + + const policyFailures: string[] = []; + if (policy !== null) { + if (nonWaitCount < policy.minNonWaitActions) { + policyFailures.push("minimum non-Wait action count was not reached"); + } + for (const kind of policy.requiredActionKinds) { + if ((actionCounts.get(kind) ?? 0) === 0) { + policyFailures.push(`required action kind ${kind} was not observed`); + } + } + for (const name of policy.requiredNamedSnapshots) { + if (!snapshots.has(name)) { + policyFailures.push(`required named snapshot ${name} was not observed`); + } + } + for (const [name, minimum] of Object.entries(policy.minDistinctNamedSnapshotValues)) { + if ((snapshots.get(name)?.values.size ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its distinct-value minimum`); + } + } + for (const [name, minimum] of Object.entries( + policy.minNamedSnapshotChangesAfterNonWait, + )) { + if ((snapshots.get(name)?.changeAfterNonWaitCount ?? 0) < minimum) { + policyFailures.push( + `named snapshot ${name} did not reach its post-non-Wait change minimum`, + ); + } + } + if (policy.requireStableTargetUrl && !stableTarget) { + policyFailures.push("the browser did not remain on the exact target URL"); + } + } + const traceBytes = await readFile(options.tracePath); + return Object.freeze({ + schema: "direct.bombadil-exploration-summary/v1", + trace: Object.freeze({ + bytes: metadata.size, + lineCount, + sha256: sha256(traceBytes), + }), + actions: Object.freeze({ + byKind: sortedCountRecord(actionCounts), + maxWaitStreak, + nonWaitCount, + targetTags: sortedCountRecord(targetTags) as Readonly>, + total: totalActions, + }), + urls: Object.freeze({ + distinctFingerprintCount: urlFingerprints.size, + fingerprintSha256: Object.freeze([...urlFingerprints].sort()), + observationCount: lineCount, + stableTarget, + }), + transitions: Object.freeze({ + distinctNonNullHashCount: transitionHashes.size, + nonNullHashCount, + }), + namedSnapshots: Object.freeze([...snapshots.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, entry]) => Object.freeze({ + changeAfterNonWaitCount: entry.changeAfterNonWaitCount, + distinctValueCount: entry.values.size, + distinctValueSha256: Object.freeze([...entry.values].sort()), + name, + observationCount: entry.observationCount, + }))), + propertyViolations: Object.freeze({ + byName: sortedCountRecord(propertyViolations) as Readonly>, + total: [...propertyViolations.values()].reduce((total, value) => total + value, 0), + }), + resourceHighWaterMarks: Object.freeze(resources), + policy: Object.freeze({ + configured: policy !== null, + failures: Object.freeze(policyFailures), + satisfied: policyFailures.length === 0, + }), + }); +} + export function parseDirectBombadilFuzzArguments( arguments_: readonly string[], defaultBaseUrl: string, @@ -714,6 +1350,155 @@ function validateTargetQuery(value: unknown): Readonly> { return Object.freeze(validated); } +function validateSnapshotName(value: unknown, label: string): string { + if ( + typeof value !== "string" + || value.length === 0 + || value.length > 128 + || !SNAPSHOT_NAME_PATTERN.test(value) + || PROTOTYPE_PROPERTY_NAMES.has(value) + || hasControlCharacters(value) + ) { + throw new Error(`${label} must be a safe bounded snapshot name`); + } + return value; +} + +function validateViewport(value: unknown): ValidatedViewport { + if (value === undefined) { + return Object.freeze({ + deviceScaleFactor: DEFAULT_DEVICE_SCALE_FACTOR, + height: DEFAULT_VIEWPORT_HEIGHT, + width: DEFAULT_VIEWPORT_WIDTH, + }); + } + if (!isRecord(value) || !Object.keys(value).every((key) => VIEWPORT_KEYS.has(key))) { + throw new Error("viewport must contain only width, height, and deviceScaleFactor"); + } + const width = value.width ?? DEFAULT_VIEWPORT_WIDTH; + const height = value.height ?? DEFAULT_VIEWPORT_HEIGHT; + const deviceScaleFactor = value.deviceScaleFactor ?? DEFAULT_DEVICE_SCALE_FACTOR; + for (const [name, dimension] of [["width", width], ["height", height]] as const) { + if ( + typeof dimension !== "number" + || !Number.isSafeInteger(dimension) + || dimension < 1 + || dimension > 65_535 + ) { + throw new Error(`viewport.${name} must be an integer between 1 and 65535`); + } + } + if ( + typeof deviceScaleFactor !== "number" + || !Number.isFinite(deviceScaleFactor) + || deviceScaleFactor < 0.1 + || deviceScaleFactor > 10 + ) { + throw new Error("viewport.deviceScaleFactor must be a finite number between 0.1 and 10"); + } + return Object.freeze({ deviceScaleFactor, height, width }); +} + +function validateSnapshotMinimumMap(options: { + readonly label: string; + readonly maximum: number; + readonly value: unknown; +}): Readonly> { + if (!isRecord(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated: Record = {}; + for (const [rawName, minimum] of Object.entries(options.value).sort(([left], [right]) => + left.localeCompare(right) + )) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if ( + typeof minimum !== "number" + || !Number.isSafeInteger(minimum) + || minimum < 1 + || minimum > options.maximum + ) { + throw new Error( + `${options.label} ${name} must be an integer between 1 and ${String(options.maximum)}`, + ); + } + validated[name] = minimum; + } + return Object.freeze(validated); +} + +function validateExplorationPolicy( + value: unknown, +): ValidatedExplorationPolicy | null { + if (value === undefined) return null; + if ( + !isRecord(value) + || !Object.keys(value).every((key) => EXPLORATION_POLICY_KEYS.has(key)) + ) { + throw new Error("explorationPolicy contains an unknown field"); + } + const minNonWaitActions = value.minNonWaitActions ?? 0; + if ( + typeof minNonWaitActions !== "number" + || !Number.isSafeInteger(minNonWaitActions) + || minNonWaitActions < 0 + || minNonWaitActions > TRACE_MAX_LINES + ) { + throw new Error( + `explorationPolicy.minNonWaitActions must be an integer between 0 and ${String(TRACE_MAX_LINES)}`, + ); + } + const requiredActionKindsInput = value.requiredActionKinds ?? []; + if (!Array.isArray(requiredActionKindsInput) || requiredActionKindsInput.length > ACTION_KINDS.length) { + throw new Error("explorationPolicy.requiredActionKinds must be a bounded array"); + } + const requiredActionKinds = [...requiredActionKindsInput]; + if ( + !requiredActionKinds.every((kind): kind is DirectBombadilActionKind => + typeof kind === "string" && ACTION_KIND_SET.has(kind) + ) + || new Set(requiredActionKinds).size !== requiredActionKinds.length + ) { + throw new Error("explorationPolicy.requiredActionKinds contains an unknown or duplicate kind"); + } + requiredActionKinds.sort(); + + const requiredNamedSnapshotsInput = value.requiredNamedSnapshots ?? []; + if (!Array.isArray(requiredNamedSnapshotsInput) || requiredNamedSnapshotsInput.length > 32) { + throw new Error("explorationPolicy.requiredNamedSnapshots must be a bounded array"); + } + const requiredNamedSnapshots = requiredNamedSnapshotsInput.map((name) => + validateSnapshotName(name, "explorationPolicy.requiredNamedSnapshots entry") + ); + if (new Set(requiredNamedSnapshots).size !== requiredNamedSnapshots.length) { + throw new Error("explorationPolicy.requiredNamedSnapshots contains a duplicate name"); + } + requiredNamedSnapshots.sort(); + + const minDistinctNamedSnapshotValues = validateSnapshotMinimumMap({ + label: "explorationPolicy.minDistinctNamedSnapshotValues", + maximum: TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME, + value: value.minDistinctNamedSnapshotValues ?? {}, + }); + const minNamedSnapshotChangesAfterNonWait = validateSnapshotMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterNonWait", + maximum: TRACE_MAX_LINES, + value: value.minNamedSnapshotChangesAfterNonWait ?? {}, + }); + const requireStableTargetUrl = value.requireStableTargetUrl ?? false; + if (typeof requireStableTargetUrl !== "boolean") { + throw new Error("explorationPolicy.requireStableTargetUrl must be a boolean"); + } + return Object.freeze({ + minDistinctNamedSnapshotValues, + minNamedSnapshotChangesAfterNonWait, + minNonWaitActions, + requireStableTargetUrl, + requiredActionKinds: Object.freeze(requiredActionKinds), + requiredNamedSnapshots: Object.freeze(requiredNamedSnapshots), + }); +} + export function validateDirectBombadilFuzzConfig( config: DirectBombadilFuzzConfig, baseUrlOverride?: string, @@ -782,6 +1567,8 @@ export function validateDirectBombadilFuzzConfig( const entryPath = config.entryPath ?? "/"; validateEntryPath(entryPath); const targetQuery = validateTargetQuery(config.targetQuery ?? {}); + const viewport = validateViewport(config.viewport); + const explorationPolicy = validateExplorationPolicy(config.explorationPolicy); const startupTimeoutMs = config.server.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; if ( !Number.isSafeInteger(startupTimeoutMs) @@ -803,8 +1590,10 @@ export function validateDirectBombadilFuzzConfig( artifactRoot: join(repositoryRoot, "artifacts", "direct-bombadil", config.artifactName), bombadilExecutable: bombadilNativeBinary(repositoryRoot), entryPath, + explorationPolicy, port, targetQuery, + viewport, server: { ...config.server, cwd: serverCwd, @@ -834,7 +1623,9 @@ export function createDirectBombadilInvocation(options: { readonly specificationPath: string; readonly targetQuery?: Readonly>; readonly timeLimitSeconds: number; + readonly viewport?: DirectBombadilViewportConfig; }): DirectBombadilInvocation { + const viewport = validateViewport(options.viewport); const target = new URL(options.entryPath ?? "/", `${options.baseUrl}/`); target.searchParams.set(SCENARIO_QUERY_KEY, options.scenario); for (const [name, value] of Object.entries(options.targetQuery ?? {})) { @@ -850,6 +1641,12 @@ export function createDirectBombadilInvocation(options: { options.outputPath, "--headless", "--instrument-javascript=", + "--width", + String(viewport.width), + "--height", + String(viewport.height), + "--device-scale-factor", + String(viewport.deviceScaleFactor), ]; if (options.replayPath === null) { command.push( @@ -1181,6 +1978,100 @@ function helpText(defaultBaseUrl: string): string { ].join("\n"); } +function parseMatrixCampaignArgument(arguments_: readonly string[]): { + readonly arguments: readonly string[]; + readonly campaignId: string | null; + readonly help: boolean; +} { + const forwarded: string[] = []; + let campaignId: string | null = null; + let help = false; + for (let index = 0; index < arguments_.length; index += 1) { + const argument = arguments_[index]; + if (argument === undefined) continue; + if (argument === "--help" || argument === "-h") help = true; + if (argument === "--campaign" || argument.startsWith("--campaign=")) { + if (campaignId !== null) throw new Error("--campaign may be provided only once"); + if (argument === "--campaign") { + const next = readOptionValue(arguments_, index, "--campaign"); + campaignId = next.value; + index = next.index; + } else { + campaignId = argument.slice("--campaign=".length); + } + if (campaignId.length === 0) throw new Error("--campaign requires a value"); + continue; + } + forwarded.push(argument); + } + return { arguments: Object.freeze(forwarded), campaignId, help }; +} + +function validateCampaignMatrix( + campaigns: readonly DirectBombadilFuzzCampaign[], +): readonly DirectBombadilFuzzCampaign[] { + if (campaigns.length === 0 || campaigns.length > 32) { + throw new Error("Bombadil campaign matrix must contain 1-32 campaigns"); + } + const ids = new Set(); + for (const campaign of campaigns) { + if (!ARTIFACT_NAME_PATTERN.test(campaign.id) || ids.has(campaign.id)) { + throw new Error("Bombadil campaign IDs must be unique lowercase kebab identifiers"); + } + ids.add(campaign.id); + } + return campaigns; +} + +/** Runs a bounded product-owned campaign matrix serially. */ +export async function runDirectBombadilFuzzMatrix( + campaignsInput: readonly DirectBombadilFuzzCampaign[], + arguments_: readonly string[] = process.argv.slice(2), + dependencyOverrides: Partial = {}, +): Promise { + const campaigns = validateCampaignMatrix(campaignsInput); + const parsed = parseMatrixCampaignArgument(arguments_); + if (parsed.help) { + process.stdout.write(`${[ + helpText(campaigns[0]?.config.baseUrl ?? ""), + " --campaign Run one campaign; required with --replay", + "", + `Campaigns: ${campaigns.map((campaign) => campaign.id).join(", ")}`, + ].join("\n")}\n`); + return { kind: "help" }; + } + const selected = parsed.campaignId === null + ? campaigns + : campaigns.filter((campaign) => campaign.id === parsed.campaignId); + if (selected.length === 0) { + throw new Error(`Unknown Bombadil campaign ${parsed.campaignId ?? ""}`); + } + if ( + parsed.campaignId === null + && parsed.arguments.some((argument) => + argument === "--replay" || argument.startsWith("--replay=") + ) + ) { + throw new Error("--replay requires exactly one --campaign in matrix mode"); + } + const results: Array<{ + readonly campaignId: string; + readonly result: Extract; + }> = []; + for (const campaign of selected) { + const result = await runDirectBombadilFuzz( + campaign.config, + parsed.arguments, + dependencyOverrides, + ); + if (result.kind !== "run") { + throw new Error("Bombadil campaign unexpectedly returned help during matrix execution"); + } + results.push({ campaignId: campaign.id, result }); + } + return { kind: "matrix", results: Object.freeze(results) }; +} + /** Runs one bounded diagnostic Bombadil campaign and always releases its server lease. */ export async function runDirectBombadilFuzz( config: DirectBombadilFuzzConfig, @@ -1224,6 +2115,7 @@ export async function runDirectBombadilFuzz( specificationPath: validated.specificationPath, targetQuery: validated.targetQuery, timeLimitSeconds: parsed.timeLimitSeconds, + viewport: validated.viewport, }); const abortableInvocation = { ...invocation, abortSignal: abortController.signal }; const serverCommand = validated.server.command.map((argument) => @@ -1236,6 +2128,8 @@ export async function runDirectBombadilFuzz( let processResult: BombadilProcessResult | null = null; let attestation: DirectBombadilTraceAttestation | null = null; let attestationFailure: unknown = null; + let explorationSummary: DirectBombadilExplorationSummary | null = null; + let explorationSummaryFailure: unknown = null; let rawTracePath: string | null = null; let serverOutput = ""; let serverOutputFailure: unknown = null; @@ -1291,6 +2185,17 @@ export async function runDirectBombadilFuzz( } catch (error) { attestationFailure = error; } + try { + explorationSummary = await summarizeDirectBombadilTrace({ + ...(validated.explorationPolicy === null + ? {} + : { explorationPolicy: validated.explorationPolicy }), + targetUrl: invocation.targetUrl, + tracePath, + }); + } catch (error) { + explorationSummaryFailure = error; + } if (processFailure !== null) { throw processFailure instanceof Error ? processFailure @@ -1313,6 +2218,16 @@ export async function runDirectBombadilFuzz( ? attestationFailure : new Error(renderUnknown(attestationFailure)); } + if (explorationSummaryFailure !== null) { + throw explorationSummaryFailure instanceof Error + ? explorationSummaryFailure + : new Error(renderUnknown(explorationSummaryFailure)); + } + if (explorationSummary?.policy.satisfied !== true) { + throw new Error( + `Bombadil exploration policy was not satisfied: ${explorationSummary?.policy.failures.join("; ") ?? "summary unavailable"}`, + ); + } } catch (error) { failure = error; } @@ -1351,6 +2266,10 @@ export async function runDirectBombadilFuzz( const status = failure === null ? "passed" : "failed"; const logPath = join(artifactRun.runDirectory, "bombadil.log"); const serverLogPath = join(artifactRun.runDirectory, "server.log"); + const explorationSummaryPath = join( + artifactRun.runDirectory, + "exploration-summary.json", + ); const record = { schema: ARTIFACT_SCHEMA, evidenceClass: "diagnostic-fuzz", @@ -1366,6 +2285,8 @@ export async function runDirectBombadilFuzz( entryPath: validated.entryPath, targetQuery: validated.targetQuery, targetUrl: invocation.targetUrl, + viewport: validated.viewport, + explorationPolicy: validated.explorationPolicy, specificationPath: validated.specificationPath, replayPath, timeLimitSeconds: replayPath === null ? parsed.timeLimitSeconds : null, @@ -1387,6 +2308,11 @@ export async function runDirectBombadilFuzz( }, attestation, attestationFailure: attestationFailure === null ? null : renderUnknown(attestationFailure), + explorationSummary, + explorationSummaryPath: explorationSummary === null ? null : explorationSummaryPath, + explorationSummaryFailure: explorationSummaryFailure === null + ? null + : renderUnknown(explorationSummaryFailure), initialDirect: attestation?.initial ?? null, interruptedSignal: capturedSignal, failure: failure === null ? null : renderUnknown(failure), @@ -1401,6 +2327,9 @@ export async function runDirectBombadilFuzz( `${serverOutput}${serverOutput.length > 0 ? "\n" : ""}`, "utf8", ); + if (explorationSummary !== null) { + await writeJsonAtomically(explorationSummaryPath, explorationSummary); + } await writeJsonAtomically(join(artifactRun.runDirectory, "run.json"), record); await writeJsonAtomically(artifactRun.manifestPath, record); diff --git a/src/tooling/bombadil.ts b/src/tooling/bombadil.ts index 96f2b64..ec72ce2 100644 --- a/src/tooling/bombadil.ts +++ b/src/tooling/bombadil.ts @@ -1,15 +1,22 @@ import { attestDirectBombadilTrace as attestTrace, runDirectBombadilFuzz as runFuzz, + runDirectBombadilFuzzMatrix as runMatrix, + summarizeDirectBombadilTrace as summarizeTrace, } from "./bombadil-runner.js"; import type { DirectBombadilFuzzConfig, + DirectBombadilFuzzCampaign, + DirectBombadilFuzzMatrixResult, DirectBombadilFuzzResult, } from "./bombadil-runner.js"; /** Host-side exact attestation for one bounded Bombadil 0.7.2 JSONL trace. */ export const attestDirectBombadilTrace: typeof attestTrace = attestTrace; +/** Derives bounded diagnostic navigation metadata without replacing the raw trace. */ +export const summarizeDirectBombadilTrace: typeof summarizeTrace = summarizeTrace; + /** Runs one bounded local Bombadil campaign and preserves diagnostic artifacts. */ export function runDirectBombadilFuzz( config: DirectBombadilFuzzConfig, @@ -18,10 +25,24 @@ export function runDirectBombadilFuzz( return arguments_ === undefined ? runFuzz(config) : runFuzz(config, arguments_); } +/** Runs a bounded product campaign matrix serially and selects one for replay. */ +export function runDirectBombadilFuzzMatrix( + campaigns: readonly DirectBombadilFuzzCampaign[], + arguments_?: readonly string[], +): Promise { + return arguments_ === undefined ? runMatrix(campaigns) : runMatrix(campaigns, arguments_); +} + export type { + DirectBombadilActionKind, + DirectBombadilExplorationPolicy, + DirectBombadilExplorationSummary, + DirectBombadilFuzzCampaign, DirectBombadilFuzzConfig, + DirectBombadilFuzzMatrixResult, DirectBombadilFuzzResult, DirectBombadilServerConfig, DirectBombadilTraceAttestation, DirectBombadilTraceBinding, + DirectBombadilViewportConfig, } from "./bombadil-runner.js"; From b17332687e79aa53b17d266303ce2a0fa072343d Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:23:09 -0400 Subject: [PATCH 02/25] release: prepare Direct 0.7.6 --- README.md | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ee5cecd..47d4845 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Copy this prompt into Codex, Claude Code, or another coding agent: ```text Use $direct to install hraness/direct from -the npm registry at the exact 0.7.5 version. Follow the repository README, add +the npm registry at the exact 0.7.6 version. Follow the repository README, add `@hraness/direct` to devDependencies only, and verify that the production dependency graph excludes Direct. Do not add a fixture composition until I ask. @@ -66,7 +66,7 @@ Pin the public npm package to an exact immutable version: ```json { "devDependencies": { - "@hraness/direct": "0.7.5" + "@hraness/direct": "0.7.6" } } ``` diff --git a/package.json b/package.json index b993123..74e3254 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hraness/direct", - "version": "0.7.5", + "version": "0.7.6", "description": "A general harness for repeatable app states.", "license": "MIT", "type": "module", From 19852e4b84a462c084b5e5ca3b2bc949ac121b53 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:31:45 -0400 Subject: [PATCH 03/25] feat: report Bombadil exploration sufficiency --- src/tooling/bombadil-runner.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index f2a2ef2..8be7621 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -2333,7 +2333,22 @@ export async function runDirectBombadilFuzz( await writeJsonAtomically(join(artifactRun.runDirectory, "run.json"), record); await writeJsonAtomically(artifactRun.manifestPath, record); - const summary = `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}; artifacts: ${artifactRun.runDirectory}; log: ${logPath}`; + const exploration = explorationSummary === null + ? "exploration=unavailable" + : [ + `nonWait=${String(explorationSummary.actions.nonWaitCount)}`, + `maxWaitStreak=${String(explorationSummary.actions.maxWaitStreak)}`, + `namedChanges=${explorationSummary.namedSnapshots + .map((snapshot) => `${snapshot.name}:${String(snapshot.changeAfterNonWaitCount)}`) + .join(",") || "none"}`, + `policy=${explorationSummary.policy.satisfied ? "satisfied" : "failed"}`, + ].join("; "); + const summary = [ + `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}`, + exploration, + `artifacts: ${artifactRun.runDirectory}`, + `log: ${logPath}`, + ].join("; "); (status === "passed" ? process.stdout : process.stderr).write(`${summary}\n`); if (failure !== null) { From 2347b0c9414029376ba3e33a7646d101a0a8fa60 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:35:02 -0400 Subject: [PATCH 04/25] fix: exclude destructive generic Bombadil clicks --- src/tooling/bombadil-campaign.test.ts | 6 ++++++ src/tooling/bombadil-campaign.ts | 23 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 63ac1cb..56a6b25 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -365,6 +365,11 @@ describe("Direct Bombadil actions", () => { [2, { value: clickAction({ textContent: "ReSeT" }) }], [2, { value: clickAction({ accessibleName: "RESET", textContent: "Again" }) }], [2, { value: clickAction({ accessibleName: "", textContent: "Reset" }) }], + [2, { value: clickAction({ textContent: "Delete track" }) }], + [2, { value: clickAction({ accessibleName: "Close editor", textContent: "Done" }) }], + [2, { value: clickAction({ textContent: "Sign-out" }) }], + [2, { value: clickAction({ textContent: "Remove from playlist" }) }], + [2, { value: clickAction({ textContent: "Unclear status" }) }], [1, { branches: [ [9, { value: clickAction({ tag: "input", inputType: "text" }) }], @@ -383,6 +388,7 @@ describe("Direct Bombadil actions", () => { expect(filtered?.generate()).toEqual({ branches: [ [7, { value: clickAction() }], + [2, { value: clickAction({ textContent: "Unclear status" }) }], [1, { branches: [ [9, { value: clickAction({ tag: "input", inputType: "text" }) }], diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index 5650e12..abb31b3 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -30,6 +30,18 @@ const DIRECT_PROBE_SCHEMA = "direct.probe/v1"; const MAX_RAW_CONTRACT_CHARACTERS = 2_000_000; const BRIDGE_KEYS = new Set(["manifest", "reset", "schema", "snapshot"]); const UNSAFE_CLICK_INPUT_TYPES = new Set(["image", "reset", "submit"]); +const UNSAFE_CLICK_LABEL_PHRASES = [ + "clear", + "close", + "delete", + "discard", + "erase", + "log out", + "remove", + "reset", + "sign out", + "unlink", +] as const; const SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; const RESOURCE_LEAK_OPTION_KEYS = new Set(["growthLimit", "metric", "windowMillis"]); const RESOURCE_METRICS = [ @@ -86,11 +98,18 @@ function safeClickAction(action: ActionTemplate): boolean { const inputType = fingerprint.inputType?.toLowerCase() ?? ""; const labels = [fingerprint.accessibleName, fingerprint.textContent] .filter((label): label is string => label !== null) - .map((label) => label.trim().toLowerCase()); + .map((label) => label + .trim() + .toLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, " ")); + const hasUnsafeLabel = labels.some((label) => { + const padded = ` ${label} `; + return UNSAFE_CLICK_LABEL_PHRASES.some((phrase) => padded.includes(` ${phrase} `)); + }); return fingerprint.href === null && tag !== "a" && fingerprint.role?.toLowerCase() !== "link" - && !labels.includes("reset") + && !hasUnsafeLabel && !UNSAFE_CLICK_INPUT_TYPES.has(inputType) && (tag !== "button" || inputType === "button"); } From 83f4fa9305da4af1d053e174dfae3df15a40ef14 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:40:23 -0400 Subject: [PATCH 05/25] docs: define conservative Bombadil click boundaries --- docs/verification.md | 10 ++++++---- src/tooling/bombadil-campaign.test.ts | 2 +- src/tooling/bombadil-campaign.ts | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/verification.md b/docs/verification.md index c4ae45a..5a6f696 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -291,11 +291,13 @@ export const no_dom_node_leak = createDirectBombadilResourceLeakProperty({ ``` The Direct action generator deliberately excludes reload, history traversal, -visible links, anchors, href targets, form submission, reset controls, and the -Enter key. It retains ordinary buttons, text input, scrolling, and an +visible links, anchors, href targets, form submission, reset controls, +destructive labels such as delete, remove, clear, discard, unlink, and close, +and the Enter key. It retains ordinary buttons, text input, scrolling, and an always-eligible low-weight wait. This preserves the post-handshake contract -within one document. Add product actions only when their navigation and form -effects are understood, and keep product-specific assertions in the campaign. +within one document. Add product actions only when their navigation, form, and +destructive effects are understood, and keep product-specific assertions in +the campaign. Guard domain actions on the state that makes them valid, then weight valuable state-changing actions above Wait. Do not increase throughput by admitting reload, navigation, submission, destructive controls, or arbitrary generated diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 56a6b25..cd77767 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -351,7 +351,7 @@ function clickAction(overrides: Readonly> = {}): u } describe("Direct Bombadil actions", () => { - test("prunes nested visible navigation and submission clicks", () => { + test("prunes nested navigation, submission, and destructive clicks", () => { fakeClicks.generate = () => ({ branches: [ [7, { value: clickAction() }], diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index abb31b3..a9eb154 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -138,7 +138,8 @@ function pruneActionTree( /** * Builds a browser action generator without reload/history actions or visible - * navigation and submission click targets, keeping Direct continuously bound. + * navigation, submission, reset, or destructive click targets, keeping Direct + * continuously bound. */ export function createDirectBombadilActions(): ActionGenerator { const safeClicks = actions(() => pruneActionTree(clicks.generate(), safeClickAction) ?? []); From 40f53bc3182fbf13a1732f8003866ab84fbf5d09 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:49:26 -0400 Subject: [PATCH 06/25] docs: separate responsive and interaction evidence --- README.md | 4 ++++ docs/verification.md | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 47d4845..db5e358 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,10 @@ promote the smallest readable failure to a deterministic product regression. When a campaign must exercise an interaction, require a named product value to change after a non-Wait action so bootstrap and idle transitions do not satisfy the exploration policy. +If the full product snapshot includes viewport dimensions, put that requirement +on a separate interaction snapshot without viewport fields and require an +opposite-size `SetViewport` independently. Latch the first ready product state +for initial-world properties so later actions cannot repair a bad initial state. The raw trace remains authoritative and may contain screenshots, URLs, typed text, accessible labels, and local paths; treat it as potentially sensitive. Summary counts and hashes help triage exploration but are not Direct coverage. diff --git a/docs/verification.md b/docs/verification.md index 5a6f696..f39983f 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -404,6 +404,15 @@ sufficiency check, not a coverage claim. Start with observed stable behavior and raise thresholds only when the action generator makes them reliably reachable. +Do not let responsive evidence stand in for product interaction. When a full +product snapshot includes viewport dimensions, name a second compact +interaction snapshot that excludes them and put the distinct-value and +post-action-change requirements on that snapshot. Require `SetViewport` +separately, and generate only a width or height different from the current +viewport. Likewise, latch the first product-ready observation for initial-world +properties so a later generated action cannot repair an incorrect initial +state before the bounded formula completes. + Run random exploration for 12 to 300 seconds. The default is 20 seconds: ```sh From 2977c6241582b33bd38a8c82eb4a8e0fe3936420 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:59:05 -0400 Subject: [PATCH 07/25] fix: preserve validated viewport number types --- src/tooling/bombadil-runner.test.ts | 8 ++++++++ src/tooling/bombadil-runner.ts | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index a164b38..82e9595 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -435,6 +435,14 @@ describe("Direct Bombadil configuration and invocation", () => { height: 768, width: 1_024, }); + expect(validateDirectBombadilFuzzConfig({ + ...config, + viewport: { deviceScaleFactor: 1.5, height: 720, width: 1_280 }, + }).viewport).toEqual({ + deviceScaleFactor: 1.5, + height: 720, + width: 1_280, + }); expect(validated.bombadilExecutable).toEndWith( `node_modules/@antithesishq/bombadil/binaries/${nativeBinaryName()}`, ); diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index 8be7621..a76deaf 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -1375,19 +1375,20 @@ function validateViewport(value: unknown): ValidatedViewport { if (!isRecord(value) || !Object.keys(value).every((key) => VIEWPORT_KEYS.has(key))) { throw new Error("viewport must contain only width, height, and deviceScaleFactor"); } - const width = value.width ?? DEFAULT_VIEWPORT_WIDTH; - const height = value.height ?? DEFAULT_VIEWPORT_HEIGHT; - const deviceScaleFactor = value.deviceScaleFactor ?? DEFAULT_DEVICE_SCALE_FACTOR; - for (const [name, dimension] of [["width", width], ["height", height]] as const) { + const validateDimension = (name: "height" | "width", input: unknown): number => { if ( - typeof dimension !== "number" - || !Number.isSafeInteger(dimension) - || dimension < 1 - || dimension > 65_535 + typeof input !== "number" + || !Number.isSafeInteger(input) + || input < 1 + || input > 65_535 ) { throw new Error(`viewport.${name} must be an integer between 1 and 65535`); } - } + return input; + }; + const width = validateDimension("width", value.width ?? DEFAULT_VIEWPORT_WIDTH); + const height = validateDimension("height", value.height ?? DEFAULT_VIEWPORT_HEIGHT); + const deviceScaleFactor = value.deviceScaleFactor ?? DEFAULT_DEVICE_SCALE_FACTOR; if ( typeof deviceScaleFactor !== "number" || !Number.isFinite(deviceScaleFactor) From 9cdf1e9cd9bf716f670654fd4f7fe68b643ef8bd Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 12:03:10 -0400 Subject: [PATCH 08/25] chore: review Direct package growth --- scripts/package-artifact.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/package-artifact.ts b/scripts/package-artifact.ts index c8b667f..08f043c 100644 --- a/scripts/package-artifact.ts +++ b/scripts/package-artifact.ts @@ -10,7 +10,7 @@ const packageBudget = Object.freeze({ entryCount: { min: 50, max: 120 }, fileCount: { min: 50, max: 60 }, packedBytes: { min: 140_000, max: 180_000 }, - unpackedBytes: { min: 650_000, max: 750_000 }, + unpackedBytes: { min: 650_000, max: 800_000 }, }); const requiredPaths = Object.freeze([ From 7e27258a875c3587874156f2f2be922709e97fb7 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 12:05:17 -0400 Subject: [PATCH 09/25] docs: publish Direct 0.7.6 install pins --- README.md | 4 ++-- skills/direct/references/install.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index db5e358..747025c 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ state with predictable local stand-ins. it does not click through the browser or test the systems it replaces. ```sh -bun add --dev @hraness/direct@0.7.5 +bun add --dev @hraness/direct@0.7.6 # or -npm install --save-dev @hraness/direct@0.7.5 +npm install --save-dev @hraness/direct@0.7.6 ``` [overview](https://hraness.com/direct) diff --git a/skills/direct/references/install.md b/skills/direct/references/install.md index 90c38b9..6d240e2 100644 --- a/skills/direct/references/install.md +++ b/skills/direct/references/install.md @@ -21,9 +21,9 @@ global `direct` CLI. For a new installation, pin the reviewed public release: ```sh -bun add --dev @hraness/direct@0.7.5 +bun add --dev @hraness/direct@0.7.6 # or, in an npm project -npm install --save-dev @hraness/direct@0.7.5 +npm install --save-dev @hraness/direct@0.7.6 ``` The equivalent manifest entry is: @@ -31,7 +31,7 @@ The equivalent manifest entry is: ```json { "devDependencies": { - "@hraness/direct": "0.7.5" + "@hraness/direct": "0.7.6" } } ``` From a15ce86eac6192c0e4d80ccec04c0114f743abe0 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 12:50:47 -0400 Subject: [PATCH 10/25] fix: make Bombadil evidence fail closed --- README.md | 6 + docs/verification.md | 37 +++-- src/tooling/bombadil-campaign.test.ts | 214 +++++++++++++++++++------- src/tooling/bombadil-campaign.ts | 168 +++++++++++++++----- src/tooling/bombadil-runner.test.ts | 137 +++++++++++++++++ src/tooling/bombadil-runner.ts | 47 ++++-- 6 files changed, 490 insertions(+), 119 deletions(-) diff --git a/README.md b/README.md index 747025c..b461193 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,7 @@ export * from "@antithesishq/bombadil/browser/defaults/properties"; const direct = createDirectBombadilProperties(); export const direct_safe_actions = createDirectBombadilActions(); +export const direct_startup_contract = direct.startupContract; export const direct_exact_contract = direct.exactContract; export const direct_stable_catalog = direct.stableCatalog; export const direct_no_declared_violations = direct.noDeclaredViolations; @@ -277,6 +278,11 @@ and releases its owned processes. Use `runDirectBombadilFuzzMatrix` when a product owns several scenarios; it runs them serially and requires one exact campaign selector for replay. +Startup is the only repairable contract phase. It must reach one exact Direct +observation within ten seconds. From that sample onward, activation identity, +route, scenario, catalog, and zero declared violations are immediate safety +invariants; only quiescence remains bounded liveness. + Keep liveness formulas time-bounded. Prefer guarded product actions with explicit weights over unrestricted browser actions, and name small JSON snapshots that expose semantic state without retaining page content. Run short diff --git a/docs/verification.md b/docs/verification.md index f39983f..cb765f9 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -272,10 +272,12 @@ const direct = createDirectBombadilProperties(); const phase = createDirectBombadilNamedSnapshot({ fallback: "unavailable", name: "todos.phase", + parse: (value) => typeof value === "string" ? value : null, read: ({ window }) => Reflect.get(window, "__todosPhase"), }); export const direct_safe_actions = createDirectBombadilActions(); +export const direct_startup_contract = direct.startupContract; export const direct_exact_contract = direct.exactContract; export const direct_stable_catalog = direct.stableCatalog; export const direct_no_declared_violations = direct.noDeclaredViolations; @@ -293,7 +295,9 @@ export const no_dom_node_leak = createDirectBombadilResourceLeakProperty({ The Direct action generator deliberately excludes reload, history traversal, visible links, anchors, href targets, form submission, reset controls, destructive labels such as delete, remove, clear, discard, unlink, and close, -and the Enter key. It retains ordinary buttons, text input, scrolling, and an +all labels because their fingerprints do not expose the associated control's +submit, reset, or button type, and the Enter key. It retains ordinary buttons, +text input, scrolling, and an always-eligible low-weight wait. This preserves the post-handshake contract within one document. Add product actions only when their navigation, form, and destructive effects are understood, and keep product-specific assertions in @@ -305,12 +309,13 @@ input. A generated action sequence is useful only while it preserves the scenario boundary and exercises behavior the product can interpret. `createDirectBombadilNamedSnapshot` gives product properties and post-run -diagnostics a small semantic signal. It requires a safe bounded name, clones no -more than two million JSON characters, and fails closed to the explicit -fallback when a page getter throws or returns unsuitable data. Extract state, -not page content or credentials. Named values are represented only by canonical -SHA-256 hashes in the exploration summary; the authoritative raw trace still -contains the original values. +diagnostics a small semantic signal. It requires a safe bounded name distinct +from `direct` and JavaScript prototype names, an explicit product parser, at +most 64 JSON levels, and at most 2 MiB of UTF-8 JSON. It fails closed to the +parsed fallback when a page getter throws or returns unsuitable data. Extract +state, not page content or credentials. Named values are represented only by +canonical SHA-256 hashes in the exploration summary; the authoritative raw +trace still contains the original values. Bombadil's 0.7.2 manual documents a sliding-window resource property at `@antithesishq/bombadil/browser/extras/resources`, but the published npm @@ -459,14 +464,16 @@ and server configuration for replay. Bombadil rejects a replay that diverges, so reproduction is strong debugging evidence but not guaranteed after the product changes. -The browser formulas require an exact scenario contract, a nonempty catalog, -zero declared violations, and quiescence to recur within ten seconds from every -observed state. Keep every liveness obligation bounded to a product latency -budget. A finite run cannot decide an unbounded future obligation, and nesting -unbounded `always` inside bounded `eventually` does not make it decidable. -Continuous identity and catalog binding therefore stay in the host attestation -below. A formula result and Bombadil exit status are not sufficient evidence -because a short or incomplete trace could otherwise pass vacuously. +The startup formula requires an exact scenario contract within ten seconds. +After that first exact sample, the browser formulas require continuous +activation identity, catalog identity, and zero declared violations. Only +quiescence may recover within a bounded ten-second liveness window. Keep every +liveness obligation bounded to a product latency budget. A finite run cannot +decide an unbounded future obligation, and nesting unbounded `always` inside +bounded `eventually` does not make it decidable. The host attestation below +rechecks the full trace independently. A formula result and Bombadil exit status +are not sufficient evidence because a short or incomplete trace could otherwise +pass vacuously. After every random run or replay, the host runner streams the bounded 0.7.2 JSONL trace from foreign input and requires one named `direct` observation per diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index cd77767..86c94cd 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -1,7 +1,13 @@ import { describe, expect, mock, test } from "bun:test"; +import type { JSON as BombadilJson } from "@antithesishq/bombadil"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { defineDirect } from "@hraness/direct"; import { createDirectSession } from "@hraness/direct/testing"; +import { summarizeDirectBombadilTrace } from "./bombadil-runner.js"; + interface FakeFormula { readonly body: unknown; readonly kind: "always" | "eventually"; @@ -239,6 +245,13 @@ describe("Direct Bombadil named snapshots", () => { const snapshot = createDirectBombadilNamedSnapshot({ fallback: { status: "unavailable" }, name: "product.phase", + parse: (value) => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return null; + } + const status = Reflect.get(value, "status"); + return typeof status === "string" ? { status } : null; + }, read: (state) => Reflect.get(state.window, "phase"), }) as unknown as FakeCell; expect(snapshot.name).toBe("product.phase"); @@ -254,21 +267,115 @@ describe("Direct Bombadil named snapshots", () => { }); expect(snapshot.read({ window: hostile })).toEqual({ status: "unavailable" }); expect(snapshot.read({ - window: { phase: "x".repeat(2_000_001) }, + window: { phase: { status: "é".repeat(1_100_000) } }, })).toEqual({ status: "unavailable" }); + expect(snapshot.read({ window: { phase: null } })).toEqual({ + status: "unavailable", + }); + }); + + test("produces a named value accepted by the host summary contract", async () => { + const snapshot = createDirectBombadilNamedSnapshot({ + fallback: { status: "unavailable" }, + name: "product.compat", + parse: (value) => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return null; + } + const status = Reflect.get(value, "status"); + return typeof status === "string" ? { status } : null; + }, + read: (state) => Reflect.get(state.window, "phase"), + }) as unknown as FakeCell; + const value = snapshot.read({ window: { phase: { status: "ready" } } }); + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-helper-summary-")); + const tracePath = join(directory, "trace.jsonl"); + try { + await writeFile(tracePath, `${JSON.stringify({ + action: null, + snapshots: [ + { + index: 0, + name: "direct", + time: 1, + value: readDirectBombadilObservation(contractFixture()), + }, + { index: 1, name: snapshot.name, time: 1, value }, + ], + state: { + hash_current: 1, + hash_previous: null, + resources: { + documents: 1, + dom_nodes: 1, + js_event_listeners: 1, + js_heap_total: 1, + js_heap_used: 1, + layout_objects: 1, + script_duration: 0, + task_duration: 0, + thread_time: 0, + timestamp: 1, + }, + screenshot: "1.png", + url: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + }, + timestamp: 1, + violations: [], + })}\n`, "utf8"); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.find(({ name }) => name === snapshot.name)) + .toMatchObject({ distinctValueCount: 1, observationCount: 1 }); + } finally { + await rm(directory, { force: true, recursive: true }); + } + }); + + test("matches the host name, UTF-8 size, and JSON depth boundary", () => { + for (const name of [ + "direct", + "__proto__", + "constructor", + "prototype", + "unsafe name", + ]) { + expect(() => createDirectBombadilNamedSnapshot({ + fallback: null, + name, + parse: (value) => value, + read: () => null, + })).toThrow("safe, unreserved"); + } + + const snapshot = createDirectBombadilNamedSnapshot({ + fallback: null, + name: "safe", + parse: (value) => value, + read: (state) => Reflect.get(state.window, "phase"), + }) as unknown as FakeCell; + let atLimit: BombadilJson = null; + for (let index = 0; index < 64; index += 1) atLimit = [atLimit]; + expect(snapshot.read({ window: { phase: atLimit } })).toEqual(atLimit); + const beyondLimit: BombadilJson = [atLimit]; + expect(snapshot.read({ window: { phase: beyondLimit } })).toBeNull(); }); - test("rejects unsafe names and non-JSON fallbacks before registering an extractor", () => { + test("rejects non-JSON or parser-invalid fallbacks before registering an extractor", () => { expect(() => createDirectBombadilNamedSnapshot({ fallback: null, - name: "unsafe name", + name: "safe", + parse: () => null, read: () => null, - })).toThrow("safe 1-128 character identifier"); - expect(() => createDirectBombadilNamedSnapshot({ + })).toThrow("accepted by parse"); + expect(() => createDirectBombadilNamedSnapshot({ fallback: undefined as never, name: "safe", + parse: (value) => value, read: () => null, - })).toThrow("fallback must be bounded JSON"); + })).toThrow("fallback must be bounded JSON accepted by parse"); }); }); @@ -374,6 +481,7 @@ describe("Direct Bombadil actions", () => { branches: [ [9, { value: clickAction({ tag: "input", inputType: "text" }) }], [8, { value: clickAction({ tag: "A" }) }], + [7, { value: clickAction({ tag: "label", textContent: "Continue" }) }], ], }], ], @@ -430,25 +538,38 @@ describe("Direct Bombadil actions", () => { }); describe("Direct Bombadil formulas", () => { - test("returns four recurring bounded health formulas", () => { + test("splits bounded startup from strict recurring health", () => { const properties = createDirectBombadilProperties() as unknown as Record; const cell = cells.at(-1); if (cell === undefined) throw new Error("Expected one Direct extractor cell"); expect(cell.name).toBe("direct"); - cell.current = readDirectBombadilObservation(contractFixture()); + const sample = (window: unknown): void => { + cell.current = cell.read({ window }); + }; + sample({}); expect(Object.keys(properties).sort()).toEqual([ "eventualQuiescence", "exactContract", "noDeclaredViolations", "stableCatalog", + "startupContract", ]); - const exactContract = properties.exactContract; - expect(exactContract?.kind).toBe("always"); - const exactEventually = requireFormula(exactContract?.body as FakeFormula); - expect(exactEventually.kind).toBe("eventually"); - expect(exactEventually.milliseconds).toBe(10_000); - expect(evaluate(exactEventually.body)).toBeTrue(); + expect(properties.startupContract?.kind).toBe("eventually"); + expect(properties.startupContract?.milliseconds).toBe(10_000); + expect(evaluate(properties.startupContract?.body)).toBeFalse(); + expect(properties.exactContract?.kind).toBe("always"); + expect(properties.stableCatalog?.kind).toBe("always"); + expect(properties.noDeclaredViolations?.kind).toBe("always"); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); + + sample(contractFixture()); + expect(evaluate(properties.startupContract?.body)).toBeTrue(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); const initial = cell.current as Readonly>; for (const [key, value] of [ ["activeScenario", ""], @@ -457,29 +578,18 @@ describe("Direct Bombadil formulas", () => { ["activeSource", "fixture"], ] as const) { cell.current = { ...initial, [key]: value }; - expect(evaluate(exactEventually.body), key).toBeFalse(); + expect(evaluate(properties.exactContract?.body), key).toBeFalse(); } cell.current = initial; - const noDeclaredViolations = properties.noDeclaredViolations; - expect(noDeclaredViolations?.kind).toBe("always"); - const violationsEventually = requireFormula( - noDeclaredViolations?.body as FakeFormula, - ); - expect(violationsEventually.kind).toBe("eventually"); - expect(violationsEventually.milliseconds).toBe(10_000); - expect(evaluate(violationsEventually.body)).toBeTrue(); - - const stableAlways = properties.stableCatalog; - expect(stableAlways?.kind).toBe("always"); - const stableEventually = requireFormula(stableAlways?.body as FakeFormula); - expect(stableEventually.kind).toBe("eventually"); - expect(stableEventually.milliseconds).toBe(10_000); - expect(evaluate(stableEventually.body)).toBeTrue(); - cell.current = readDirectBombadilObservation(contractFixture({ - catalogHash: "", + sample(contractFixture({ + catalogHash: "fnv1a-64:ffffffffffffffff", })); - expect(evaluate(stableEventually.body)).toBeFalse(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeFalse(); + + sample(contractFixture({ violations: { console: 1 } })); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeFalse(); const outerAlways = properties.eventualQuiescence; expect(outerAlways?.kind).toBe("always"); @@ -492,32 +602,28 @@ describe("Direct Bombadil formulas", () => { const properties = createDirectBombadilProperties() as unknown as Record; const cell = cells.at(-1); if (cell === undefined) throw new Error("Expected one Direct extractor cell"); - cell.current = readDirectBombadilObservation({}); - - const exactEventually = requireFormula(properties.exactContract?.body as FakeFormula); - const violationsEventually = requireFormula( - properties.noDeclaredViolations?.body as FakeFormula, - ); - const stableEventually = requireFormula(properties.stableCatalog?.body as FakeFormula); - expect(evaluate(exactEventually.body)).toBeFalse(); - expect(evaluate(violationsEventually.body)).toBeFalse(); - expect(evaluate(stableEventually.body)).toBeFalse(); + const sample = (window: unknown): void => { + cell.current = cell.read({ window }); + }; + sample({}); + expect(evaluate(properties.startupContract?.body)).toBeFalse(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); const eventual = properties.eventualQuiescence?.body as FakeFormula; expect(evaluate(eventual.body)).toBeFalse(); - cell.current = readDirectBombadilObservation(contractFixture()); - expect(evaluate(exactEventually.body)).toBeTrue(); - expect(evaluate(violationsEventually.body)).toBeTrue(); - expect(evaluate(stableEventually.body)).toBeTrue(); + sample(contractFixture()); + expect(evaluate(properties.startupContract?.body)).toBeTrue(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); expect(evaluate(eventual.body)).toBeTrue(); - cell.current = readDirectBombadilObservation(contractFixture({ - catalogHash: "", - violations: { console: 1 }, - })); - expect(evaluate(exactEventually.body)).toBeFalse(); - expect(evaluate(stableEventually.body)).toBeFalse(); - expect(evaluate(violationsEventually.body)).toBeFalse(); + sample({}); + expect(evaluate(properties.exactContract?.body)).toBeFalse(); + expect(evaluate(properties.stableCatalog?.body)).toBeFalse(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeFalse(); }); }); diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index a9eb154..8cf4512 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -28,6 +28,8 @@ const DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; const DIRECT_SESSION_MANIFEST_SCHEMA = "direct.session-manifest/v1"; const DIRECT_PROBE_SCHEMA = "direct.probe/v1"; const MAX_RAW_CONTRACT_CHARACTERS = 2_000_000; +const MAX_NAMED_SNAPSHOT_CANONICAL_BYTES = 2 * 1024 * 1024; +const MAX_NAMED_SNAPSHOT_JSON_DEPTH = 64; const BRIDGE_KEYS = new Set(["manifest", "reset", "schema", "snapshot"]); const UNSAFE_CLICK_INPUT_TYPES = new Set(["image", "reset", "submit"]); const UNSAFE_CLICK_LABEL_PHRASES = [ @@ -43,6 +45,12 @@ const UNSAFE_CLICK_LABEL_PHRASES = [ "unlink", ] as const; const SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +const RESERVED_SNAPSHOT_NAMES = new Set([ + "direct", + "__proto__", + "constructor", + "prototype", +]); const RESOURCE_LEAK_OPTION_KEYS = new Set(["growthLimit", "metric", "windowMillis"]); const RESOURCE_METRICS = [ "dom_nodes", @@ -71,12 +79,24 @@ export interface DirectBombadilObservation { } export interface DirectBombadilProperties { + readonly startupContract: Formula; readonly exactContract: Formula; readonly stableCatalog: Formula; readonly noDeclaredViolations: Formula; readonly eventualQuiescence: Formula; } +function observationHasExactContract( + observation: DirectBombadilObservation, +): boolean { + return observation.contractValid + && observation.activeSource === "scenario" + && observation.activeScenario.length > 0 + && observation.activeRoute.length > 0 + && observation.activationHash.length > 0 + && observation.catalogHash.length > 0; +} + export type DirectBombadilResourceMetric = (typeof RESOURCE_METRICS)[number]; export interface DirectBombadilResourceLeakOptions { @@ -108,6 +128,7 @@ function safeClickAction(action: ActionTemplate): boolean { }); return fingerprint.href === null && tag !== "a" + && tag !== "label" && fingerprint.role?.toLowerCase() !== "link" && !hasUnsafeLabel && !UNSAFE_CLICK_INPUT_TYPES.has(inputType) @@ -255,10 +276,53 @@ function boundedJsonClone(value: unknown): BombadilJson | null { return JSON.parse(source) as BombadilJson; } -function optionalBoundedJsonClone(value: unknown): BombadilJson | undefined { - const source = JSON.stringify(value); - if (source === undefined || source.length > MAX_RAW_CONTRACT_CHARACTERS) return undefined; - return JSON.parse(source) as BombadilJson; +function cloneNamedSnapshotJson( + value: unknown, + depth = 0, + ancestors: WeakSet = new WeakSet(), +): BombadilJson | undefined { + if (depth > MAX_NAMED_SNAPSHOT_JSON_DEPTH) return undefined; + if (value === null || typeof value === "boolean" || typeof value === "string") { + return value; + } + if (typeof value === "number") return Number.isFinite(value) ? value : undefined; + if (typeof value !== "object") return undefined; + if (ancestors.has(value)) return undefined; + ancestors.add(value); + try { + if (Array.isArray(value)) { + const cloned: BombadilJson[] = []; + for (const entry of value) { + const child = cloneNamedSnapshotJson(entry, depth + 1, ancestors); + if (child === undefined) return undefined; + cloned.push(child); + } + return cloned; + } + const clonedEntries: [string, BombadilJson][] = []; + for (const key of Object.keys(value)) { + const child = cloneNamedSnapshotJson( + Reflect.get(value, key), + depth + 1, + ancestors, + ); + if (child === undefined) return undefined; + clonedEntries.push([key, child]); + } + return Object.fromEntries(clonedEntries) as BombadilJson; + } finally { + ancestors.delete(value); + } +} + +function boundedNamedSnapshotJson(value: unknown): BombadilJson | undefined { + const cloned = cloneNamedSnapshotJson(value); + if (cloned === undefined) return undefined; + const source = JSON.stringify(cloned); + return new TextEncoder().encode(source).byteLength + <= MAX_NAMED_SNAPSHOT_CANONICAL_BYTES + ? cloned + : undefined; } /** @@ -268,24 +332,43 @@ function optionalBoundedJsonClone(value: unknown): BombadilJson | undefined { export function createDirectBombadilNamedSnapshot(options: { readonly fallback: T; readonly name: string; + readonly parse: (value: BombadilJson) => T | null; readonly read: (state: BombadilBrowserState) => unknown; }): Cell { if ( options.name.length === 0 || options.name.length > 128 || !SNAPSHOT_NAME_PATTERN.test(options.name) + || RESERVED_SNAPSHOT_NAMES.has(options.name) ) { - throw new Error("Bombadil snapshot name must be a safe 1-128 character identifier"); + throw new Error( + "Bombadil snapshot name must be a safe, unreserved 1-128 character identifier", + ); + } + const parse = (value: unknown): T | null => { + const cloned = boundedNamedSnapshotJson(value); + if (cloned === undefined) return null; + const parsed = options.parse(cloned); + return parsed !== null && boundedNamedSnapshotJson(parsed) !== undefined + ? parsed + : null; + }; + let fallback: T | null = null; + try { + fallback = parse(options.fallback); + } catch { + fallback = null; } - const fallback = optionalBoundedJsonClone(options.fallback); - if (fallback === undefined) { - throw new Error("Bombadil snapshot fallback must be bounded JSON"); + if (fallback === null) { + throw new Error( + "Bombadil snapshot fallback must be bounded JSON accepted by parse", + ); } return extract((state) => { try { - return (optionalBoundedJsonClone(options.read(state)) ?? fallback) as T; + return parse(options.read(state)) ?? fallback; } catch { - return fallback as T; + return fallback; } }).named(options.name); } @@ -387,39 +470,50 @@ export function readDirectBombadilObservation( } } -/** Builds the four Direct invariants used by Bombadil browser campaigns. */ +/** Builds bounded startup plus strict recurring Direct browser invariants. */ export function createDirectBombadilProperties(): DirectBombadilProperties { - const direct = extract((state) => - readDirectBombadilObservation(state.window) - ).named("direct"); + let initial: Readonly<{ + activationHash: string; + activeRoute: string; + activeScenario: string; + catalogHash: string; + }> | null = null; + const direct = extract((state) => { + const observation = readDirectBombadilObservation(state.window); + if (initial === null && observationHasExactContract(observation)) { + initial = Object.freeze({ + activationHash: observation.activationHash, + activeRoute: observation.activeRoute, + activeScenario: observation.activeScenario, + catalogHash: observation.catalogHash, + }); + } + return observation; + }).named("direct"); - const exactContract = always( - eventually(() => - direct.current.contractValid - && direct.current.activeSource === "scenario" - && direct.current.activeScenario.length > 0 - && direct.current.activeRoute.length > 0 - && direct.current.activationHash.length > 0 - ).within(10, "seconds"), - ); - const stableCatalog = always( - eventually(() => - direct.current.contractValid - && direct.current.catalogHash.length > 0 - ).within(10, "seconds"), - ); - const noDeclaredViolations = always( - eventually(() => - direct.current.contractValid - && direct.current.violationsValid - && direct.current.violations.every((value: number) => value === 0) - ).within(10, "seconds"), - ); + const startupContract = eventually(() => initial !== null).within(10, "seconds"); + const exactContract = always(() => initial === null || ( + observationHasExactContract(direct.current) + && direct.current.activationHash === initial.activationHash + && direct.current.activeRoute === initial.activeRoute + && direct.current.activeScenario === initial.activeScenario + )); + const stableCatalog = always(() => initial === null || ( + observationHasExactContract(direct.current) + && direct.current.catalogHash === initial.catalogHash + )); + const noDeclaredViolations = always(() => initial === null || ( + observationHasExactContract(direct.current) + && direct.current.violationsValid + && direct.current.violations.every((value: number) => value === 0) + )); const eventualQuiescence = always( - eventually(() => direct.current.isQuiescent).within(10, "seconds"), + eventually(() => initial !== null && direct.current.isQuiescent) + .within(10, "seconds"), ); return Object.freeze({ + startupContract, exactContract, stableCatalog, noDeclaredViolations, diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index 82e9595..d7711f9 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -448,6 +448,46 @@ describe("Direct Bombadil configuration and invocation", () => { ); }); + test("orders query and policy artifacts by explicit code units", async () => { + const { config } = await fixture(); + const validated = validateDirectBombadilFuzzConfig({ + ...config, + targetQuery: { z: "4", "a.": "3", A: "1", "a-": "2" }, + explorationPolicy: { + minDistinctNamedSnapshotValues: { + z: 4, + "a.": 3, + A: 1, + "a-": 2, + }, + minNamedSnapshotChangesAfterNonWait: { + z: 4, + "a.": 3, + A: 1, + "a-": 2, + }, + requiredActionKinds: ["Wait", "TypeText", "Click"], + requiredNamedSnapshots: ["z", "a.", "A", "a-"], + }, + }); + const expectedNames = ["A", "a-", "a.", "z"]; + expect(Object.keys(validated.targetQuery)).toEqual(expectedNames); + expect(validated.explorationPolicy?.requiredActionKinds).toEqual([ + "Click", + "TypeText", + "Wait", + ]); + expect(validated.explorationPolicy?.requiredNamedSnapshots).toEqual( + expectedNames, + ); + expect(Object.keys( + validated.explorationPolicy?.minDistinctNamedSnapshotValues ?? {}, + )).toEqual(expectedNames); + expect(Object.keys( + validated.explorationPolicy?.minNamedSnapshotChangesAfterNonWait ?? {}, + )).toEqual(expectedNames); + }); + test("rejects unsafe artifact, scenario, route, path, readiness, and server command inputs", async () => { const { config, repositoryRoot } = await fixture(); expect(() => validateDirectBombadilFuzzConfig({ @@ -925,6 +965,103 @@ describe("Direct Bombadil exploration summary", () => { }); }); + test("uses the first exact Direct observation only as the policy baseline", async () => { + const tracePath = await summaryTrace([ + traceLine(absentObservation(), 1, { + namedSnapshots: [{ name: "phase", value: "loading" }], + }), + traceLine(absentObservation(), 2, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: "pre-handshake-click" }], + }), + traceLine(directObservation(), 3, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: "ready" }], + }), + ]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + minNonWaitActions: 1, + requiredActionKinds: ["Click"], + requiredNamedSnapshots: ["phase"], + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.actions).toMatchObject({ + byKind: {}, + nonWaitCount: 0, + total: 0, + }); + expect(summary.urls.observationCount).toBe(1); + expect(summary.transitions).toEqual({ + distinctNonNullHashCount: 1, + nonNullHashCount: 1, + }); + expect(summary.namedSnapshots.find((entry) => entry.name === "phase")) + .toMatchObject({ + changeAfterNonWaitCount: 0, + distinctValueCount: 1, + observationCount: 1, + }); + expect(summary.policy).toMatchObject({ + failures: [ + expect.stringContaining("minimum non-Wait"), + expect.stringContaining("required action kind Click"), + expect.stringContaining("distinct-value minimum"), + expect.stringContaining("post-non-Wait change minimum"), + ], + satisfied: false, + }); + }); + + test("orders mixed-case and punctuation summary names by code unit", async () => { + const observation = directObservation(); + const tracePath = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: [ + { name: "z", value: 4 }, + { name: "a.", value: 3 }, + { name: "A", value: 1 }, + { name: "a-", value: 2 }, + ], + violations: [ + { name: "z", violation: { False: {} } }, + { name: "a.", violation: { False: {} } }, + { name: "A", violation: { False: {} } }, + { name: "a-", violation: { False: {} } }, + ], + })]); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.map(({ name }) => name)).toEqual([ + "A", + "a-", + "a.", + "direct", + "z", + ]); + expect(Object.keys(summary.propertyViolations.byName)).toEqual([ + "A", + "a-", + "a.", + "z", + ]); + }); + test("rejects hostile envelopes, action targets, and excessively deep snapshot JSON", async () => { const observation = directObservation(); const badTarget = await summaryTrace([traceLine(observation, 1, { diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index a76deaf..b53c3a7 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -457,6 +457,12 @@ function hasExactKeys( return keys.length === expected.size && keys.every((key) => expected.has(key)); } +function compareCodeUnits(left: string, right: string): number { + if (left < right) return -1; + if (left > right) return 1; + return 0; +} + function parseTraceDirectObservation(value: unknown): TraceDirectObservation { if (!isRecord(value) || !hasExactKeys(value, DIRECT_OBSERVATION_KEYS)) { throw new Error("Bombadil trace has an invalid named direct observation"); @@ -650,7 +656,7 @@ function canonicalJson(value: unknown, depth = 0): string { return `[${value.map((entry) => canonicalJson(entry, depth + 1)).join(",")}]`; } if (!isRecord(value)) throw new Error("Bombadil named snapshot is not JSON"); - const entries = Object.keys(value).sort().map((key) => + const entries = Object.keys(value).sort(compareCodeUnits).map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key], depth + 1)}` ); return `{${entries.join(",")}}`; @@ -975,7 +981,7 @@ function sortedCountRecord( values: ReadonlyMap, ): Readonly>> { return Object.freeze(Object.fromEntries( - [...values.entries()].sort(([left], [right]) => left.localeCompare(right)), + [...values.entries()].sort(([left], [right]) => compareCodeUnits(left, right)), ) as Partial>); } @@ -1030,6 +1036,9 @@ export async function summarizeDirectBombadilTrace(options: { let waitStreak = 0; let maxWaitStreak = 0; let nonNullHashCount = 0; + let policyObservationCount = 0; + let reachedExactObservation = false; + let previousObservationWasExact = false; let stableTarget = true; const stream = createReadStream(options.tracePath, { encoding: "utf8" }); const lines = createInterface({ input: stream, crlfDelay: Infinity }); @@ -1043,7 +1052,17 @@ export async function summarizeDirectBombadilTrace(options: { throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); } const parsed = parseTraceLine(line, lineCount); - if (parsed.action !== null) { + const currentObservationIsExact = + exactTraceDirectObservation(parsed.directObservation) !== null; + if (!reachedExactObservation && !currentObservationIsExact) { + previousObservationWasExact = false; + continue; + } + if (!reachedExactObservation) reachedExactObservation = true; + policyObservationCount += 1; + const actionFollowsExactObservation = previousObservationWasExact; + + if (actionFollowsExactObservation && parsed.action !== null) { totalActions += 1; actionCounts.set(parsed.action.kind, (actionCounts.get(parsed.action.kind) ?? 0) + 1); if (parsed.action.kind === "Wait") { @@ -1062,7 +1081,7 @@ export async function summarizeDirectBombadilTrace(options: { (targetTags.get(parsed.action.targetTag) ?? 0) + 1, ); } - } else { + } else if (actionFollowsExactObservation) { waitStreak = 0; } @@ -1096,7 +1115,8 @@ export async function summarizeDirectBombadilTrace(options: { snapshots.set(snapshot.name, entry); } if ( - parsed.action !== null + actionFollowsExactObservation + && parsed.action !== null && parsed.action.kind !== "Wait" && entry.lastValueSha256 !== null && entry.lastValueSha256 !== snapshot.valueSha256 @@ -1126,6 +1146,7 @@ export async function summarizeDirectBombadilTrace(options: { parsed.state.resources[sourceName as keyof typeof RESOURCE_FIELD_MAP], ); } + previousObservationWasExact = currentObservationIsExact; } } finally { lines.close(); @@ -1183,8 +1204,8 @@ export async function summarizeDirectBombadilTrace(options: { }), urls: Object.freeze({ distinctFingerprintCount: urlFingerprints.size, - fingerprintSha256: Object.freeze([...urlFingerprints].sort()), - observationCount: lineCount, + fingerprintSha256: Object.freeze([...urlFingerprints].sort(compareCodeUnits)), + observationCount: policyObservationCount, stableTarget, }), transitions: Object.freeze({ @@ -1192,11 +1213,11 @@ export async function summarizeDirectBombadilTrace(options: { nonNullHashCount, }), namedSnapshots: Object.freeze([...snapshots.entries()] - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => compareCodeUnits(left, right)) .map(([name, entry]) => Object.freeze({ changeAfterNonWaitCount: entry.changeAfterNonWaitCount, distinctValueCount: entry.values.size, - distinctValueSha256: Object.freeze([...entry.values].sort()), + distinctValueSha256: Object.freeze([...entry.values].sort(compareCodeUnits)), name, observationCount: entry.observationCount, }))), @@ -1325,7 +1346,7 @@ function validateTargetQuery(value: unknown): Readonly> { } const validated: Record = {}; for (const [name, queryValue] of [...entries].sort(([left], [right]) => - left.localeCompare(right) + compareCodeUnits(left, right) )) { if ( name.length === 0 @@ -1410,7 +1431,7 @@ function validateSnapshotMinimumMap(options: { } const validated: Record = {}; for (const [rawName, minimum] of Object.entries(options.value).sort(([left], [right]) => - left.localeCompare(right) + compareCodeUnits(left, right) )) { const name = validateSnapshotName(rawName, `${options.label} key`); if ( @@ -1462,7 +1483,7 @@ function validateExplorationPolicy( ) { throw new Error("explorationPolicy.requiredActionKinds contains an unknown or duplicate kind"); } - requiredActionKinds.sort(); + requiredActionKinds.sort(compareCodeUnits); const requiredNamedSnapshotsInput = value.requiredNamedSnapshots ?? []; if (!Array.isArray(requiredNamedSnapshotsInput) || requiredNamedSnapshotsInput.length > 32) { @@ -1474,7 +1495,7 @@ function validateExplorationPolicy( if (new Set(requiredNamedSnapshots).size !== requiredNamedSnapshots.length) { throw new Error("explorationPolicy.requiredNamedSnapshots contains a duplicate name"); } - requiredNamedSnapshots.sort(); + requiredNamedSnapshots.sort(compareCodeUnits); const minDistinctNamedSnapshotValues = validateSnapshotMinimumMap({ label: "explorationPolicy.minDistinctNamedSnapshotValues", From 6ccc6bb4cd3b75cb9a57415eddf8af048f183cce Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 12:53:19 -0400 Subject: [PATCH 11/25] fix: remove stale Bombadil test helper --- src/tooling/bombadil-campaign.test.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 86c94cd..2f47923 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -167,11 +167,6 @@ function evaluate(body: unknown): boolean { return (body as () => boolean)(); } -function requireFormula(value: boolean | FakeFormula): FakeFormula { - if (typeof value === "boolean") throw new Error("Expected a nested formula"); - return value; -} - describe("Direct Bombadil observation", () => { test("accepts the exact current bridge, manifest, and probe contract", () => { const observation = readDirectBombadilObservation(contractFixture()); From 89dfd30e1036ad0fc52c355a944315a4d131b423 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:00:51 -0400 Subject: [PATCH 12/25] fix: accept null Bombadil snapshot values --- docs/verification.md | 2 +- src/tooling/bombadil-campaign.test.ts | 10 +++++----- src/tooling/bombadil-campaign.ts | 16 ++++++++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/verification.md b/docs/verification.md index cb765f9..2aa45be 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -272,7 +272,7 @@ const direct = createDirectBombadilProperties(); const phase = createDirectBombadilNamedSnapshot({ fallback: "unavailable", name: "todos.phase", - parse: (value) => typeof value === "string" ? value : null, + parse: (value) => typeof value === "string" ? value : undefined, read: ({ window }) => Reflect.get(window, "__todosPhase"), }); diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 2f47923..027bc8f 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -242,10 +242,10 @@ describe("Direct Bombadil named snapshots", () => { name: "product.phase", parse: (value) => { if (typeof value !== "object" || value === null || Array.isArray(value)) { - return null; + return undefined; } const status = Reflect.get(value, "status"); - return typeof status === "string" ? { status } : null; + return typeof status === "string" ? { status } : undefined; }, read: (state) => Reflect.get(state.window, "phase"), }) as unknown as FakeCell; @@ -275,10 +275,10 @@ describe("Direct Bombadil named snapshots", () => { name: "product.compat", parse: (value) => { if (typeof value !== "object" || value === null || Array.isArray(value)) { - return null; + return undefined; } const status = Reflect.get(value, "status"); - return typeof status === "string" ? { status } : null; + return typeof status === "string" ? { status } : undefined; }, read: (state) => Reflect.get(state.window, "phase"), }) as unknown as FakeCell; @@ -362,7 +362,7 @@ describe("Direct Bombadil named snapshots", () => { expect(() => createDirectBombadilNamedSnapshot({ fallback: null, name: "safe", - parse: () => null, + parse: () => undefined, read: () => null, })).toThrow("accepted by parse"); expect(() => createDirectBombadilNamedSnapshot({ diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index 8cf4512..6d44f2b 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -332,7 +332,7 @@ function boundedNamedSnapshotJson(value: unknown): BombadilJson | undefined { export function createDirectBombadilNamedSnapshot(options: { readonly fallback: T; readonly name: string; - readonly parse: (value: BombadilJson) => T | null; + readonly parse: (value: BombadilJson) => T | undefined; readonly read: (state: BombadilBrowserState) => unknown; }): Cell { if ( @@ -345,21 +345,21 @@ export function createDirectBombadilNamedSnapshot(option "Bombadil snapshot name must be a safe, unreserved 1-128 character identifier", ); } - const parse = (value: unknown): T | null => { + const parse = (value: unknown): T | undefined => { const cloned = boundedNamedSnapshotJson(value); - if (cloned === undefined) return null; + if (cloned === undefined) return undefined; const parsed = options.parse(cloned); - return parsed !== null && boundedNamedSnapshotJson(parsed) !== undefined + return parsed !== undefined && boundedNamedSnapshotJson(parsed) !== undefined ? parsed - : null; + : undefined; }; - let fallback: T | null = null; + let fallback: T | undefined; try { fallback = parse(options.fallback); } catch { - fallback = null; + fallback = undefined; } - if (fallback === null) { + if (fallback === undefined) { throw new Error( "Bombadil snapshot fallback must be bounded JSON accepted by parse", ); From c06b641aa5aaac59ae47831e51756955123d1aa6 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:02:16 -0400 Subject: [PATCH 13/25] fix: own parsed Bombadil snapshots --- src/tooling/bombadil-campaign.test.ts | 19 +++++++++++++++++-- src/tooling/bombadil-campaign.ts | 6 +++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 027bc8f..c9d4f2d 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -270,6 +270,14 @@ describe("Direct Bombadil named snapshots", () => { }); test("produces a named value accepted by the host summary contract", async () => { + class MutableParsedValue { + status = "ready"; + + toJSON() { + return { status: "é".repeat(1_100_000) }; + } + } + const parserOutput = new MutableParsedValue(); const snapshot = createDirectBombadilNamedSnapshot({ fallback: { status: "unavailable" }, name: "product.compat", @@ -278,11 +286,18 @@ describe("Direct Bombadil named snapshots", () => { return undefined; } const status = Reflect.get(value, "status"); - return typeof status === "string" ? { status } : undefined; + return typeof status === "string" ? parserOutput : undefined; }, read: (state) => Reflect.get(state.window, "phase"), }) as unknown as FakeCell; - const value = snapshot.read({ window: { phase: { status: "ready" } } }); + const value = snapshot.read({ + window: { phase: { status: "ready" } }, + }) as { status: string }; + expect(value).toEqual({ status: "ready" }); + expect(value).not.toBe(parserOutput); + expect(Object.getPrototypeOf(value)).toBe(Object.prototype); + parserOutput.status = "mutated"; + expect(value).toEqual({ status: "ready" }); const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-helper-summary-")); const tracePath = join(directory, "trace.jsonl"); try { diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index 6d44f2b..c564f76 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -349,9 +349,9 @@ export function createDirectBombadilNamedSnapshot(option const cloned = boundedNamedSnapshotJson(value); if (cloned === undefined) return undefined; const parsed = options.parse(cloned); - return parsed !== undefined && boundedNamedSnapshotJson(parsed) !== undefined - ? parsed - : undefined; + if (parsed === undefined) return undefined; + const owned = boundedNamedSnapshotJson(parsed); + return owned === undefined ? undefined : owned as T; }; let fallback: T | undefined; try { From 9c63cd4672b544580b70da6d37feec2310d64ad7 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:05:09 -0400 Subject: [PATCH 14/25] fix: validate owned Bombadil JSON --- docs/verification.md | 2 +- src/tooling/bombadil-campaign.test.ts | 50 +++++++++++++-------------- src/tooling/bombadil-campaign.ts | 18 ++++------ 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/docs/verification.md b/docs/verification.md index 2aa45be..0ac8180 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -272,8 +272,8 @@ const direct = createDirectBombadilProperties(); const phase = createDirectBombadilNamedSnapshot({ fallback: "unavailable", name: "todos.phase", - parse: (value) => typeof value === "string" ? value : undefined, read: ({ window }) => Reflect.get(window, "__todosPhase"), + validate: (value): value is string => typeof value === "string", }); export const direct_safe_actions = createDirectBombadilActions(); diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index c9d4f2d..a469a25 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -240,14 +240,13 @@ describe("Direct Bombadil named snapshots", () => { const snapshot = createDirectBombadilNamedSnapshot({ fallback: { status: "unavailable" }, name: "product.phase", - parse: (value) => { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - return undefined; - } - const status = Reflect.get(value, "status"); - return typeof status === "string" ? { status } : undefined; - }, read: (state) => Reflect.get(state.window, "phase"), + validate: (value): value is { readonly status: string } => ( + typeof value === "object" + && value !== null + && !Array.isArray(value) + && typeof Reflect.get(value, "status") === "string" + ), }) as unknown as FakeCell; expect(snapshot.name).toBe("product.phase"); expect(snapshot.read({ window: { phase: { status: "ready" } } })).toEqual({ @@ -270,33 +269,32 @@ describe("Direct Bombadil named snapshots", () => { }); test("produces a named value accepted by the host summary contract", async () => { - class MutableParsedValue { + class MutablePageValue { status = "ready"; toJSON() { return { status: "é".repeat(1_100_000) }; } } - const parserOutput = new MutableParsedValue(); + const pageValue = new MutablePageValue(); const snapshot = createDirectBombadilNamedSnapshot({ fallback: { status: "unavailable" }, name: "product.compat", - parse: (value) => { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - return undefined; - } - const status = Reflect.get(value, "status"); - return typeof status === "string" ? parserOutput : undefined; - }, - read: (state) => Reflect.get(state.window, "phase"), + read: () => pageValue, + validate: (value): value is { readonly status: string } => ( + typeof value === "object" + && value !== null + && !Array.isArray(value) + && typeof Reflect.get(value, "status") === "string" + ), }) as unknown as FakeCell; const value = snapshot.read({ window: { phase: { status: "ready" } }, }) as { status: string }; expect(value).toEqual({ status: "ready" }); - expect(value).not.toBe(parserOutput); + expect(value).not.toBe(pageValue); expect(Object.getPrototypeOf(value)).toBe(Object.prototype); - parserOutput.status = "mutated"; + pageValue.status = "mutated"; expect(value).toEqual({ status: "ready" }); const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-helper-summary-")); const tracePath = join(directory, "trace.jsonl"); @@ -355,16 +353,16 @@ describe("Direct Bombadil named snapshots", () => { expect(() => createDirectBombadilNamedSnapshot({ fallback: null, name, - parse: (value) => value, read: () => null, + validate: (_value): _value is BombadilJson => true, })).toThrow("safe, unreserved"); } const snapshot = createDirectBombadilNamedSnapshot({ fallback: null, name: "safe", - parse: (value) => value, read: (state) => Reflect.get(state.window, "phase"), + validate: (_value): _value is BombadilJson => true, }) as unknown as FakeCell; let atLimit: BombadilJson = null; for (let index = 0; index < 64; index += 1) atLimit = [atLimit]; @@ -373,19 +371,19 @@ describe("Direct Bombadil named snapshots", () => { expect(snapshot.read({ window: { phase: beyondLimit } })).toBeNull(); }); - test("rejects non-JSON or parser-invalid fallbacks before registering an extractor", () => { + test("rejects non-JSON or predicate-invalid fallbacks before registering an extractor", () => { expect(() => createDirectBombadilNamedSnapshot({ fallback: null, name: "safe", - parse: () => undefined, read: () => null, - })).toThrow("accepted by parse"); + validate: (_value): _value is never => false, + })).toThrow("accepted by validate"); expect(() => createDirectBombadilNamedSnapshot({ fallback: undefined as never, name: "safe", - parse: (value) => value, read: () => null, - })).toThrow("fallback must be bounded JSON accepted by parse"); + validate: (_value): _value is BombadilJson => true, + })).toThrow("fallback must be bounded JSON accepted by validate"); }); }); diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index c564f76..85d3010 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -332,8 +332,8 @@ function boundedNamedSnapshotJson(value: unknown): BombadilJson | undefined { export function createDirectBombadilNamedSnapshot(options: { readonly fallback: T; readonly name: string; - readonly parse: (value: BombadilJson) => T | undefined; readonly read: (state: BombadilBrowserState) => unknown; + readonly validate: (value: BombadilJson) => value is T; }): Cell { if ( options.name.length === 0 @@ -345,28 +345,24 @@ export function createDirectBombadilNamedSnapshot(option "Bombadil snapshot name must be a safe, unreserved 1-128 character identifier", ); } - const parse = (value: unknown): T | undefined => { - const cloned = boundedNamedSnapshotJson(value); - if (cloned === undefined) return undefined; - const parsed = options.parse(cloned); - if (parsed === undefined) return undefined; - const owned = boundedNamedSnapshotJson(parsed); - return owned === undefined ? undefined : owned as T; + const validate = (value: unknown): T | undefined => { + const owned = boundedNamedSnapshotJson(value); + return owned !== undefined && options.validate(owned) ? owned : undefined; }; let fallback: T | undefined; try { - fallback = parse(options.fallback); + fallback = validate(options.fallback); } catch { fallback = undefined; } if (fallback === undefined) { throw new Error( - "Bombadil snapshot fallback must be bounded JSON accepted by parse", + "Bombadil snapshot fallback must be bounded JSON accepted by validate", ); } return extract((state) => { try { - return parse(options.read(state)) ?? fallback; + return validate(options.read(state)) ?? fallback; } catch { return fallback; } From 08a6cd8a0596a01800e17c8553ecac509eb98b60 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:11:27 -0400 Subject: [PATCH 15/25] fix: name Bombadil test predicates --- src/tooling/bombadil-campaign.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index a469a25..812be74 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -167,6 +167,14 @@ function evaluate(body: unknown): boolean { return (body as () => boolean)(); } +function acceptBombadilJson(value: BombadilJson): value is BombadilJson { + return value !== undefined; +} + +function rejectBombadilJson(value: BombadilJson): value is never { + return value === undefined; +} + describe("Direct Bombadil observation", () => { test("accepts the exact current bridge, manifest, and probe contract", () => { const observation = readDirectBombadilObservation(contractFixture()); @@ -354,7 +362,7 @@ describe("Direct Bombadil named snapshots", () => { fallback: null, name, read: () => null, - validate: (_value): _value is BombadilJson => true, + validate: acceptBombadilJson, })).toThrow("safe, unreserved"); } @@ -362,7 +370,7 @@ describe("Direct Bombadil named snapshots", () => { fallback: null, name: "safe", read: (state) => Reflect.get(state.window, "phase"), - validate: (_value): _value is BombadilJson => true, + validate: acceptBombadilJson, }) as unknown as FakeCell; let atLimit: BombadilJson = null; for (let index = 0; index < 64; index += 1) atLimit = [atLimit]; @@ -376,13 +384,13 @@ describe("Direct Bombadil named snapshots", () => { fallback: null, name: "safe", read: () => null, - validate: (_value): _value is never => false, + validate: rejectBombadilJson, })).toThrow("accepted by validate"); expect(() => createDirectBombadilNamedSnapshot({ fallback: undefined as never, name: "safe", read: () => null, - validate: (_value): _value is BombadilJson => true, + validate: acceptBombadilJson, })).toThrow("fallback must be bounded JSON accepted by validate"); }); }); From ffee66bf5f8a6cc69b72f5856c39b9482c230d01 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:29:17 -0400 Subject: [PATCH 16/25] feat: attribute Bombadil exploration actions --- README.md | 6 +- docs/verification.md | 29 ++- src/tooling/bombadil-runner.test.ts | 297 ++++++++++++++++++++++- src/tooling/bombadil-runner.ts | 363 ++++++++++++++++++++++++---- 4 files changed, 628 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index b461193..0f1f7be 100644 --- a/README.md +++ b/README.md @@ -290,8 +290,10 @@ snapshots that expose semantic state without retaining page content. Run short scheduled diagnostic lane. Inspect and replay a retained failing trace, then promote the smallest readable failure to a deterministic product regression. When a campaign must exercise an interaction, require a named product value to -change after a non-Wait action so bootstrap and idle transitions do not satisfy -the exploration policy. +change after the intended action kind, as well as after a non-Wait action, so +bootstrap, idle, prerequisite, and unrelated transitions do not satisfy the +exploration policy. Attribution requires adjacent exact Direct observations; +it is temporal response evidence rather than proof of causality. If the full product snapshot includes viewport dimensions, put that requirement on a separate interaction snapshot without viewport fields and require an opposite-size `SetViewport` independently. Latch the first ready product state diff --git a/docs/verification.md b/docs/verification.md index 0ac8180..f2a4252 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -310,9 +310,10 @@ scenario boundary and exercises behavior the product can interpret. `createDirectBombadilNamedSnapshot` gives product properties and post-run diagnostics a small semantic signal. It requires a safe bounded name distinct -from `direct` and JavaScript prototype names, an explicit product parser, at +from `direct` and JavaScript prototype names, an explicit product type +predicate over Direct's owned plain-JSON clone, at most 64 JSON levels, and at most 2 MiB of UTF-8 JSON. It fails closed to the -parsed fallback when a page getter throws or returns unsuitable data. Extract +validated fallback when a page getter throws or returns unsuitable data. Extract state, not page content or credentials. Named values are represented only by canonical SHA-256 hashes in the exploration summary; the authoritative raw trace still contains the original values. @@ -355,6 +356,9 @@ await runDirectBombadilFuzz({ minNonWaitActions: 1, requiredNamedSnapshots: ["direct", "todos.phase"], minDistinctNamedSnapshotValues: { "todos.phase": 2 }, + minNamedSnapshotChangesAfterActionKind: { + "todos.phase": { Click: 1 }, + }, minNamedSnapshotChangesAfterNonWait: { "todos.phase": 1 }, requireStableTargetUrl: true, }, @@ -399,11 +403,15 @@ responsive behavior that needs separate exploration. `explorationPolicy` is optional. When present, it can require a minimum count of non-Wait actions, particular action kinds and named snapshots, minimum distinct hashed values for named snapshots, named-value changes observed after -a non-Wait action, and an exact stable target URL. The change requirement keeps +a non-Wait action or one particular action kind, and an exact stable target +URL. The change requirement keeps bootstrap or Wait-only transitions from satisfying a product-interaction -threshold. The trace records the last action with the resulting state, so this -count identifies a named value transition associated with an active product -action. It does not prove causality. The policy detects a campaign that passed +threshold. Per-kind attribution requires exact Direct observations and the +same named snapshot in both immediately adjacent samples. The trace records +the last action with the resulting state, so this count identifies a named +value transition associated with an active product action. It does not prove +causality. The runner strictly validates every 0.7.2 action payload before +crediting its kind. The policy detects a campaign that passed its properties without doing the intended exploration. It is a diagnostic sufficiency check, not a coverage claim. Start with observed stable behavior and raise thresholds only when the action generator makes them reliably @@ -500,12 +508,15 @@ Each attempt writes `run.json`, `exploration-summary.json`, `bombadil.log`, and `artifacts/direct-bombadil///`, including failures. The rolling `manifest.json` points to the latest record. `rawTracePath` reports a regular nonempty trace even if attestation fails; `tracePath` is present only -after exact attestation. The summary strictly parses the 0.7.2 envelopes and +after exact attestation. The v2 summary strictly parses the 0.7.2 envelopes and records the raw trace SHA-256, action-kind and safe target-tag counts, non-Wait count and longest Wait streak, origin-relative URL fingerprints, non-null transition-hash cardinality, canonical named-snapshot value hashes, property -violation names and counts, named-value changes after non-Wait actions, and -browser resource high-water marks. It excludes +violation names and counts, named-value changes after non-Wait actions and +specific action kinds, and browser resource high-water marks. Action, URL, +transition, and named-snapshot policy evidence starts at the first exact Direct +observation. Separate raw URL and transition hashes, property violations, and +resource maxima retain strictly parsed startup-prefix diagnostics. It excludes typed text, accessible names, snapshot values, URLs, screenshots, and absolute paths. These diagnostics describe what Bombadil happened to explore. They do not measure code, state, interaction, or Direct catalog coverage, and the raw diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index d7711f9..ace1c47 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -460,6 +460,12 @@ describe("Direct Bombadil configuration and invocation", () => { A: 1, "a-": 2, }, + minNamedSnapshotChangesAfterActionKind: { + z: { Wait: 4, Click: 3 }, + "a.": { SetViewport: 3 }, + A: { Wait: 1, Click: 2 }, + "a-": { TypeText: 2 }, + }, minNamedSnapshotChangesAfterNonWait: { z: 4, "a.": 3, @@ -483,6 +489,12 @@ describe("Direct Bombadil configuration and invocation", () => { expect(Object.keys( validated.explorationPolicy?.minDistinctNamedSnapshotValues ?? {}, )).toEqual(expectedNames); + expect(Object.keys( + validated.explorationPolicy?.minNamedSnapshotChangesAfterActionKind ?? {}, + )).toEqual(expectedNames); + expect(Object.keys( + validated.explorationPolicy?.minNamedSnapshotChangesAfterActionKind.A ?? {}, + )).toEqual(["Click", "Wait"]); expect(Object.keys( validated.explorationPolicy?.minNamedSnapshotChangesAfterNonWait ?? {}, )).toEqual(expectedNames); @@ -546,6 +558,30 @@ describe("Direct Bombadil configuration and invocation", () => { ...config, explorationPolicy: { minNamedSnapshotChangesAfterNonWait: { phase: 10_001 } }, })).toThrow("between 1 and 10000"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { phase: {} }, + }, + })).toThrow("bounded action map"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { Unknown: 1 } as never, + }, + }, + })).toThrow("unknown action kind"); + for (const minimum of [0, 10_001]) { + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { Click: minimum }, + }, + }, + })).toThrow("between 1 and 10000"); + } }); test("rejects specification, server cwd, and replay symlinks that escape the repository", async () => { @@ -871,6 +907,7 @@ describe("Direct Bombadil exploration summary", () => { const options = { explorationPolicy: { minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterActionKind: { phase: { Click: 1 } }, minNamedSnapshotChangesAfterNonWait: { phase: 1 }, minNonWaitActions: 2, requireStableTargetUrl: true, @@ -884,7 +921,7 @@ describe("Direct Bombadil exploration summary", () => { const second = await summarizeDirectBombadilTrace(options); expect(second).toEqual(first); expect(first).toMatchObject({ - schema: "direct.bombadil-exploration-summary/v1", + schema: "direct.bombadil-exploration-summary/v2", actions: { byKind: { Click: 1, TypeText: 1, Wait: 1 }, maxWaitStreak: 1, @@ -895,9 +932,16 @@ describe("Direct Bombadil exploration summary", () => { urls: { distinctFingerprintCount: 1, observationCount: 4, + rawDistinctFingerprintCount: 1, + rawObservationCount: 4, stableTarget: true, }, - transitions: { distinctNonNullHashCount: 4, nonNullHashCount: 4 }, + transitions: { + distinctNonNullHashCount: 4, + nonNullHashCount: 4, + rawDistinctNonNullHashCount: 4, + rawNonNullHashCount: 4, + }, propertyViolations: { byName: { noConsoleErrors: 1 }, total: 1 }, resourceHighWaterMarks: { domNodes: 4, @@ -906,6 +950,7 @@ describe("Direct Bombadil exploration summary", () => { policy: { configured: true, failures: [], satisfied: true }, }); expect(first.namedSnapshots.find((entry) => entry.name === "phase")).toMatchObject({ + changeAfterActionKind: { Click: 1 }, changeAfterNonWaitCount: 1, distinctValueCount: 2, observationCount: 4, @@ -919,6 +964,82 @@ describe("Direct Bombadil exploration summary", () => { expect(serialized).not.toContain("/tmp/"); }); + test("accepts every exact Bombadil 0.7.2 browser action payload", async () => { + const actions = [ + "Back", + { + Click: { + fingerprint: { accessible_name: "private click label", tag: "button" }, + point: { x: 1, y: 2 }, + }, + }, + { + DoubleClick: { + delay_millis: 10, + fingerprint: { + structural_path: "private/path", + tag: "x-private.widget_\u00e9", + }, + point: { x: 2, y: 3 }, + }, + }, + "Forward", + { + MouseDrag: { + delay_millis: 5, + from: { x: 3, y: 4 }, + steps: 10, + to: { x: 30, y: 40 }, + }, + }, + { PressKey: { code: 13 } }, + "Reload", + { ScrollDown: { distance: 100, origin: { x: 5, y: 6 } } }, + { ScrollUp: { distance: 100, origin: { x: 5, y: 6 } } }, + { + SetFileInputFiles: { + files: ["/private/file.txt"], + selector: "#private-file-input", + }, + }, + { SetViewport: { height: 720, width: 1_280 } }, + { TypeText: { delay_millis: 5, text: "private typed text" } }, + "Wait", + ] as const; + const observation = directObservation(); + const tracePath = await summaryTrace([ + traceLine(observation, 1), + ...actions.map((action, index) => traceLine(observation, index + 2, { + action, + })), + ]); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(Object.keys(summary.actions.byKind)).toEqual([ + "Back", + "Click", + "DoubleClick", + "Forward", + "MouseDrag", + "PressKey", + "Reload", + "ScrollDown", + "ScrollUp", + "SetFileInputFiles", + "SetViewport", + "TypeText", + "Wait", + ]); + expect(summary.actions.total).toBe(actions.length); + expect(Object.keys(summary.actions.targetTags)).toContain("button"); + expect(Object.keys(summary.actions.targetTags).some((tag) => + /^sha256:[a-f0-9]{64}$/u.test(tag) + )).toBeTrue(); + expect(JSON.stringify(summary)).not.toContain("private"); + }); + test("reports strict policy misses while keeping the diagnostic summary", async () => { const tracePath = await summaryTrace([traceLine(directObservation(), 1)]); const summary = await summarizeDirectBombadilTrace({ @@ -965,10 +1086,133 @@ describe("Direct Bombadil exploration summary", () => { }); }); + test("attributes named changes to the exact action kind", async () => { + const observation = directObservation(); + const noOpClickTrace = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(observation, 2, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(observation, 3, { + action: { SetViewport: { height: 720, width: 1_280 } }, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + ]); + const failed = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { Click: 1, SetViewport: 1 }, + }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: noOpClickTrace, + }); + expect(failed.namedSnapshots.find((entry) => entry.name === "phase")) + .toMatchObject({ + changeAfterActionKind: { SetViewport: 1 }, + changeAfterNonWaitCount: 1, + }); + expect(failed.policy).toMatchObject({ + failures: [expect.stringContaining("post-Click change minimum")], + satisfied: false, + }); + + const attributedTrace = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(observation, 2, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + traceLine(observation, 3, { + action: { SetViewport: { height: 720, width: 1_280 } }, + namedSnapshots: [{ name: "phase", value: 2 }], + }), + ]); + const attributed = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { SetViewport: 1, Click: 1 }, + }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: attributedTrace, + }); + const phase = attributed.namedSnapshots.find((entry) => entry.name === "phase"); + expect(phase?.changeAfterActionKind).toEqual({ Click: 1, SetViewport: 1 }); + expect(phase?.changeAfterNonWaitCount).toBe(2); + expect(phase?.changeAfterNonWaitCount).toBe( + Object.entries(phase?.changeAfterActionKind ?? {}) + .filter(([kind]) => kind !== "Wait") + .reduce((total, [, count]) => total + count, 0), + ); + expect(attributed.policy.satisfied).toBeTrue(); + }); + + test("requires adjacent exact observations for action attribution", async () => { + const exact = directObservation(); + const click = { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + } as const; + const missingSnapshotTrace = await summaryTrace([ + traceLine(exact, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(exact, 2), + traceLine(exact, 3, { + action: click, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + ]); + const invalidCurrentTrace = await summaryTrace([ + traceLine(exact, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(absentObservation(), 2, { + action: click, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + ]); + for (const tracePath of [missingSnapshotTrace, invalidCurrentTrace]) { + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { phase: { Click: 1 } }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.find((entry) => entry.name === "phase") + ?.changeAfterActionKind).toEqual({}); + expect(summary.policy.satisfied).toBeFalse(); + } + }); + test("uses the first exact Direct observation only as the policy baseline", async () => { const tracePath = await summaryTrace([ - traceLine(absentObservation(), 1, { + traceLine(absentObservation(), 20, { namedSnapshots: [{ name: "phase", value: "loading" }], + violations: [{ + name: "startup_contract", + violation: { False: {} }, + }], }), traceLine(absentObservation(), 2, { action: { @@ -992,6 +1236,7 @@ describe("Direct Bombadil exploration summary", () => { const summary = await summarizeDirectBombadilTrace({ explorationPolicy: { minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterActionKind: { phase: { Click: 1 } }, minNamedSnapshotChangesAfterNonWait: { phase: 1 }, minNonWaitActions: 1, requiredActionKinds: ["Click"], @@ -1006,10 +1251,18 @@ describe("Direct Bombadil exploration summary", () => { total: 0, }); expect(summary.urls.observationCount).toBe(1); + expect(summary.urls.rawObservationCount).toBe(3); expect(summary.transitions).toEqual({ distinctNonNullHashCount: 1, nonNullHashCount: 1, + rawDistinctNonNullHashCount: 3, + rawNonNullHashCount: 3, + }); + expect(summary.propertyViolations).toEqual({ + byName: { startup_contract: 1 }, + total: 1, }); + expect(summary.resourceHighWaterMarks.domNodes).toBe(20); expect(summary.namedSnapshots.find((entry) => entry.name === "phase")) .toMatchObject({ changeAfterNonWaitCount: 0, @@ -1022,6 +1275,7 @@ describe("Direct Bombadil exploration summary", () => { expect.stringContaining("required action kind Click"), expect.stringContaining("distinct-value minimum"), expect.stringContaining("post-non-Wait change minimum"), + expect.stringContaining("post-Click change minimum"), ], satisfied: false, }); @@ -1065,13 +1319,42 @@ describe("Direct Bombadil exploration summary", () => { test("rejects hostile envelopes, action targets, and excessively deep snapshot JSON", async () => { const observation = directObservation(); const badTarget = await summaryTrace([traceLine(observation, 1, { - action: { Click: { fingerprint: { tag: "unsafe tag" } } }, + action: { + Click: { + fingerprint: { tag: "unsafe tag" }, + point: { x: 1, y: 1 }, + }, + }, })]); expect((await rejection(summarizeDirectBombadilTrace({ targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", tracePath: badTarget, }))).message).toContain("invalid action target tag"); + for (const action of [ + { PressKey: {} }, + { PressKey: { code: 256 } }, + { SetViewport: {} }, + { SetViewport: { height: 720, width: 0 } }, + { + MouseDrag: { + delay_millis: 0, + from: { x: 0, y: 0 }, + steps: 0, + to: { x: 1, y: 1 }, + }, + }, + { TypeText: { delay_millis: 0, text: "safe", unexpected: true } }, + ]) { + const malformedAction = await summaryTrace([traceLine(observation, 1, { + action, + })]); + expect((await rejection(summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: malformedAction, + }))).message).toContain("invalid action"); + } + let deep: unknown = null; for (let index = 0; index < 70; index += 1) deep = [deep]; const deepTrace = await summaryTrace([traceLine(observation, 1, { @@ -1218,7 +1501,7 @@ describe("Direct Bombadil run lifecycle", () => { initialDirect: { source: "scenario", scenario: "surface.ready", route: "/surface" }, server: { logPresent: true }, explorationSummary: { - schema: "direct.bombadil-exploration-summary/v1", + schema: "direct.bombadil-exploration-summary/v2", policy: { configured: false, satisfied: true }, }, viewport: { deviceScaleFactor: 2, height: 768, width: 1_024 }, @@ -1234,7 +1517,7 @@ describe("Direct Bombadil run lifecycle", () => { expect(await readFile(String(server.logPath), "utf8")).toContain("server output"); const summaryPath = String(manifest.explorationSummaryPath); expect(JSON.parse(await readFile(summaryPath, "utf8"))).toMatchObject({ - schema: "direct.bombadil-exploration-summary/v1", + schema: "direct.bombadil-exploration-summary/v2", trace: { lineCount: 2 }, }); }); @@ -1260,7 +1543,7 @@ describe("Direct Bombadil run lifecycle", () => { }, }); expect(await readFile(String(manifest.explorationSummaryPath), "utf8")) - .toContain("direct.bombadil-exploration-summary/v1"); + .toContain("direct.bombadil-exploration-summary/v2"); }); test("retains an attested failure artifact and server log for a nonzero exit", async () => { diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index b53c3a7..ddad076 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -79,9 +79,41 @@ const TRACE_RESOURCE_KEYS = new Set([ "timestamp", ]); const TRACE_VIOLATION_KEYS = new Set(["name", "violation"]); +const TRACE_POINT_KEYS = new Set(["x", "y"]); +const TRACE_FINGERPRINT_KEYS = new Set([ + "accessible_name", + "href", + "id", + "input_type", + "name_attr", + "placeholder", + "role", + "structural_path", + "tag", + "test_id", + "text_content", +]); +const TRACE_CLICK_ACTION_KEYS = new Set(["fingerprint", "point"]); +const TRACE_DOUBLE_CLICK_ACTION_KEYS = new Set([ + "delay_millis", + "fingerprint", + "point", +]); +const TRACE_TYPE_TEXT_ACTION_KEYS = new Set(["delay_millis", "text"]); +const TRACE_PRESS_KEY_ACTION_KEYS = new Set(["code"]); +const TRACE_SCROLL_ACTION_KEYS = new Set(["distance", "origin"]); +const TRACE_FILE_INPUT_ACTION_KEYS = new Set(["files", "selector"]); +const TRACE_MOUSE_DRAG_ACTION_KEYS = new Set([ + "delay_millis", + "from", + "steps", + "to", +]); +const TRACE_VIEWPORT_ACTION_KEYS = new Set(["height", "width"]); const VIEWPORT_KEYS = new Set(["deviceScaleFactor", "height", "width"]); const EXPLORATION_POLICY_KEYS = new Set([ "minDistinctNamedSnapshotValues", + "minNamedSnapshotChangesAfterActionKind", "minNamedSnapshotChangesAfterNonWait", "minNonWaitActions", "requireStableTargetUrl", @@ -115,10 +147,6 @@ const UNIT_ACTION_KINDS = new Set([ "Reload", "Wait", ]); -const TARGETED_ACTION_KINDS = new Set([ - "Click", - "DoubleClick", -]); const DIRECT_OBSERVATION_KEYS = new Set([ "activationHash", "activeRoute", @@ -153,6 +181,10 @@ export interface DirectBombadilViewportConfig { export interface DirectBombadilExplorationPolicy { readonly minDistinctNamedSnapshotValues?: Readonly>; + readonly minNamedSnapshotChangesAfterActionKind?: Readonly>> + >>; readonly minNamedSnapshotChangesAfterNonWait?: Readonly>; readonly minNonWaitActions?: number; readonly requireStableTargetUrl?: boolean; @@ -161,7 +193,7 @@ export interface DirectBombadilExplorationPolicy { } export interface DirectBombadilExplorationSummary { - readonly schema: "direct.bombadil-exploration-summary/v1"; + readonly schema: "direct.bombadil-exploration-summary/v2"; readonly trace: { readonly bytes: number; readonly lineCount: number; @@ -178,13 +210,21 @@ export interface DirectBombadilExplorationSummary { readonly distinctFingerprintCount: number; readonly fingerprintSha256: readonly string[]; readonly observationCount: number; + readonly rawDistinctFingerprintCount: number; + readonly rawFingerprintSha256: readonly string[]; + readonly rawObservationCount: number; readonly stableTarget: boolean; }; readonly transitions: { readonly distinctNonNullHashCount: number; readonly nonNullHashCount: number; + readonly rawDistinctNonNullHashCount: number; + readonly rawNonNullHashCount: number; }; readonly namedSnapshots: readonly { + readonly changeAfterActionKind: Readonly< + Partial> + >; readonly changeAfterNonWaitCount: number; readonly distinctValueCount: number; readonly distinctValueSha256: readonly string[]; @@ -364,6 +404,10 @@ interface ValidatedViewport { interface ValidatedExplorationPolicy { readonly minDistinctNamedSnapshotValues: Readonly>; + readonly minNamedSnapshotChangesAfterActionKind: Readonly>> + >>; readonly minNamedSnapshotChangesAfterNonWait: Readonly>; readonly minNonWaitActions: number; readonly requireStableTargetUrl: boolean; @@ -676,43 +720,163 @@ function namedSnapshotValueSha256(value: unknown): string { return sha256(canonical); } +function validTracePoint(value: unknown): boolean { + return isRecord(value) + && hasExactKeys(value, TRACE_POINT_KEYS) + && typeof value.x === "number" + && Number.isFinite(value.x) + && typeof value.y === "number" + && Number.isFinite(value.y); +} + +function parseTraceFingerprintTag(value: unknown, lineNumber: number): string { + if ( + !isRecord(value) + || !Object.keys(value).every((key) => TRACE_FINGERPRINT_KEYS.has(key)) + ) { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target`, + ); + } + for (const [key, candidate] of Object.entries(value)) { + if (key !== "tag" && typeof candidate !== "string") { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target`, + ); + } + } + const tag = value.tag; + if ( + typeof tag !== "string" + || tag.length === 0 + ) { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target tag`, + ); + } + if ( + typeof value.structural_path === "string" + && Object.keys(value).some((key) => ( + key !== "tag" && key !== "structural_path" + )) + ) { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target`, + ); + } + return tag.length <= 64 && TARGET_TAG_PATTERN.test(tag) + ? tag + : `sha256:${sha256(tag)}`; +} + +function isSafeIntegerBetween( + value: unknown, + minimum: number, + maximum: number, +): value is number { + return typeof value === "number" + && Number.isSafeInteger(value) + && value >= minimum + && value <= maximum; +} + +function invalidTraceAction(lineNumber: number): never { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action`, + ); +} + function parseTraceAction(value: unknown, lineNumber: number): ParsedTraceAction | null { if (value === null) return null; if (typeof value === "string") { if (!ACTION_KIND_SET.has(value) || !UNIT_ACTION_KINDS.has(value as DirectBombadilActionKind)) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + return invalidTraceAction(lineNumber); } return { kind: value as DirectBombadilActionKind, targetTag: null }; } if (!isRecord(value) || Object.keys(value).length !== 1) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + return invalidTraceAction(lineNumber); } const kind = Object.keys(value)[0]; + const payload = kind === undefined ? undefined : value[kind]; if ( kind === undefined || !ACTION_KIND_SET.has(kind) || UNIT_ACTION_KINDS.has(kind as DirectBombadilActionKind) - || !isRecord(value[kind]) + || !isRecord(payload) ) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + return invalidTraceAction(lineNumber); } const actionKind = kind as DirectBombadilActionKind; let targetTag: string | null = null; - if (TARGETED_ACTION_KINDS.has(actionKind)) { - const fingerprint = value[kind].fingerprint; - if (!isRecord(fingerprint)) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); - } - const tag = fingerprint.tag; - if ( - typeof tag !== "string" - || tag.length === 0 - || tag.length > 64 - || !TARGET_TAG_PATTERN.test(tag) - ) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target tag`); - } - targetTag = tag; + switch (actionKind) { + case "Click": + if ( + !hasExactKeys(payload, TRACE_CLICK_ACTION_KEYS) + || !validTracePoint(payload.point) + ) { + return invalidTraceAction(lineNumber); + } + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "DoubleClick": + if ( + !hasExactKeys(payload, TRACE_DOUBLE_CLICK_ACTION_KEYS) + || !isSafeIntegerBetween(payload.delay_millis, 0, 1_000) + || !validTracePoint(payload.point) + ) return invalidTraceAction(lineNumber); + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "TypeText": + if ( + !hasExactKeys(payload, TRACE_TYPE_TEXT_ACTION_KEYS) + || !isSafeIntegerBetween(payload.delay_millis, 0, Number.MAX_SAFE_INTEGER) + || typeof payload.text !== "string" + ) return invalidTraceAction(lineNumber); + break; + case "PressKey": + if ( + !hasExactKeys(payload, TRACE_PRESS_KEY_ACTION_KEYS) + || !isSafeIntegerBetween(payload.code, 0, 255) + ) { + return invalidTraceAction(lineNumber); + } + break; + case "ScrollDown": + case "ScrollUp": + if ( + !hasExactKeys(payload, TRACE_SCROLL_ACTION_KEYS) + || typeof payload.distance !== "number" + || !Number.isFinite(payload.distance) + || !validTracePoint(payload.origin) + ) return invalidTraceAction(lineNumber); + break; + case "SetFileInputFiles": + if ( + !hasExactKeys(payload, TRACE_FILE_INPUT_ACTION_KEYS) + || typeof payload.selector !== "string" + || !Array.isArray(payload.files) + || !payload.files.every((file) => typeof file === "string") + ) return invalidTraceAction(lineNumber); + break; + case "MouseDrag": + if ( + !hasExactKeys(payload, TRACE_MOUSE_DRAG_ACTION_KEYS) + || !isSafeIntegerBetween(payload.delay_millis, 0, 1_000) + || !isSafeIntegerBetween(payload.steps, 1, 255) + || !validTracePoint(payload.from) + || !validTracePoint(payload.to) + ) return invalidTraceAction(lineNumber); + break; + case "SetViewport": + if ( + !hasExactKeys(payload, TRACE_VIEWPORT_ACTION_KEYS) + || !isSafeIntegerBetween(payload.height, 1, 10_000) + || !isSafeIntegerBetween(payload.width, 1, 10_000) + ) return invalidTraceAction(lineNumber); + break; + default: + return invalidTraceAction(lineNumber); } return { kind: actionKind, targetTag }; } @@ -1011,9 +1175,13 @@ export async function summarizeDirectBombadilTrace(options: { const actionCounts = new Map(); const targetTags = new Map(); const urlFingerprints = new Set(); + const rawUrlFingerprints = new Set(); const transitionHashes = new Set(); + const rawTransitionHashes = new Set(); const snapshots = new Map; changeAfterNonWaitCount: number; + lastObservationIndex: number | null; lastValueSha256: string | null; observationCount: number; readonly values: Set; @@ -1036,8 +1204,8 @@ export async function summarizeDirectBombadilTrace(options: { let waitStreak = 0; let maxWaitStreak = 0; let nonNullHashCount = 0; + let rawNonNullHashCount = 0; let policyObservationCount = 0; - let reachedExactObservation = false; let previousObservationWasExact = false; let stableTarget = true; const stream = createReadStream(options.tracePath, { encoding: "utf8" }); @@ -1052,15 +1220,43 @@ export async function summarizeDirectBombadilTrace(options: { throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); } const parsed = parseTraceLine(line, lineCount); + const rawRelativeUrl = + `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + rawUrlFingerprints.add(sha256(rawRelativeUrl)); + if (rawUrlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct raw URL fingerprints`, + ); + } + if (parsed.state.currentHash !== null) { + rawNonNullHashCount += 1; + rawTransitionHashes.add(String(parsed.state.currentHash)); + } + for (const name of parsed.propertyViolationNames) { + if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`, + ); + } + propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); + } + for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { + resources[outputName] = Math.max( + resources[outputName], + parsed.state.resources[sourceName as keyof typeof RESOURCE_FIELD_MAP], + ); + } const currentObservationIsExact = exactTraceDirectObservation(parsed.directObservation) !== null; - if (!reachedExactObservation && !currentObservationIsExact) { + if (!currentObservationIsExact) { previousObservationWasExact = false; continue; } - if (!reachedExactObservation) reachedExactObservation = true; policyObservationCount += 1; const actionFollowsExactObservation = previousObservationWasExact; + const recordedActionKind = actionFollowsExactObservation + ? parsed.action?.kind ?? null + : null; if (actionFollowsExactObservation && parsed.action !== null) { totalActions += 1; @@ -1107,22 +1303,30 @@ export async function summarizeDirectBombadilTrace(options: { ); } entry = { + changeAfterActionKind: new Map(), changeAfterNonWaitCount: 0, + lastObservationIndex: null, lastValueSha256: null, observationCount: 0, values: new Set(), }; snapshots.set(snapshot.name, entry); } - if ( - actionFollowsExactObservation - && parsed.action !== null - && parsed.action.kind !== "Wait" + const changedAfterRecordedAction = + recordedActionKind !== null + && entry.lastObservationIndex === policyObservationCount - 1 && entry.lastValueSha256 !== null - && entry.lastValueSha256 !== snapshot.valueSha256 - ) { + && entry.lastValueSha256 !== snapshot.valueSha256; + if (changedAfterRecordedAction) { + entry.changeAfterActionKind.set( + recordedActionKind, + (entry.changeAfterActionKind.get(recordedActionKind) ?? 0) + 1, + ); + } + if (changedAfterRecordedAction && recordedActionKind !== "Wait") { entry.changeAfterNonWaitCount += 1; } + entry.lastObservationIndex = policyObservationCount; entry.lastValueSha256 = snapshot.valueSha256; entry.observationCount += 1; entry.values.add(snapshot.valueSha256); @@ -1132,21 +1336,7 @@ export async function summarizeDirectBombadilTrace(options: { ); } } - for (const name of parsed.propertyViolationNames) { - if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { - throw new Error( - `Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`, - ); - } - propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); - } - for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { - resources[outputName] = Math.max( - resources[outputName], - parsed.state.resources[sourceName as keyof typeof RESOURCE_FIELD_MAP], - ); - } - previousObservationWasExact = currentObservationIsExact; + previousObservationWasExact = true; } } finally { lines.close(); @@ -1183,13 +1373,27 @@ export async function summarizeDirectBombadilTrace(options: { ); } } + for (const [name, minimumByKind] of Object.entries( + policy.minNamedSnapshotChangesAfterActionKind, + )) { + for (const [kind, minimum] of Object.entries(minimumByKind) as [ + DirectBombadilActionKind, + number, + ][]) { + if ((snapshots.get(name)?.changeAfterActionKind.get(kind) ?? 0) < minimum) { + policyFailures.push( + `named snapshot ${name} did not reach its post-${kind} change minimum`, + ); + } + } + } if (policy.requireStableTargetUrl && !stableTarget) { policyFailures.push("the browser did not remain on the exact target URL"); } } const traceBytes = await readFile(options.tracePath); return Object.freeze({ - schema: "direct.bombadil-exploration-summary/v1", + schema: "direct.bombadil-exploration-summary/v2", trace: Object.freeze({ bytes: metadata.size, lineCount, @@ -1206,15 +1410,23 @@ export async function summarizeDirectBombadilTrace(options: { distinctFingerprintCount: urlFingerprints.size, fingerprintSha256: Object.freeze([...urlFingerprints].sort(compareCodeUnits)), observationCount: policyObservationCount, + rawDistinctFingerprintCount: rawUrlFingerprints.size, + rawFingerprintSha256: Object.freeze( + [...rawUrlFingerprints].sort(compareCodeUnits), + ), + rawObservationCount: lineCount, stableTarget, }), transitions: Object.freeze({ distinctNonNullHashCount: transitionHashes.size, nonNullHashCount, + rawDistinctNonNullHashCount: rawTransitionHashes.size, + rawNonNullHashCount, }), namedSnapshots: Object.freeze([...snapshots.entries()] .sort(([left], [right]) => compareCodeUnits(left, right)) .map(([name, entry]) => Object.freeze({ + changeAfterActionKind: sortedCountRecord(entry.changeAfterActionKind), changeAfterNonWaitCount: entry.changeAfterNonWaitCount, distinctValueCount: entry.values.size, distinctValueSha256: Object.freeze([...entry.values].sort(compareCodeUnits)), @@ -1449,6 +1661,53 @@ function validateSnapshotMinimumMap(options: { return Object.freeze(validated); } +function validateSnapshotActionMinimumMap(options: { + readonly label: string; + readonly value: unknown; +}): Readonly>> +>> { + if (!isRecord(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated: Record< + string, + Readonly>> + > = {}; + for (const [rawName, rawMinimumByKind] of Object.entries(options.value) + .sort(([left], [right]) => compareCodeUnits(left, right))) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if ( + !isRecord(rawMinimumByKind) + || Object.keys(rawMinimumByKind).length === 0 + || Object.keys(rawMinimumByKind).length > ACTION_KINDS.length + ) { + throw new Error(`${options.label} ${name} must be a bounded action map`); + } + const minimumByKind: Partial> = {}; + for (const [rawKind, minimum] of Object.entries(rawMinimumByKind) + .sort(([left], [right]) => compareCodeUnits(left, right))) { + if (!ACTION_KIND_SET.has(rawKind)) { + throw new Error(`${options.label} ${name} contains an unknown action kind`); + } + if ( + typeof minimum !== "number" + || !Number.isSafeInteger(minimum) + || minimum < 1 + || minimum > TRACE_MAX_LINES + ) { + throw new Error( + `${options.label} ${name}.${rawKind} must be an integer between 1 and ${String(TRACE_MAX_LINES)}`, + ); + } + minimumByKind[rawKind as DirectBombadilActionKind] = minimum; + } + validated[name] = Object.freeze(minimumByKind); + } + return Object.freeze(validated); +} + function validateExplorationPolicy( value: unknown, ): ValidatedExplorationPolicy | null { @@ -1502,6 +1761,11 @@ function validateExplorationPolicy( maximum: TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME, value: value.minDistinctNamedSnapshotValues ?? {}, }); + const minNamedSnapshotChangesAfterActionKind = + validateSnapshotActionMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterActionKind", + value: value.minNamedSnapshotChangesAfterActionKind ?? {}, + }); const minNamedSnapshotChangesAfterNonWait = validateSnapshotMinimumMap({ label: "explorationPolicy.minNamedSnapshotChangesAfterNonWait", maximum: TRACE_MAX_LINES, @@ -1513,6 +1777,7 @@ function validateExplorationPolicy( } return Object.freeze({ minDistinctNamedSnapshotValues, + minNamedSnapshotChangesAfterActionKind, minNamedSnapshotChangesAfterNonWait, minNonWaitActions, requireStableTargetUrl, From 82353396640765c077cd4b98db2d3bc8fe9fd92f Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:31:26 -0400 Subject: [PATCH 17/25] test: reject empty Bombadil target tags --- src/tooling/bombadil-runner.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index ace1c47..fb2ccf4 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -1321,7 +1321,7 @@ describe("Direct Bombadil exploration summary", () => { const badTarget = await summaryTrace([traceLine(observation, 1, { action: { Click: { - fingerprint: { tag: "unsafe tag" }, + fingerprint: { tag: "" }, point: { x: 1, y: 1 }, }, }, From be71a7fc4bc68e948aba5786a4b51097e811d3bd Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 14:29:56 -0400 Subject: [PATCH 18/25] build: refresh Bombadil tooling bundle --- dist/tooling/bombadil.js | 816 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 810 insertions(+), 6 deletions(-) diff --git a/dist/tooling/bombadil.js b/dist/tooling/bombadil.js index 3e4afaf..9f094fd 100644 --- a/dist/tooling/bombadil.js +++ b/dist/tooling/bombadil.js @@ -5,6 +5,7 @@ import { readFile, realpath, stat, writeFile as writeFile2 } from "fs/promises"; import { isAbsolute, join as join2, relative, resolve } from "path"; import process2 from "process"; import { createInterface } from "readline"; +import { createHash } from "crypto"; // src/core/result.ts function ok(value) { @@ -1134,6 +1135,12 @@ var TRACE_MAX_BYTES = 64 * 1024 * 1024; var TRACE_MAX_LINE_BYTES = 16 * 1024 * 1024; var TRACE_MAX_LINES = 1e4; var TRACE_MAX_SNAPSHOTS_PER_LINE = 4096; +var TRACE_MAX_NAMED_SNAPSHOT_NAMES = 128; +var TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME = 1024; +var TRACE_MAX_DISTINCT_URLS = 1024; +var TRACE_MAX_PROPERTY_NAMES = 128; +var TRACE_MAX_CANONICAL_SNAPSHOT_BYTES = 2 * 1024 * 1024; +var TRACE_MAX_JSON_DEPTH = 64; var RANDOM_RUN_OVERHEAD_MS = 30000; var REPLAY_WALL_CLOCK_TIMEOUT_MS = MAX_TIME_LIMIT_SECONDS * 1000 + RANDOM_RUN_OVERHEAD_MS; var PROCESS_TERMINATION_GRACE_MS = 5000; @@ -1142,6 +1149,94 @@ var SERVER_OUTPUT_TIMEOUT_MS = 3000; var DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; var TRACE_LINE_KEYS = new Set(["action", "snapshots", "state", "timestamp", "violations"]); var TRACE_SNAPSHOT_KEYS = new Set(["index", "name", "time", "value"]); +var TRACE_STATE_KEYS = new Set([ + "hash_current", + "hash_previous", + "resources", + "screenshot", + "url" +]); +var TRACE_RESOURCE_KEYS = new Set([ + "documents", + "dom_nodes", + "js_event_listeners", + "js_heap_total", + "js_heap_used", + "layout_objects", + "script_duration", + "task_duration", + "thread_time", + "timestamp" +]); +var TRACE_VIOLATION_KEYS = new Set(["name", "violation"]); +var TRACE_POINT_KEYS = new Set(["x", "y"]); +var TRACE_FINGERPRINT_KEYS = new Set([ + "accessible_name", + "href", + "id", + "input_type", + "name_attr", + "placeholder", + "role", + "structural_path", + "tag", + "test_id", + "text_content" +]); +var TRACE_CLICK_ACTION_KEYS = new Set(["fingerprint", "point"]); +var TRACE_DOUBLE_CLICK_ACTION_KEYS = new Set([ + "delay_millis", + "fingerprint", + "point" +]); +var TRACE_TYPE_TEXT_ACTION_KEYS = new Set(["delay_millis", "text"]); +var TRACE_PRESS_KEY_ACTION_KEYS = new Set(["code"]); +var TRACE_SCROLL_ACTION_KEYS = new Set(["distance", "origin"]); +var TRACE_FILE_INPUT_ACTION_KEYS = new Set(["files", "selector"]); +var TRACE_MOUSE_DRAG_ACTION_KEYS = new Set([ + "delay_millis", + "from", + "steps", + "to" +]); +var TRACE_VIEWPORT_ACTION_KEYS = new Set(["height", "width"]); +var VIEWPORT_KEYS = new Set(["deviceScaleFactor", "height", "width"]); +var EXPLORATION_POLICY_KEYS = new Set([ + "minDistinctNamedSnapshotValues", + "minNamedSnapshotChangesAfterActionKind", + "minNamedSnapshotChangesAfterNonWait", + "minNonWaitActions", + "requireStableTargetUrl", + "requiredActionKinds", + "requiredNamedSnapshots" +]); +var DEFAULT_VIEWPORT_WIDTH = 1024; +var DEFAULT_VIEWPORT_HEIGHT = 768; +var DEFAULT_DEVICE_SCALE_FACTOR = 2; +var SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +var TARGET_TAG_PATTERN = /^[a-z][a-z0-9-]*$/u; +var ACTION_KINDS = [ + "Back", + "Click", + "DoubleClick", + "Forward", + "MouseDrag", + "PressKey", + "Reload", + "ScrollDown", + "ScrollUp", + "SetFileInputFiles", + "SetViewport", + "TypeText", + "Wait" +]; +var ACTION_KIND_SET = new Set(ACTION_KINDS); +var UNIT_ACTION_KINDS = new Set([ + "Back", + "Forward", + "Reload", + "Wait" +]); var DIRECT_OBSERVATION_KEYS = new Set([ "activationHash", "activeRoute", @@ -1217,6 +1312,13 @@ function hasExactKeys(value, expected) { const keys = Object.keys(value); return keys.length === expected.size && keys.every((key) => expected.has(key)); } +function compareCodeUnits(left, right) { + if (left < right) + return -1; + if (left > right) + return 1; + return 0; +} function parseTraceDirectObservation(value) { if (!isRecord2(value) || !hasExactKeys(value, DIRECT_OBSERVATION_KEYS)) { throw new Error("Bombadil trace has an invalid named direct observation"); @@ -1311,6 +1413,181 @@ function exactTraceDirectObservation(observation) { isQuiescent: probe2.value.isQuiescent }; } +var RESOURCE_FIELD_MAP = { + documents: "documents", + dom_nodes: "domNodes", + js_event_listeners: "jsEventListeners", + js_heap_total: "jsHeapTotalBytes", + js_heap_used: "jsHeapUsedBytes", + layout_objects: "layoutObjects", + script_duration: "scriptDurationSeconds", + task_duration: "taskDurationSeconds", + thread_time: "threadTimeSeconds" +}; +function canonicalJson2(value, depth = 0) { + if (depth > TRACE_MAX_JSON_DEPTH) { + throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(TRACE_MAX_JSON_DEPTH)}`); + } + if (value === null || typeof value === "boolean" || typeof value === "string") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new Error("Bombadil named snapshot has a non-finite number"); + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((entry) => canonicalJson2(entry, depth + 1)).join(",")}]`; + } + if (!isRecord2(value)) + throw new Error("Bombadil named snapshot is not JSON"); + const entries = Object.keys(value).sort(compareCodeUnits).map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key], depth + 1)}`); + return `{${entries.join(",")}}`; +} +function sha256(value) { + return createHash("sha256").update(value).digest("hex"); +} +function namedSnapshotValueSha256(value) { + const canonical = canonicalJson2(value); + if (Buffer.byteLength(canonical, "utf8") > TRACE_MAX_CANONICAL_SNAPSHOT_BYTES) { + throw new Error(`Bombadil named snapshot exceeds ${String(TRACE_MAX_CANONICAL_SNAPSHOT_BYTES)} canonical bytes`); + } + return sha256(canonical); +} +function validTracePoint(value) { + return isRecord2(value) && hasExactKeys(value, TRACE_POINT_KEYS) && typeof value.x === "number" && Number.isFinite(value.x) && typeof value.y === "number" && Number.isFinite(value.y); +} +function parseTraceFingerprintTag(value, lineNumber) { + if (!isRecord2(value) || !Object.keys(value).every((key) => TRACE_FINGERPRINT_KEYS.has(key))) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + for (const [key, candidate] of Object.entries(value)) { + if (key !== "tag" && typeof candidate !== "string") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + } + const tag = value.tag; + if (typeof tag !== "string" || tag.length === 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target tag`); + } + if (typeof value.structural_path === "string" && Object.keys(value).some((key) => key !== "tag" && key !== "structural_path")) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + return tag.length <= 64 && TARGET_TAG_PATTERN.test(tag) ? tag : `sha256:${sha256(tag)}`; +} +function isSafeIntegerBetween(value, minimum, maximum) { + return typeof value === "number" && Number.isSafeInteger(value) && value >= minimum && value <= maximum; +} +function invalidTraceAction(lineNumber) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); +} +function parseTraceAction(value, lineNumber) { + if (value === null) + return null; + if (typeof value === "string") { + if (!ACTION_KIND_SET.has(value) || !UNIT_ACTION_KINDS.has(value)) { + return invalidTraceAction(lineNumber); + } + return { kind: value, targetTag: null }; + } + if (!isRecord2(value) || Object.keys(value).length !== 1) { + return invalidTraceAction(lineNumber); + } + const kind = Object.keys(value)[0]; + const payload = kind === undefined ? undefined : value[kind]; + if (kind === undefined || !ACTION_KIND_SET.has(kind) || UNIT_ACTION_KINDS.has(kind) || !isRecord2(payload)) { + return invalidTraceAction(lineNumber); + } + const actionKind = kind; + let targetTag = null; + switch (actionKind) { + case "Click": + if (!hasExactKeys(payload, TRACE_CLICK_ACTION_KEYS) || !validTracePoint(payload.point)) { + return invalidTraceAction(lineNumber); + } + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "DoubleClick": + if (!hasExactKeys(payload, TRACE_DOUBLE_CLICK_ACTION_KEYS) || !isSafeIntegerBetween(payload.delay_millis, 0, 1000) || !validTracePoint(payload.point)) + return invalidTraceAction(lineNumber); + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "TypeText": + if (!hasExactKeys(payload, TRACE_TYPE_TEXT_ACTION_KEYS) || !isSafeIntegerBetween(payload.delay_millis, 0, Number.MAX_SAFE_INTEGER) || typeof payload.text !== "string") + return invalidTraceAction(lineNumber); + break; + case "PressKey": + if (!hasExactKeys(payload, TRACE_PRESS_KEY_ACTION_KEYS) || !isSafeIntegerBetween(payload.code, 0, 255)) { + return invalidTraceAction(lineNumber); + } + break; + case "ScrollDown": + case "ScrollUp": + if (!hasExactKeys(payload, TRACE_SCROLL_ACTION_KEYS) || typeof payload.distance !== "number" || !Number.isFinite(payload.distance) || !validTracePoint(payload.origin)) + return invalidTraceAction(lineNumber); + break; + case "SetFileInputFiles": + if (!hasExactKeys(payload, TRACE_FILE_INPUT_ACTION_KEYS) || typeof payload.selector !== "string" || !Array.isArray(payload.files) || !payload.files.every((file) => typeof file === "string")) + return invalidTraceAction(lineNumber); + break; + case "MouseDrag": + if (!hasExactKeys(payload, TRACE_MOUSE_DRAG_ACTION_KEYS) || !isSafeIntegerBetween(payload.delay_millis, 0, 1000) || !isSafeIntegerBetween(payload.steps, 1, 255) || !validTracePoint(payload.from) || !validTracePoint(payload.to)) + return invalidTraceAction(lineNumber); + break; + case "SetViewport": + if (!hasExactKeys(payload, TRACE_VIEWPORT_ACTION_KEYS) || !isSafeIntegerBetween(payload.height, 1, 1e4) || !isSafeIntegerBetween(payload.width, 1, 1e4)) + return invalidTraceAction(lineNumber); + break; + default: + return invalidTraceAction(lineNumber); + } + return { kind: actionKind, targetTag }; +} +function parseNonNegativeFiniteNumber(value, lineNumber, field) { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + return value; +} +function parseTraceState(value, lineNumber) { + if (!isRecord2(value) || !hasExactKeys(value, TRACE_STATE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser state`); + } + if (typeof value.url !== "string" || value.url.length === 0 || value.url.length > 8192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + let url; + try { + url = new URL(value.url); + } catch { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL protocol`); + } + if (typeof value.screenshot !== "string" || value.screenshot.length > 8192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid screenshot path`); + } + for (const field of ["hash_previous", "hash_current"]) { + const hash = value[field]; + if (hash !== null && (typeof hash !== "number" || !Number.isFinite(hash) || !Number.isInteger(hash) || hash < 0)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + } + if (!isRecord2(value.resources) || !hasExactKeys(value.resources, TRACE_RESOURCE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid browser resources`); + } + const resources = {}; + for (const field of Object.keys(RESOURCE_FIELD_MAP)) { + resources[field] = parseNonNegativeFiniteNumber(value.resources[field], lineNumber, `resources.${field}`); + } + parseNonNegativeFiniteNumber(value.resources.timestamp, lineNumber, "resources.timestamp"); + return { + currentHash: value.hash_current, + resources, + url + }; +} function parseTraceLine(line, lineNumber) { let input; try { @@ -1324,7 +1601,27 @@ function parseTraceLine(line, lineNumber) { if (!Number.isSafeInteger(input.timestamp) || typeof input.timestamp !== "number" || input.timestamp < 0 || !Array.isArray(input.snapshots) || input.snapshots.length > TRACE_MAX_SNAPSHOTS_PER_LINE || !Array.isArray(input.violations)) { throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid state fields`); } + const state = parseTraceState(input.state, lineNumber); + const action = parseTraceAction(input.action, lineNumber); const snapshots = input.snapshots; + const namedSnapshots = []; + const namedSnapshotNames = new Set; + for (const snapshotValue of snapshots) { + if (!isRecord2(snapshotValue) || !hasExactKeys(snapshotValue, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshotValue.index) || !Number.isSafeInteger(snapshotValue.time) || snapshotValue.index < 0 || snapshotValue.time < 0 || snapshotValue.name !== null && typeof snapshotValue.name !== "string") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid snapshot`); + } + if (snapshotValue.name !== null) { + const name = validateSnapshotName(snapshotValue.name, `Bombadil trace line ${String(lineNumber)} snapshot name`); + if (namedSnapshotNames.has(name)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} repeats named snapshot ${name}`); + } + namedSnapshotNames.add(name); + namedSnapshots.push({ + name, + valueSha256: namedSnapshotValueSha256(snapshotValue.value) + }); + } + } const directSnapshots = snapshots.filter((snapshot2) => isRecord2(snapshot2) && snapshot2.name === "direct"); if (directSnapshots.length !== 1) { throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); @@ -1333,7 +1630,23 @@ function parseTraceLine(line, lineNumber) { if (snapshot === undefined || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshot.index) || !Number.isSafeInteger(snapshot.time) || snapshot.index < 0 || snapshot.time < 0) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); } - return parseTraceDirectObservation(snapshot.value); + const propertyViolationNames = []; + for (const violation of input.violations) { + if (!isRecord2(violation) || !hasExactKeys(violation, TRACE_VIOLATION_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + propertyViolationNames.push(validateSnapshotName(violation.name, `Bombadil trace line ${String(lineNumber)} property violation name`)); + if (!isRecord2(violation.violation) || Object.keys(violation.violation).length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + } + return { + action, + directObservation: parseTraceDirectObservation(snapshot.value), + namedSnapshots, + propertyViolationNames, + state + }; } async function attestDirectBombadilTrace(options) { const metadata = await stat(options.tracePath).catch(() => null); @@ -1360,7 +1673,7 @@ async function attestDirectBombadilTrace(options) { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); } - const observation = parseTraceLine(line, observationCount); + const observation = parseTraceLine(line, observationCount).directObservation; const exact = exactTraceDirectObservation(observation); if (exact === null) { if (initial !== null) { @@ -1428,6 +1741,245 @@ async function attestDirectBombadilTrace(options) { validObservationCount }; } +function sortedCountRecord(values) { + return Object.freeze(Object.fromEntries([...values.entries()].sort(([left], [right]) => compareCodeUnits(left, right)))); +} +async function summarizeDirectBombadilTrace(options) { + const metadata = await stat(options.tracePath).catch(() => null); + if (metadata === null || !metadata.isFile() || metadata.size === 0) { + throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + } + if (metadata.size > TRACE_MAX_BYTES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_BYTES)} bytes`); + } + let targetUrl; + try { + targetUrl = new URL(options.targetUrl); + } catch { + throw new Error("targetUrl must be an absolute URL"); + } + const policy = validateExplorationPolicy(options.explorationPolicy); + const actionCounts = new Map; + const targetTags = new Map; + const urlFingerprints = new Set; + const rawUrlFingerprints = new Set; + const transitionHashes = new Set; + const rawTransitionHashes = new Set; + const snapshots = new Map; + const propertyViolations = new Map; + const resources = { + documents: 0, + domNodes: 0, + jsEventListeners: 0, + jsHeapTotalBytes: 0, + jsHeapUsedBytes: 0, + layoutObjects: 0, + scriptDurationSeconds: 0, + taskDurationSeconds: 0, + threadTimeSeconds: 0 + }; + let lineCount = 0; + let totalActions = 0; + let nonWaitCount = 0; + let waitStreak = 0; + let maxWaitStreak = 0; + let nonNullHashCount = 0; + let rawNonNullHashCount = 0; + let policyObservationCount = 0; + let previousObservationWasExact = false; + let stableTarget = true; + const stream = createReadStream(options.tracePath, { encoding: "utf8" }); + const lines = createInterface({ input: stream, crlfDelay: Infinity }); + try { + for await (const line of lines) { + lineCount += 1; + if (lineCount > TRACE_MAX_LINES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); + } + if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { + throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); + } + const parsed = parseTraceLine(line, lineCount); + const rawRelativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + rawUrlFingerprints.add(sha256(rawRelativeUrl)); + if (rawUrlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct raw URL fingerprints`); + } + if (parsed.state.currentHash !== null) { + rawNonNullHashCount += 1; + rawTransitionHashes.add(String(parsed.state.currentHash)); + } + for (const name of parsed.propertyViolationNames) { + if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`); + } + propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); + } + for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { + resources[outputName] = Math.max(resources[outputName], parsed.state.resources[sourceName]); + } + const currentObservationIsExact = exactTraceDirectObservation(parsed.directObservation) !== null; + if (!currentObservationIsExact) { + previousObservationWasExact = false; + continue; + } + policyObservationCount += 1; + const actionFollowsExactObservation = previousObservationWasExact; + const recordedActionKind = actionFollowsExactObservation ? parsed.action?.kind ?? null : null; + if (actionFollowsExactObservation && parsed.action !== null) { + totalActions += 1; + actionCounts.set(parsed.action.kind, (actionCounts.get(parsed.action.kind) ?? 0) + 1); + if (parsed.action.kind === "Wait") { + waitStreak += 1; + maxWaitStreak = Math.max(maxWaitStreak, waitStreak); + } else { + nonWaitCount += 1; + waitStreak = 0; + } + if (parsed.action.targetTag !== null) { + if (!targetTags.has(parsed.action.targetTag) && targetTags.size >= 128) { + throw new Error("Bombadil trace exceeds 128 distinct action target tags"); + } + targetTags.set(parsed.action.targetTag, (targetTags.get(parsed.action.targetTag) ?? 0) + 1); + } + } else if (actionFollowsExactObservation) { + waitStreak = 0; + } + const relativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + urlFingerprints.add(sha256(relativeUrl)); + if (urlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct URL fingerprints`); + } + stableTarget &&= parsed.state.url.href === targetUrl.href; + if (parsed.state.currentHash !== null) { + nonNullHashCount += 1; + transitionHashes.add(String(parsed.state.currentHash)); + } + for (const snapshot of parsed.namedSnapshots) { + let entry = snapshots.get(snapshot.name); + if (entry === undefined) { + if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`); + } + entry = { + changeAfterActionKind: new Map, + changeAfterNonWaitCount: 0, + lastObservationIndex: null, + lastValueSha256: null, + observationCount: 0, + values: new Set + }; + snapshots.set(snapshot.name, entry); + } + const changedAfterRecordedAction = recordedActionKind !== null && entry.lastObservationIndex === policyObservationCount - 1 && entry.lastValueSha256 !== null && entry.lastValueSha256 !== snapshot.valueSha256; + if (changedAfterRecordedAction) { + entry.changeAfterActionKind.set(recordedActionKind, (entry.changeAfterActionKind.get(recordedActionKind) ?? 0) + 1); + } + if (changedAfterRecordedAction && recordedActionKind !== "Wait") { + entry.changeAfterNonWaitCount += 1; + } + entry.lastObservationIndex = policyObservationCount; + entry.lastValueSha256 = snapshot.valueSha256; + entry.observationCount += 1; + entry.values.add(snapshot.valueSha256); + if (entry.values.size > TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { + throw new Error(`Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`); + } + } + previousObservationWasExact = true; + } + } finally { + lines.close(); + stream.destroy(); + } + if (lineCount === 0) + throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + const policyFailures = []; + if (policy !== null) { + if (nonWaitCount < policy.minNonWaitActions) { + policyFailures.push("minimum non-Wait action count was not reached"); + } + for (const kind of policy.requiredActionKinds) { + if ((actionCounts.get(kind) ?? 0) === 0) { + policyFailures.push(`required action kind ${kind} was not observed`); + } + } + for (const name of policy.requiredNamedSnapshots) { + if (!snapshots.has(name)) { + policyFailures.push(`required named snapshot ${name} was not observed`); + } + } + for (const [name, minimum] of Object.entries(policy.minDistinctNamedSnapshotValues)) { + if ((snapshots.get(name)?.values.size ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its distinct-value minimum`); + } + } + for (const [name, minimum] of Object.entries(policy.minNamedSnapshotChangesAfterNonWait)) { + if ((snapshots.get(name)?.changeAfterNonWaitCount ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its post-non-Wait change minimum`); + } + } + for (const [name, minimumByKind] of Object.entries(policy.minNamedSnapshotChangesAfterActionKind)) { + for (const [kind, minimum] of Object.entries(minimumByKind)) { + if ((snapshots.get(name)?.changeAfterActionKind.get(kind) ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its post-${kind} change minimum`); + } + } + } + if (policy.requireStableTargetUrl && !stableTarget) { + policyFailures.push("the browser did not remain on the exact target URL"); + } + } + const traceBytes = await readFile(options.tracePath); + return Object.freeze({ + schema: "direct.bombadil-exploration-summary/v2", + trace: Object.freeze({ + bytes: metadata.size, + lineCount, + sha256: sha256(traceBytes) + }), + actions: Object.freeze({ + byKind: sortedCountRecord(actionCounts), + maxWaitStreak, + nonWaitCount, + targetTags: sortedCountRecord(targetTags), + total: totalActions + }), + urls: Object.freeze({ + distinctFingerprintCount: urlFingerprints.size, + fingerprintSha256: Object.freeze([...urlFingerprints].sort(compareCodeUnits)), + observationCount: policyObservationCount, + rawDistinctFingerprintCount: rawUrlFingerprints.size, + rawFingerprintSha256: Object.freeze([...rawUrlFingerprints].sort(compareCodeUnits)), + rawObservationCount: lineCount, + stableTarget + }), + transitions: Object.freeze({ + distinctNonNullHashCount: transitionHashes.size, + nonNullHashCount, + rawDistinctNonNullHashCount: rawTransitionHashes.size, + rawNonNullHashCount + }), + namedSnapshots: Object.freeze([...snapshots.entries()].sort(([left], [right]) => compareCodeUnits(left, right)).map(([name, entry]) => Object.freeze({ + changeAfterActionKind: sortedCountRecord(entry.changeAfterActionKind), + changeAfterNonWaitCount: entry.changeAfterNonWaitCount, + distinctValueCount: entry.values.size, + distinctValueSha256: Object.freeze([...entry.values].sort(compareCodeUnits)), + name, + observationCount: entry.observationCount + }))), + propertyViolations: Object.freeze({ + byName: sortedCountRecord(propertyViolations), + total: [...propertyViolations.values()].reduce((total, value) => total + value, 0) + }), + resourceHighWaterMarks: Object.freeze(resources), + policy: Object.freeze({ + configured: policy !== null, + failures: Object.freeze(policyFailures), + satisfied: policyFailures.length === 0 + }) + }); +} function parseDirectBombadilFuzzArguments(arguments_, defaultBaseUrl) { let baseUrl = defaultBaseUrl; let timeLimitSeconds = DEFAULT_TIME_LIMIT_SECONDS; @@ -1527,7 +2079,7 @@ function validateTargetQuery(value) { throw new Error("targetQuery may contain at most 16 parameters"); } const validated = {}; - for (const [name, queryValue] of [...entries].sort(([left], [right]) => left.localeCompare(right))) { + for (const [name, queryValue] of [...entries].sort(([left], [right]) => compareCodeUnits(left, right))) { if (name.length === 0 || name.length > 128 || !QUERY_PARAMETER_NAME_PATTERN.test(name) || PROTOTYPE_PROPERTY_NAMES.has(name) || hasControlCharacters3(name) || name === SCENARIO_QUERY_KEY2 || name === FIXTURE_QUERY_KEY2) { throw new Error("targetQuery contains an invalid or reserved parameter name"); } @@ -1538,6 +2090,131 @@ function validateTargetQuery(value) { } return Object.freeze(validated); } +function validateSnapshotName(value, label) { + if (typeof value !== "string" || value.length === 0 || value.length > 128 || !SNAPSHOT_NAME_PATTERN.test(value) || PROTOTYPE_PROPERTY_NAMES.has(value) || hasControlCharacters3(value)) { + throw new Error(`${label} must be a safe bounded snapshot name`); + } + return value; +} +function validateViewport(value) { + if (value === undefined) { + return Object.freeze({ + deviceScaleFactor: DEFAULT_DEVICE_SCALE_FACTOR, + height: DEFAULT_VIEWPORT_HEIGHT, + width: DEFAULT_VIEWPORT_WIDTH + }); + } + if (!isRecord2(value) || !Object.keys(value).every((key) => VIEWPORT_KEYS.has(key))) { + throw new Error("viewport must contain only width, height, and deviceScaleFactor"); + } + const validateDimension = (name, input) => { + if (typeof input !== "number" || !Number.isSafeInteger(input) || input < 1 || input > 65535) { + throw new Error(`viewport.${name} must be an integer between 1 and 65535`); + } + return input; + }; + const width = validateDimension("width", value.width ?? DEFAULT_VIEWPORT_WIDTH); + const height = validateDimension("height", value.height ?? DEFAULT_VIEWPORT_HEIGHT); + const deviceScaleFactor = value.deviceScaleFactor ?? DEFAULT_DEVICE_SCALE_FACTOR; + if (typeof deviceScaleFactor !== "number" || !Number.isFinite(deviceScaleFactor) || deviceScaleFactor < 0.1 || deviceScaleFactor > 10) { + throw new Error("viewport.deviceScaleFactor must be a finite number between 0.1 and 10"); + } + return Object.freeze({ deviceScaleFactor, height, width }); +} +function validateSnapshotMinimumMap(options) { + if (!isRecord2(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated = {}; + for (const [rawName, minimum] of Object.entries(options.value).sort(([left], [right]) => compareCodeUnits(left, right))) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if (typeof minimum !== "number" || !Number.isSafeInteger(minimum) || minimum < 1 || minimum > options.maximum) { + throw new Error(`${options.label} ${name} must be an integer between 1 and ${String(options.maximum)}`); + } + validated[name] = minimum; + } + return Object.freeze(validated); +} +function validateSnapshotActionMinimumMap(options) { + if (!isRecord2(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated = {}; + for (const [rawName, rawMinimumByKind] of Object.entries(options.value).sort(([left], [right]) => compareCodeUnits(left, right))) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if (!isRecord2(rawMinimumByKind) || Object.keys(rawMinimumByKind).length === 0 || Object.keys(rawMinimumByKind).length > ACTION_KINDS.length) { + throw new Error(`${options.label} ${name} must be a bounded action map`); + } + const minimumByKind = {}; + for (const [rawKind, minimum] of Object.entries(rawMinimumByKind).sort(([left], [right]) => compareCodeUnits(left, right))) { + if (!ACTION_KIND_SET.has(rawKind)) { + throw new Error(`${options.label} ${name} contains an unknown action kind`); + } + if (typeof minimum !== "number" || !Number.isSafeInteger(minimum) || minimum < 1 || minimum > TRACE_MAX_LINES) { + throw new Error(`${options.label} ${name}.${rawKind} must be an integer between 1 and ${String(TRACE_MAX_LINES)}`); + } + minimumByKind[rawKind] = minimum; + } + validated[name] = Object.freeze(minimumByKind); + } + return Object.freeze(validated); +} +function validateExplorationPolicy(value) { + if (value === undefined) + return null; + if (!isRecord2(value) || !Object.keys(value).every((key) => EXPLORATION_POLICY_KEYS.has(key))) { + throw new Error("explorationPolicy contains an unknown field"); + } + const minNonWaitActions = value.minNonWaitActions ?? 0; + if (typeof minNonWaitActions !== "number" || !Number.isSafeInteger(minNonWaitActions) || minNonWaitActions < 0 || minNonWaitActions > TRACE_MAX_LINES) { + throw new Error(`explorationPolicy.minNonWaitActions must be an integer between 0 and ${String(TRACE_MAX_LINES)}`); + } + const requiredActionKindsInput = value.requiredActionKinds ?? []; + if (!Array.isArray(requiredActionKindsInput) || requiredActionKindsInput.length > ACTION_KINDS.length) { + throw new Error("explorationPolicy.requiredActionKinds must be a bounded array"); + } + const requiredActionKinds = [...requiredActionKindsInput]; + if (!requiredActionKinds.every((kind) => typeof kind === "string" && ACTION_KIND_SET.has(kind)) || new Set(requiredActionKinds).size !== requiredActionKinds.length) { + throw new Error("explorationPolicy.requiredActionKinds contains an unknown or duplicate kind"); + } + requiredActionKinds.sort(compareCodeUnits); + const requiredNamedSnapshotsInput = value.requiredNamedSnapshots ?? []; + if (!Array.isArray(requiredNamedSnapshotsInput) || requiredNamedSnapshotsInput.length > 32) { + throw new Error("explorationPolicy.requiredNamedSnapshots must be a bounded array"); + } + const requiredNamedSnapshots = requiredNamedSnapshotsInput.map((name) => validateSnapshotName(name, "explorationPolicy.requiredNamedSnapshots entry")); + if (new Set(requiredNamedSnapshots).size !== requiredNamedSnapshots.length) { + throw new Error("explorationPolicy.requiredNamedSnapshots contains a duplicate name"); + } + requiredNamedSnapshots.sort(compareCodeUnits); + const minDistinctNamedSnapshotValues = validateSnapshotMinimumMap({ + label: "explorationPolicy.minDistinctNamedSnapshotValues", + maximum: TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME, + value: value.minDistinctNamedSnapshotValues ?? {} + }); + const minNamedSnapshotChangesAfterActionKind = validateSnapshotActionMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterActionKind", + value: value.minNamedSnapshotChangesAfterActionKind ?? {} + }); + const minNamedSnapshotChangesAfterNonWait = validateSnapshotMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterNonWait", + maximum: TRACE_MAX_LINES, + value: value.minNamedSnapshotChangesAfterNonWait ?? {} + }); + const requireStableTargetUrl = value.requireStableTargetUrl ?? false; + if (typeof requireStableTargetUrl !== "boolean") { + throw new Error("explorationPolicy.requireStableTargetUrl must be a boolean"); + } + return Object.freeze({ + minDistinctNamedSnapshotValues, + minNamedSnapshotChangesAfterActionKind, + minNamedSnapshotChangesAfterNonWait, + minNonWaitActions, + requireStableTargetUrl, + requiredActionKinds: Object.freeze(requiredActionKinds), + requiredNamedSnapshots: Object.freeze(requiredNamedSnapshots) + }); +} function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { const repositoryRoot = resolve(config.repositoryRoot); if (!isAbsolute(config.repositoryRoot) || repositoryRoot !== config.repositoryRoot) { @@ -1590,6 +2267,8 @@ function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { const entryPath = config.entryPath ?? "/"; validateEntryPath(entryPath); const targetQuery = validateTargetQuery(config.targetQuery ?? {}); + const viewport = validateViewport(config.viewport); + const explorationPolicy = validateExplorationPolicy(config.explorationPolicy); const startupTimeoutMs = config.server.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; if (!Number.isSafeInteger(startupTimeoutMs) || startupTimeoutMs < 1000 || startupTimeoutMs > MAX_STARTUP_TIMEOUT_MS) { throw new Error(`server.startupTimeoutMs must be an integer between 1000 and ${String(MAX_STARTUP_TIMEOUT_MS)}`); @@ -1604,8 +2283,10 @@ function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { artifactRoot: join2(repositoryRoot, "artifacts", "direct-bombadil", config.artifactName), bombadilExecutable: bombadilNativeBinary(repositoryRoot), entryPath, + explorationPolicy, port, targetQuery, + viewport, server: { ...config.server, cwd: serverCwd, @@ -1624,6 +2305,7 @@ function resolveReplayPath(repositoryRoot, replayPath) { return resolved; } function createDirectBombadilInvocation(options) { + const viewport = validateViewport(options.viewport); const target = new URL(options.entryPath ?? "/", `${options.baseUrl}/`); target.searchParams.set(SCENARIO_QUERY_KEY2, options.scenario); for (const [name, value] of Object.entries(options.targetQuery ?? {})) { @@ -1638,7 +2320,13 @@ function createDirectBombadilInvocation(options) { "--output-path", options.outputPath, "--headless", - "--instrument-javascript=" + "--instrument-javascript=", + "--width", + String(viewport.width), + "--height", + String(viewport.height), + "--device-scale-factor", + String(viewport.deviceScaleFactor) ]; if (options.replayPath === null) { command.push("--exit-on-violation", "--time-limit", `${String(options.timeLimitSeconds)}s`); @@ -1892,6 +2580,78 @@ function helpText(defaultBaseUrl) { ].join(` `); } +function parseMatrixCampaignArgument(arguments_) { + const forwarded = []; + let campaignId = null; + let help = false; + for (let index = 0;index < arguments_.length; index += 1) { + const argument = arguments_[index]; + if (argument === undefined) + continue; + if (argument === "--help" || argument === "-h") + help = true; + if (argument === "--campaign" || argument.startsWith("--campaign=")) { + if (campaignId !== null) + throw new Error("--campaign may be provided only once"); + if (argument === "--campaign") { + const next = readOptionValue(arguments_, index, "--campaign"); + campaignId = next.value; + index = next.index; + } else { + campaignId = argument.slice("--campaign=".length); + } + if (campaignId.length === 0) + throw new Error("--campaign requires a value"); + continue; + } + forwarded.push(argument); + } + return { arguments: Object.freeze(forwarded), campaignId, help }; +} +function validateCampaignMatrix(campaigns) { + if (campaigns.length === 0 || campaigns.length > 32) { + throw new Error("Bombadil campaign matrix must contain 1-32 campaigns"); + } + const ids = new Set; + for (const campaign of campaigns) { + if (!ARTIFACT_NAME_PATTERN.test(campaign.id) || ids.has(campaign.id)) { + throw new Error("Bombadil campaign IDs must be unique lowercase kebab identifiers"); + } + ids.add(campaign.id); + } + return campaigns; +} +async function runDirectBombadilFuzzMatrix(campaignsInput, arguments_ = process2.argv.slice(2), dependencyOverrides = {}) { + const campaigns = validateCampaignMatrix(campaignsInput); + const parsed = parseMatrixCampaignArgument(arguments_); + if (parsed.help) { + process2.stdout.write(`${[ + helpText(campaigns[0]?.config.baseUrl ?? ""), + " --campaign Run one campaign; required with --replay", + "", + `Campaigns: ${campaigns.map((campaign) => campaign.id).join(", ")}` + ].join(` +`)} +`); + return { kind: "help" }; + } + const selected = parsed.campaignId === null ? campaigns : campaigns.filter((campaign) => campaign.id === parsed.campaignId); + if (selected.length === 0) { + throw new Error(`Unknown Bombadil campaign ${parsed.campaignId ?? ""}`); + } + if (parsed.campaignId === null && parsed.arguments.some((argument) => argument === "--replay" || argument.startsWith("--replay="))) { + throw new Error("--replay requires exactly one --campaign in matrix mode"); + } + const results = []; + for (const campaign of selected) { + const result = await runDirectBombadilFuzz(campaign.config, parsed.arguments, dependencyOverrides); + if (result.kind !== "run") { + throw new Error("Bombadil campaign unexpectedly returned help during matrix execution"); + } + results.push({ campaignId: campaign.id, result }); + } + return { kind: "matrix", results: Object.freeze(results) }; +} async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2), dependencyOverrides = {}) { const parsed = parseDirectBombadilFuzzArguments(arguments_, config.baseUrl); if (parsed.kind === "help") { @@ -1923,7 +2683,8 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) scenario: validated.scenario, specificationPath: validated.specificationPath, targetQuery: validated.targetQuery, - timeLimitSeconds: parsed.timeLimitSeconds + timeLimitSeconds: parsed.timeLimitSeconds, + viewport: validated.viewport }); const abortableInvocation = { ...invocation, abortSignal: abortController.signal }; const serverCommand = validated.server.command.map((argument) => argument === "{port}" ? validated.port : argument); @@ -1933,6 +2694,8 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) let processResult = null; let attestation = null; let attestationFailure = null; + let explorationSummary = null; + let explorationSummaryFailure = null; let rawTracePath = null; let serverOutput = ""; let serverOutputFailure = null; @@ -1988,6 +2751,15 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) } catch (error) { attestationFailure = error; } + try { + explorationSummary = await summarizeDirectBombadilTrace({ + ...validated.explorationPolicy === null ? {} : { explorationPolicy: validated.explorationPolicy }, + targetUrl: invocation.targetUrl, + tracePath + }); + } catch (error) { + explorationSummaryFailure = error; + } if (processFailure !== null) { throw processFailure instanceof Error ? processFailure : new Error(renderUnknown(processFailure)); } @@ -2005,6 +2777,12 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) if (attestationFailure !== null) { throw attestationFailure instanceof Error ? attestationFailure : new Error(renderUnknown(attestationFailure)); } + if (explorationSummaryFailure !== null) { + throw explorationSummaryFailure instanceof Error ? explorationSummaryFailure : new Error(renderUnknown(explorationSummaryFailure)); + } + if (explorationSummary?.policy.satisfied !== true) { + throw new Error(`Bombadil exploration policy was not satisfied: ${explorationSummary?.policy.failures.join("; ") ?? "summary unavailable"}`); + } } catch (error) { failure = error; } @@ -2038,6 +2816,7 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) const status = failure === null ? "passed" : "failed"; const logPath = join2(artifactRun.runDirectory, "bombadil.log"); const serverLogPath = join2(artifactRun.runDirectory, "server.log"); + const explorationSummaryPath = join2(artifactRun.runDirectory, "exploration-summary.json"); const record = { schema: ARTIFACT_SCHEMA, evidenceClass: "diagnostic-fuzz", @@ -2053,6 +2832,8 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) entryPath: validated.entryPath, targetQuery: validated.targetQuery, targetUrl: invocation.targetUrl, + viewport: validated.viewport, + explorationPolicy: validated.explorationPolicy, specificationPath: validated.specificationPath, replayPath, timeLimitSeconds: replayPath === null ? parsed.timeLimitSeconds : null, @@ -2074,6 +2855,9 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) }, attestation, attestationFailure: attestationFailure === null ? null : renderUnknown(attestationFailure), + explorationSummary, + explorationSummaryPath: explorationSummary === null ? null : explorationSummaryPath, + explorationSummaryFailure: explorationSummaryFailure === null ? null : renderUnknown(explorationSummaryFailure), initialDirect: attestation?.initial ?? null, interruptedSignal: capturedSignal, failure: failure === null ? null : renderUnknown(failure) @@ -2085,9 +2869,23 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) ` : ""}`, "utf8"); await writeFile2(serverLogPath, `${serverOutput}${serverOutput.length > 0 ? ` ` : ""}`, "utf8"); + if (explorationSummary !== null) { + await writeJsonAtomically(explorationSummaryPath, explorationSummary); + } await writeJsonAtomically(join2(artifactRun.runDirectory, "run.json"), record); await writeJsonAtomically(artifactRun.manifestPath, record); - const summary = `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}; artifacts: ${artifactRun.runDirectory}; log: ${logPath}`; + const exploration = explorationSummary === null ? "exploration=unavailable" : [ + `nonWait=${String(explorationSummary.actions.nonWaitCount)}`, + `maxWaitStreak=${String(explorationSummary.actions.maxWaitStreak)}`, + `namedChanges=${explorationSummary.namedSnapshots.map((snapshot) => `${snapshot.name}:${String(snapshot.changeAfterNonWaitCount)}`).join(",") || "none"}`, + `policy=${explorationSummary.policy.satisfied ? "satisfied" : "failed"}` + ].join("; "); + const summary = [ + `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}`, + exploration, + `artifacts: ${artifactRun.runDirectory}`, + `log: ${logPath}` + ].join("; "); (status === "passed" ? process2.stdout : process2.stderr).write(`${summary} `); if (failure !== null) { @@ -2108,10 +2906,16 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) // src/tooling/bombadil.ts var attestDirectBombadilTrace2 = attestDirectBombadilTrace; +var summarizeDirectBombadilTrace2 = summarizeDirectBombadilTrace; function runDirectBombadilFuzz2(config, arguments_) { return arguments_ === undefined ? runDirectBombadilFuzz(config) : runDirectBombadilFuzz(config, arguments_); } +function runDirectBombadilFuzzMatrix2(campaigns, arguments_) { + return arguments_ === undefined ? runDirectBombadilFuzzMatrix(campaigns) : runDirectBombadilFuzzMatrix(campaigns, arguments_); +} export { + summarizeDirectBombadilTrace2 as summarizeDirectBombadilTrace, + runDirectBombadilFuzzMatrix2 as runDirectBombadilFuzzMatrix, runDirectBombadilFuzz2 as runDirectBombadilFuzz, attestDirectBombadilTrace2 as attestDirectBombadilTrace }; From a4fa0492457c1a6d25b373bc1f8dbe145243b82e Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 15:39:33 -0400 Subject: [PATCH 19/25] fix: preserve Bombadil trace and cancellation contracts --- dist/tooling/bombadil.js | 252 ++++++++++---- dist/tooling/browser-verification-entry.js | 42 ++- src/tooling/bombadil-runner.test.ts | 364 ++++++++++++++++++++- src/tooling/bombadil-runner.ts | 278 ++++++++++++---- src/tooling/browser-verification.test.ts | 62 ++++ src/tooling/browser-verification.ts | 60 +++- 6 files changed, 920 insertions(+), 138 deletions(-) diff --git a/dist/tooling/bombadil.js b/dist/tooling/bombadil.js index 9f094fd..341bc26 100644 --- a/dist/tooling/bombadil.js +++ b/dist/tooling/bombadil.js @@ -1046,23 +1046,54 @@ async function stopVerificationServerWithOutput(server, stopTimeoutMs = DEFAULT_ async function stopVerificationServer(server, stopTimeoutMs = DEFAULT_STOP_TIMEOUT_MS) { await stopVerificationServerWithOutput(server, stopTimeoutMs); } +function verificationServerAcquisitionAbortError() { + return new Error("Verification server acquisition was aborted"); +} +function throwIfVerificationServerAcquisitionAborted(signal) { + if (signal?.aborted === true) + throw verificationServerAcquisitionAbortError(); +} +async function waitForVerificationServerAcquisitionStep(promise, signal) { + if (signal === undefined) + return await promise; + throwIfVerificationServerAcquisitionAborted(signal); + let abortListener; + const aborted = new Promise((_resolve, reject) => { + abortListener = () => reject(verificationServerAcquisitionAbortError()); + signal.addEventListener("abort", abortListener, { once: true }); + if (signal.aborted) + abortListener(); + }); + let value; + try { + value = await Promise.race([promise, aborted]); + } finally { + if (abortListener !== undefined) + signal.removeEventListener("abort", abortListener); + } + throwIfVerificationServerAcquisitionAborted(signal); + return value; +} async function acquireVerificationServer(options) { const probeTimeoutMs = options.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; const readinessPath = options.readinessPath ?? "/"; const isReachable = options.isReachable ?? serverIsReachable; const canStartLocally = canAutomaticallyStartLocalServer(options.baseUrl, options.localHosts); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { if (canStartLocally && options.reuseExistingLocalServer === false) { throw new Error(`A local server is already reachable at ${options.baseUrl}; ` + "verification will not reuse a server whose worktree ownership is unknown"); } - await Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS), options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "reused" }; } } if (!canStartLocally) { throw new Error(`No server is reachable at ${options.baseUrl}; automatic startup is limited to local HTTP URLs`); } + throwIfVerificationServerAcquisitionAborted(options.abortSignal); const server = options.startServer(); let exitedWithCode = null; try { @@ -1073,10 +1104,11 @@ async function acquireVerificationServer(options) { exitedWithCode = exitCode; break; } - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "started", server }; } - await Bun.sleep(options.pollIntervalMs ?? 200); + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.pollIntervalMs ?? 200), options.abortSignal); } } catch (error) { await stopVerificationServer(server); @@ -1424,9 +1456,9 @@ var RESOURCE_FIELD_MAP = { task_duration: "taskDurationSeconds", thread_time: "threadTimeSeconds" }; -function canonicalJson2(value, depth = 0) { - if (depth > TRACE_MAX_JSON_DEPTH) { - throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(TRACE_MAX_JSON_DEPTH)}`); +function canonicalJson2(value, depth = 0, maximumDepth = TRACE_MAX_JSON_DEPTH) { + if (depth > maximumDepth) { + throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(maximumDepth)}`); } if (value === null || typeof value === "boolean" || typeof value === "string") { return JSON.stringify(value); @@ -1437,20 +1469,21 @@ function canonicalJson2(value, depth = 0) { return JSON.stringify(value); } if (Array.isArray(value)) { - return `[${value.map((entry) => canonicalJson2(entry, depth + 1)).join(",")}]`; + return `[${value.map((entry) => canonicalJson2(entry, depth + 1, maximumDepth)).join(",")}]`; } if (!isRecord2(value)) throw new Error("Bombadil named snapshot is not JSON"); - const entries = Object.keys(value).sort(compareCodeUnits).map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key], depth + 1)}`); + const entries = Object.keys(value).sort(compareCodeUnits).map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key], depth + 1, maximumDepth)}`); return `{${entries.join(",")}}`; } function sha256(value) { return createHash("sha256").update(value).digest("hex"); } -function namedSnapshotValueSha256(value) { - const canonical = canonicalJson2(value); - if (Buffer.byteLength(canonical, "utf8") > TRACE_MAX_CANONICAL_SNAPSHOT_BYTES) { - throw new Error(`Bombadil named snapshot exceeds ${String(TRACE_MAX_CANONICAL_SNAPSHOT_BYTES)} canonical bytes`); +function namedSnapshotValueSha256(value, options = {}) { + const maximumBytes = options.maximumBytes ?? TRACE_MAX_CANONICAL_SNAPSHOT_BYTES; + const canonical = canonicalJson2(value, 0, options.maximumDepth ?? TRACE_MAX_JSON_DEPTH); + if (Buffer.byteLength(canonical, "utf8") > maximumBytes) { + throw new Error(`Bombadil named snapshot exceeds ${String(maximumBytes)} canonical bytes`); } return sha256(canonical); } @@ -1588,7 +1621,7 @@ function parseTraceState(value, lineNumber) { url }; } -function parseTraceLine(line, lineNumber) { +function parseTraceEnvelope(line, lineNumber) { let input; try { input = JSON.parse(line); @@ -1601,37 +1634,82 @@ function parseTraceLine(line, lineNumber) { if (!Number.isSafeInteger(input.timestamp) || typeof input.timestamp !== "number" || input.timestamp < 0 || !Array.isArray(input.snapshots) || input.snapshots.length > TRACE_MAX_SNAPSHOTS_PER_LINE || !Array.isArray(input.violations)) { throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid state fields`); } - const state = parseTraceState(input.state, lineNumber); - const action = parseTraceAction(input.action, lineNumber); - const snapshots = input.snapshots; - const namedSnapshots = []; - const namedSnapshotNames = new Set; + return { + action: input.action, + snapshots: input.snapshots, + state: input.state, + timestamp: input.timestamp, + violations: input.violations + }; +} +function parseDirectTraceObservation(envelope, lineNumber) { + const directSnapshots = envelope.snapshots.filter((snapshot2) => isRecord2(snapshot2) && snapshot2.name === "direct"); + if (directSnapshots.length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); + } + const snapshot = directSnapshots[0]; + if (snapshot === undefined || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshot.index) || !Number.isSafeInteger(snapshot.time) || snapshot.index < 0 || snapshot.time < 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); + } + return { + observation: parseTraceDirectObservation(snapshot.value), + value: snapshot.value + }; +} +function parseDirectTraceLine(line, lineNumber) { + return parseDirectTraceObservation(parseTraceEnvelope(line, lineNumber), lineNumber).observation; +} +function parseTraceLine(line, lineNumber, strictDiagnosticSnapshotNames) { + const envelope = parseTraceEnvelope(line, lineNumber); + const state = parseTraceState(envelope.state, lineNumber); + const action = parseTraceAction(envelope.action, lineNumber); + const snapshots = envelope.snapshots; + const direct = parseDirectTraceObservation(envelope, lineNumber); + const namedSnapshots = [{ + name: "direct", + valueSha256: namedSnapshotValueSha256(direct.value, { + maximumBytes: TRACE_MAX_LINE_BYTES, + maximumDepth: TRACE_MAX_JSON_DEPTH + 4 + }) + }]; + const diagnosticSnapshotValues = new Map; for (const snapshotValue of snapshots) { if (!isRecord2(snapshotValue) || !hasExactKeys(snapshotValue, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshotValue.index) || !Number.isSafeInteger(snapshotValue.time) || snapshotValue.index < 0 || snapshotValue.time < 0 || snapshotValue.name !== null && typeof snapshotValue.name !== "string") { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid snapshot`); } - if (snapshotValue.name !== null) { - const name = validateSnapshotName(snapshotValue.name, `Bombadil trace line ${String(lineNumber)} snapshot name`); - if (namedSnapshotNames.has(name)) { + if (snapshotValue.name === null || snapshotValue.name === "direct") + continue; + let name; + try { + name = validateSnapshotName(snapshotValue.name, `Bombadil trace line ${String(lineNumber)} snapshot name`); + } catch (error) { + if (strictDiagnosticSnapshotNames.has(snapshotValue.name)) + throw error; + continue; + } + const values = diagnosticSnapshotValues.get(name) ?? []; + values.push(snapshotValue.value); + diagnosticSnapshotValues.set(name, values); + } + for (const [name, values] of diagnosticSnapshotValues) { + if (values.length !== 1) { + if (strictDiagnosticSnapshotNames.has(name)) { throw new Error(`Bombadil trace line ${String(lineNumber)} repeats named snapshot ${name}`); } - namedSnapshotNames.add(name); + continue; + } + try { namedSnapshots.push({ name, - valueSha256: namedSnapshotValueSha256(snapshotValue.value) + valueSha256: namedSnapshotValueSha256(values[0]) }); + } catch (error) { + if (strictDiagnosticSnapshotNames.has(name)) + throw error; } } - const directSnapshots = snapshots.filter((snapshot2) => isRecord2(snapshot2) && snapshot2.name === "direct"); - if (directSnapshots.length !== 1) { - throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); - } - const snapshot = directSnapshots[0]; - if (snapshot === undefined || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshot.index) || !Number.isSafeInteger(snapshot.time) || snapshot.index < 0 || snapshot.time < 0) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); - } const propertyViolationNames = []; - for (const violation of input.violations) { + for (const violation of envelope.violations) { if (!isRecord2(violation) || !hasExactKeys(violation, TRACE_VIOLATION_KEYS)) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); } @@ -1642,7 +1720,7 @@ function parseTraceLine(line, lineNumber) { } return { action, - directObservation: parseTraceDirectObservation(snapshot.value), + directObservation: direct.observation, namedSnapshots, propertyViolationNames, state @@ -1673,7 +1751,7 @@ async function attestDirectBombadilTrace(options) { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); } - const observation = parseTraceLine(line, observationCount).directObservation; + const observation = parseDirectTraceLine(line, observationCount); const exact = exactTraceDirectObservation(observation); if (exact === null) { if (initial !== null) { @@ -1759,6 +1837,7 @@ async function summarizeDirectBombadilTrace(options) { throw new Error("targetUrl must be an absolute URL"); } const policy = validateExplorationPolicy(options.explorationPolicy); + const strictDiagnosticSnapshotNames = explorationPolicySnapshotNames(policy); const actionCounts = new Map; const targetTags = new Map; const urlFingerprints = new Set; @@ -1788,6 +1867,8 @@ async function summarizeDirectBombadilTrace(options) { let policyObservationCount = 0; let previousObservationWasExact = false; let stableTarget = true; + let trackedUnrelatedSnapshotNameCount = 0; + const unrelatedSnapshotNameLimit = Math.max(0, TRACE_MAX_NAMED_SNAPSHOT_NAMES - strictDiagnosticSnapshotNames.size); const stream = createReadStream(options.tracePath, { encoding: "utf8" }); const lines = createInterface({ input: stream, crlfDelay: Infinity }); try { @@ -1799,7 +1880,7 @@ async function summarizeDirectBombadilTrace(options) { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); } - const parsed = parseTraceLine(line, lineCount); + const parsed = parseTraceLine(line, lineCount, strictDiagnosticSnapshotNames); const rawRelativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; rawUrlFingerprints.add(sha256(rawRelativeUrl)); if (rawUrlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { @@ -1858,6 +1939,10 @@ async function summarizeDirectBombadilTrace(options) { for (const snapshot of parsed.namedSnapshots) { let entry = snapshots.get(snapshot.name); if (entry === undefined) { + const isStrictSnapshot = snapshot.name === "direct" || strictDiagnosticSnapshotNames.has(snapshot.name); + if (!isStrictSnapshot && trackedUnrelatedSnapshotNameCount >= unrelatedSnapshotNameLimit) { + continue; + } if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`); } @@ -1870,6 +1955,14 @@ async function summarizeDirectBombadilTrace(options) { values: new Set }; snapshots.set(snapshot.name, entry); + if (!isStrictSnapshot) + trackedUnrelatedSnapshotNameCount += 1; + } + if (!entry.values.has(snapshot.valueSha256) && entry.values.size >= TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { + if (snapshot.name === "direct" || strictDiagnosticSnapshotNames.has(snapshot.name)) { + throw new Error(`Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`); + } + continue; } const changedAfterRecordedAction = recordedActionKind !== null && entry.lastObservationIndex === policyObservationCount - 1 && entry.lastValueSha256 !== null && entry.lastValueSha256 !== snapshot.valueSha256; if (changedAfterRecordedAction) { @@ -1882,9 +1975,6 @@ async function summarizeDirectBombadilTrace(options) { entry.lastValueSha256 = snapshot.valueSha256; entry.observationCount += 1; entry.values.add(snapshot.valueSha256); - if (entry.values.size > TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { - throw new Error(`Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`); - } } previousObservationWasExact = true; } @@ -2159,6 +2249,20 @@ function validateSnapshotActionMinimumMap(options) { } return Object.freeze(validated); } +function explorationPolicySnapshotNames(policy) { + const names = new Set(["direct"]); + if (policy === null) + return names; + for (const name of policy.requiredNamedSnapshots) + names.add(name); + for (const name of Object.keys(policy.minDistinctNamedSnapshotValues)) + names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterNonWait)) + names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterActionKind)) + names.add(name); + return names; +} function validateExplorationPolicy(value) { if (value === undefined) return null; @@ -2205,7 +2309,7 @@ function validateExplorationPolicy(value) { if (typeof requireStableTargetUrl !== "boolean") { throw new Error("explorationPolicy.requireStableTargetUrl must be a boolean"); } - return Object.freeze({ + const validated = Object.freeze({ minDistinctNamedSnapshotValues, minNamedSnapshotChangesAfterActionKind, minNamedSnapshotChangesAfterNonWait, @@ -2214,6 +2318,10 @@ function validateExplorationPolicy(value) { requiredActionKinds: Object.freeze(requiredActionKinds), requiredNamedSnapshots: Object.freeze(requiredNamedSnapshots) }); + if (explorationPolicySnapshotNames(validated).size > TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error(`explorationPolicy may reference at most ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES - 1)} distinct non-Direct snapshots`); + } + return validated; } function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { const repositoryRoot = resolve(config.repositoryRoot); @@ -2450,6 +2558,7 @@ async function runBombadilNativeProcess(invocation) { } var defaultDependencies = { acquireServer: acquireVerificationServer, + createAbortController: () => new AbortController, now: () => new Date, runBombadil: runBombadilNativeProcess, serverOutputTimeoutMs: SERVER_OUTPUT_TIMEOUT_MS, @@ -2652,6 +2761,17 @@ async function runDirectBombadilFuzzMatrix(campaignsInput, arguments_ = process2 } return { kind: "matrix", results: Object.freeze(results) }; } +function throwIfBombadilRunAborted(signal) { + if (signal.aborted) + throw new Error("Bombadil fuzzing was interrupted"); +} +function terminateAbortedOwnedServer(signal, server) { + if (!signal.aborted) + return; + if (server.exitCode() === null) + server.terminate(); + throwIfBombadilRunAborted(signal); +} async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2), dependencyOverrides = {}) { const parsed = parseDirectBombadilFuzzArguments(arguments_, config.baseUrl); if (parsed.kind === "help") { @@ -2672,7 +2792,7 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) }); const outputPath = join2(artifactRun.runDirectory, "bombadil"); const tracePath = join2(outputPath, "trace.jsonl"); - const abortController = new AbortController; + const abortController = dependencies.createAbortController?.() ?? new AbortController; const invocation = createDirectBombadilInvocation({ baseUrl: validated.baseUrl, bombadilExecutable: validated.bombadilExecutable, @@ -2715,23 +2835,37 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) try { await requireRegularFile(validated.bombadilExecutable, "The root Bombadil executable"); bombadilVersion = await readExactBombadilVersion(validated.repositoryRoot); - if (abortController.signal.aborted) - throw new Error("Bombadil fuzzing was interrupted"); - lease = await dependencies.acquireServer({ - baseUrl: validated.baseUrl, - label: validated.label, - readinessPath: validated.server.readinessPath, - reuseExistingLocalServer: false, - startupTimeoutMs: validated.server.startupTimeoutMs, - startServer: () => { - ownedServer = dependencies.spawnServer({ - command: serverCommand, - cwd: validated.server.cwd, - ...validated.server.env === undefined ? {} : { env: validated.server.env } - }); - return ownedServer; - } - }); + throwIfBombadilRunAborted(abortController.signal); + try { + lease = await dependencies.acquireServer({ + abortSignal: abortController.signal, + baseUrl: validated.baseUrl, + label: validated.label, + readinessPath: validated.server.readinessPath, + reuseExistingLocalServer: false, + startupTimeoutMs: validated.server.startupTimeoutMs, + startServer: () => { + throwIfBombadilRunAborted(abortController.signal); + ownedServer = dependencies.spawnServer({ + command: serverCommand, + cwd: validated.server.cwd, + ...validated.server.env === undefined ? {} : { env: validated.server.env } + }); + terminateAbortedOwnedServer(abortController.signal, ownedServer); + return ownedServer; + } + }); + } catch (error) { + if (abortController.signal.aborted) + throwIfBombadilRunAborted(abortController.signal); + throw error; + } + if (abortController.signal.aborted) { + const acquiredOwnedServer = ownedServer; + if (acquiredOwnedServer?.exitCode() === null) + acquiredOwnedServer.terminate(); + throwIfBombadilRunAborted(abortController.signal); + } let processFailure = null; try { processResult = await dependencies.runBombadil(abortableInvocation); diff --git a/dist/tooling/browser-verification-entry.js b/dist/tooling/browser-verification-entry.js index d8a4582..77fa758 100644 --- a/dist/tooling/browser-verification-entry.js +++ b/dist/tooling/browser-verification-entry.js @@ -1393,23 +1393,54 @@ async function stopVerificationServerWithOutput(server, stopTimeoutMs = DEFAULT_ async function stopVerificationServer(server, stopTimeoutMs = DEFAULT_STOP_TIMEOUT_MS) { await stopVerificationServerWithOutput(server, stopTimeoutMs); } +function verificationServerAcquisitionAbortError() { + return new Error("Verification server acquisition was aborted"); +} +function throwIfVerificationServerAcquisitionAborted(signal) { + if (signal?.aborted === true) + throw verificationServerAcquisitionAbortError(); +} +async function waitForVerificationServerAcquisitionStep(promise, signal) { + if (signal === undefined) + return await promise; + throwIfVerificationServerAcquisitionAborted(signal); + let abortListener; + const aborted = new Promise((_resolve, reject) => { + abortListener = () => reject(verificationServerAcquisitionAbortError()); + signal.addEventListener("abort", abortListener, { once: true }); + if (signal.aborted) + abortListener(); + }); + let value; + try { + value = await Promise.race([promise, aborted]); + } finally { + if (abortListener !== undefined) + signal.removeEventListener("abort", abortListener); + } + throwIfVerificationServerAcquisitionAborted(signal); + return value; +} async function acquireVerificationServer(options) { const probeTimeoutMs = options.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; const readinessPath = options.readinessPath ?? "/"; const isReachable = options.isReachable ?? serverIsReachable; const canStartLocally = canAutomaticallyStartLocalServer(options.baseUrl, options.localHosts); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { if (canStartLocally && options.reuseExistingLocalServer === false) { throw new Error(`A local server is already reachable at ${options.baseUrl}; ` + "verification will not reuse a server whose worktree ownership is unknown"); } - await Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS), options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "reused" }; } } if (!canStartLocally) { throw new Error(`No server is reachable at ${options.baseUrl}; automatic startup is limited to local HTTP URLs`); } + throwIfVerificationServerAcquisitionAborted(options.abortSignal); const server = options.startServer(); let exitedWithCode = null; try { @@ -1420,10 +1451,11 @@ async function acquireVerificationServer(options) { exitedWithCode = exitCode; break; } - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "started", server }; } - await Bun.sleep(options.pollIntervalMs ?? 200); + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.pollIntervalMs ?? 200), options.abortSignal); } } catch (error) { await stopVerificationServer(server); diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index fb2ccf4..45e668c 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -171,6 +171,67 @@ function directObservation(options: { }; } +function largeDirectObservation(): Record { + const scenarioIds = Array.from({ length: 256 }, (_, index) => + `s${String(index).padStart(3, "0")}.${"x".repeat(115)}` + ); + const firstScenario = scenarioIds[0]; + if (firstScenario === undefined) throw new Error("large Direct fixture needs a scenario"); + const citedScenarios: [string, ...string[]] = [firstScenario, ...scenarioIds.slice(1)]; + const definition = defineDirect({ + parseWorld: (input) => { + if ( + typeof input !== "object" + || input === null + || Array.isArray(input) + || typeof Reflect.get(input, "count") !== "number" + ) { + throw new Error("count is required"); + } + return { count: Reflect.get(input, "count") as number }; + }, + defaultScenario: firstScenario, + scenarios: scenarioIds.map((id, index) => ({ + description: "d".repeat(2_000), + id, + route: `/surface/${String(index)}`, + title: "t".repeat(160), + world: { count: index }, + })), + coverage: Array.from({ length: 256 }, (_, index) => ({ + claim: "c".repeat(1_000), + key: `coverage.${String(index)}`, + mode: "fixture" as const, + scenarios: citedScenarios, + })), + }); + const session = createDirectSession({ + definition, + activation: { kind: "scenario", scenario: firstScenario }, + create: () => ({}), + }); + if (!session.ok) throw new Error(session.error.message); + const snapshot = session.value.probe.snapshot(); + if (!snapshot.ok) throw new Error(snapshot.error.message); + const manifest = jsonClone(session.value.manifest); + const probe = jsonClone(snapshot.value); + return { + activationHash: manifest.active.activationHash, + activeRoute: manifest.active.route, + activeScenario: manifest.active.scenario, + activeSource: manifest.active.source, + bridgePresent: true, + bridgeSchema: "direct.browser-bridge/v2", + catalogHash: manifest.catalogHash, + contractValid: true, + isQuiescent: probe.isQuiescent, + manifest, + probe, + violations: Object.values(probe.violations), + violationsValid: true, + }; +} + function absentObservation(): Record { return { activationHash: "", @@ -238,15 +299,24 @@ function traceLine( async function writeTrace( tracePath: string, observations: readonly unknown[], + lineOptions: readonly TraceLineOptions[] = [], ): Promise { await mkdir(join(tracePath, ".."), { recursive: true }); await writeFile( tracePath, - `${observations.map((observation, index) => traceLine(observation, index + 1)).join("\n")}\n`, + `${observations.map((observation, index) => + traceLine(observation, index + 1, lineOptions[index]) + ).join("\n")}\n`, "utf8", ); } +function nestedJson(depth: number): unknown { + let value: unknown = null; + for (let index = 0; index < depth; index += 1) value = [value]; + return value; +} + function fakeServer( calls: string[], output: Promise = Promise.resolve("server output"), @@ -288,6 +358,7 @@ function dependencies(options: { readonly noTrace?: boolean; readonly neverServerOutput?: boolean; readonly observations?: readonly unknown[]; + readonly traceLineOptions?: readonly TraceLineOptions[]; readonly serverOutputTimeoutMs?: number; readonly stopFailure?: boolean; readonly termination?: "aborted" | "timeout"; @@ -336,6 +407,7 @@ function dependencies(options: { await writeTrace( join(invocation.outputPath, "trace.jsonl"), options.observations ?? [absentObservation(), directObservation()], + options.traceLineOptions, ); } return { @@ -354,7 +426,7 @@ function dependencies(options: { if (options.stopFailure === true) { throw new Error("server cleanup failed"); } - ownedServer.terminate(); + if (ownedServer.exitCode() === null) ownedServer.terminate(); await ownedServer.exited; }, }, @@ -582,6 +654,31 @@ describe("Direct Bombadil configuration and invocation", () => { }, })).toThrow("between 1 and 10000"); } + const snapshotNames = (prefix: string): string[] => + Array.from({ length: 32 }, (_, index) => `${prefix}${String(index)}`); + const maximumPolicy = { + minDistinctNamedSnapshotValues: Object.fromEntries( + snapshotNames("d").map((name) => [name, 1]), + ), + minNamedSnapshotChangesAfterActionKind: Object.fromEntries( + snapshotNames("a").map((name) => [name, { Click: 1 }]), + ), + minNamedSnapshotChangesAfterNonWait: Object.fromEntries( + snapshotNames("n").map((name) => [name, 1]), + ), + requiredNamedSnapshots: snapshotNames("r").slice(0, 31), + } as const; + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: maximumPolicy, + })).not.toThrow(); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + ...maximumPolicy, + requiredNamedSnapshots: snapshotNames("r"), + }, + })).toThrow("at most 127 distinct non-Direct snapshots"); }); test("rejects specification, server cwd, and replay symlinks that escape the repository", async () => { @@ -740,11 +837,14 @@ describe("Direct Bombadil campaign matrix", () => { }); describe("Direct Bombadil trace attestation", () => { - async function attest(observations: readonly unknown[]) { + async function attest( + observations: readonly unknown[], + lineOptions: readonly TraceLineOptions[] = [], + ) { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-trace-")); temporaryDirectories.push(directory); const tracePath = join(directory, "trace.jsonl"); - await writeTrace(tracePath, observations); + await writeTrace(tracePath, observations, lineOptions); return attestDirectBombadilTrace({ expectedRoute: "/surface", expectedScenario: "surface.ready", @@ -774,6 +874,72 @@ describe("Direct Bombadil trace attestation", () => { }); }); + test("ignores unrelated Bombadil snapshots outside the Direct attestation contract", async () => { + const oversizedValue = "x".repeat(2 * 1024 * 1024 + 1); + const cases: readonly { + readonly label: string; + readonly snapshots: NonNullable; + }[] = [{ + label: "arbitrary name", + snapshots: [{ name: "phase state", value: "ready" }], + }, { + label: "duplicate unrelated name", + snapshots: [ + { name: "phase", value: "loading" }, + { name: "phase", value: "ready" }, + ], + }, { + label: "depth 65 value", + snapshots: [{ name: "deep", value: nestedJson(65) }], + }, { + label: "value above two MiB", + snapshots: [{ name: "large", value: oversizedValue }], + }]; + + for (const testCase of cases) { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-compatible-trace-")); + temporaryDirectories.push(directory); + const tracePath = join(directory, "trace.jsonl"); + await writeTrace(tracePath, [directObservation()], [{ + namedSnapshots: testCase.snapshots, + }]); + const result = await attestDirectBombadilTrace({ + expectedRoute: "/surface", + expectedScenario: "surface.ready", + tracePath, + }); + expect(result.validObservationCount, testCase.label).toBe(1); + } + }); + + test("keeps exact Direct uniqueness and schema bounds independent of diagnostic limits", async () => { + const duplicateError = await rejection(attest([directObservation()], [{ + namedSnapshots: [{ name: "direct", value: directObservation() }], + }])); + expect(duplicateError.message).toContain("must contain one named direct snapshot"); + + const observation = largeDirectObservation(); + expect(Buffer.byteLength(JSON.stringify(observation), "utf8")) + .toBeGreaterThan(2 * 1024 * 1024); + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-large-direct-trace-")); + temporaryDirectories.push(directory); + const tracePath = join(directory, "trace.jsonl"); + await writeTrace(tracePath, [observation]); + const result = await attestDirectBombadilTrace({ + expectedRoute: String(observation.activeRoute), + expectedScenario: String(observation.activeScenario), + tracePath, + }); + expect(result.validObservationCount).toBe(1); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots).toEqual([ + expect.objectContaining({ name: "direct", observationCount: 1 }), + ]); + }); + test("rejects a vacuous, wrongly activated, or post-activation missing trace", async () => { expect((await rejection(attest([absentObservation()]))).message) .toContain("never reached a valid Direct contract"); @@ -964,6 +1130,60 @@ describe("Direct Bombadil exploration summary", () => { expect(serialized).not.toContain("/tmp/"); }); + test("omits out-of-contract unrelated snapshots while keeping policy snapshots strict", async () => { + const observation = directObservation(); + const oversizedValue = "x".repeat(2 * 1024 * 1024 + 1); + const tracePath = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: [ + { name: "owned", value: "ready" }, + { name: "phase state", value: "ready" }, + { name: "duplicate", value: 0 }, + { name: "duplicate", value: 1 }, + { name: "deep", value: nestedJson(65) }, + { name: "large", value: oversizedValue }, + ], + })]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { requiredNamedSnapshots: ["owned"] }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.map(({ name }) => name)).toEqual(["direct", "owned"]); + expect(summary.policy).toMatchObject({ configured: true, failures: [], satisfied: true }); + + const strictCases: readonly { + readonly label: string; + readonly snapshots: NonNullable; + readonly expected: string; + }[] = [{ + label: "duplicate", + snapshots: [ + { name: "owned", value: 0 }, + { name: "owned", value: 1 }, + ], + expected: "repeats named snapshot owned", + }, { + label: "depth", + snapshots: [{ name: "owned", value: nestedJson(65) }], + expected: "exceeds JSON depth", + }, { + label: "size", + snapshots: [{ name: "owned", value: oversizedValue }], + expected: "exceeds 2097152 canonical bytes", + }]; + for (const testCase of strictCases) { + const strictTracePath = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: testCase.snapshots, + })]); + const error = await rejection(summarizeDirectBombadilTrace({ + explorationPolicy: { requiredNamedSnapshots: ["owned"] }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: strictTracePath, + })); + expect(error.message, testCase.label).toContain(testCase.expected); + } + }); + test("accepts every exact Bombadil 0.7.2 browser action payload", async () => { const actions = [ "Back", @@ -1361,6 +1581,7 @@ describe("Direct Bombadil exploration summary", () => { namedSnapshots: [{ name: "deep", value: deep }], })]); expect((await rejection(summarizeDirectBombadilTrace({ + explorationPolicy: { requiredNamedSnapshots: ["deep"] }, targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", tracePath: deepTrace, }))).message).toContain("exceeds JSON depth"); @@ -1522,6 +1743,141 @@ describe("Direct Bombadil run lifecycle", () => { }); }); + test("runs with policy-owned evidence despite arbitrary unrelated named snapshots", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies({ + observations: [directObservation()], + traceLineOptions: [{ + namedSnapshots: [ + { name: "owned", value: "ready" }, + { name: "phase state", value: "ready" }, + { name: "duplicate", value: 0 }, + { name: "duplicate", value: 1 }, + { name: "deep", value: nestedJson(65) }, + { name: "large", value: "x".repeat(2 * 1024 * 1024 + 1) }, + ], + }], + }); + const result = await runDirectBombadilFuzz({ + ...config, + explorationPolicy: { requiredNamedSnapshots: ["owned"] }, + }, [], runtime.overrides); + expect(result.kind).toBe("run"); + const manifest = JSON.parse(await readFile( + join(repositoryRoot, "artifacts", "direct-bombadil", "fixture-product", "manifest.json"), + "utf8", + )) as Record; + expect(manifest).toMatchObject({ + status: "passed", + attestation: { validObservationCount: 1 }, + explorationSummary: { + namedSnapshots: [{ name: "direct" }, { name: "owned" }], + policy: { configured: true, failures: [], satisfied: true }, + }, + }); + }); + + test("does not spawn when cancellation wins before server startup", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + const error = await rejection(runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + acquireServer: (options): Promise => { + runtime.calls.push("acquire-server"); + controller.abort(); + return Promise.resolve({ source: "started", server: options.startServer() }); + }, + })); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual(["acquire-server"]); + }); + + test("terminates a server when cancellation wins during spawn", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + const server = fakeServer(runtime.calls); + const error = await rejection(runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + spawnServer: () => { + runtime.calls.push("spawn-server"); + controller.abort(); + return server; + }, + })); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual([ + "acquire-server", + "spawn-server", + "terminate", + "stop-server", + ]); + expect(server.exitCode()).toBe(0); + }); + + test("cleans a just-acquired server before Bombadil can run", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + const error = await rejection(runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + acquireServer: (options): Promise => { + runtime.calls.push("acquire-server"); + const server = options.startServer(); + controller.abort(); + return Promise.resolve({ source: "started", server }); + }, + })); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual([ + "acquire-server", + "spawn-server", + "terminate", + "stop-server", + ]); + expect(runtime.calls).not.toContain("run-bombadil"); + }); + + test("unwinds a pending server acquisition when cancellation arrives", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + let markAcquiring!: () => void; + const acquiring = new Promise((resolve) => { + markAcquiring = resolve; + }); + const run = runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + acquireServer: async (options): Promise => { + runtime.calls.push("acquire-server"); + const server = options.startServer(); + markAcquiring(); + await new Promise((_resolve, reject) => { + const abort = (): void => reject(new Error("acquisition aborted")); + options.abortSignal?.addEventListener("abort", abort, { once: true }); + if (options.abortSignal?.aborted === true) abort(); + }); + return { source: "started", server }; + }, + }); + await acquiring; + controller.abort(); + const error = await rejection(run); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual([ + "acquire-server", + "spawn-server", + "stop-server", + "terminate", + ]); + expect(runtime.calls).not.toContain("run-bombadil"); + }); + test("fails a configured exploration policy while retaining the derived sidecar", async () => { const { config, repositoryRoot } = await fixture(); const runtime = dependencies(); diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index ddad076..57107e8 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -354,6 +354,7 @@ export interface DirectBombadilTraceAttestation { export interface DirectBombadilRunnerDependencies { readonly acquireServer: typeof acquireVerificationServer; + readonly createAbortController?: () => AbortController; readonly now: () => Date; readonly runBombadil: ( invocation: DirectBombadilInvocation, @@ -673,6 +674,19 @@ interface ParsedTraceLine { readonly state: ParsedTraceState; } +interface ParsedTraceEnvelope { + readonly action: unknown; + readonly snapshots: readonly unknown[]; + readonly state: unknown; + readonly timestamp: number; + readonly violations: readonly unknown[]; +} + +interface ParsedDirectTraceObservation { + readonly observation: TraceDirectObservation; + readonly value: unknown; +} + const RESOURCE_FIELD_MAP = { documents: "documents", dom_nodes: "domNodes", @@ -685,9 +699,13 @@ const RESOURCE_FIELD_MAP = { thread_time: "threadTimeSeconds", } as const; -function canonicalJson(value: unknown, depth = 0): string { - if (depth > TRACE_MAX_JSON_DEPTH) { - throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(TRACE_MAX_JSON_DEPTH)}`); +function canonicalJson( + value: unknown, + depth = 0, + maximumDepth = TRACE_MAX_JSON_DEPTH, +): string { + if (depth > maximumDepth) { + throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(maximumDepth)}`); } if (value === null || typeof value === "boolean" || typeof value === "string") { return JSON.stringify(value); @@ -697,11 +715,11 @@ function canonicalJson(value: unknown, depth = 0): string { return JSON.stringify(value); } if (Array.isArray(value)) { - return `[${value.map((entry) => canonicalJson(entry, depth + 1)).join(",")}]`; + return `[${value.map((entry) => canonicalJson(entry, depth + 1, maximumDepth)).join(",")}]`; } if (!isRecord(value)) throw new Error("Bombadil named snapshot is not JSON"); const entries = Object.keys(value).sort(compareCodeUnits).map((key) => - `${JSON.stringify(key)}:${canonicalJson(value[key], depth + 1)}` + `${JSON.stringify(key)}:${canonicalJson(value[key], depth + 1, maximumDepth)}` ); return `{${entries.join(",")}}`; } @@ -710,11 +728,22 @@ function sha256(value: string | Uint8Array): string { return createHash("sha256").update(value).digest("hex"); } -function namedSnapshotValueSha256(value: unknown): string { - const canonical = canonicalJson(value); - if (Buffer.byteLength(canonical, "utf8") > TRACE_MAX_CANONICAL_SNAPSHOT_BYTES) { +function namedSnapshotValueSha256( + value: unknown, + options: { + readonly maximumBytes?: number; + readonly maximumDepth?: number; + } = {}, +): string { + const maximumBytes = options.maximumBytes ?? TRACE_MAX_CANONICAL_SNAPSHOT_BYTES; + const canonical = canonicalJson( + value, + 0, + options.maximumDepth ?? TRACE_MAX_JSON_DEPTH, + ); + if (Buffer.byteLength(canonical, "utf8") > maximumBytes) { throw new Error( - `Bombadil named snapshot exceeds ${String(TRACE_MAX_CANONICAL_SNAPSHOT_BYTES)} canonical bytes`, + `Bombadil named snapshot exceeds ${String(maximumBytes)} canonical bytes`, ); } return sha256(canonical); @@ -941,7 +970,7 @@ function parseTraceState(value: unknown, lineNumber: number): ParsedTraceState { }; } -function parseTraceLine(line: string, lineNumber: number): ParsedTraceLine { +function parseTraceEnvelope(line: string, lineNumber: number): ParsedTraceEnvelope { let input: unknown; try { input = JSON.parse(line) as unknown; @@ -961,11 +990,65 @@ function parseTraceLine(line: string, lineNumber: number): ParsedTraceLine { ) { throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid state fields`); } - const state = parseTraceState(input.state, lineNumber); - const action = parseTraceAction(input.action, lineNumber); - const snapshots = input.snapshots as unknown[]; - const namedSnapshots: Array<{ readonly name: string; readonly valueSha256: string }> = []; - const namedSnapshotNames = new Set(); + return { + action: input.action, + snapshots: input.snapshots as unknown[], + state: input.state, + timestamp: input.timestamp, + violations: input.violations, + }; +} + +function parseDirectTraceObservation( + envelope: ParsedTraceEnvelope, + lineNumber: number, +): ParsedDirectTraceObservation { + const directSnapshots = envelope.snapshots.filter( + (snapshot): snapshot is Readonly> => + isRecord(snapshot) && snapshot.name === "direct", + ); + if (directSnapshots.length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); + } + const snapshot = directSnapshots[0]; + if ( + snapshot === undefined + || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) + || !Number.isSafeInteger(snapshot.index) + || !Number.isSafeInteger(snapshot.time) + || (snapshot.index as number) < 0 + || (snapshot.time as number) < 0 + ) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); + } + return { + observation: parseTraceDirectObservation(snapshot.value), + value: snapshot.value, + }; +} + +function parseDirectTraceLine(line: string, lineNumber: number): TraceDirectObservation { + return parseDirectTraceObservation(parseTraceEnvelope(line, lineNumber), lineNumber).observation; +} + +function parseTraceLine( + line: string, + lineNumber: number, + strictDiagnosticSnapshotNames: ReadonlySet, +): ParsedTraceLine { + const envelope = parseTraceEnvelope(line, lineNumber); + const state = parseTraceState(envelope.state, lineNumber); + const action = parseTraceAction(envelope.action, lineNumber); + const snapshots = envelope.snapshots; + const direct = parseDirectTraceObservation(envelope, lineNumber); + const namedSnapshots: Array<{ readonly name: string; readonly valueSha256: string }> = [{ + name: "direct", + valueSha256: namedSnapshotValueSha256(direct.value, { + maximumBytes: TRACE_MAX_LINE_BYTES, + maximumDepth: TRACE_MAX_JSON_DEPTH + 4, + }), + }]; + const diagnosticSnapshotValues = new Map(); for (const snapshotValue of snapshots) { if ( !isRecord(snapshotValue) @@ -978,40 +1061,39 @@ function parseTraceLine(line: string, lineNumber: number): ParsedTraceLine { ) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid snapshot`); } - if (snapshotValue.name !== null) { - const name = validateSnapshotName( + if (snapshotValue.name === null || snapshotValue.name === "direct") continue; + let name: string; + try { + name = validateSnapshotName( snapshotValue.name, `Bombadil trace line ${String(lineNumber)} snapshot name`, ); - if (namedSnapshotNames.has(name)) { + } catch (error) { + if (strictDiagnosticSnapshotNames.has(snapshotValue.name)) throw error; + continue; + } + const values = diagnosticSnapshotValues.get(name) ?? []; + values.push(snapshotValue.value); + diagnosticSnapshotValues.set(name, values); + } + for (const [name, values] of diagnosticSnapshotValues) { + if (values.length !== 1) { + if (strictDiagnosticSnapshotNames.has(name)) { throw new Error(`Bombadil trace line ${String(lineNumber)} repeats named snapshot ${name}`); } - namedSnapshotNames.add(name); + continue; + } + try { namedSnapshots.push({ name, - valueSha256: namedSnapshotValueSha256(snapshotValue.value), + valueSha256: namedSnapshotValueSha256(values[0]), }); + } catch (error) { + if (strictDiagnosticSnapshotNames.has(name)) throw error; } } - const directSnapshots = snapshots.filter((snapshot): snapshot is Readonly> => - isRecord(snapshot) && snapshot.name === "direct" - ); - if (directSnapshots.length !== 1) { - throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); - } - const snapshot = directSnapshots[0]; - if ( - snapshot === undefined - || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) - || !Number.isSafeInteger(snapshot.index) - || !Number.isSafeInteger(snapshot.time) - || (snapshot.index as number) < 0 - || (snapshot.time as number) < 0 - ) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); - } const propertyViolationNames: string[] = []; - for (const violation of input.violations) { + for (const violation of envelope.violations) { if (!isRecord(violation) || !hasExactKeys(violation, TRACE_VIOLATION_KEYS)) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); } @@ -1025,7 +1107,7 @@ function parseTraceLine(line: string, lineNumber: number): ParsedTraceLine { } return { action, - directObservation: parseTraceDirectObservation(snapshot.value), + directObservation: direct.observation, namedSnapshots, propertyViolationNames, state, @@ -1063,7 +1145,7 @@ export async function attestDirectBombadilTrace(options: { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); } - const observation = parseTraceLine(line, observationCount).directObservation; + const observation = parseDirectTraceLine(line, observationCount); const exact = exactTraceDirectObservation(observation); if (exact === null) { if (initial !== null) { @@ -1172,6 +1254,7 @@ export async function summarizeDirectBombadilTrace(options: { throw new Error("targetUrl must be an absolute URL"); } const policy = validateExplorationPolicy(options.explorationPolicy); + const strictDiagnosticSnapshotNames = explorationPolicySnapshotNames(policy); const actionCounts = new Map(); const targetTags = new Map(); const urlFingerprints = new Set(); @@ -1208,6 +1291,11 @@ export async function summarizeDirectBombadilTrace(options: { let policyObservationCount = 0; let previousObservationWasExact = false; let stableTarget = true; + let trackedUnrelatedSnapshotNameCount = 0; + const unrelatedSnapshotNameLimit = Math.max( + 0, + TRACE_MAX_NAMED_SNAPSHOT_NAMES - strictDiagnosticSnapshotNames.size, + ); const stream = createReadStream(options.tracePath, { encoding: "utf8" }); const lines = createInterface({ input: stream, crlfDelay: Infinity }); try { @@ -1219,7 +1307,7 @@ export async function summarizeDirectBombadilTrace(options: { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); } - const parsed = parseTraceLine(line, lineCount); + const parsed = parseTraceLine(line, lineCount, strictDiagnosticSnapshotNames); const rawRelativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; rawUrlFingerprints.add(sha256(rawRelativeUrl)); @@ -1297,6 +1385,11 @@ export async function summarizeDirectBombadilTrace(options: { for (const snapshot of parsed.namedSnapshots) { let entry = snapshots.get(snapshot.name); if (entry === undefined) { + const isStrictSnapshot = snapshot.name === "direct" + || strictDiagnosticSnapshotNames.has(snapshot.name); + if (!isStrictSnapshot && trackedUnrelatedSnapshotNameCount >= unrelatedSnapshotNameLimit) { + continue; + } if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { throw new Error( `Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`, @@ -1311,6 +1404,21 @@ export async function summarizeDirectBombadilTrace(options: { values: new Set(), }; snapshots.set(snapshot.name, entry); + if (!isStrictSnapshot) trackedUnrelatedSnapshotNameCount += 1; + } + if ( + !entry.values.has(snapshot.valueSha256) + && entry.values.size >= TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME + ) { + if ( + snapshot.name === "direct" + || strictDiagnosticSnapshotNames.has(snapshot.name) + ) { + throw new Error( + `Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`, + ); + } + continue; } const changedAfterRecordedAction = recordedActionKind !== null @@ -1330,11 +1438,6 @@ export async function summarizeDirectBombadilTrace(options: { entry.lastValueSha256 = snapshot.valueSha256; entry.observationCount += 1; entry.values.add(snapshot.valueSha256); - if (entry.values.size > TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { - throw new Error( - `Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`, - ); - } } previousObservationWasExact = true; } @@ -1708,6 +1811,18 @@ function validateSnapshotActionMinimumMap(options: { return Object.freeze(validated); } +function explorationPolicySnapshotNames( + policy: ValidatedExplorationPolicy | null, +): ReadonlySet { + const names = new Set(["direct"]); + if (policy === null) return names; + for (const name of policy.requiredNamedSnapshots) names.add(name); + for (const name of Object.keys(policy.minDistinctNamedSnapshotValues)) names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterNonWait)) names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterActionKind)) names.add(name); + return names; +} + function validateExplorationPolicy( value: unknown, ): ValidatedExplorationPolicy | null { @@ -1775,7 +1890,7 @@ function validateExplorationPolicy( if (typeof requireStableTargetUrl !== "boolean") { throw new Error("explorationPolicy.requireStableTargetUrl must be a boolean"); } - return Object.freeze({ + const validated: ValidatedExplorationPolicy = Object.freeze({ minDistinctNamedSnapshotValues, minNamedSnapshotChangesAfterActionKind, minNamedSnapshotChangesAfterNonWait, @@ -1784,6 +1899,12 @@ function validateExplorationPolicy( requiredActionKinds: Object.freeze(requiredActionKinds), requiredNamedSnapshots: Object.freeze(requiredNamedSnapshots), }); + if (explorationPolicySnapshotNames(validated).size > TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error( + `explorationPolicy may reference at most ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES - 1)} distinct non-Direct snapshots`, + ); + } + return validated; } export function validateDirectBombadilFuzzConfig( @@ -2091,6 +2212,7 @@ export async function runBombadilNativeProcess( const defaultDependencies: DirectBombadilRunnerDependencies = { acquireServer: acquireVerificationServer, + createAbortController: () => new AbortController(), now: () => new Date(), runBombadil: runBombadilNativeProcess, serverOutputTimeoutMs: SERVER_OUTPUT_TIMEOUT_MS, @@ -2359,6 +2481,19 @@ export async function runDirectBombadilFuzzMatrix( return { kind: "matrix", results: Object.freeze(results) }; } +function throwIfBombadilRunAborted(signal: AbortSignal): void { + if (signal.aborted) throw new Error("Bombadil fuzzing was interrupted"); +} + +function terminateAbortedOwnedServer( + signal: AbortSignal, + server: ManagedVerificationServer, +): void { + if (!signal.aborted) return; + if (server.exitCode() === null) server.terminate(); + throwIfBombadilRunAborted(signal); +} + /** Runs one bounded diagnostic Bombadil campaign and always releases its server lease. */ export async function runDirectBombadilFuzz( config: DirectBombadilFuzzConfig, @@ -2390,7 +2525,7 @@ export async function runDirectBombadilFuzz( }); const outputPath = join(artifactRun.runDirectory, "bombadil"); const tracePath = join(outputPath, "trace.jsonl"); - const abortController = new AbortController(); + const abortController = dependencies.createAbortController?.() ?? new AbortController(); const invocation = createDirectBombadilInvocation({ baseUrl: validated.baseUrl, bombadilExecutable: validated.bombadilExecutable, @@ -2436,23 +2571,36 @@ export async function runDirectBombadilFuzz( try { await requireRegularFile(validated.bombadilExecutable, "The root Bombadil executable"); bombadilVersion = await readExactBombadilVersion(validated.repositoryRoot); - if (abortController.signal.aborted) throw new Error("Bombadil fuzzing was interrupted"); - - lease = await dependencies.acquireServer({ - baseUrl: validated.baseUrl, - label: validated.label, - readinessPath: validated.server.readinessPath, - reuseExistingLocalServer: false, - startupTimeoutMs: validated.server.startupTimeoutMs, - startServer: () => { - ownedServer = dependencies.spawnServer({ - command: serverCommand, - cwd: validated.server.cwd, - ...(validated.server.env === undefined ? {} : { env: validated.server.env }), - }); - return ownedServer; - }, - }); + throwIfBombadilRunAborted(abortController.signal); + + try { + lease = await dependencies.acquireServer({ + abortSignal: abortController.signal, + baseUrl: validated.baseUrl, + label: validated.label, + readinessPath: validated.server.readinessPath, + reuseExistingLocalServer: false, + startupTimeoutMs: validated.server.startupTimeoutMs, + startServer: () => { + throwIfBombadilRunAborted(abortController.signal); + ownedServer = dependencies.spawnServer({ + command: serverCommand, + cwd: validated.server.cwd, + ...(validated.server.env === undefined ? {} : { env: validated.server.env }), + }); + terminateAbortedOwnedServer(abortController.signal, ownedServer); + return ownedServer; + }, + }); + } catch (error) { + if (abortController.signal.aborted) throwIfBombadilRunAborted(abortController.signal); + throw error; + } + if (abortController.signal.aborted) { + const acquiredOwnedServer = ownedServer as ManagedVerificationServer | null; + if (acquiredOwnedServer?.exitCode() === null) acquiredOwnedServer.terminate(); + throwIfBombadilRunAborted(abortController.signal); + } let processFailure: unknown = null; try { processResult = await dependencies.runBombadil(abortableInvocation); diff --git a/src/tooling/browser-verification.test.ts b/src/tooling/browser-verification.test.ts index 2e84b3d..13e36e6 100644 --- a/src/tooling/browser-verification.test.ts +++ b/src/tooling/browser-verification.test.ts @@ -730,6 +730,68 @@ describe("server leases", () => { expect(fixture.calls).toEqual(["terminate"]); }); + test("aborts a pending readiness probe and terminates the owned server", async () => { + const fixture = fakeServer(); + const controller = new AbortController(); + let markPendingProbe!: () => void; + const pendingProbe = new Promise((resolve) => { + markPendingProbe = resolve; + }); + const neverReachable = new Promise(() => undefined); + let probes = 0; + const acquisition = acquireVerificationServer({ + abortSignal: controller.signal, + baseUrl: "http://localhost:8080", + label: "Fixture server", + startupTimeoutMs: 120_000, + startServer: () => fixture.server, + isReachable: () => { + probes += 1; + if (probes === 1) return false; + markPendingProbe(); + return neverReachable; + }, + }); + await pendingProbe; + controller.abort(); + const failure = await rejection(acquisition); + expect(failure.message).toBe("Verification server acquisition was aborted"); + expect(fixture.calls).toEqual(["terminate"]); + }, 1_000); + + test("does not return a lease when cancellation follows readiness", async () => { + const fixture = fakeServer(); + const controller = new AbortController(); + let markPendingProbe!: () => void; + const pendingProbe = new Promise((resolve) => { + markPendingProbe = resolve; + }); + let resolveReachability!: (reachable: boolean) => void; + const reachability = new Promise((resolve) => { + resolveReachability = resolve; + }); + let probes = 0; + const acquisition = acquireVerificationServer({ + abortSignal: controller.signal, + baseUrl: "http://localhost:8080", + label: "Fixture server", + startupTimeoutMs: 120_000, + startServer: () => fixture.server, + isReachable: () => { + probes += 1; + if (probes === 1) return false; + markPendingProbe(); + return reachability; + }, + }); + await pendingProbe; + resolveReachability(true); + queueMicrotask(() => controller.abort()); + const failure = await rejection(acquisition); + expect(failure.message).toBe("Verification server acquisition was aborted"); + expect(fixture.calls).toEqual(["terminate"]); + }); + test("bounds cleanup when a server never exits after SIGKILL", async () => { const calls: string[] = []; const never = new Promise(() => undefined); diff --git a/src/tooling/browser-verification.ts b/src/tooling/browser-verification.ts index 95da4e9..46ca7f0 100644 --- a/src/tooling/browser-verification.ts +++ b/src/tooling/browser-verification.ts @@ -813,7 +813,38 @@ export async function stopVerificationServer( await stopVerificationServerWithOutput(server, stopTimeoutMs); } +function verificationServerAcquisitionAbortError(): Error { + return new Error("Verification server acquisition was aborted"); +} + +function throwIfVerificationServerAcquisitionAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted === true) throw verificationServerAcquisitionAbortError(); +} + +async function waitForVerificationServerAcquisitionStep( + promise: Promise, + signal: AbortSignal | undefined, +): Promise { + if (signal === undefined) return await promise; + throwIfVerificationServerAcquisitionAborted(signal); + let abortListener: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + abortListener = () => reject(verificationServerAcquisitionAbortError()); + signal.addEventListener("abort", abortListener, { once: true }); + if (signal.aborted) abortListener(); + }); + let value: Value; + try { + value = await Promise.race([promise, aborted]); + } finally { + if (abortListener !== undefined) signal.removeEventListener("abort", abortListener); + } + throwIfVerificationServerAcquisitionAborted(signal); + return value; +} + export async function acquireVerificationServer(options: { + readonly abortSignal?: AbortSignal; readonly baseUrl: string; readonly label: string; readonly localHosts?: ReadonlySet; @@ -837,7 +868,11 @@ export async function acquireVerificationServer(options: { options.baseUrl, options.localHosts, ); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); + if (await waitForVerificationServerAcquisitionStep( + Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), + options.abortSignal, + )) { if (canStartLocally && options.reuseExistingLocalServer === false) { throw new Error( `A local server is already reachable at ${options.baseUrl}; ` @@ -847,8 +882,15 @@ export async function acquireVerificationServer(options: { // A verifier-owned command can exit before its child listener has finished // shutting down. Require the listener to survive a bounded interval before // another verifier trusts it as independently managed infrastructure. - await Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + await waitForVerificationServerAcquisitionStep( + Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS), + options.abortSignal, + ); + if (await waitForVerificationServerAcquisitionStep( + Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), + options.abortSignal, + )) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "reused" }; } } @@ -858,6 +900,7 @@ export async function acquireVerificationServer(options: { ); } + throwIfVerificationServerAcquisitionAborted(options.abortSignal); const server = options.startServer(); let exitedWithCode: number | null = null; try { @@ -868,10 +911,17 @@ export async function acquireVerificationServer(options: { exitedWithCode = exitCode; break; } - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + if (await waitForVerificationServerAcquisitionStep( + Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), + options.abortSignal, + )) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "started", server }; } - await Bun.sleep(options.pollIntervalMs ?? 200); + await waitForVerificationServerAcquisitionStep( + Bun.sleep(options.pollIntervalMs ?? 200), + options.abortSignal, + ); } } catch (error) { await stopVerificationServer(server); From 5d2f6343e73d5d3e0b4edee6a620e2c983f04895 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 17:39:03 -0400 Subject: [PATCH 20/25] Fix npm publish package budget --- .github/workflows/npm-publish.yml | 2 +- scripts/npm-publish-workflow.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 8f172af..6bd70e6 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -275,7 +275,7 @@ jobs: const minimumPackedBytes = 140_000; const maximumPackedBytes = 180_000; const minimumUnpackedBytes = 650_000; - const maximumUnpackedBytes = 750_000; + const maximumUnpackedBytes = 810_000; const maximumMetadataBytes = 250_000; const expectedName = "@hraness/direct"; const expectedVersion = process.env.EXPECTED_VERSION; diff --git a/scripts/npm-publish-workflow.test.ts b/scripts/npm-publish-workflow.test.ts index efe7abf..91deb0d 100644 --- a/scripts/npm-publish-workflow.test.ts +++ b/scripts/npm-publish-workflow.test.ts @@ -267,7 +267,7 @@ describe("npm release workflows", () => { "const minimumPackedBytes = 140_000", "const maximumPackedBytes = 180_000", "const minimumUnpackedBytes = 650_000", - "const maximumUnpackedBytes = 750_000", + "const maximumUnpackedBytes = 810_000", "record.files.length !== record.entryCount", "unpackedSize !== record.unpackedSize", 'createHash("sha1")', From c0d3734f84ccb16c5920f8638983ea1476c1199f Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sat, 29 Aug 2026 04:52:42 -0400 Subject: [PATCH 21/25] feat: harden Bombadil artifact delivery --- .github/workflows/npm-publish.yml | 4 +- AGENTS.md | 1 + README.md | 30 +- docs/publishing.md | 6 +- docs/verification.md | 113 +- package.json | 2 +- scripts/npm-publish-workflow.test.ts | 30 +- scripts/package-artifact.ts | 4 +- scripts/package-smoke.ts | 128 +- skills/direct/references/install.md | 6 +- skills/direct/references/verification.md | 23 +- src/exports.test.ts | 61 +- src/tooling/bombadil-runner.test.ts | 1233 +++++++- src/tooling/bombadil-runner.ts | 3581 +++++++++++++++++++--- src/tooling/bombadil.ts | 51 +- src/tooling/browser-verification.test.ts | 44 + src/tooling/browser-verification.ts | 62 +- 17 files changed, 4956 insertions(+), 423 deletions(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 6bd70e6..b9dd0fb 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -273,9 +273,9 @@ jobs: const minimumFiles = 50; const maximumFiles = 60; const minimumPackedBytes = 140_000; - const maximumPackedBytes = 180_000; + const maximumPackedBytes = 220_000; const minimumUnpackedBytes = 650_000; - const maximumUnpackedBytes = 810_000; + const maximumUnpackedBytes = 1_010_000; const maximumMetadataBytes = 250_000; const expectedName = "@hraness/direct"; const expectedVersion = process.env.EXPECTED_VERSION; diff --git a/AGENTS.md b/AGENTS.md index c12ca1f..694ef45 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,7 @@ - Keep core code product-, platform-, and framework-neutral. Put React, browser globals, and Node-only tooling behind explicit subpaths. - Build Bun host `@hraness/direct/tooling/*` entries separately. Keep every development-only export out of the default, core, React, testing, and web graphs, and prove the separation through the packed-consumer boundary gate. Ship the Bombadil campaign subpath as TypeScript source because 0.7.2 resolves no package export conditions, and keep it free of filesystem and process APIs because its compiler loads that subpath into a browser specification. - Pin optional browser tools exactly. The Bombadil integration supports 0.7.2 only, treats its JSONL trace as foreign bounded input, and must attest the canonical Direct manifest and probe after every run rather than trust a zero exit status. +- Constrain every Bombadil run to exclusive UUID leaves, owned process groups, bounded files and totals, a final descriptor-bound inventory, and a sanitized receipt. Public CI may upload only the exact receipt/summary leaf; raw traces and diagnostics require explicit bounded private vetting. Give each product-owned named snapshot an exact fail-closed parser or predicate. - Keep React Native and Expo imports in the reference example; `@hraness/direct/react` remains the platform-neutral React binding. - Keep `.js` extensions on relative TypeScript import and export specifiers; the published source type surface must compile under both Bundler and NodeNext resolution. - Treat this repository as the complete project. Files and Git prose may use only its public names, paths, commands, and examples; do not refer to or infer any non-public source, system, product, package, path, or implementation detail. diff --git a/README.md b/README.md index 84c4353..7ef1d19 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ state with predictable local fixtures. it does not click through the browser or test the systems it replaces. ```sh -bun add --dev @hraness/direct@0.7.7 +bun add --dev @hraness/direct@0.7.8 # or -npm install --save-dev @hraness/direct@0.7.7 +npm install --save-dev @hraness/direct@0.7.8 ``` [Install @hraness/direct from npm](https://www.npmjs.com/package/@hraness/direct) · @@ -53,7 +53,7 @@ Copy this prompt into Codex, Claude Code, or another coding agent: ```text Use $direct to install hraness/direct from -the npm registry at the exact 0.7.7 version. Follow the repository README, add +the npm registry at the exact 0.7.8 version. Follow the repository README, add `@hraness/direct` to devDependencies only, and verify that the production dependency graph excludes Direct. Do not add a fixture composition until I ask. @@ -69,7 +69,7 @@ Pin the public npm package to an exact immutable version: ```json { "devDependencies": { - "@hraness/direct": "0.7.7" + "@hraness/direct": "0.7.8" } } ``` @@ -279,7 +279,21 @@ the exact native 0.7.2 binary, attests the bounded trace with Direct's canonical parsers, writes pass or failure artifacts plus a compact exploration summary, and releases its owned processes. Use `runDirectBombadilFuzzMatrix` when a product owns several scenarios; it runs them serially and requires one exact -campaign selector for replay. +campaign selector for replay. Matrix upload plans are public-summary only and +publish one atomic parent leaf; run a selected campaign directly for bounded +access-controlled private diagnostics. + +Scheduled wrappers should precompute one lowercase UUID and pass it through +the runner's `artifactRun` option. Resolve the exact leaf with +`resolveDirectBombadilUploadLeaf` and upload only that leaf with `if: always()`. +Its default +public mode contains a bounded sanitized receipt and summary, including for +rejected or failed runs. Raw traces, logs, screenshots, paths, labels, typed +values, queries, and foreign errors stay local unless an access-controlled job +explicitly selects the bounded `private-vetted` mode. +Parse retained JSON from `unknown` with the four exported +`parseDirectBombadil*Receipt` and `parseDirectBombadil*Summary` functions; +never cast `JSON.parse` output to an evidence type. Startup is the only repairable contract phase. It must reach one exact Direct observation within ten seconds. From that sample onward, activation identity, @@ -292,6 +306,12 @@ snapshots that expose semantic state without retaining page content. Run short 12–30 second campaigns while editing and longer 60–300 second matrices in a scheduled diagnostic lane. Inspect and replay a retained failing trace, then promote the smallest readable failure to a deterministic product regression. +Give every product-owned named snapshot an exact fail-closed parser or type +predicate. A local random walk discovers reachable surprises; an Antithesis +environment supplies deterministic simulation and reproducibility around the +same bounded properties. Do not treat either one as a replacement for Direct's +deterministic scenarios, semantic assertions, production-boundary checks, or +ordinary browser gates. When a campaign must exercise an interaction, require a named product value to change after the intended action kind, as well as after a non-Wait action, so bootstrap, idle, prerequisite, and unrelated transitions do not satisfy the diff --git a/docs/publishing.md b/docs/publishing.md index 64c18bd..7a05ab9 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -62,8 +62,8 @@ cookie, one-time password, recovery code, or write token to GitHub. `v`. ```sh - git tag v0.7.7 - git push origin refs/tags/v0.7.7 + git tag v0.7.8 + git push origin refs/tags/v0.7.8 ``` 3. Wait for **Release**. The workflow runs these boundaries in order: @@ -123,7 +123,7 @@ Merge the fix to `main`, then dispatch **Release** from current `main` with the exact existing stable tag: ```sh -gh workflow run release.yml --ref main -f tag=v0.7.7 +gh workflow run release.yml --ref main -f tag=v0.7.8 ``` The recovery path skips npm publication. It accepts only the newest stable diff --git a/docs/verification.md b/docs/verification.md index f2a4252..b3c7bbb 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -334,15 +334,16 @@ Create a Bun wrapper such as `direct/fuzz-browser.ts`: ```ts #!/usr/bin/env bun +import { realpath } from "node:fs/promises"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { runDirectBombadilFuzz } from "@hraness/direct/tooling/bombadil"; const directRoot = fileURLToPath(new URL(".", import.meta.url)); -const repositoryRoot = resolve(directRoot, "../../.."); +const repositoryRoot = await realpath(resolve(directRoot, "../../..")); -await runDirectBombadilFuzz({ +const config = { artifactName: "todos", baseUrl: "http://127.0.0.1:5173", entryPath: "/direct/", @@ -379,7 +380,19 @@ await runDirectBombadilFuzz({ readinessPath: "/direct/", startupTimeoutMs: 30_000, }, -}, process.argv.slice(2)); +} as const; + +const runId = process.env.DIRECT_BOMBADIL_RUN_ID; +await runDirectBombadilFuzz(config, runId === undefined + ? process.argv.slice(2) + : { + arguments: process.argv.slice(2), + artifactRun: { + repositoryRoot, + runId, + uploadMode: "public-summary", + }, + }); ``` `baseUrl` must be an HTTP root origin on `127.0.0.1` or `localhost` with an @@ -432,6 +445,25 @@ Run random exploration for 12 to 300 seconds. The default is 20 seconds: bun direct/fuzz-browser.ts --time-limit 20s ``` +For a scheduled run, create one lowercase RFC 4122 UUID before starting the +wrapper and publish its exact upload leaf as a job output. Pass that UUID in +`artifactRun`; then an `if: always()` upload step can target only +`artifacts/direct-bombadil-upload/`, including terminal failures after +the runner accepts a valid options envelope and establishes the canonical +plan's unclaimed upload session. The runner strips +`DIRECT_BOMBADIL_RUN_ID` from native and server children. Never upload the +artifact-name root or +the whole repository artifact directory because either can include older raw +runs. + +```sh +run_id="$(bun -e 'console.log(crypto.randomUUID())')" +repository_root="$(git rev-parse --show-toplevel)" +repository_root="$(cd "$repository_root" && pwd -P)" +printf 'path=%s/artifacts/direct-bombadil-upload/%s\n' "$repository_root" "$run_id" >> "$GITHUB_OUTPUT" +DIRECT_BOMBADIL_RUN_ID="$run_id" bun direct/fuzz-browser.ts --time-limit 20s +``` + Use 12–30 seconds in the edit loop. A scheduled diagnostic lane can run each campaign for 60–300 seconds, serially, with an outer job timeout and retained failure artifacts. Random exploration should supplement the deterministic @@ -441,17 +473,29 @@ particular random path. For multiple product scenarios, pass a bounded matrix to the shared runner: ```ts -import { runDirectBombadilFuzzMatrix } from "@hraness/direct/tooling/bombadil"; +import { + resolveDirectBombadilUploadLeaf, + runDirectBombadilFuzzMatrix, +} from "@hraness/direct/tooling/bombadil"; + +const runId = process.env.DIRECT_BOMBADIL_RUN_ID; +if (runId === undefined) throw new Error("DIRECT_BOMBADIL_RUN_ID is required"); +const artifactRun = { repositoryRoot, runId, uploadMode: "public-summary" } as const; +console.log(resolveDirectBombadilUploadLeaf(artifactRun)); await runDirectBombadilFuzzMatrix([ { id: "populated", config: populatedCampaign }, { id: "empty", config: emptyCampaign }, -], process.argv.slice(2)); +], { arguments: process.argv.slice(2), artifactRun }); ``` Without `--campaign`, the matrix runs every unique campaign serially. Select one for focused work with `--campaign empty`. Replay is intentionally rejected without that selector so a trace cannot be applied to the wrong scenario. +Matrices accept only `public-summary` upload plans and publish one atomic parent +leaf after every selected campaign reaches a terminal state. Run one selected +campaign directly when access-controlled `private-vetted` diagnostics are +required. Use `--base-url` to select another local root origin. Use `--replay` with a repository-local `.jsonl` trace instead of `--time-limit` to reproduce a prior @@ -498,17 +542,21 @@ The runner invokes the exact native binary at the consumer repository root with headless mode, JavaScript instrumentation disabled, a bounded output directory, and exit-on-violation for random exploration. An outer wall-clock deadline covers the native process. Timeout, interruption, or exit triggers -bounded process-group cleanup; timeout and interruption use TERM then KILL, -while a completed leader cannot leave descendants holding output pipes. The +bounded process-group cleanup; timeout, interruption, and artifact quota +breaches use immediate KILL, and a completed leader cannot +leave descendants holding output pipes. The configured local server is always stopped through the shared browser-verification lease helpers, and its output drain remains bounded even when cleanup itself fails. -Each attempt writes `run.json`, `exploration-summary.json`, `bombadil.log`, and `server.log` below -`artifacts/direct-bombadil///`, including failures. The -rolling `manifest.json` points to the latest record. `rawTracePath` reports a -regular nonempty trace even if attestation fails; `tracePath` is present only -after exact attestation. The v2 summary strictly parses the 0.7.2 envelopes and +Once the run leaf exists, local diagnostics are written below +`artifacts/direct-bombadil///`. They can include +`run.json`, `exploration-summary.json`, `bombadil.log`, `server.log`, and the +native output. Early configuration rejection can precede that local leaf. The +rolling `manifest.json` is a convenience pointer, not authoritative evidence. +The exclusive UUID receipt and upload leaf are authoritative. `rawTracePath` +reports a regular nonempty trace even if attestation fails; `tracePath` is +present only after exact attestation. The v2 summary strictly parses the 0.7.2 envelopes and records the raw trace SHA-256, action-kind and safe target-tag counts, non-Wait count and longest Wait streak, origin-relative URL fingerprints, non-null transition-hash cardinality, canonical named-snapshot value hashes, property @@ -522,15 +570,38 @@ paths. These diagnostics describe what Bombadil happened to explore. They do not measure code, state, interaction, or Direct catalog coverage, and the raw trace remains authoritative. -Keep all generated artifacts out of source control by default. Upload a failed -scheduled run to access-controlled CI storage with a bounded retention period; -the raw trace can contain screenshots, query values, typed text, accessibility -labels, extracted values, and local paths. Preserve it long enough to inspect -and replay. Once the defect is understood, add the smallest deterministic -regression at the owning parser, reducer, port, component, semantic browser, or -Direct scenario boundary. Verify that regression fails before the fix and -passes after it. Retain a reviewed trace fixture only when replay itself adds -durable value; otherwise remove the sensitive trace after promotion. +Keep all generated artifacts out of source control by default. The default +`public-summary` upload leaf contains only a newly constructed bounded receipt +and sanitized summary. It excludes raw traces, screenshots, URLs, query +values, typed text, accessible labels, logs, absolute paths, and foreign error +messages. `private-vetted` is an explicit opt-in for access-controlled CI; it +descriptor-copies only allowlisted native files that pass the campaign count, +depth, path, per-file, and aggregate-byte quotas. Two host logs have separate +fixed capture bounds. The resulting upload must then match a newly constructed +exact-tree inventory and hashes. Symlinks, hard links, special files, unstable +files, and unapproved extensions fail closed. Live polling is best-effort +disk-pressure containment. The final post-cleanup inventory and +descriptor-bound hashes are the authoritative retained-artifact gate. +`diagnosticsRetained` says whether a private leaf retained vetted raw files. +Parse disk JSON from `unknown` with `parseDirectBombadilArtifactReceipt`, +`parseDirectBombadilSanitizedRunSummary`, `parseDirectBombadilMatrixReceipt`, +or `parseDirectBombadilMatrixSummary`; never cast `JSON.parse` output. The last +synchronous interruption check precedes atomic rename dispatch. A later signal +belongs to the caller after terminal publication. + +Bombadil and the configured server run in owned process groups, which the host +settles before the final scan. Node does not expose `openat`, so this boundary +does not claim containment against a hostile concurrent process running as the +same user. Repository scheduling and exclusive run leaves remain required. +If either writer group cannot be proven absent, the runner skips raw inventory, +attestation, and private copying and publishes only a sanitized +`writer-settlement` failure receipt. +Preserve private diagnostics only long enough to inspect and replay. Once the +defect is understood, add the smallest deterministic regression at the owning +parser, reducer, port, component, semantic browser, or Direct scenario +boundary. Verify that regression fails before the fix and passes after it. +Retain a reviewed trace fixture only when replay itself adds durable value; +otherwise remove the sensitive trace after promotion. ## Report coverage without promotion diff --git a/package.json b/package.json index b53f7d6..7cdb514 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hraness/direct", - "version": "0.7.7", + "version": "0.7.8", "description": "A TypeScript harness for deterministic frontend development with repeatable scenarios, local fixtures, and browser verification for coding agents.", "license": "MIT", "type": "module", diff --git a/scripts/npm-publish-workflow.test.ts b/scripts/npm-publish-workflow.test.ts index 91deb0d..d947980 100644 --- a/scripts/npm-publish-workflow.test.ts +++ b/scripts/npm-publish-workflow.test.ts @@ -159,7 +159,7 @@ describe("npm release workflows", () => { readonly version?: unknown; }; expect(manifest).toEqual(expect.objectContaining({ - version: "0.7.7", + version: "0.7.8", description: "A TypeScript harness for deterministic frontend development with repeatable scenarios, local fixtures, and browser verification for coding agents.", keywords: [ "frontend-development", @@ -265,9 +265,9 @@ describe("npm release workflows", () => { "const minimumFiles = 50", "const maximumFiles = 60", "const minimumPackedBytes = 140_000", - "const maximumPackedBytes = 180_000", + "const maximumPackedBytes = 220_000", "const minimumUnpackedBytes = 650_000", - "const maximumUnpackedBytes = 810_000", + "const maximumUnpackedBytes = 1_010_000", "record.files.length !== record.entryCount", "unpackedSize !== record.unpackedSize", 'createHash("sha1")', @@ -357,7 +357,7 @@ describe("npm release workflows", () => { const binaryDirectory = join(directory, "bin"); const commandLog = join(directory, "commands.log"); const publishMarker = join(directory, "published.txt"); - const tarball = join(directory, "hraness-direct-0.7.7.tgz"); + const tarball = join(directory, "hraness-direct-0.7.8.tgz"); const metadata = join(directory, "npm-pack.json"); const digest = join(directory, "npm-package.sha256"); const sourceSha = "b".repeat(40); @@ -375,7 +375,7 @@ describe("npm release workflows", () => { writeFile(metadata, "reviewed metadata fixture\n", "utf8"), writeFile(digest, "reviewed digest fixture\n", "utf8"), ]); - await writeFile(gitStub, `#!/bin/bash\nset -euo pipefail\nprintf 'git %s\\n' "$*" >> "$COMMAND_LOG"\ncase "$*" in\n *"rev-parse refs/heads/main"*) printf '%s\\n' "$DEFAULT_SHA" ;;\n *"rev-parse refs/tags/v0.7.7^{commit}"*) printf '%s\\n' "$TAG_SHA" ;;\n *"merge-base --is-ancestor"*) [[ "$ANCESTRY_STATE" == ancestor ]] ;;\n *"tag --list v*"*) printf '%s\\n' "$REMOTE_TAGS" ;;\nesac\n`, "utf8"); + await writeFile(gitStub, `#!/bin/bash\nset -euo pipefail\nprintf 'git %s\\n' "$*" >> "$COMMAND_LOG"\ncase "$*" in\n *"rev-parse refs/heads/main"*) printf '%s\\n' "$DEFAULT_SHA" ;;\n *"rev-parse refs/tags/v0.7.8^{commit}"*) printf '%s\\n' "$TAG_SHA" ;;\n *"merge-base --is-ancestor"*) [[ "$ANCESTRY_STATE" == ancestor ]] ;;\n *"tag --list v*"*) printf '%s\\n' "$REMOTE_TAGS" ;;\nesac\n`, "utf8"); await writeFile(sha256Stub, `#!/bin/bash\nset -euo pipefail\nprintf 'sha256sum %s\\n' "$*" >> "$COMMAND_LOG"\ncase "$1" in\n "$TARBALL") value="$EXPECTED_ARCHIVE_SHA256" ;;\n "$METADATA") value="$EXPECTED_METADATA_SHA256" ;;\n "$DIGEST") value="$EXPECTED_DIGEST_SHA256" ;;\n *) echo "unexpected hash target: $1" >&2; exit 1 ;;\nesac\nprintf '%s %s\\n' "$value" "$1"\n`, "utf8"); await writeFile(npmStub, `#!/bin/bash\nset -euo pipefail\nprintf 'npm %s\\n' "$*" >> "$COMMAND_LOG"\nif [[ "\${1-}" == view ]]; then\n printf '%s\\n' "$PUBLISHED_VERSIONS_JSON"\n exit 0\nfi\nprintf 'published\\n' > "$PUBLISH_MARKER"\n`, "utf8"); await Promise.all([chmod(gitStub, 0o755), chmod(npmStub, 0o755), chmod(sha256Stub, 0o755)]); @@ -391,15 +391,15 @@ describe("npm release workflows", () => { EXPECTED_DIGEST_SHA256: digestSha256, EXPECTED_METADATA_SHA256: metadataSha256, EXPECTED_SOURCE_SHA: sourceSha, - EXPECTED_VERSION: "0.7.7", - GITHUB_REF: "refs/tags/v0.7.7", + EXPECTED_VERSION: "0.7.8", + GITHUB_REF: "refs/tags/v0.7.8", GITHUB_REPOSITORY: "hraness/direct", GITHUB_SHA: sourceSha, METADATA: metadata, PATH: `${binaryDirectory}:${process.env.PATH ?? ""}`, - PUBLISHED_VERSIONS_JSON: '["0.7.4","0.7.5","0.7.6"]', + PUBLISHED_VERSIONS_JSON: '["0.7.4","0.7.5","0.7.6","0.7.7"]', PUBLISH_MARKER: publishMarker, - REMOTE_TAGS: "v0.7.4\nv0.7.5\nv0.7.6\nv0.7.7", + REMOTE_TAGS: "v0.7.4\nv0.7.5\nv0.7.6\nv0.7.7\nv0.7.8", RUNNER_TEMP: directory, TAG_SHA: sourceSha, TARBALL: tarball, @@ -430,7 +430,7 @@ describe("npm release workflows", () => { }); expect(moved.exitCode).not.toBe(0); expect(`${moved.stdout}${moved.stderr}`).toContain( - "Tag v0.7.7 changed after artifact verification", + "Tag v0.7.8 changed after artifact verification", ); expect(await Bun.file(publishMarker).exists()).toBe(false); @@ -441,18 +441,18 @@ describe("npm release workflows", () => { }); expect(detached.exitCode).not.toBe(0); expect(`${detached.stdout}${detached.stderr}`).toContain( - "Tag v0.7.7 is no longer reachable from main", + "Tag v0.7.8 is no longer reachable from main", ); expect(await Bun.file(publishMarker).exists()).toBe(false); await rm(commandLog, { force: true }); const superseded = await runWorkflowScript(script, { ...baseEnvironment, - REMOTE_TAGS: "v0.7.4\nv0.7.5\nv0.7.6\nv0.7.7\nv0.7.8", + REMOTE_TAGS: "v0.7.4\nv0.7.5\nv0.7.6\nv0.7.7\nv0.7.8\nv0.7.9", }); expect(superseded.exitCode).not.toBe(0); expect(`${superseded.stdout}${superseded.stderr}`).toContain( - "Tag v0.7.7 is not the newest stable tag v0.7.8", + "Tag v0.7.8 is not the newest stable tag v0.7.9", ); expect(await readFile(commandLog, "utf8")).not.toContain("npm publish"); expect(await Bun.file(publishMarker).exists()).toBe(false); @@ -460,11 +460,11 @@ describe("npm release workflows", () => { await rm(commandLog, { force: true }); const staleVersion = await runWorkflowScript(script, { ...baseEnvironment, - PUBLISHED_VERSIONS_JSON: '["0.7.4","0.7.8"]', + PUBLISHED_VERSIONS_JSON: '["0.7.4","0.7.9"]', }); expect(staleVersion.exitCode).not.toBe(0); expect(`${staleVersion.stdout}${staleVersion.stderr}`).toContain( - "@hraness/direct@0.7.7 is not newer than published stable 0.7.8", + "@hraness/direct@0.7.8 is not newer than published stable 0.7.9", ); expect(await Bun.file(publishMarker).exists()).toBe(false); diff --git a/scripts/package-artifact.ts b/scripts/package-artifact.ts index 759445b..f15fc4f 100644 --- a/scripts/package-artifact.ts +++ b/scripts/package-artifact.ts @@ -9,8 +9,8 @@ const maximumTarBytes = 2_000_000; const packageBudget = Object.freeze({ entryCount: { min: 50, max: 120 }, fileCount: { min: 50, max: 60 }, - packedBytes: { min: 140_000, max: 180_000 }, - unpackedBytes: { min: 650_000, max: 810_000 }, + packedBytes: { min: 140_000, max: 220_000 }, + unpackedBytes: { min: 650_000, max: 1_010_000 }, }); const requiredPaths = Object.freeze([ diff --git a/scripts/package-smoke.ts b/scripts/package-smoke.ts index e126a56..5687d7a 100644 --- a/scripts/package-smoke.ts +++ b/scripts/package-smoke.ts @@ -388,10 +388,38 @@ try { await writeFile(join(consumer, "runtime-index.ts"), typeImportSource(runtimeImportSpecifiers)); await writeFile(join(consumer, "tooling-index.ts"), `${typeImportSource(toolingTypeImportSpecifiers)} type BombadilRunnerArity = Parameters["length"]; + type BombadilRunnerInput = Parameters[1]; + type BombadilMatrixInput = Parameters[1]; const supportedBombadilRunnerArities: readonly BombadilRunnerArity[] = [1, 2]; + const supportedBombadilArguments = ["--time-limit=12s"] as const; + const supportedBombadilRunnerInput: BombadilRunnerInput = { + arguments: supportedBombadilArguments, + artifactRun: { + repositoryRoot: "/absolute/repository", + runId: "00000000-0000-4000-8000-000000000001", + uploadMode: "public-summary", + }, + }; + const supportedBombadilTupleInput: BombadilRunnerInput = supportedBombadilArguments; + const supportedBombadilMatrixInput: BombadilMatrixInput = { + arguments: supportedBombadilArguments, + artifactRun: { + repositoryRoot: "/absolute/repository", + runId: "00000000-0000-4000-8000-000000000002", + uploadMode: "public-summary", + }, + }; + const unsupportedPrivateBombadilMatrixInput: BombadilMatrixInput = { + artifactRun: { + repositoryRoot: "/absolute/repository", + runId: "00000000-0000-4000-8000-000000000003", + // @ts-expect-error Packaged matrix uploads are public-summary only. + uploadMode: "private-vetted", + }, + }; // @ts-expect-error Public tooling does not expose dependency injection. const unsupportedBombadilRunnerArity: BombadilRunnerArity = 3; - void [supportedBombadilRunnerArities, unsupportedBombadilRunnerArity]; + void [supportedBombadilMatrixInput, supportedBombadilRunnerArities, supportedBombadilRunnerInput, supportedBombadilTupleInput, unsupportedBombadilRunnerArity, unsupportedPrivateBombadilMatrixInput]; `); await writeFile(join(consumer, "tsconfig.bundler.json"), typeScriptConfig({ include: "runtime-index.ts", @@ -445,6 +473,11 @@ try { readDirectBrowserContract, } from "@hraness/direct/tooling/browser-verification"; import { + parseDirectBombadilArtifactReceipt, + parseDirectBombadilMatrixReceipt, + parseDirectBombadilMatrixSummary, + parseDirectBombadilSanitizedRunSummary, + resolveDirectBombadilUploadLeaf, runDirectBombadilFuzz, } from "@hraness/direct/tooling/bombadil"; import { findForbiddenMarkers } from "@hraness/direct/tooling/bundle-boundary"; @@ -465,6 +498,95 @@ try { if (typeof runDirectBombadilFuzz !== "function") { throw new Error("Bombadil host tooling runner is missing"); } + const sha256 = "a".repeat(64); + const policy = { + maxDepth: 32, + maxEntries: 4096, + maxFileBytes: 67108864, + maxFiles: 2048, + maxPathBytes: 4096, + maxTotalBytes: 134217728, + }; + const runId = "00000000-0000-4000-8000-000000000001"; + const receipt = { + schema: "direct.bombadil-artifact-receipt/v1", + completedAt: "2026-08-29T00:00:00.000Z", + diagnosticsRetained: false, + failureCode: null, + inventory: { entryCount: 1, fileCount: 1, inventorySha256: sha256, totalBytes: 1 }, + mode: "public-summary", + policy, + runId, + status: "passed", + }; + const summary = { + schema: "direct.bombadil-upload-summary/v1", + artifactName: "package-smoke", + attestation: { invalidObservationCount: 0, observationCount: 1, validObservationCount: 1 }, + exploration: { + actionCount: 0, + nonWaitActionCount: 0, + policySatisfied: true, + traceBytes: 1, + traceLineCount: 1, + traceSha256: sha256, + }, + failureCode: null, + scenario: "package.ready", + status: "passed", + }; + const matrixReceipt = { + schema: "direct.bombadil-matrix-receipt/v1", + campaigns: [{ + campaignId: "package-smoke", + index: 0, + receipt: "campaigns/package-smoke/receipt.json", + status: "passed", + }], + completedAt: "2026-08-29T00:00:00.000Z", + failureCode: null, + mode: "public-summary", + omittedCampaignCount: 0, + runId, + status: "passed", + }; + const matrixSummary = { + schema: "direct.bombadil-matrix-summary/v1", + campaigns: { + failed: 0, + notRun: 0, + notSelected: 0, + omitted: 0, + passed: 1, + rejected: 0, + total: 1, + }, + failureCode: null, + status: "passed", + }; + if ( + !parseDirectBombadilArtifactReceipt(receipt).ok + || !parseDirectBombadilSanitizedRunSummary(summary).ok + || !parseDirectBombadilMatrixReceipt(matrixReceipt).ok + || !parseDirectBombadilMatrixSummary(matrixSummary).ok + ) { + throw new Error("Bombadil package evidence parsers rejected exact valid fixtures"); + } + if ( + parseDirectBombadilArtifactReceipt({ ...receipt, extra: true }).ok + || parseDirectBombadilMatrixReceipt({ ...matrixReceipt, schema: "wrong" }).ok + || parseDirectBombadilSanitizedRunSummary({ ...summary, failureCode: "unknown" }).ok + ) { + throw new Error("Bombadil package evidence parsers accepted malformed fixtures"); + } + const uploadLeaf = resolveDirectBombadilUploadLeaf({ + repositoryRoot: "/absolute/repository", + runId, + uploadMode: "public-summary", + }); + if (uploadLeaf !== "/absolute/repository/artifacts/direct-bombadil-upload/" + runId) { + throw new Error("Bombadil upload-leaf resolver returned an unexpected path"); + } type CampaignProperties = DirectBombadilProperties; void (undefined as unknown as CampaignProperties); `); @@ -501,6 +623,10 @@ try { "browser-verification", "@antithesishq/bombadil", "direct.bombadil-run/v1", + "direct.bombadil-artifact-receipt/v1", + "direct.bombadil-upload-summary/v1", + "direct.bombadil-matrix-receipt/v1", + "direct.bombadil-matrix-summary/v1", "bundle-boundary", "node:crypto", "node:fs", diff --git a/skills/direct/references/install.md b/skills/direct/references/install.md index 854b885..1567f3c 100644 --- a/skills/direct/references/install.md +++ b/skills/direct/references/install.md @@ -21,9 +21,9 @@ global `direct` CLI. For a new installation, pin the reviewed public release: ```sh -bun add --dev @hraness/direct@0.7.7 +bun add --dev @hraness/direct@0.7.8 # or, in an npm project -npm install --save-dev @hraness/direct@0.7.7 +npm install --save-dev @hraness/direct@0.7.8 ``` The equivalent manifest entry is: @@ -31,7 +31,7 @@ The equivalent manifest entry is: ```json { "devDependencies": { - "@hraness/direct": "0.7.7" + "@hraness/direct": "0.7.8" } } ``` diff --git a/skills/direct/references/verification.md b/skills/direct/references/verification.md index 7a395ec..7506c91 100644 --- a/skills/direct/references/verification.md +++ b/skills/direct/references/verification.md @@ -41,9 +41,26 @@ consumer. Keep the default browser properties, exported Direct formulas, and conservative Direct action generator in the campaign; keep product-specific actions and assertions local. Random runs must be 12 to 300 seconds. Require the runner's canonical post-run trace attestation even when Bombadil exits -zero, and retain raw trace, process log, server log, and failure artifacts. -Treat the result as diagnostic fuzz evidence, not as a semantic product check -or proof of any replaced system. +zero. Give every product-owned named snapshot an exact fail-closed parser or +type predicate. Treat local random walks as diagnostic exploration, not as the +deterministic simulated workload used by an Antithesis environment and not as +a semantic product check or proof of any replaced system. + +Precompute a lowercase UUID for scheduled runs and pass one exact +`artifactRun` plan. Use `resolveDirectBombadilUploadLeaf`; point `if: always()` +only at that leaf. Public CI may retain only the +bounded sanitized receipt and summary. Raw traces, screenshots, logs, paths, +foreign messages, queries, labels, and typed values require explicit +`private-vetted` access-controlled storage. Keep quotas fail-closed, require +the final descriptor-bound inventory after both process groups settle, and +never upload an artifact root that can sweep another run. +Parse retained JSON from `unknown` with `parseDirectBombadilArtifactReceipt`, +`parseDirectBombadilSanitizedRunSummary`, `parseDirectBombadilMatrixReceipt`, +or `parseDirectBombadilMatrixSummary`; never cast `JSON.parse` output. + +Campaign matrices are public-summary only and publish one atomic parent leaf +after every selected child is terminal. Run one selected campaign directly +when bounded private diagnostics are required. For the agent-browser path, use one task-owned local Chromium session and process for a sequential batch of at most eight scenarios. Before each scenario, call `window new` for a fresh diff --git a/src/exports.test.ts b/src/exports.test.ts index 6672cd7..11a73c8 100644 --- a/src/exports.test.ts +++ b/src/exports.test.ts @@ -7,6 +7,12 @@ import * as testing from "@hraness/direct/testing"; import * as browserVerification from "@hraness/direct/tooling/browser-verification"; import * as bombadil from "@hraness/direct/tooling/bombadil"; import type { DirectBombadilProperties } from "@hraness/direct/tooling/bombadil-campaign"; +import type { + DirectBombadilFuzzMatrixResult, + DirectBombadilFuzzResult, + DirectBombadilFuzzRunInput, + DirectBombadilMatrixRunInput, +} from "@hraness/direct/tooling/bombadil"; import * as bundleBoundary from "@hraness/direct/tooling/bundle-boundary"; import * as web from "@hraness/direct/web"; @@ -60,6 +66,11 @@ describe("public package exports", () => { test("host tooling stays behind explicit subpaths", () => { expect(Object.keys(bombadil).toSorted()).toEqual([ "attestDirectBombadilTrace", + "parseDirectBombadilArtifactReceipt", + "parseDirectBombadilMatrixReceipt", + "parseDirectBombadilMatrixSummary", + "parseDirectBombadilSanitizedRunSummary", + "resolveDirectBombadilUploadLeaf", "runDirectBombadilFuzz", "runDirectBombadilFuzzMatrix", "summarizeDirectBombadilTrace", @@ -69,6 +80,11 @@ describe("public package exports", () => { expect(typeof browserVerification.readDirectBrowserContract).toBe("function"); expect(typeof bombadil.runDirectBombadilFuzz).toBe("function"); expect(typeof bombadil.attestDirectBombadilTrace).toBe("function"); + expect(typeof bombadil.parseDirectBombadilArtifactReceipt).toBe("function"); + expect(typeof bombadil.parseDirectBombadilMatrixReceipt).toBe("function"); + expect(typeof bombadil.parseDirectBombadilMatrixSummary).toBe("function"); + expect(typeof bombadil.parseDirectBombadilSanitizedRunSummary).toBe("function"); + expect(typeof bombadil.resolveDirectBombadilUploadLeaf).toBe("function"); expect(typeof bombadil.runDirectBombadilFuzzMatrix).toBe("function"); expect(typeof bombadil.summarizeDirectBombadilTrace).toBe("function"); expect(typeof bundleBoundary.checkBundleBoundary).toBe("function"); @@ -79,10 +95,53 @@ describe("public package exports", () => { expect("checkBundleBoundary" in testing).toBeFalse(); type PublicRunnerArity = Parameters["length"]; const supportedRunnerArities: readonly PublicRunnerArity[] = [1, 2]; + const supportedRunOptions: DirectBombadilFuzzRunInput = { + arguments: ["--time-limit=12s"], + artifactRun: { + repositoryRoot: "/absolute/repository", + runId: "00000000-0000-4000-8000-000000000001", + uploadMode: "public-summary", + }, + }; + const supportedArgumentTuple = ["--time-limit=12s"] as const; + const supportedTupleInput: DirectBombadilFuzzRunInput = supportedArgumentTuple; + const legacyRunResult: DirectBombadilFuzzResult = { + artifactDirectory: "/absolute/repository/artifacts/direct-bombadil/package/run", + kind: "run", + manifestPath: "/absolute/repository/artifacts/direct-bombadil/package/manifest.json", + status: "passed", + }; + const legacyMatrixResult: DirectBombadilFuzzMatrixResult = { + kind: "matrix", + results: [{ campaignId: "package", result: legacyRunResult }], + }; + const supportedMatrixOptions: DirectBombadilMatrixRunInput = { + arguments: supportedArgumentTuple, + artifactRun: { + repositoryRoot: "/absolute/repository", + runId: "00000000-0000-4000-8000-000000000002", + uploadMode: "public-summary", + }, + }; + const unsupportedPrivateMatrixOptions: DirectBombadilMatrixRunInput = { + artifactRun: { + repositoryRoot: "/absolute/repository", + runId: "00000000-0000-4000-8000-000000000003", + // @ts-expect-error Matrix uploads are always sanitized public summaries. + uploadMode: "private-vetted", + }, + }; // @ts-expect-error Dependency injection stays internal to package tests. const unsupportedRunnerArity: PublicRunnerArity = 3; expect(supportedRunnerArities).toEqual([1, 2]); - void unsupportedRunnerArity; + void [ + supportedMatrixOptions, + legacyMatrixResult, + supportedRunOptions, + supportedTupleInput, + unsupportedPrivateMatrixOptions, + unsupportedRunnerArity, + ]; type CampaignProperties = DirectBombadilProperties; void (undefined as unknown as CampaignProperties); }); diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index 45e668c..c7ea967 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -1,14 +1,33 @@ import { afterEach, describe, expect, test } from "bun:test"; import { defineDirect } from "@hraness/direct"; import { createDirectSession } from "@hraness/direct/testing"; -import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { getEventListeners } from "node:events"; +import { + chmod, + link, + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { attestDirectBombadilTrace, + closeBombadilArtifactCopyHandles, createDirectBombadilInvocation, + inspectBombadilArtifactTreeForTest, + parseDirectBombadilArtifactReceipt, parseDirectBombadilFuzzArguments, + parseDirectBombadilMatrixReceipt, + parseDirectBombadilMatrixSummary, + parseDirectBombadilSanitizedRunSummary, + resolveDirectBombadilUploadLeaf, runBombadilNativeProcess, runDirectBombadilFuzz, runDirectBombadilFuzzMatrix, @@ -16,6 +35,7 @@ import { validateDirectBombadilFuzzConfig, type DirectBombadilFuzzConfig, type DirectBombadilInvocation, + type DirectBombadilMatrixRunInput, type DirectBombadilRunnerDependencies, } from "./bombadil-runner.js"; import type { @@ -25,6 +45,20 @@ import type { const temporaryDirectories: string[] = []; +function artifactRunPlan< + UploadMode extends "private-vetted" | "public-summary" = "public-summary", +>( + repositoryRoot: string, + suffix: number, + uploadMode: UploadMode = "public-summary" as UploadMode, +) { + return { + repositoryRoot, + runId: `00000000-0000-4000-8000-${String(suffix).padStart(12, "0")}`, + uploadMode, + } as const; +} + afterEach(async () => { await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true }) @@ -48,7 +82,9 @@ async function fixture(): Promise<{ readonly config: DirectBombadilFuzzConfig; readonly repositoryRoot: string; }> { - const repositoryRoot = await mkdtemp(join(tmpdir(), "direct-bombadil-runner-")); + const repositoryRoot = await realpath( + await mkdtemp(join(tmpdir(), "direct-bombadil-runner-")), + ); temporaryDirectories.push(repositoryRoot); const productRoot = join(repositoryRoot, "projects", "fixture"); const specificationPath = join(productRoot, "direct", "bombadil-campaign.ts"); @@ -352,7 +388,43 @@ async function rejection(promise: Promise): Promise { throw new Error("Expected the operation to reject"); } +function controllableSignals(): { + readonly controller: DirectBombadilRunnerDependencies["signalController"]; + readonly emit: (signal: NodeJS.Signals) => void; + readonly forwarded: NodeJS.Signals[]; + readonly listenerCount: () => number; +} { + const listeners = new Map void>>(); + const forwarded: NodeJS.Signals[] = []; + return { + controller: { + forward: (signal) => { + forwarded.push(signal); + }, + once: (signal, listener) => { + const signalListeners = listeners.get(signal) ?? new Set(); + signalListeners.add(listener); + listeners.set(signal, signalListeners); + }, + removeListener: (signal, listener) => { + listeners.get(signal)?.delete(listener); + }, + }, + emit: (signal) => { + const signalListeners = [...(listeners.get(signal) ?? [])]; + listeners.delete(signal); + for (const listener of signalListeners) listener(signal); + }, + forwarded, + listenerCount: () => [...listeners.values()].reduce( + (total, signalListeners) => total + signalListeners.size, + 0, + ), + }; +} + function dependencies(options: { + readonly afterTrace?: (invocation: DirectBombadilInvocation) => Promise; readonly exitCode?: number; readonly failAcquire?: boolean; readonly noTrace?: boolean; @@ -410,6 +482,7 @@ function dependencies(options: { options.traceLineOptions, ); } + await options.afterTrace?.(invocation); return { exitCode: options.exitCode ?? 0, stdout: "bombadil stdout", @@ -578,10 +651,18 @@ describe("Direct Bombadil configuration and invocation", () => { ...config, artifactName: "../escape", })).toThrow("artifactName"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + artifactName: "a".repeat(81), + })).toThrow("artifactName"); expect(() => validateDirectBombadilFuzzConfig({ ...config, scenario: "Unsafe Scenario", })).toThrow("scenario"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + scenario: "a".repeat(121), + })).toThrow("scenario"); expect(() => validateDirectBombadilFuzzConfig({ ...config, expectedRoute: "", @@ -788,27 +869,43 @@ describe("Direct Bombadil configuration and invocation", () => { describe("Direct Bombadil campaign matrix", () => { test("runs unique bounded campaigns serially and selects exactly one", async () => { - const { config } = await fixture(); + const { config, repositoryRoot } = await fixture(); const campaigns = [{ id: "primary", config }, { id: "secondary", config: { ...config, artifactName: "fixture-secondary" }, }] as const; const allRuntime = dependencies(); + const allSignals = controllableSignals(); + const matrixController = new AbortController(); + let controllerCount = 0; const all = await runDirectBombadilFuzzMatrix( campaigns, ["--time-limit=12s"], - allRuntime.overrides, + { + ...allRuntime.overrides, + createAbortController: () => { + controllerCount += 1; + return controllerCount === 1 ? matrixController : new AbortController(); + }, + signalController: allSignals.controller, + }, ); expect(all).toMatchObject({ kind: "matrix", results: [{ campaignId: "primary" }, { campaignId: "secondary" }], }); expect(allRuntime.calls.filter((call) => call === "run-bombadil")).toHaveLength(2); + expect(allSignals.listenerCount()).toBe(0); + expect(getEventListeners(matrixController.signal, "abort")).toHaveLength(0); const selectedRuntime = dependencies(); + const selectedPlan = artifactRunPlan(repositoryRoot, 20); const selected = await runDirectBombadilFuzzMatrix( campaigns, - ["--campaign=secondary", "--time-limit=12s"], + { + arguments: ["--campaign=secondary", "--time-limit=12s"], + artifactRun: selectedPlan, + }, selectedRuntime.overrides, ); expect(selected).toMatchObject({ @@ -816,6 +913,16 @@ describe("Direct Bombadil campaign matrix", () => { results: [{ campaignId: "secondary" }], }); expect(selectedRuntime.calls.filter((call) => call === "run-bombadil")).toHaveLength(1); + expect(JSON.parse(await readFile(selected.receiptPath, "utf8"))).toMatchObject({ + campaigns: [ + { campaignId: "primary", receipt: null, status: "not-selected" }, + { + campaignId: "secondary", + receipt: "campaigns/secondary/receipt.json", + status: "passed", + }, + ], + }); }); test("rejects ambiguous replay, duplicate IDs, and unknown selection", async () => { @@ -834,6 +941,521 @@ describe("Direct Bombadil campaign matrix", () => { { id: "same", config: { ...config, artifactName: "other" } }, ], []))).message).toContain("unique lowercase kebab"); }); + + test("publishes rejected and partially executed matrix terminal states", async () => { + const { config, repositoryRoot } = await fixture(); + const duplicatePlan = artifactRunPlan(repositoryRoot, 21); + await rejection(runDirectBombadilFuzzMatrix([ + { id: "same", config }, + { id: "same", config: { ...config, artifactName: "other" } }, + ], { arguments: [], artifactRun: duplicatePlan })); + const duplicateReceipt = JSON.parse(await readFile(join( + repositoryRoot, + "artifacts", + "direct-bombadil-upload", + duplicatePlan.runId, + "receipt.json", + ), "utf8")) as Record; + expect(duplicateReceipt).toMatchObject({ + schema: "direct.bombadil-matrix-receipt/v1", + failureCode: "configuration-rejected", + status: "failed", + campaigns: [{ status: "rejected" }, { status: "rejected" }], + }); + + const runtime = dependencies(); + const baseRunBombadil = runtime.overrides.runBombadil; + if (baseRunBombadil === undefined) throw new Error("Expected fixture Bombadil dependency"); + let invocationCount = 0; + const partialPlan = artifactRunPlan(repositoryRoot, 22); + await rejection(runDirectBombadilFuzzMatrix([ + { id: "first", config }, + { id: "second", config: { ...config, artifactName: "fixture-second" } }, + { id: "third", config: { ...config, artifactName: "fixture-third" } }, + ], { arguments: [], artifactRun: partialPlan }, { + ...runtime.overrides, + runBombadil: async (invocation) => { + invocationCount += 1; + const result = await baseRunBombadil(invocation); + return invocationCount === 2 ? { ...result, exitCode: 9 } : result; + }, + })); + expect(invocationCount).toBe(2); + const partialReceipt = JSON.parse(await readFile(join( + repositoryRoot, + "artifacts", + "direct-bombadil-upload", + partialPlan.runId, + "receipt.json", + ), "utf8")) as Record; + expect(partialReceipt).toMatchObject({ + schema: "direct.bombadil-matrix-receipt/v1", + status: "failed", + campaigns: [ + { campaignId: "first", status: "passed" }, + { campaignId: "second", status: "failed" }, + { campaignId: "third", status: "not-run" }, + ], + }); + }); + + test("bounds rejected matrices and rejects private matrix uploads with a public receipt", async () => { + const { config, repositoryRoot } = await fixture(); + const campaigns = Array.from({ length: 34 }, (_, index) => ({ + config, + id: `campaign-${String(index)}`, + })); + const oversizedPlan = artifactRunPlan(repositoryRoot, 23); + await rejection(runDirectBombadilFuzzMatrix(campaigns, { + arguments: [], + artifactRun: oversizedPlan, + })); + const oversizedReceipt = JSON.parse(await readFile(join( + repositoryRoot, + "artifacts", + "direct-bombadil-upload", + oversizedPlan.runId, + "receipt.json", + ), "utf8")) as Record; + expect(oversizedReceipt).toMatchObject({ + mode: "public-summary", + omittedCampaignCount: 2, + status: "failed", + }); + expect(oversizedReceipt.campaigns).toHaveLength(32); + + const privatePlan = artifactRunPlan(repositoryRoot, 24, "private-vetted"); + const privateInput = { + arguments: [], + artifactRun: privatePlan, + } as unknown as DirectBombadilMatrixRunInput; + const privateError = await rejection(runDirectBombadilFuzzMatrix( + [{ id: "primary", config }], + privateInput, + )); + expect(privateError.message).toContain("public-summary"); + const privateReceipt = JSON.parse(await readFile(join( + repositoryRoot, + "artifacts", + "direct-bombadil-upload", + privatePlan.runId, + "receipt.json", + ), "utf8")) as Record; + expect(privateReceipt).toMatchObject({ + failureCode: "configuration-rejected", + mode: "public-summary", + status: "failed", + }); + + const longIdPlan = artifactRunPlan(repositoryRoot, 25); + await rejection(runDirectBombadilFuzzMatrix([ + { id: "a".repeat(81), config }, + ], { arguments: [], artifactRun: longIdPlan })); + expect(JSON.parse(await readFile(join( + repositoryRoot, + "artifacts", + "direct-bombadil-upload", + longIdPlan.runId, + "receipt.json", + ), "utf8"))).toMatchObject({ + campaigns: [{ campaignId: null, status: "rejected" }], + }); + }); + + test("publishes one interrupted matrix leaf before forwarding its signal", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies(); + const baseRunBombadil = runtime.overrides.runBombadil; + if (baseRunBombadil === undefined) throw new Error("Expected fixture Bombadil dependency"); + const signals = controllableSignals(); + const plan = artifactRunPlan(repositoryRoot, 26); + const error = await rejection(runDirectBombadilFuzzMatrix( + [{ id: "primary", config }, { + id: "secondary", + config: { ...config, artifactName: "fixture-secondary" }, + }], + { arguments: [], artifactRun: plan }, + { + ...runtime.overrides, + runBombadil: async (invocation) => { + const result = await baseRunBombadil(invocation); + signals.emit("SIGTERM"); + return result; + }, + signalController: signals.controller, + }, + )); + expect(error.message).toContain("SIGTERM"); + expect(signals.forwarded).toEqual(["SIGTERM"]); + expect(signals.listenerCount()).toBe(0); + expect(JSON.parse(await readFile(join( + repositoryRoot, + "artifacts", + "direct-bombadil-upload", + plan.runId, + "receipt.json", + ), "utf8"))).toMatchObject({ + failureCode: "interrupted", + campaigns: [ + { campaignId: "primary", status: "failed" }, + { campaignId: "secondary", status: "not-run" }, + ], + status: "failed", + }); + }); + + test("preserves a child configuration rejection in the parent receipt", async () => { + const { config, repositoryRoot } = await fixture(); + const mutableConfig = { ...config }; + let controllerCount = 0; + const plan = artifactRunPlan(repositoryRoot, 27); + await rejection(runDirectBombadilFuzzMatrix( + [{ id: "primary", config: mutableConfig }, { + id: "secondary", + config: { ...config, artifactName: "fixture-secondary" }, + }], + { arguments: [], artifactRun: plan }, + { + ...dependencies().overrides, + createAbortController: () => { + controllerCount += 1; + if (controllerCount === 2) mutableConfig.artifactName = "../rejected"; + return new AbortController(); + }, + }, + )); + expect(JSON.parse(await readFile(join( + resolveDirectBombadilUploadLeaf(plan), + "receipt.json", + ), "utf8"))).toMatchObject({ + failureCode: "configuration-rejected", + campaigns: [ + { campaignId: "primary", status: "rejected" }, + { campaignId: "secondary", status: "not-run" }, + ], + status: "failed", + }); + expect(JSON.parse(await readFile(join( + resolveDirectBombadilUploadLeaf(plan), + "campaigns", + "primary", + "receipt.json", + ), "utf8"))).toMatchObject({ + failureCode: "configuration-rejected", + status: "rejected", + }); + expect(await readFile(join( + resolveDirectBombadilUploadLeaf(plan), + "campaigns", + "primary", + "summary.json", + ), "utf8")).toContain("direct.bombadil-upload-summary/v1"); + }); + + test("interrupts a child before acquisition and leaves no signal listeners", async () => { + const { config, repositoryRoot } = await fixture(); + const signals = controllableSignals(); + const runtime = dependencies(); + const plan = artifactRunPlan(repositoryRoot, 28); + let runIdCount = 0; + await rejection(runDirectBombadilFuzzMatrix( + [{ id: "primary", config }, { + id: "secondary", + config: { ...config, artifactName: "fixture-secondary" }, + }], + { arguments: [], artifactRun: plan }, + { + ...runtime.overrides, + createRunId: () => { + runIdCount += 1; + if (runIdCount === 1) signals.emit("SIGTERM"); + return `10000000-0000-4000-8000-${String(runIdCount).padStart(12, "0")}`; + }, + signalController: signals.controller, + }, + )); + expect(runtime.calls).toEqual([]); + expect(signals.forwarded).toEqual(["SIGTERM"]); + expect(signals.listenerCount()).toBe(0); + expect(JSON.parse(await readFile(join( + resolveDirectBombadilUploadLeaf(plan), + "receipt.json", + ), "utf8"))).toMatchObject({ + failureCode: "interrupted", + campaigns: [ + { campaignId: "primary", status: "failed" }, + { campaignId: "secondary", status: "not-run" }, + ], + }); + }); + + test("converts a parent-publication interruption and releases child abort listeners", async () => { + const { config, repositoryRoot } = await fixture(); + const signals = controllableSignals(); + const matrixController = new AbortController(); + let controllerCount = 0; + let commitCount = 0; + const plan = artifactRunPlan(repositoryRoot, 29); + const error = await rejection(runDirectBombadilFuzzMatrix( + [{ id: "primary", config }, { + id: "secondary", + config: { ...config, artifactName: "fixture-secondary" }, + }], + { arguments: [], artifactRun: plan }, + { + ...dependencies().overrides, + beforeArtifactCommit: () => { + commitCount += 1; + signals.emit("SIGTERM"); + }, + createAbortController: () => { + controllerCount += 1; + return controllerCount === 1 ? matrixController : new AbortController(); + }, + signalController: signals.controller, + }, + )); + expect(error.message).toContain("SIGTERM"); + expect(commitCount).toBe(1); + expect(signals.forwarded).toEqual(["SIGTERM"]); + expect(signals.listenerCount()).toBe(0); + expect(getEventListeners(matrixController.signal, "abort")).toHaveLength(0); + expect(JSON.parse(await readFile(join( + resolveDirectBombadilUploadLeaf(plan), + "receipt.json", + ), "utf8"))).toMatchObject({ + failureCode: "interrupted", + campaigns: [{ status: "passed" }, { status: "passed" }], + status: "failed", + }); + }); + + test("removes a failed matrix publication staging leaf", async () => { + const { config, repositoryRoot } = await fixture(); + const plan = artifactRunPlan(repositoryRoot, 42); + const error = await rejection(runDirectBombadilFuzzMatrix( + [{ id: "primary", config }], + { arguments: [], artifactRun: plan }, + { + ...dependencies().overrides, + beforeArtifactCommit: () => { + throw new Error("matrix precommit rejected"); + }, + }, + )); + expect(error.message).toContain("matrix precommit rejected"); + expect(await readdir(dirname(resolveDirectBombadilUploadLeaf(plan)))).toEqual([]); + }); +}); + +describe("Direct Bombadil sanitized evidence contracts", () => { + test("closes the source descriptor even when destination cleanup fails", async () => { + const calls: string[] = []; + const destinationError = new Error("destination close failed"); + const error = await rejection(closeBombadilArtifactCopyHandles( + { + close: async () => { + calls.push("destination"); + throw destinationError; + }, + }, + { + close: async () => { + calls.push("source"); + }, + }, + )); + expect(error).toBe(destinationError); + expect(calls).toEqual(["destination", "source"]); + + const both = await rejection(closeBombadilArtifactCopyHandles( + { close: async () => { throw new Error("destination"); } }, + { close: async () => { throw new Error("source"); } }, + )); + expect(both).toBeInstanceOf(AggregateError); + expect((both as AggregateError).errors).toHaveLength(2); + }); + + test("round-trips all four emitted evidence files and resolves the exact upload leaf", async () => { + const { config, repositoryRoot } = await fixture(); + const runPlan = artifactRunPlan(repositoryRoot, 31); + const run = await runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: runPlan, + }, dependencies().overrides); + if (run.kind !== "run") throw new Error("Expected a run result"); + expect(run.uploadArtifactPath).toBe(resolveDirectBombadilUploadLeaf(runPlan)); + const runReceipt = parseDirectBombadilArtifactReceipt(JSON.parse(await readFile( + join(run.uploadArtifactPath, "receipt.json"), + "utf8", + ))); + const runSummary = parseDirectBombadilSanitizedRunSummary(JSON.parse(await readFile( + join(run.uploadArtifactPath, "summary.json"), + "utf8", + ))); + + const matrixPlan = artifactRunPlan(repositoryRoot, 32); + const matrix = await runDirectBombadilFuzzMatrix( + [{ id: "primary", config }], + { arguments: [], artifactRun: matrixPlan }, + dependencies().overrides, + ); + if (matrix.kind !== "matrix") throw new Error("Expected a matrix result"); + const matrixReceipt = parseDirectBombadilMatrixReceipt(JSON.parse(await readFile( + join(matrix.uploadArtifactPath, "receipt.json"), + "utf8", + ))); + const matrixSummary = parseDirectBombadilMatrixSummary(JSON.parse(await readFile( + join(matrix.uploadArtifactPath, "summary.json"), + "utf8", + ))); + for (const parsed of [runReceipt, runSummary, matrixReceipt, matrixSummary]) { + expect(parsed.ok).toBeTrue(); + if (parsed.ok) expect(Object.isFrozen(parsed.value)).toBeTrue(); + } + if (!runReceipt.ok || !matrixReceipt.ok || !matrixSummary.ok) { + throw new Error("Expected parsed Bombadil evidence"); + } + expect(Object.isFrozen(runReceipt.value.inventory)).toBeTrue(); + expect(Object.isFrozen(runReceipt.value.policy)).toBeTrue(); + expect(Object.isFrozen(matrixReceipt.value.campaigns)).toBeTrue(); + expect(Object.isFrozen(matrixSummary.value.campaigns)).toBeTrue(); + expect(resolveDirectBombadilUploadLeaf({ + repositoryRoot, + runId: runPlan.runId, + })).toBe(run.uploadArtifactPath); + expect(resolveDirectBombadilUploadLeaf({ + ...runPlan, + uploadMode: "private-vetted", + })).toBe(run.uploadArtifactPath); + expect(() => resolveDirectBombadilUploadLeaf({ + ...runPlan, + repositoryRoot: `${repositoryRoot}/../invalid`, + })).toThrow("absolute normalized path"); + expect(() => resolveDirectBombadilUploadLeaf({ + ...runPlan, + runId: "not-a-uuid", + })).toThrow("lowercase RFC 4122 UUID"); + expect(() => resolveDirectBombadilUploadLeaf({ + ...runPlan, + uploadMode: "invalid" as "public-summary", + })).toThrow("public-summary or private-vetted"); + }); + + test("rejects hostile values, exact-key tampering, and impossible terminal states", async () => { + const { config, repositoryRoot } = await fixture(); + const runPlan = artifactRunPlan(repositoryRoot, 33); + const run = await runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: runPlan, + }, dependencies().overrides); + if (run.kind !== "run") throw new Error("Expected a run result"); + const receipt = record(JSON.parse(await readFile( + join(run.uploadArtifactPath, "receipt.json"), + "utf8", + )), "run receipt"); + const summary = record(JSON.parse(await readFile( + join(run.uploadArtifactPath, "summary.json"), + "utf8", + )), "run summary"); + expect(parseDirectBombadilArtifactReceipt({ ...receipt, extra: true }).ok).toBeFalse(); + expect(parseDirectBombadilArtifactReceipt({ ...receipt, schema: "wrong" }).ok).toBeFalse(); + expect(parseDirectBombadilArtifactReceipt({ + ...receipt, + inventory: { + ...record(receipt.inventory, "run receipt inventory"), + fileCount: 0, + }, + }).ok).toBeFalse(); + expect(parseDirectBombadilArtifactReceipt({ + ...receipt, + inventory: { + entryCount: 0, + fileCount: 0, + inventorySha256: null, + totalBytes: 0, + }, + }).ok).toBeFalse(); + expect(parseDirectBombadilArtifactReceipt({ + ...receipt, + diagnosticsRetained: true, + failureCode: "interrupted", + mode: "private-vetted", + status: "failed", + }).ok).toBeFalse(); + expect(parseDirectBombadilArtifactReceipt({ + ...receipt, + failureCode: "writer-settlement", + status: "failed", + }).ok).toBeFalse(); + const accessorReceipt = Object.defineProperty({ ...receipt }, "status", { + enumerable: true, + get: () => "passed", + }); + expect(parseDirectBombadilArtifactReceipt(accessorReceipt).ok).toBeFalse(); + expect(parseDirectBombadilArtifactReceipt(new Proxy(receipt, { + ownKeys: () => { + throw new Error("hostile proxy"); + }, + })).ok).toBeFalse(); + expect(parseDirectBombadilSanitizedRunSummary({ + ...summary, + attestation: null, + }).ok).toBeFalse(); + const failedSummary = { ...summary, failureCode: "unknown", status: "failed" }; + expect(parseDirectBombadilSanitizedRunSummary({ + ...failedSummary, + attestation: { + invalidObservationCount: 0, + observationCount: 0, + validObservationCount: 0, + }, + }).ok).toBeFalse(); + expect(parseDirectBombadilSanitizedRunSummary({ + ...summary, + failureCode: "writer-settlement", + status: "failed", + }).ok).toBeFalse(); + expect(parseDirectBombadilSanitizedRunSummary({ + ...failedSummary, + exploration: { + ...record(summary.exploration, "run summary exploration"), + traceLineCount: 1, + }, + }).ok).toBeFalse(); + + const matrixPlan = artifactRunPlan(repositoryRoot, 34); + const matrix = await runDirectBombadilFuzzMatrix( + [{ id: "primary", config }], + { arguments: [], artifactRun: matrixPlan }, + dependencies().overrides, + ); + if (matrix.kind !== "matrix") throw new Error("Expected a matrix result"); + const matrixReceipt = record(JSON.parse(await readFile( + join(matrix.uploadArtifactPath, "receipt.json"), + "utf8", + )), "matrix receipt"); + const matrixSummary = record(JSON.parse(await readFile( + join(matrix.uploadArtifactPath, "summary.json"), + "utf8", + )), "matrix summary"); + const campaigns = matrixReceipt.campaigns; + if (!Array.isArray(campaigns)) throw new Error("Expected matrix campaigns"); + expect(parseDirectBombadilMatrixReceipt({ + ...matrixReceipt, + campaigns: campaigns.map((campaign) => ({ + ...record(campaign, "matrix campaign"), + receipt: "campaigns/primary/other.json", + })), + }).ok).toBeFalse(); + expect(parseDirectBombadilMatrixSummary({ + ...matrixSummary, + campaigns: { + ...record(matrixSummary.campaigns, "matrix summary campaigns"), + omitted: 1, + }, + }).ok).toBeFalse(); + }); }); describe("Direct Bombadil trace attestation", () => { @@ -1589,6 +2211,151 @@ describe("Direct Bombadil exploration summary", () => { }); describe("Direct Bombadil process lifecycle", () => { + test("tolerates only live-scan entry disappearance and fails final proof closed", async () => { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-entry-race-")); + temporaryDirectories.push(directory); + const transientPath = join(directory, "transient.log"); + const policy = { + maxDepth: 4, + maxEntries: 8, + maxFileBytes: 1_024, + maxFiles: 4, + maxPathBytes: 256, + maxTotalBytes: 2_048, + }; + await writeFile(transientPath, "transient\n"); + const transient = await rejection(inspectBombadilArtifactTreeForTest({ + allowTransientEntryAbsence: true, + beforeEntryInspect: async (path) => { + await rm(path); + }, + hashFiles: false, + policy, + root: directory, + })); + expect((transient as NodeJS.ErrnoException).code).toBe("ENOENT"); + + await writeFile(transientPath, "final\n"); + const final = await rejection(inspectBombadilArtifactTreeForTest({ + beforeEntryInspect: async (path) => { + await rm(path); + }, + hashFiles: true, + policy, + root: directory, + })); + expect(final.name).toBe("BombadilArtifactPolicyError"); + expect(final.message).toContain("could not be inspected safely"); + + const nested = join(directory, "nested"); + await mkdir(nested); + await writeFile(join(nested, "trace.log"), "transient\n"); + const nestedTransient = await rejection(inspectBombadilArtifactTreeForTest({ + allowTransientEntryAbsence: true, + beforeDirectoryOpen: async (path) => { + if (path === nested) await rm(path, { recursive: true }); + }, + hashFiles: false, + policy, + root: directory, + })); + expect((nestedTransient as NodeJS.ErrnoException).code).toBe("ENOENT"); + + await mkdir(nested); + await writeFile(join(nested, "trace.log"), "final\n"); + const nestedFinal = await rejection(inspectBombadilArtifactTreeForTest({ + beforeDirectoryOpen: async (path) => { + if (path === nested) await rm(path, { recursive: true }); + }, + hashFiles: true, + policy, + root: directory, + })); + expect(nestedFinal.name).toBe("BombadilArtifactPolicyError"); + expect(nestedFinal.message).toContain("could not be opened safely"); + }); + + test("omits the upload coordination UUID from the native process environment", async () => { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-native-environment-")); + temporaryDirectories.push(directory); + const previous = process.env.DIRECT_BOMBADIL_RUN_ID; + process.env.DIRECT_BOMBADIL_RUN_ID = "child-visible-secret"; + const running = runBombadilNativeProcess({ + command: [ + process.execPath, + "-e", + "console.log(process.env.DIRECT_BOMBADIL_RUN_ID ?? 'absent')", + ], + cwd: directory, + outputPath: directory, + targetUrl: "http://127.0.0.1:4919/", + wallClockTimeoutMs: 5_000, + }); + if (previous === undefined) delete process.env.DIRECT_BOMBADIL_RUN_ID; + else process.env.DIRECT_BOMBADIL_RUN_ID = previous; + const result = await running; + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe("absent"); + }); + + test("aborts the owned process group when live artifacts exceed quota", async () => { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-artifact-quota-")); + temporaryDirectories.push(directory); + const overflowPath = join(directory, "overflow.log"); + const source = [ + "const fs = require('node:fs');", + `fs.writeFileSync(${JSON.stringify(overflowPath)}, Buffer.alloc(4096));`, + "process.on('SIGTERM', () => {});", + "setInterval(() => {}, 1000);", + ].join(" "); + const startedAt = Date.now(); + const error = await rejection(runBombadilNativeProcess({ + artifactPolicy: { + maxDepth: 4, + maxEntries: 8, + maxFileBytes: 1_024, + maxFiles: 4, + maxPathBytes: 256, + maxTotalBytes: 2_048, + }, + command: [process.execPath, "-e", source], + cwd: directory, + outputPath: directory, + targetUrl: "http://127.0.0.1:4919/", + terminationGraceMs: 50, + wallClockTimeoutMs: 5_000, + })); + expect(error.message).toContain("per-file byte quota"); + expect(Date.now() - startedAt).toBeLessThan(2_000); + }); + + test("promotes an artifact-policy result that races a clean process exit", async () => { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-artifact-exit-race-")); + temporaryDirectories.push(directory); + const overflowPath = join(directory, "overflow.log"); + const error = await rejection(runBombadilNativeProcess({ + artifactPolicy: { + maxDepth: 4, + maxEntries: 8, + maxFileBytes: 1_024, + maxFiles: 4, + maxPathBytes: 256, + maxTotalBytes: 2_048, + }, + command: [ + process.execPath, + "-e", + `require("node:fs").writeFileSync(${JSON.stringify(overflowPath)}, Buffer.alloc(4096));`, + ], + cwd: directory, + outputPath: directory, + targetUrl: "http://127.0.0.1:4919/", + terminationGraceMs: 50, + wallClockTimeoutMs: 5_000, + })); + expect(error.message).toContain("per-file byte quota"); + }); + test("cleans descendants and inherited pipes after a normal leader exit", async () => { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-normal-exit-")); temporaryDirectories.push(directory); @@ -1660,7 +2427,7 @@ describe("Direct Bombadil process lifecycle", () => { expect(Date.now() - startedAt).toBeLessThan(2_000); }); - test("aborts and escalates an uncooperative native child promptly", async () => { + test("kills an uncooperative native child immediately on abort", async () => { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-abort-")); temporaryDirectories.push(directory); const controller = new AbortController(); @@ -1683,6 +2450,58 @@ describe("Direct Bombadil process lifecycle", () => { expect(result.stdout).toContain("abort output"); expect(Date.now() - startedAt).toBeLessThan(2_000); }); + + test("gives an aborted artifact writer no quota-growing TERM grace", async () => { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-abort-quota-")); + temporaryDirectories.push(directory); + const growingPath = join(directory, "growing.log"); + const controller = new AbortController(); + const source = [ + "const fs = require('node:fs');", + `const path = ${JSON.stringify(growingPath)};`, + "process.on('SIGTERM', () => fs.appendFileSync(path, Buffer.alloc(8192)));", + "fs.writeFileSync(path, Buffer.alloc(256));", + "setInterval(() => {}, 1000);", + ].join(" "); + const running = runBombadilNativeProcess({ + abortSignal: controller.signal, + artifactPolicy: { + maxDepth: 4, + maxEntries: 8, + maxFileBytes: 4_096, + maxFiles: 4, + maxPathBytes: 256, + maxTotalBytes: 8_192, + }, + command: [process.execPath, "-e", source], + cwd: directory, + outputPath: directory, + targetUrl: "http://127.0.0.1:4919/", + terminationGraceMs: 2_000, + wallClockTimeoutMs: 5_000, + }); + let ready = false; + for (let attempt = 0; attempt < 100; attempt += 1) { + try { + await readFile(growingPath); + ready = true; + break; + } catch { + await Bun.sleep(10); + } + } + if (!ready) { + controller.abort(); + await running; + throw new Error("Bombadil writer did not publish its readiness file"); + } + const startedAt = Date.now(); + controller.abort(); + const result = await running; + expect(result.termination).toBe("aborted"); + expect((await readFile(growingPath)).byteLength).toBeLessThanOrEqual(4_096); + expect(Date.now() - startedAt).toBeLessThan(1_500); + }); }); describe("Direct Bombadil run lifecycle", () => { @@ -1741,6 +2560,190 @@ describe("Direct Bombadil run lifecycle", () => { schema: "direct.bombadil-exploration-summary/v2", trace: { lineCount: 2 }, }); + if (result.kind !== "run") throw new Error("Expected a Bombadil run result"); + expect((await readdir(result.uploadArtifactPath)).sort()).toEqual([ + "receipt.json", + "summary.json", + ]); + expect(JSON.parse(await readFile(result.receiptPath, "utf8"))).toMatchObject({ + schema: "direct.bombadil-artifact-receipt/v1", + failureCode: null, + mode: "public-summary", + status: "passed", + inventory: { fileCount: 1 }, + }); + }); + + test("publishes only sanitized files publicly and descriptor-vetted files privately", async () => { + const { config, repositoryRoot } = await fixture(); + const sentinel = "secret-query-and-log-sentinel"; + const publicRuntime = dependencies(); + const publicResult = await runDirectBombadilFuzz({ + ...config, + targetQuery: { token: sentinel }, + }, { + arguments: [], + artifactRun: artifactRunPlan(repositoryRoot, 11), + }, publicRuntime.overrides); + if (publicResult.kind !== "run") throw new Error("Expected a public Bombadil run result"); + const publicPayload = (await Promise.all((await readdir(publicResult.uploadArtifactPath)).map( + async (name) => await readFile(join(publicResult.uploadArtifactPath, name), "utf8"), + ))).join("\n"); + expect(publicPayload).not.toContain(sentinel); + expect(publicPayload).not.toContain(repositoryRoot); + expect(publicPayload).not.toContain("bombadil stdout"); + expect(JSON.parse(await readFile(publicResult.receiptPath, "utf8"))).toMatchObject({ + diagnosticsRetained: false, + mode: "public-summary", + }); + + const privateRuntime = dependencies(); + const privateResult = await runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: artifactRunPlan(repositoryRoot, 12, "private-vetted"), + }, privateRuntime.overrides); + if (privateResult.kind !== "run") throw new Error("Expected a private Bombadil run result"); + expect((await readdir(privateResult.uploadArtifactPath)).sort()).toEqual([ + "diagnostics", + "receipt.json", + "summary.json", + ]); + expect(await readFile( + join(privateResult.uploadArtifactPath, "diagnostics", "bombadil-output", "trace.jsonl"), + "utf8", + )).toContain('"name":"direct"'); + expect(await readFile( + join(privateResult.uploadArtifactPath, "diagnostics", "host", "bombadil.log"), + "utf8", + )).toContain("bombadil stdout"); + expect(JSON.parse(await readFile(privateResult.receiptPath, "utf8"))).toMatchObject({ + diagnosticsRetained: true, + mode: "private-vetted", + }); + }); + + test("rejects symlink artifacts and publishes a receipt without raw diagnostics", async () => { + const { config, repositoryRoot } = await fixture(); + const outside = join(repositoryRoot, "outside.txt"); + await writeFile(outside, "do not copy\n"); + const runtime = dependencies({ + afterTrace: async (invocation) => { + await symlink(outside, join(invocation.outputPath, "escaped.txt")); + }, + }); + const plan = artifactRunPlan(repositoryRoot, 13, "private-vetted"); + const error = await rejection(runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: plan, + }, runtime.overrides)); + expect(error.message).toContain("symbolic link"); + const upload = join(repositoryRoot, "artifacts", "direct-bombadil-upload", plan.runId); + expect((await readdir(upload)).sort()).toEqual(["receipt.json", "summary.json"]); + expect(JSON.parse(await readFile(join(upload, "receipt.json"), "utf8"))).toMatchObject({ + diagnosticsRetained: false, + failureCode: "artifact-policy", + status: "failed", + }); + }); + + test("rejects multiply-linked artifacts before private copying", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies({ + afterTrace: async (invocation) => { + await link( + join(invocation.outputPath, "trace.jsonl"), + join(invocation.outputPath, "duplicate.jsonl"), + ); + }, + }); + const plan = artifactRunPlan(repositoryRoot, 15, "private-vetted"); + const error = await rejection(runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: plan, + }, runtime.overrides)); + expect(error.message).toContain("multiply-linked"); + const upload = join(repositoryRoot, "artifacts", "direct-bombadil-upload", plan.runId); + expect((await readdir(upload)).sort()).toEqual(["receipt.json", "summary.json"]); + }); + + test("includes tagged empty directories in the authoritative inventory hash", async () => { + const { config, repositoryRoot } = await fixture(); + const baselinePlan = artifactRunPlan(repositoryRoot, 16); + await runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: baselinePlan, + }, dependencies().overrides); + const baselineReceipt = record(JSON.parse(await readFile(join( + repositoryRoot, + "artifacts", + "direct-bombadil-upload", + baselinePlan.runId, + "receipt.json", + ), "utf8")), "baseline receipt"); + + const emptyDirectoryPlan = artifactRunPlan(repositoryRoot, 17); + await runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: emptyDirectoryPlan, + }, dependencies({ + afterTrace: async (invocation) => { + await mkdir(join(invocation.outputPath, "empty")); + }, + }).overrides); + const emptyDirectoryReceipt = record(JSON.parse(await readFile(join( + repositoryRoot, + "artifacts", + "direct-bombadil-upload", + emptyDirectoryPlan.runId, + "receipt.json", + ), "utf8")), "empty-directory receipt"); + const baselineInventory = record(baselineReceipt.inventory, "baseline inventory"); + const emptyDirectoryInventory = record( + emptyDirectoryReceipt.inventory, + "empty-directory inventory", + ); + expect(emptyDirectoryInventory).toMatchObject({ entryCount: 2, fileCount: 1 }); + expect(emptyDirectoryInventory.inventorySha256).not.toBe( + baselineInventory.inventorySha256, + ); + }); + + test("preserves the primary failure when sanitized receipt publication also fails", async () => { + const { config, repositoryRoot } = await fixture(); + const plan = artifactRunPlan(repositoryRoot, 18); + await mkdir(join( + repositoryRoot, + "artifacts", + "direct-bombadil-upload", + `.staging-${plan.runId}`, + ), { recursive: true }); + const error = await rejection(runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: plan, + }, dependencies({ exitCode: 9 }).overrides)); + expect(error).toBeInstanceOf(AggregateError); + expect(error.message).toContain("exited with status 9"); + expect(error.message).toContain("receipt publication also failed"); + }); + + test("publishes a sanitized rejection receipt before invalid configuration can spawn", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies(); + const plan = artifactRunPlan(repositoryRoot, 14); + const error = await rejection(runDirectBombadilFuzz({ + ...config, + artifactName: "../escape", + targetQuery: { token: "configuration-secret" }, + }, { arguments: [], artifactRun: plan }, runtime.overrides)); + expect(error.message).toContain("artifactName"); + expect(runtime.calls).toEqual([]); + const upload = join(repositoryRoot, "artifacts", "direct-bombadil-upload", plan.runId); + const payload = await readFile(join(upload, "receipt.json"), "utf8"); + expect(payload).not.toContain("configuration-secret"); + expect(JSON.parse(payload)).toMatchObject({ + failureCode: "configuration-rejected", + status: "rejected", + }); }); test("runs with policy-owned evidence despite arbitrary unrelated named snapshots", async () => { @@ -1777,6 +2780,111 @@ describe("Direct Bombadil run lifecycle", () => { }); }); + test("publishes an interrupted receipt when a signal wins during preflight", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies(); + const signals = controllableSignals(); + const runId = "00000000-0000-4000-8000-000000000035"; + let runIdCount = 0; + const error = await rejection(runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createRunId: () => { + runIdCount += 1; + signals.emit("SIGTERM"); + return runId; + }, + signalController: signals.controller, + })); + expect(error.message).toContain("interrupted"); + expect(runtime.calls).toEqual([]); + expect(signals.forwarded).toEqual(["SIGTERM"]); + expect(signals.listenerCount()).toBe(0); + expect(runIdCount).toBe(1); + expect(JSON.parse(await readFile(join( + resolveDirectBombadilUploadLeaf({ repositoryRoot, runId }), + "receipt.json", + ), "utf8"))).toMatchObject({ + diagnosticsRetained: false, + failureCode: "interrupted", + status: "failed", + }); + }); + + test("publishes an interrupted receipt when a signal wins during acquisition", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies(); + const signals = controllableSignals(); + const plan = artifactRunPlan(repositoryRoot, 36); + await rejection(runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: plan, + }, { + ...runtime.overrides, + acquireServer: async () => { + runtime.calls.push("acquire-server"); + signals.emit("SIGINT"); + throw new Error("acquisition interrupted"); + }, + signalController: signals.controller, + })); + expect(runtime.calls).toEqual(["acquire-server"]); + expect(signals.forwarded).toEqual(["SIGINT"]); + expect(signals.listenerCount()).toBe(0); + expect(JSON.parse(await readFile(join( + resolveDirectBombadilUploadLeaf(plan), + "receipt.json", + ), "utf8"))).toMatchObject({ + failureCode: "interrupted", + status: "failed", + }); + }); + + test("converts a pre-commit signal into one interrupted immutable leaf", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies(); + const signals = controllableSignals(); + const plan = artifactRunPlan(repositoryRoot, 37, "private-vetted"); + let commitCount = 0; + const error = await rejection(runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: plan, + }, { + ...runtime.overrides, + beforeArtifactCommit: () => { + commitCount += 1; + signals.emit("SIGTERM"); + }, + signalController: signals.controller, + })); + expect(error.message).toContain("SIGTERM"); + expect(commitCount).toBe(1); + expect(signals.forwarded).toEqual(["SIGTERM"]); + expect(signals.listenerCount()).toBe(0); + const upload = resolveDirectBombadilUploadLeaf(plan); + expect((await readdir(upload)).sort()).toEqual(["receipt.json", "summary.json"]); + expect(JSON.parse(await readFile(join(upload, "receipt.json"), "utf8"))).toMatchObject({ + diagnosticsRetained: false, + failureCode: "interrupted", + status: "failed", + }); + }); + + test("removes private diagnostics when a publication precheck rejects", async () => { + const { config, repositoryRoot } = await fixture(); + const plan = artifactRunPlan(repositoryRoot, 41, "private-vetted"); + const error = await rejection(runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: plan, + }, { + ...dependencies().overrides, + beforeArtifactCommit: () => { + throw new Error("run precommit rejected"); + }, + })); + expect(error.message).toContain("run precommit rejected"); + expect(await readdir(dirname(resolveDirectBombadilUploadLeaf(plan)))).toEqual([]); + }); + test("does not spawn when cancellation wins before server startup", async () => { const { config } = await fixture(); const runtime = dependencies(); @@ -1999,16 +3107,102 @@ describe("Direct Bombadil run lifecycle", () => { expect(await readFile(String(server.logPath), "utf8")).toContain("server output"); }); - test("bounds server output after cleanup fails and still writes artifacts", async () => { + test("bounds a server-output drain independently of writer settlement", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies({ + neverServerOutput: true, + serverOutputTimeoutMs: 10, + }); + const plan = artifactRunPlan(repositoryRoot, 38); + const startedAt = Date.now(); + const error = await rejection(runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: plan, + }, runtime.overrides)); + expect(error.message).toContain("server output did not settle"); + expect(Date.now() - startedAt).toBeLessThan(1_000); + const manifest = record(JSON.parse(await readFile(join( + repositoryRoot, + "artifacts", + "direct-bombadil", + "fixture-product", + "manifest.json", + ), "utf8")), "manifest"); + expect(record(manifest.server, "server").outputFailure).toBeString(); + expect(JSON.parse(await readFile(join( + resolveDirectBombadilUploadLeaf(plan), + "receipt.json", + ), "utf8"))).toMatchObject({ + failureCode: "server", + status: "failed", + }); + }); + + test("classifies local evidence-write failure as persistence", async () => { + const { config, repositoryRoot } = await fixture(); + const plan = artifactRunPlan(repositoryRoot, 39, "private-vetted"); + const runtime = dependencies({ + afterTrace: async (invocation) => { + await mkdir(join(dirname(invocation.outputPath), "bombadil.log")); + }, + }); + const error = await rejection(runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: plan, + }, runtime.overrides)); + expect(error.message).toContain("local diagnostic logs could not be persisted"); + const upload = resolveDirectBombadilUploadLeaf(plan); + expect((await readdir(upload)).sort()).toEqual([ + "diagnostics", + "receipt.json", + "summary.json", + ]); + expect(JSON.parse(await readFile(join( + upload, + "receipt.json", + ), "utf8"))).toMatchObject({ + diagnosticsRetained: true, + failureCode: "persistence", + mode: "private-vetted", + status: "failed", + }); + }); + + test("classifies an unreadable allowlisted output as artifact-policy", async () => { + const { config, repositoryRoot } = await fixture(); + const plan = artifactRunPlan(repositoryRoot, 40); + const runtime = dependencies({ + afterTrace: async (invocation) => { + await chmod(join(invocation.outputPath, "trace.jsonl"), 0o000); + }, + }); + const error = await rejection(runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: plan, + }, runtime.overrides)); + expect(error.message).toContain("inventory could not be proven safe"); + expect(JSON.parse(await readFile(join( + resolveDirectBombadilUploadLeaf(plan), + "receipt.json", + ), "utf8"))).toMatchObject({ + failureCode: "artifact-policy", + status: "failed", + }); + }); + + test("suppresses artifact inspection and private copying when server cleanup fails", async () => { const { config, repositoryRoot } = await fixture(); const runtime = dependencies({ neverServerOutput: true, serverOutputTimeoutMs: 10, stopFailure: true, }); + const plan = artifactRunPlan(repositoryRoot, 19, "private-vetted"); const startedAt = Date.now(); - expect((await rejection(runDirectBombadilFuzz(config, [], runtime.overrides))).message) - .toContain("server cleanup failed"); + expect((await rejection(runDirectBombadilFuzz(config, { + arguments: [], + artifactRun: plan, + }, runtime.overrides))).message).toContain("writers were not proven absent"); expect(Date.now() - startedAt).toBeLessThan(1_000); const manifest = JSON.parse(await readFile( join(repositoryRoot, "artifacts", "direct-bombadil", "fixture-product", "manifest.json"), @@ -2016,13 +3210,28 @@ describe("Direct Bombadil run lifecycle", () => { )) as Record; expect(manifest).toMatchObject({ status: "failed", - failure: "Error: server cleanup failed", + failure: expect.stringContaining("BombadilWriterSettlementError"), server: { logPresent: false, - outputFailure: expect.stringContaining("did not settle within 10ms"), + outputFailure: null, }, }); const server = record(manifest.server, "server"); expect(await readFile(String(server.logPath), "utf8")).toBe(""); + const upload = join(repositoryRoot, "artifacts", "direct-bombadil-upload", plan.runId); + expect((await readdir(upload)).sort()).toEqual(["receipt.json", "summary.json"]); + expect(JSON.parse(await readFile(join(upload, "receipt.json"), "utf8"))).toMatchObject({ + failureCode: "writer-settlement", + inventory: { entryCount: 0, fileCount: 0, inventorySha256: null }, + status: "failed", + }); + expect(parseDirectBombadilArtifactReceipt(JSON.parse(await readFile( + join(upload, "receipt.json"), + "utf8", + ))).ok).toBeTrue(); + expect(parseDirectBombadilSanitizedRunSummary(JSON.parse(await readFile( + join(upload, "summary.json"), + "utf8", + ))).ok).toBeTrue(); }); }); diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index 57107e8..3135468 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -1,9 +1,18 @@ -import { createReadStream } from "node:fs"; -import { readFile, realpath, stat, writeFile } from "node:fs/promises"; -import { isAbsolute, join, relative, resolve } from "node:path"; +import { constants as fileSystemConstants, type BigIntStats } from "node:fs"; +import { + lstat, + mkdir, + open, + opendir, + readFile, + realpath, + rename, + rm, + stat, +} from "node:fs/promises"; +import { extname, isAbsolute, join, relative, resolve } from "node:path"; import process from "node:process"; -import { createInterface } from "node:readline"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { parseDirectProbeSnapshot, @@ -13,11 +22,12 @@ import { FIXTURE_QUERY_KEY, SCENARIO_QUERY_KEY, } from "@hraness/direct"; +import { parseJsonValue } from "../core/json.js"; +import { err, ok, type Result } from "../core/result.js"; import { acquireVerificationServer, canAutomaticallyStartLocalServer, - createArtifactRun, normalizeRootHttpOrigin, renderUnknown, spawnVerificationServer, @@ -36,8 +46,111 @@ const DEFAULT_STARTUP_TIMEOUT_MS = 60_000; const MAX_STARTUP_TIMEOUT_MS = 120_000; const LOG_LIMIT = 24_000; const ARTIFACT_SCHEMA = "direct.bombadil-run/v1"; +const ARTIFACT_RECEIPT_SCHEMA = "direct.bombadil-artifact-receipt/v1"; +const ARTIFACT_SUMMARY_SCHEMA = "direct.bombadil-upload-summary/v1"; +const MATRIX_RECEIPT_SCHEMA = "direct.bombadil-matrix-receipt/v1"; +const MATRIX_SUMMARY_SCHEMA = "direct.bombadil-matrix-summary/v1"; +const ARTIFACT_FAILURE_CODES = new Set([ + "artifact-policy", + "configuration-rejected", + "exploration-policy", + "interrupted", + "persistence", + "process", + "server", + "trace-attestation", + "writer-settlement", + "unknown", +]); +const ARTIFACT_RECEIPT_KEYS = new Set([ + "completedAt", + "diagnosticsRetained", + "failureCode", + "inventory", + "mode", + "policy", + "runId", + "schema", + "status", +]); +const ARTIFACT_RECEIPT_INVENTORY_KEYS = new Set([ + "entryCount", + "fileCount", + "inventorySha256", + "totalBytes", +]); +const ARTIFACT_POLICY_RECEIPT_KEYS = new Set([ + "maxDepth", + "maxEntries", + "maxFileBytes", + "maxFiles", + "maxPathBytes", + "maxTotalBytes", +]); +const RUN_SUMMARY_KEYS = new Set([ + "artifactName", + "attestation", + "exploration", + "failureCode", + "scenario", + "schema", + "status", +]); +const RUN_SUMMARY_ATTESTATION_KEYS = new Set([ + "invalidObservationCount", + "observationCount", + "validObservationCount", +]); +const RUN_SUMMARY_EXPLORATION_KEYS = new Set([ + "actionCount", + "nonWaitActionCount", + "policySatisfied", + "traceBytes", + "traceLineCount", + "traceSha256", +]); +const MATRIX_RECEIPT_KEYS = new Set([ + "campaigns", + "completedAt", + "failureCode", + "mode", + "omittedCampaignCount", + "runId", + "schema", + "status", +]); +const MATRIX_CAMPAIGN_RECEIPT_KEYS = new Set([ + "campaignId", + "index", + "receipt", + "status", +]); +const MATRIX_SUMMARY_KEYS = new Set([ + "campaigns", + "failureCode", + "schema", + "status", +]); +const MATRIX_SUMMARY_CAMPAIGNS_KEYS = new Set([ + "failed", + "notRun", + "notSelected", + "omitted", + "passed", + "rejected", + "total", +]); +const SHA256_PATTERN = /^[0-9a-f]{64}$/u; +const ARTIFACT_EVIDENCE_JSON_LIMITS = Object.freeze({ + maxDepth: 8, + maxNodes: 2_048, + maxStringBytes: 64 * 1024, +}); const SCENARIO_PATTERN = /^[a-z][a-z0-9]*(?:[._/-][a-z0-9]+)*$/u; const ARTIFACT_NAME_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u; +const MAX_ARTIFACT_IDENTIFIER_LENGTH = 80; +const MAX_MATRIX_CAMPAIGNS = 32; +const ARTIFACT_COORDINATION_ENVIRONMENT = "DIRECT_BOMBADIL_RUN_ID"; const ENVIRONMENT_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/u; const QUERY_PARAMETER_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_.-]*$/u; const PROTOTYPE_PROPERTY_NAMES = new Set(["__proto__", "constructor", "prototype"]); @@ -56,6 +169,31 @@ const REPLAY_WALL_CLOCK_TIMEOUT_MS = MAX_TIME_LIMIT_SECONDS * 1_000 + RANDOM_RUN const PROCESS_TERMINATION_GRACE_MS = 5_000; const MIN_PROCESS_OUTPUT_DRAIN_MS = 500; const SERVER_OUTPUT_TIMEOUT_MS = 3_000; +const ARTIFACT_MONITOR_INTERVAL_MS = 100; +const DEFAULT_ARTIFACT_MAX_ENTRIES = 4_096; +const DEFAULT_ARTIFACT_MAX_FILES = 2_048; +const DEFAULT_ARTIFACT_MAX_TOTAL_BYTES = 128 * 1024 * 1024; +const DEFAULT_ARTIFACT_MAX_FILE_BYTES = 64 * 1024 * 1024; +const DEFAULT_ARTIFACT_MAX_DEPTH = 32; +const DEFAULT_ARTIFACT_MAX_PATH_BYTES = 4_096; +const MAX_ARTIFACT_ENTRIES = 16_384; +const MAX_ARTIFACT_FILES = 8_192; +const MAX_ARTIFACT_TOTAL_BYTES = 256 * 1024 * 1024; +const MAX_ARTIFACT_FILE_BYTES = 64 * 1024 * 1024; +const MAX_ARTIFACT_DEPTH = 64; +const MAX_ARTIFACT_PATH_BYTES = 4_096; +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; +const ARTIFACT_PATH_PART_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u; +const PRIVATE_DIAGNOSTIC_EXTENSIONS = new Set([ + ".jpeg", + ".jpg", + ".json", + ".jsonl", + ".log", + ".png", + ".txt", + ".webp", +]); const DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; const TRACE_LINE_KEYS = new Set(["action", "snapshots", "state", "timestamp", "violations"]); const TRACE_SNAPSHOT_KEYS = new Set(["index", "name", "time", "value"]); @@ -171,6 +309,138 @@ export interface DirectBombadilServerConfig { readonly startupTimeoutMs?: number; } +export interface DirectBombadilArtifactPolicy { + readonly maxDepth?: number; + readonly maxEntries?: number; + readonly maxFileBytes?: number; + readonly maxFiles?: number; + readonly maxPathBytes?: number; + readonly maxTotalBytes?: number; +} + +export type DirectBombadilUploadMode = "private-vetted" | "public-summary"; + +export interface DirectBombadilArtifactRunPlan { + readonly repositoryRoot: string; + readonly runId: string; + readonly uploadMode?: DirectBombadilUploadMode; +} + +export interface DirectBombadilFuzzRunOptions { + readonly arguments?: readonly string[]; + readonly artifactRun?: DirectBombadilArtifactRunPlan; +} + +export type DirectBombadilFuzzRunInput = + | readonly string[] + | DirectBombadilFuzzRunOptions; + +export interface DirectBombadilMatrixRunOptions { + readonly arguments?: readonly string[]; + readonly artifactRun?: Omit & { + readonly uploadMode?: "public-summary"; + }; +} + +export type DirectBombadilMatrixRunInput = + | readonly string[] + | DirectBombadilMatrixRunOptions; + +export type DirectBombadilArtifactFailureCode = + | "artifact-policy" + | "configuration-rejected" + | "exploration-policy" + | "interrupted" + | "persistence" + | "process" + | "server" + | "trace-attestation" + | "writer-settlement" + | "unknown"; + +export interface DirectBombadilArtifactReceipt { + readonly schema: typeof ARTIFACT_RECEIPT_SCHEMA; + readonly completedAt: string; + readonly diagnosticsRetained: boolean; + readonly failureCode: DirectBombadilArtifactFailureCode | null; + readonly inventory: { + readonly entryCount: number; + readonly fileCount: number; + readonly inventorySha256: string | null; + readonly totalBytes: number; + }; + readonly mode: DirectBombadilUploadMode; + readonly policy: Required; + readonly runId: string; + readonly status: "failed" | "passed" | "rejected"; +} + +export interface DirectBombadilArtifactParseError { + readonly code: "invalid-bombadil-artifact-evidence"; + readonly message: string; +} + +export interface DirectBombadilSanitizedRunSummary { + readonly schema: typeof ARTIFACT_SUMMARY_SCHEMA; + readonly artifactName: string; + readonly attestation: null | { + readonly invalidObservationCount: number; + readonly observationCount: number; + readonly validObservationCount: number; + }; + readonly exploration: null | { + readonly actionCount: number; + readonly nonWaitActionCount: number; + readonly policySatisfied: boolean; + readonly traceBytes: number; + readonly traceLineCount: number; + readonly traceSha256: string; + }; + readonly failureCode: DirectBombadilArtifactFailureCode | null; + readonly scenario: string; + readonly status: "failed" | "passed" | "rejected"; +} + +export type DirectBombadilMatrixCampaignStatus = + | "failed" + | "not-run" + | "not-selected" + | "passed" + | "rejected"; + +export interface DirectBombadilMatrixCampaignReceiptEntry { + readonly campaignId: string | null; + readonly index: number; + readonly receipt: string | null; + readonly status: DirectBombadilMatrixCampaignStatus; +} + +export interface DirectBombadilMatrixReceipt { + readonly schema: typeof MATRIX_RECEIPT_SCHEMA; + readonly campaigns: readonly DirectBombadilMatrixCampaignReceiptEntry[]; + readonly completedAt: string; + readonly failureCode: DirectBombadilArtifactFailureCode | null; + readonly mode: "public-summary"; + readonly omittedCampaignCount: number; + readonly runId: string; + readonly status: "failed" | "passed"; +} + +export interface DirectBombadilMatrixSummary { + readonly schema: typeof MATRIX_SUMMARY_SCHEMA; + readonly campaigns: { + readonly failed: number; + readonly notRun: number; + readonly notSelected: number; + readonly omitted: number; + readonly passed: number; + readonly rejected: number; + readonly total: number; + }; + readonly failureCode: DirectBombadilArtifactFailureCode | null; + readonly status: "failed" | "passed"; +} + export type DirectBombadilActionKind = (typeof ACTION_KINDS)[number]; export interface DirectBombadilViewportConfig { @@ -254,6 +524,7 @@ export interface DirectBombadilExplorationSummary { } export interface DirectBombadilFuzzConfig { + readonly artifactPolicy?: DirectBombadilArtifactPolicy; readonly artifactName: string; readonly baseUrl: string; readonly entryPath?: `/${string}`; @@ -286,6 +557,13 @@ export type DirectBombadilFuzzResult = readonly status: "passed"; }; +type DirectBombadilFuzzExecutionResult = + | Extract + | (Extract & { + readonly receiptPath: string; + readonly uploadArtifactPath: string; + }); + export interface DirectBombadilFuzzCampaign { readonly config: DirectBombadilFuzzConfig; readonly id: string; @@ -301,8 +579,21 @@ export type DirectBombadilFuzzMatrixResult = }[]; }; +type DirectBombadilFuzzMatrixExecutionResult = + | Extract + | { + readonly kind: "matrix"; + readonly receiptPath: string; + readonly results: readonly { + readonly campaignId: string; + readonly result: Extract; + }[]; + readonly uploadArtifactPath: string; + }; + export interface DirectBombadilInvocation { readonly abortSignal?: AbortSignal; + readonly artifactPolicy?: DirectBombadilArtifactPolicy; readonly command: readonly string[]; readonly cwd: string; readonly outputPath: string; @@ -354,158 +645,2162 @@ export interface DirectBombadilTraceAttestation { export interface DirectBombadilRunnerDependencies { readonly acquireServer: typeof acquireVerificationServer; + readonly beforeArtifactCommit?: () => Promise | void; readonly createAbortController?: () => AbortController; + readonly createRunId: () => string; readonly now: () => Date; readonly runBombadil: ( invocation: DirectBombadilInvocation, ) => Promise; + readonly signalController: ProcessSignalController; readonly serverOutputTimeoutMs: number; readonly spawnServer: (options: { readonly command: readonly string[]; readonly cwd: string; + readonly detachedProcessGroup?: boolean; readonly env?: Readonly>; + readonly omitEnvironment?: readonly string[]; }) => ManagedVerificationServer; readonly stopServer: typeof stopVerificationServer; } -interface ProcessSignalEmitter { - readonly once: ( - signal: NodeJS.Signals, - listener: (signal: NodeJS.Signals) => void, - ) => unknown; - readonly removeListener: ( - signal: NodeJS.Signals, - listener: (signal: NodeJS.Signals) => void, - ) => unknown; +interface ProcessSignalEmitter { + readonly once: ( + signal: NodeJS.Signals, + listener: (signal: NodeJS.Signals) => void, + ) => unknown; + readonly removeListener: ( + signal: NodeJS.Signals, + listener: (signal: NodeJS.Signals) => void, + ) => unknown; +} + +interface ProcessSignalController extends ProcessSignalEmitter { + readonly forward: (signal: NodeJS.Signals) => void; +} + +type ValidatedConfig = Omit< + DirectBombadilFuzzConfig, + "artifactPolicy" | "explorationPolicy" | "server" | "viewport" +> & { + readonly artifactPolicy: ValidatedArtifactPolicy; + readonly artifactRoot: string; + readonly baseUrl: string; + readonly bombadilExecutable: string; + readonly entryPath: `/${string}`; + readonly explorationPolicy: ValidatedExplorationPolicy | null; + readonly port: string; + readonly targetQuery: Readonly>; + readonly viewport: ValidatedViewport; + readonly server: DirectBombadilServerConfig & { + readonly readinessPath: `/${string}`; + readonly startupTimeoutMs: number; + }; +}; + +type ValidatedArtifactPolicy = Required; + +interface ArtifactInventoryFile { + readonly device: bigint; + readonly inode: bigint; + readonly relativePath: string; + readonly sha256: string; + readonly size: number; +} + +interface ArtifactInventory { + readonly directories: readonly string[]; + readonly entryCount: number; + readonly files: readonly ArtifactInventoryFile[]; + readonly fileCount: number; + readonly inventorySha256: string; + readonly totalBytes: number; +} + +interface ArtifactUploadSessionBase { + readonly finalDirectory: string; + readonly mode: DirectBombadilUploadMode; + readonly receiptPath: string; + readonly runId: string; +} + +interface AtomicArtifactUploadSession extends ArtifactUploadSessionBase { + readonly publication: "atomic-leaf"; + readonly stagingDirectory: string; +} + +interface DeferredArtifactUploadSession extends ArtifactUploadSessionBase { + readonly deferredPayload: { value: SanitizedRunUploadPayload | null }; + readonly publication: "deferred"; +} + +type ArtifactUploadSession = AtomicArtifactUploadSession | DeferredArtifactUploadSession; + +interface SanitizedRunUploadPayload { + readonly receipt: DirectBombadilArtifactReceipt; + readonly summary: DirectBombadilSanitizedRunSummary; +} + +interface ExpectedUploadFile { + readonly relativePath: string; + readonly sha256: string; + readonly size: number; +} + +interface NormalizedFuzzRunOptions { + readonly arguments: readonly string[]; + readonly artifactRun: DirectBombadilArtifactRunPlan | null; +} + +class BombadilArtifactPolicyError extends Error { + public constructor(message: string) { + super(message); + this.name = "BombadilArtifactPolicyError"; + } +} + +class BombadilWriterSettlementError extends Error { + public constructor(message: string, cause: unknown) { + super(message, { cause }); + this.name = "BombadilWriterSettlementError"; + } +} + +class BombadilPersistenceError extends AggregateError { + public constructor(message: string, errors: readonly unknown[]) { + super(errors, message, { cause: errors[0] }); + this.name = "BombadilPersistenceError"; + } +} + +interface ValidatedViewport { + readonly deviceScaleFactor: number; + readonly height: number; + readonly width: number; +} + +interface ValidatedExplorationPolicy { + readonly minDistinctNamedSnapshotValues: Readonly>; + readonly minNamedSnapshotChangesAfterActionKind: Readonly>> + >>; + readonly minNamedSnapshotChangesAfterNonWait: Readonly>; + readonly minNonWaitActions: number; + readonly requireStableTargetUrl: boolean; + readonly requiredActionKinds: readonly DirectBombadilActionKind[]; + readonly requiredNamedSnapshots: readonly string[]; +} + +function readOptionValue( + arguments_: readonly string[], + index: number, + option: string, +): { readonly index: number; readonly value: string } { + const value = arguments_[index + 1]; + if (value === undefined || value.startsWith("-")) { + throw new Error(`${option} requires a value`); + } + return { index: index + 1, value }; +} + +function parseTimeLimit(value: string): number { + const match = /^([1-9][0-9]*)s$/u.exec(value); + if (match === null) { + throw new Error("--time-limit must be a whole number of seconds such as 20s"); + } + const seconds = Number(match[1]); + if ( + !Number.isSafeInteger(seconds) + || seconds < MIN_TIME_LIMIT_SECONDS + || seconds > MAX_TIME_LIMIT_SECONDS + ) { + throw new Error( + `--time-limit must be between ${String(MIN_TIME_LIMIT_SECONDS)}s and ${String(MAX_TIME_LIMIT_SECONDS)}s`, + ); + } + return seconds; +} + +function bombadilNativeBinary(repositoryRoot: string): string { + let binary: string; + if (process.platform === "darwin" && process.arch === "arm64") { + binary = "bombadil-darwin-arm64"; + } else if (process.platform === "linux" && process.arch === "x64") { + binary = "bombadil-linux-x64"; + } else if (process.platform === "linux" && process.arch === "arm64") { + binary = "bombadil-linux-arm64"; + } else { + throw new Error(`Bombadil 0.7.2 does not support ${process.platform}-${process.arch}`); + } + return join( + repositoryRoot, + "node_modules", + "@antithesishq", + "bombadil", + "binaries", + binary, + ); +} + +function requireLocalRootHttpOrigin(value: string): string { + const baseUrl = normalizeRootHttpOrigin(value); + const url = new URL(baseUrl); + if (!canAutomaticallyStartLocalServer(baseUrl)) { + throw new Error("--base-url must use HTTP on 127.0.0.1 or localhost"); + } + if (url.port === "") { + throw new Error("--base-url must include an explicit local server port"); + } + if (Number(url.port) < 1) { + throw new Error("--base-url port must be between 1 and 65535"); + } + return baseUrl; +} + +function hasControlCharacters(value: string): boolean { + for (const character of value) { + const code = character.charCodeAt(0); + if (code < 32 || code === 127) return true; + } + return false; +} + +function isRecord(value: unknown): value is Readonly> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isReadonlyStringArray(value: unknown): value is readonly string[] { + return Array.isArray(value) && value.every((entry) => typeof entry === "string"); +} + +function hasExactKeys( + value: Readonly>, + expected: ReadonlySet, +): boolean { + const keys = Object.keys(value); + return keys.length === expected.size && keys.every((key) => expected.has(key)); +} + +function compareCodeUnits(left: string, right: string): number { + if (left < right) return -1; + if (left > right) return 1; + return 0; +} + +function boundedArtifactInteger(options: { + readonly label: string; + readonly maximum: number; + readonly value: number | undefined; + readonly defaultValue: number; +}): number { + const value = options.value ?? options.defaultValue; + if (!Number.isSafeInteger(value) || value < 1 || value > options.maximum) { + throw new Error(`${options.label} must be an integer between 1 and ${String(options.maximum)}`); + } + return value; +} + +function validateArtifactPolicy( + input: DirectBombadilArtifactPolicy | undefined, +): ValidatedArtifactPolicy { + const value = input ?? {}; + return Object.freeze({ + maxDepth: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_DEPTH, + label: "artifactPolicy.maxDepth", + maximum: MAX_ARTIFACT_DEPTH, + value: value.maxDepth, + }), + maxEntries: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_ENTRIES, + label: "artifactPolicy.maxEntries", + maximum: MAX_ARTIFACT_ENTRIES, + value: value.maxEntries, + }), + maxFileBytes: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_FILE_BYTES, + label: "artifactPolicy.maxFileBytes", + maximum: MAX_ARTIFACT_FILE_BYTES, + value: value.maxFileBytes, + }), + maxFiles: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_FILES, + label: "artifactPolicy.maxFiles", + maximum: MAX_ARTIFACT_FILES, + value: value.maxFiles, + }), + maxPathBytes: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_PATH_BYTES, + label: "artifactPolicy.maxPathBytes", + maximum: MAX_ARTIFACT_PATH_BYTES, + value: value.maxPathBytes, + }), + maxTotalBytes: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_TOTAL_BYTES, + label: "artifactPolicy.maxTotalBytes", + maximum: MAX_ARTIFACT_TOTAL_BYTES, + value: value.maxTotalBytes, + }), + }); +} + +function normalizeFuzzRunOptions( + input: DirectBombadilFuzzRunInput | DirectBombadilMatrixRunInput | undefined, +): NormalizedFuzzRunOptions { + if (input === undefined || isReadonlyStringArray(input)) { + return { + arguments: Object.freeze([...(input ?? [])]), + artifactRun: null, + }; + } + if (!isRecord(input)) throw new Error("Bombadil run options must be an object or argument array"); + const keys = Object.keys(input); + if (keys.some((key) => key !== "arguments" && key !== "artifactRun")) { + throw new Error("Bombadil run options contain an unknown field"); + } + const arguments_ = input.arguments ?? []; + if (!isReadonlyStringArray(arguments_)) { + throw new Error("Bombadil run options arguments must be a string array"); + } + return { + arguments: Object.freeze([...arguments_]), + artifactRun: input.artifactRun ?? null, + }; +} + +function validateArtifactRunPlan( + input: DirectBombadilArtifactRunPlan, +): DirectBombadilArtifactRunPlan & { readonly uploadMode: DirectBombadilUploadMode } { + const repositoryRoot = resolve(input.repositoryRoot); + if (!isAbsolute(input.repositoryRoot) || repositoryRoot !== input.repositoryRoot) { + throw new Error("artifactRun.repositoryRoot must be an absolute normalized path"); + } + if (!UUID_PATTERN.test(input.runId)) { + throw new Error("artifactRun.runId must be a lowercase RFC 4122 UUID"); + } + const uploadMode = input.uploadMode ?? "public-summary"; + if (uploadMode !== "public-summary" && uploadMode !== "private-vetted") { + throw new Error("artifactRun.uploadMode must be public-summary or private-vetted"); + } + return Object.freeze({ repositoryRoot, runId: input.runId, uploadMode }); +} + +function isBoundedArtifactIdentifier(value: string): boolean { + return value.length <= MAX_ARTIFACT_IDENTIFIER_LENGTH + && ARTIFACT_NAME_PATTERN.test(value); +} + +function isBoundedScenarioIdentifier(value: string): boolean { + return value.length <= 120 && SCENARIO_PATTERN.test(value); +} + +function requireEvidenceRecord( + value: unknown, + keys: ReadonlySet, + label: string, +): Readonly> { + if (!isRecord(value) || !hasExactKeys(value, keys)) { + throw new Error(`${label} must contain exactly its documented fields`); + } + return value; +} + +function requireEvidenceInteger( + value: unknown, + label: string, + maximum = Number.MAX_SAFE_INTEGER, +): number { + if (!Number.isSafeInteger(value) || (value as number) < 0 || (value as number) > maximum) { + throw new Error(`${label} must be a nonnegative safe integer no greater than ${String(maximum)}`); + } + return value as number; +} + +function requireEvidencePositiveInteger( + value: unknown, + label: string, + maximum: number, +): number { + const parsed = requireEvidenceInteger(value, label, maximum); + if (parsed === 0) throw new Error(`${label} must be greater than zero`); + return parsed; +} + +function requireEvidenceSha256(value: unknown, label: string): string { + if (typeof value !== "string" || !SHA256_PATTERN.test(value)) { + throw new Error(`${label} must be a lowercase SHA-256 digest`); + } + return value; +} + +function requireEvidenceTimestamp(value: unknown, label: string): string { + if (typeof value !== "string") throw new Error(`${label} must be an ISO timestamp`); + const milliseconds = Date.parse(value); + if (!Number.isFinite(milliseconds) || new Date(milliseconds).toISOString() !== value) { + throw new Error(`${label} must be a canonical ISO timestamp`); + } + return value; +} + +function parseEvidenceFailureCode( + value: unknown, + label: string, +): DirectBombadilArtifactFailureCode | null { + if (value === null) return null; + if (typeof value !== "string" || !ARTIFACT_FAILURE_CODES.has( + value as DirectBombadilArtifactFailureCode, + )) { + throw new Error(`${label} is not a known Bombadil failure code`); + } + return value as DirectBombadilArtifactFailureCode; +} + +function requireEvidenceStatus( + value: unknown, + label: string, +): "failed" | "passed" | "rejected" { + if (value !== "failed" && value !== "passed" && value !== "rejected") { + throw new Error(`${label} must be failed, passed, or rejected`); + } + return value; +} + +function requireFailureStatusConsistency( + status: "failed" | "passed" | "rejected", + failureCode: DirectBombadilArtifactFailureCode | null, + label: string, +): void { + if ((status === "passed") !== (failureCode === null)) { + throw new Error(`${label} status and failureCode are inconsistent`); + } + if (status === "rejected" && failureCode !== "configuration-rejected") { + throw new Error(`${label} rejected status requires configuration-rejected`); + } +} + +function parseArtifactReceiptUnchecked(input: unknown): DirectBombadilArtifactReceipt { + const value = requireEvidenceRecord(input, ARTIFACT_RECEIPT_KEYS, "Bombadil receipt"); + if (value.schema !== ARTIFACT_RECEIPT_SCHEMA) { + throw new Error("Bombadil receipt schema is unsupported"); + } + const completedAt = requireEvidenceTimestamp(value.completedAt, "Bombadil receipt completedAt"); + if (typeof value.diagnosticsRetained !== "boolean") { + throw new Error("Bombadil receipt diagnosticsRetained must be boolean"); + } + const failureCode = parseEvidenceFailureCode(value.failureCode, "Bombadil receipt failureCode"); + const status = requireEvidenceStatus(value.status, "Bombadil receipt status"); + requireFailureStatusConsistency(status, failureCode, "Bombadil receipt"); + if (value.mode !== "private-vetted" && value.mode !== "public-summary") { + throw new Error("Bombadil receipt mode is unsupported"); + } + if (value.diagnosticsRetained && value.mode !== "private-vetted") { + throw new Error("Public Bombadil receipts cannot retain diagnostics"); + } + if (typeof value.runId !== "string" || !UUID_PATTERN.test(value.runId)) { + throw new Error("Bombadil receipt runId must be a lowercase RFC 4122 UUID"); + } + const rawPolicy = requireEvidenceRecord( + value.policy, + ARTIFACT_POLICY_RECEIPT_KEYS, + "Bombadil receipt policy", + ); + const policy = Object.freeze({ + maxDepth: requireEvidencePositiveInteger( + rawPolicy.maxDepth, + "Bombadil receipt policy.maxDepth", + MAX_ARTIFACT_DEPTH, + ), + maxEntries: requireEvidencePositiveInteger( + rawPolicy.maxEntries, + "Bombadil receipt policy.maxEntries", + MAX_ARTIFACT_ENTRIES, + ), + maxFileBytes: requireEvidencePositiveInteger( + rawPolicy.maxFileBytes, + "Bombadil receipt policy.maxFileBytes", + MAX_ARTIFACT_FILE_BYTES, + ), + maxFiles: requireEvidencePositiveInteger( + rawPolicy.maxFiles, + "Bombadil receipt policy.maxFiles", + MAX_ARTIFACT_FILES, + ), + maxPathBytes: requireEvidencePositiveInteger( + rawPolicy.maxPathBytes, + "Bombadil receipt policy.maxPathBytes", + MAX_ARTIFACT_PATH_BYTES, + ), + maxTotalBytes: requireEvidencePositiveInteger( + rawPolicy.maxTotalBytes, + "Bombadil receipt policy.maxTotalBytes", + MAX_ARTIFACT_TOTAL_BYTES, + ), + }); + const rawInventory = requireEvidenceRecord( + value.inventory, + ARTIFACT_RECEIPT_INVENTORY_KEYS, + "Bombadil receipt inventory", + ); + const entryCount = requireEvidenceInteger( + rawInventory.entryCount, + "Bombadil receipt inventory.entryCount", + policy.maxEntries, + ); + const fileCount = requireEvidenceInteger( + rawInventory.fileCount, + "Bombadil receipt inventory.fileCount", + policy.maxFiles, + ); + const totalBytes = requireEvidenceInteger( + rawInventory.totalBytes, + "Bombadil receipt inventory.totalBytes", + policy.maxTotalBytes, + ); + if (fileCount > entryCount) { + throw new Error("Bombadil receipt inventory.fileCount cannot exceed entryCount"); + } + if (fileCount === 0 && totalBytes !== 0) { + throw new Error("Bombadil receipt inventory bytes require at least one file"); + } + const inventorySha256 = rawInventory.inventorySha256 === null + ? null + : requireEvidenceSha256( + rawInventory.inventorySha256, + "Bombadil receipt inventory.inventorySha256", + ); + if ( + (entryCount === 0 && (fileCount !== 0 || totalBytes !== 0 || inventorySha256 !== null)) + || (entryCount > 0 && inventorySha256 === null) + ) { + throw new Error("Bombadil receipt empty-inventory fields are inconsistent"); + } + if ( + (status === "passed" && (entryCount === 0 || fileCount === 0 || totalBytes === 0)) + || (status === "passed" && value.mode === "private-vetted" && !value.diagnosticsRetained) + || (failureCode === "interrupted" && value.diagnosticsRetained) + || (failureCode === "configuration-rejected" && status !== "rejected") + || ( + failureCode === "writer-settlement" + && (value.diagnosticsRetained || entryCount !== 0 || fileCount !== 0 || totalBytes !== 0) + ) + || ( + status === "rejected" + && (value.diagnosticsRetained || entryCount !== 0 || fileCount !== 0 || totalBytes !== 0) + ) + ) { + throw new Error("Bombadil receipt terminal state and retained evidence are inconsistent"); + } + return Object.freeze({ + schema: ARTIFACT_RECEIPT_SCHEMA, + completedAt, + diagnosticsRetained: value.diagnosticsRetained, + failureCode, + inventory: Object.freeze({ entryCount, fileCount, inventorySha256, totalBytes }), + mode: value.mode, + policy, + runId: value.runId, + status, + }); +} + +function parseRunSummaryUnchecked(input: unknown): DirectBombadilSanitizedRunSummary { + const value = requireEvidenceRecord(input, RUN_SUMMARY_KEYS, "Bombadil run summary"); + if (value.schema !== ARTIFACT_SUMMARY_SCHEMA) { + throw new Error("Bombadil run summary schema is unsupported"); + } + if (typeof value.artifactName !== "string" || !isBoundedArtifactIdentifier(value.artifactName)) { + throw new Error("Bombadil run summary artifactName is invalid"); + } + if (typeof value.scenario !== "string" || !isBoundedScenarioIdentifier(value.scenario)) { + throw new Error("Bombadil run summary scenario is invalid"); + } + const failureCode = parseEvidenceFailureCode( + value.failureCode, + "Bombadil run summary failureCode", + ); + const status = requireEvidenceStatus(value.status, "Bombadil run summary status"); + requireFailureStatusConsistency(status, failureCode, "Bombadil run summary"); + let attestation: DirectBombadilSanitizedRunSummary["attestation"] = null; + if (value.attestation !== null) { + const raw = requireEvidenceRecord( + value.attestation, + RUN_SUMMARY_ATTESTATION_KEYS, + "Bombadil run summary attestation", + ); + const observationCount = requireEvidenceInteger( + raw.observationCount, + "Bombadil run summary attestation.observationCount", + TRACE_MAX_LINES, + ); + const invalidObservationCount = requireEvidenceInteger( + raw.invalidObservationCount, + "Bombadil run summary attestation.invalidObservationCount", + observationCount, + ); + const validObservationCount = requireEvidenceInteger( + raw.validObservationCount, + "Bombadil run summary attestation.validObservationCount", + observationCount, + ); + if (invalidObservationCount + validObservationCount !== observationCount) { + throw new Error("Bombadil run summary attestation counts do not reconcile"); + } + if (observationCount === 0 || validObservationCount === 0) { + throw new Error("Bombadil run summary attestation must contain a valid observation"); + } + attestation = Object.freeze({ + invalidObservationCount, + observationCount, + validObservationCount, + }); + } + let exploration: DirectBombadilSanitizedRunSummary["exploration"] = null; + if (value.exploration !== null) { + const raw = requireEvidenceRecord( + value.exploration, + RUN_SUMMARY_EXPLORATION_KEYS, + "Bombadil run summary exploration", + ); + const traceLineCount = requireEvidenceInteger( + raw.traceLineCount, + "Bombadil run summary exploration.traceLineCount", + TRACE_MAX_LINES, + ); + const actionCount = requireEvidenceInteger( + raw.actionCount, + "Bombadil run summary exploration.actionCount", + traceLineCount, + ); + const nonWaitActionCount = requireEvidenceInteger( + raw.nonWaitActionCount, + "Bombadil run summary exploration.nonWaitActionCount", + actionCount, + ); + if (typeof raw.policySatisfied !== "boolean") { + throw new Error("Bombadil run summary exploration.policySatisfied must be boolean"); + } + exploration = Object.freeze({ + actionCount, + nonWaitActionCount, + policySatisfied: raw.policySatisfied, + traceBytes: requireEvidenceInteger( + raw.traceBytes, + "Bombadil run summary exploration.traceBytes", + TRACE_MAX_BYTES, + ), + traceLineCount, + traceSha256: requireEvidenceSha256( + raw.traceSha256, + "Bombadil run summary exploration.traceSha256", + ), + }); + if (exploration.traceBytes === 0 || exploration.traceLineCount === 0) { + throw new Error("Bombadil run summary exploration trace must be nonempty"); + } + } + if ( + status === "passed" + && ( + attestation === null + || attestation.observationCount === 0 + || attestation.validObservationCount === 0 + || exploration === null + || !exploration.policySatisfied + || attestation.observationCount !== exploration.traceLineCount + ) + ) { + throw new Error("A passed Bombadil run summary requires attested policy-satisfying evidence"); + } + if ( + attestation !== null + && exploration !== null + && attestation.observationCount !== exploration.traceLineCount + ) { + throw new Error("Bombadil run summary trace counts do not reconcile"); + } + if (status === "rejected" && (attestation !== null || exploration !== null)) { + throw new Error("A rejected Bombadil run summary cannot claim trace evidence"); + } + if (failureCode === "configuration-rejected" && status !== "rejected") { + throw new Error("A configuration-rejected Bombadil run summary must be rejected"); + } + if (failureCode === "writer-settlement" && (attestation !== null || exploration !== null)) { + throw new Error("A writer-settlement Bombadil run summary cannot claim trace evidence"); + } + return Object.freeze({ + schema: ARTIFACT_SUMMARY_SCHEMA, + artifactName: value.artifactName, + attestation, + exploration, + failureCode, + scenario: value.scenario, + status, + }); +} + +function parseMatrixReceiptUnchecked(input: unknown): DirectBombadilMatrixReceipt { + const value = requireEvidenceRecord(input, MATRIX_RECEIPT_KEYS, "Bombadil matrix receipt"); + if (value.schema !== MATRIX_RECEIPT_SCHEMA || value.mode !== "public-summary") { + throw new Error("Bombadil matrix receipt schema or mode is unsupported"); + } + const completedAt = requireEvidenceTimestamp( + value.completedAt, + "Bombadil matrix receipt completedAt", + ); + const failureCode = parseEvidenceFailureCode( + value.failureCode, + "Bombadil matrix receipt failureCode", + ); + if (value.status !== "failed" && value.status !== "passed") { + throw new Error("Bombadil matrix receipt status must be failed or passed"); + } + requireFailureStatusConsistency(value.status, failureCode, "Bombadil matrix receipt"); + if (typeof value.runId !== "string" || !UUID_PATTERN.test(value.runId)) { + throw new Error("Bombadil matrix receipt runId must be a lowercase RFC 4122 UUID"); + } + if (!Array.isArray(value.campaigns) || value.campaigns.length > MAX_MATRIX_CAMPAIGNS) { + throw new Error("Bombadil matrix receipt campaigns exceed the bounded matrix size"); + } + const campaignIds = new Set(); + const campaigns = value.campaigns.map((inputCampaign, index) => { + const campaign = requireEvidenceRecord( + inputCampaign, + MATRIX_CAMPAIGN_RECEIPT_KEYS, + `Bombadil matrix receipt campaign ${String(index)}`, + ); + if (campaign.index !== index) { + throw new Error("Bombadil matrix receipt campaign indices must be ordered and contiguous"); + } + const campaignId = campaign.campaignId; + if ( + campaignId !== null + && ( + typeof campaignId !== "string" + || !isBoundedArtifactIdentifier(campaignId) + || campaignIds.has(campaignId) + ) + ) { + throw new Error("Bombadil matrix receipt campaign IDs must be unique bounded identifiers"); + } + if (campaignId !== null) campaignIds.add(campaignId); + if ( + campaign.status !== "failed" + && campaign.status !== "not-run" + && campaign.status !== "not-selected" + && campaign.status !== "passed" + && campaign.status !== "rejected" + ) { + throw new Error("Bombadil matrix receipt campaign status is unsupported"); + } + const expectedReceipt = campaignId === null + ? null + : `campaigns/${campaignId}/receipt.json`; + if ( + campaign.receipt !== null + && (typeof campaign.receipt !== "string" || campaign.receipt !== expectedReceipt) + ) { + throw new Error("Bombadil matrix child receipt path is not canonical"); + } + if ( + ((campaign.status === "not-run" || campaign.status === "not-selected") + && campaign.receipt !== null) + || (campaign.status === "passed" && campaign.receipt !== expectedReceipt) + || (campaignId === null && (campaign.status !== "rejected" || campaign.receipt !== null)) + ) { + throw new Error("Bombadil matrix child terminal state is inconsistent"); + } + return Object.freeze({ + campaignId, + index, + receipt: campaign.receipt as string | null, + status: campaign.status, + }); + }); + const omittedCampaignCount = requireEvidenceInteger( + value.omittedCampaignCount, + "Bombadil matrix receipt omittedCampaignCount", + ); + if ( + value.status === "passed" + && ( + omittedCampaignCount !== 0 + || !campaigns.some((campaign) => campaign.status === "passed") + || campaigns.some((campaign) => + campaign.status === "failed" + || campaign.status === "not-run" + || campaign.status === "rejected" + ) + ) + ) { + throw new Error("A passed Bombadil matrix receipt has a nonterminal child"); + } + return Object.freeze({ + schema: MATRIX_RECEIPT_SCHEMA, + campaigns: Object.freeze(campaigns), + completedAt, + failureCode, + mode: "public-summary", + omittedCampaignCount, + runId: value.runId, + status: value.status, + }); +} + +function parseMatrixSummaryUnchecked(input: unknown): DirectBombadilMatrixSummary { + const value = requireEvidenceRecord(input, MATRIX_SUMMARY_KEYS, "Bombadil matrix summary"); + if (value.schema !== MATRIX_SUMMARY_SCHEMA) { + throw new Error("Bombadil matrix summary schema is unsupported"); + } + const failureCode = parseEvidenceFailureCode( + value.failureCode, + "Bombadil matrix summary failureCode", + ); + if (value.status !== "failed" && value.status !== "passed") { + throw new Error("Bombadil matrix summary status must be failed or passed"); + } + requireFailureStatusConsistency(value.status, failureCode, "Bombadil matrix summary"); + const rawCampaigns = requireEvidenceRecord( + value.campaigns, + MATRIX_SUMMARY_CAMPAIGNS_KEYS, + "Bombadil matrix summary campaigns", + ); + const total = requireEvidenceInteger( + rawCampaigns.total, + "Bombadil matrix summary campaigns.total", + MAX_MATRIX_CAMPAIGNS, + ); + const campaigns = Object.freeze({ + failed: requireEvidenceInteger(rawCampaigns.failed, "Bombadil matrix summary failed", total), + notRun: requireEvidenceInteger(rawCampaigns.notRun, "Bombadil matrix summary notRun", total), + notSelected: requireEvidenceInteger( + rawCampaigns.notSelected, + "Bombadil matrix summary notSelected", + total, + ), + omitted: requireEvidenceInteger(rawCampaigns.omitted, "Bombadil matrix summary omitted"), + passed: requireEvidenceInteger(rawCampaigns.passed, "Bombadil matrix summary passed", total), + rejected: requireEvidenceInteger(rawCampaigns.rejected, "Bombadil matrix summary rejected", total), + total, + }); + if ( + campaigns.failed + + campaigns.notRun + + campaigns.notSelected + + campaigns.passed + + campaigns.rejected + !== campaigns.total + ) { + throw new Error("Bombadil matrix summary campaign counts do not reconcile"); + } + if ( + value.status === "passed" + && ( + campaigns.failed !== 0 + || campaigns.notRun !== 0 + || campaigns.rejected !== 0 + || campaigns.omitted !== 0 + || campaigns.passed === 0 + ) + ) { + throw new Error("A passed Bombadil matrix summary contains unsuccessful campaigns"); + } + return Object.freeze({ + schema: MATRIX_SUMMARY_SCHEMA, + campaigns, + failureCode, + status: value.status, + }); +} + +function artifactEvidenceError(error: unknown): DirectBombadilArtifactParseError { + return Object.freeze({ + code: "invalid-bombadil-artifact-evidence", + message: renderUnknown(error), + }); +} + +function cloneArtifactEvidence(input: unknown): unknown { + const parsed = parseJsonValue(input, ARTIFACT_EVIDENCE_JSON_LIMITS); + if (!parsed.ok) { + throw new Error(`Bombadil artifact evidence is not bounded inert JSON: ${parsed.error.message}`); + } + return parsed.value; +} + +/** Parse, clone, and freeze a foreign sanitized Bombadil run receipt. */ +export function parseDirectBombadilArtifactReceipt( + input: unknown, +): Result { + try { + return ok(parseArtifactReceiptUnchecked(cloneArtifactEvidence(input))); + } catch (error) { + return err(artifactEvidenceError(error)); + } +} + +/** Parse, clone, and freeze a foreign sanitized Bombadil run summary. */ +export function parseDirectBombadilSanitizedRunSummary( + input: unknown, +): Result { + try { + return ok(parseRunSummaryUnchecked(cloneArtifactEvidence(input))); + } catch (error) { + return err(artifactEvidenceError(error)); + } +} + +/** Parse, clone, and freeze a foreign sanitized Bombadil matrix receipt. */ +export function parseDirectBombadilMatrixReceipt( + input: unknown, +): Result { + try { + return ok(parseMatrixReceiptUnchecked(cloneArtifactEvidence(input))); + } catch (error) { + return err(artifactEvidenceError(error)); + } +} + +/** Parse, clone, and freeze a foreign sanitized Bombadil matrix summary. */ +export function parseDirectBombadilMatrixSummary( + input: unknown, +): Result { + try { + return ok(parseMatrixSummaryUnchecked(cloneArtifactEvidence(input))); + } catch (error) { + return err(artifactEvidenceError(error)); + } +} + +/** Resolve the lexically validated exact upload leaf for an `if: always()` caller. */ +export function resolveDirectBombadilUploadLeaf( + input: DirectBombadilArtifactRunPlan, +): string { + const plan = validateArtifactRunPlan(input); + return join(plan.repositoryRoot, "artifacts", "direct-bombadil-upload", plan.runId); +} + +async function requireSafeDirectory(path: string, label: string): Promise { + let metadata; + try { + metadata = await lstat(path); + } catch { + throw new BombadilArtifactPolicyError(`${label} does not exist`); + } + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + throw new BombadilArtifactPolicyError(`${label} must be a non-symlink directory`); + } +} + +async function ensureSafeDirectoryChain( + repositoryRoot: string, + parts: readonly string[], +): Promise { + await requireSafeDirectory(repositoryRoot, "repositoryRoot"); + let current = repositoryRoot; + for (const part of parts) { + if (!ARTIFACT_PATH_PART_PATTERN.test(part) || part === "." || part === "..") { + throw new BombadilArtifactPolicyError("Artifact directory contains an unsafe path component"); + } + current = join(current, part); + try { + await mkdir(current, { mode: 0o700 }); + } catch (error) { + if (!isRecord(error) || error.code !== "EEXIST") throw error; + } + await requireSafeDirectory(current, `Artifact directory ${part}`); + const resolved = await realpath(current); + if (!isWithin(repositoryRoot, resolved) || resolved !== current) { + throw new BombadilArtifactPolicyError("Artifact directory escaped repositoryRoot"); + } + } + return current; +} + +async function createExclusiveDirectory(path: string, label: string): Promise { + try { + await mkdir(path, { mode: 0o700 }); + } catch (error) { + if (isRecord(error) && error.code === "EEXIST") { + throw new BombadilArtifactPolicyError(`${label} already exists`); + } + throw error; + } + await requireSafeDirectory(path, label); +} + +async function createBombadilArtifactRun(options: { + readonly artifactName: string; + readonly repositoryRoot: string; + readonly runId: string; +}): Promise<{ + readonly artifactRoot: string; + readonly manifestPath: string; + readonly runDirectory: string; +}> { + if (!UUID_PATTERN.test(options.runId)) { + throw new BombadilArtifactPolicyError("Bombadil raw artifact run ID must be a UUID"); + } + const artifactRoot = await ensureSafeDirectoryChain(options.repositoryRoot, [ + "artifacts", + "direct-bombadil", + options.artifactName, + ]); + const runDirectory = join(artifactRoot, options.runId); + await createExclusiveDirectory(runDirectory, "Bombadil artifact run leaf"); + return { + artifactRoot, + manifestPath: join(artifactRoot, "manifest.json"), + runDirectory, + }; +} + +async function prepareArtifactUploadSession( + planInput: DirectBombadilArtifactRunPlan, +): Promise { + const plan = validateArtifactRunPlan(planInput); + let repositoryRoot: string | null; + try { + repositoryRoot = await realpath(plan.repositoryRoot); + } catch (error) { + if (!isRecord(error) || error.code !== "ENOENT") { + throw new BombadilArtifactPolicyError( + `artifactRun.repositoryRoot could not be proven safe: ${renderUnknown(error)}`, + ); + } + repositoryRoot = null; + } + if (repositoryRoot === null || repositoryRoot !== plan.repositoryRoot) { + throw new BombadilArtifactPolicyError( + "artifactRun.repositoryRoot must resolve to its exact configured directory", + ); + } + const root = await ensureSafeDirectoryChain(repositoryRoot, [ + "artifacts", + "direct-bombadil-upload", + ]); + const finalDirectory = join(root, plan.runId); + let finalMetadata; + try { + finalMetadata = await lstat(finalDirectory); + } catch (error) { + if (!isRecord(error) || error.code !== "ENOENT") { + throw new BombadilArtifactPolicyError( + `Bombadil upload run leaf could not be inspected: ${renderUnknown(error)}`, + ); + } + finalMetadata = null; + } + if (finalMetadata !== null) { + throw new BombadilArtifactPolicyError("Bombadil upload run leaf already exists"); + } + const stagingDirectory = join(root, `.staging-${plan.runId}`); + return { + finalDirectory, + mode: plan.uploadMode, + publication: "atomic-leaf", + receiptPath: join(finalDirectory, "receipt.json"), + runId: plan.runId, + stagingDirectory, + }; +} + +async function requireArtifactUploadLeafAbsent( + session: AtomicArtifactUploadSession, +): Promise { + let existing; + try { + existing = await lstat(session.finalDirectory); + } catch (error) { + if (!isRecord(error) || error.code !== "ENOENT") { + throw new BombadilArtifactPolicyError( + `Bombadil upload run leaf could not be inspected: ${renderUnknown(error)}`, + ); + } + existing = null; + } + if (existing !== null) { + throw new BombadilArtifactPolicyError("Bombadil upload run leaf appeared before publication"); + } +} + +async function commitArtifactUploadSession( + session: AtomicArtifactUploadSession, +): Promise { + // The validated staging tree becomes immutable evidence at this dispatch. + // Nothing fallible may run after the atomic rename. + await rename(session.stagingDirectory, session.finalDirectory); +} + +function validateArtifactRelativePath( + relativePath: string, + policy: ValidatedArtifactPolicy, +): readonly string[] { + const parts = relativePath.split("/"); + if ( + relativePath.length === 0 + || relativePath.includes("\\") + || Buffer.byteLength(relativePath, "utf8") > policy.maxPathBytes + || parts.length > policy.maxDepth + || parts.some((part) => + part === "" + || part === "." + || part === ".." + || part.startsWith(".") + || !ARTIFACT_PATH_PART_PATTERN.test(part) + ) + ) { + throw new BombadilArtifactPolicyError(`Bombadil emitted unsafe artifact path ${relativePath}`); + } + return parts; +} + +function artifactOutputFileIsAllowed(relativePath: string): boolean { + return relativePath === "trace.jsonl" + || PRIVATE_DIAGNOSTIC_EXTENSIONS.has(extname(relativePath).toLowerCase()); +} + +function sameBigIntFileMetadata( + left: Readonly<{ dev: bigint; ino: bigint; size: bigint; ctimeNs: bigint; mtimeNs: bigint }>, + right: Readonly<{ dev: bigint; ino: bigint; size: bigint; ctimeNs: bigint; mtimeNs: bigint }>, +): boolean { + return left.dev === right.dev + && left.ino === right.ino + && left.size === right.size + && left.ctimeNs === right.ctimeNs + && left.mtimeNs === right.mtimeNs; +} + +async function withClosedArtifactHandle( + handle: Readonly<{ close: () => Promise }>, + operation: () => Promise, +): Promise { + let value: Value | undefined; + let operationFailure: unknown = null; + try { + value = await operation(); + } catch (error) { + operationFailure = error; + } + let closeFailure: unknown = null; + try { + await handle.close(); + } catch (error) { + closeFailure = error; + } + if (operationFailure !== null) { + if (closeFailure !== null) { + throw new AggregateError( + [operationFailure, closeFailure], + "Bombadil artifact operation and descriptor cleanup both failed", + { cause: operationFailure }, + ); + } + throw operationFailure; + } + if (closeFailure !== null) throw closeFailure; + return value as Value; +} + +async function hashBoundRegularFile(options: { + readonly expected: BigIntStats; + readonly path: string; + readonly policy: ValidatedArtifactPolicy; + readonly relativePath: string; +}): Promise { + const flags = fileSystemConstants.O_RDONLY + | fileSystemConstants.O_NOFOLLOW + | fileSystemConstants.O_NONBLOCK; + const handle = await open(options.path, flags); + return await withClosedArtifactHandle(handle, async () => { + const before = await handle.stat({ bigint: true }); + if ( + !before.isFile() + || before.nlink !== 1n + || !options.expected.isFile() + || options.expected.nlink !== 1n + || !sameBigIntFileMetadata(before, options.expected) + ) { + throw new BombadilArtifactPolicyError( + `Bombadil artifact ${options.relativePath} changed identity before inspection`, + ); + } + const size = Number(before.size); + if (!Number.isSafeInteger(size) || size > options.policy.maxFileBytes) { + throw new BombadilArtifactPolicyError( + `Bombadil artifact ${options.relativePath} exceeds the per-file byte quota`, + ); + } + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(64 * 1024); + let offset = 0; + while (offset < size) { + const length = Math.min(buffer.length, size - offset); + const read = await handle.read(buffer, 0, length, offset); + if (read.bytesRead === 0) { + throw new BombadilArtifactPolicyError( + `Bombadil artifact ${options.relativePath} changed while inspected`, + ); + } + hash.update(buffer.subarray(0, read.bytesRead)); + offset += read.bytesRead; + } + const after = await handle.stat({ bigint: true }); + if (!sameBigIntFileMetadata(before, after)) { + throw new BombadilArtifactPolicyError( + `Bombadil artifact ${options.relativePath} changed while inspected`, + ); + } + return { + device: before.dev, + inode: before.ino, + relativePath: options.relativePath, + sha256: hash.digest("hex"), + size, + }; + }); +} + +async function readBoundRegularFileBytes(options: { + readonly expected?: ArtifactInventoryFile; + readonly label: string; + readonly maximumBytes: number; + readonly path: string; +}): Promise { + const flags = fileSystemConstants.O_RDONLY + | fileSystemConstants.O_NOFOLLOW + | fileSystemConstants.O_NONBLOCK; + let handle; + try { + handle = await open(options.path, flags); + } catch { + throw new BombadilArtifactPolicyError(`${options.label} is not an openable regular file`); + } + try { + return await withClosedArtifactHandle(handle, async () => { + const before = await handle.stat({ bigint: true }); + const size = Number(before.size); + if ( + !before.isFile() + || before.nlink !== 1n + || !Number.isSafeInteger(size) + || size < 1 + || size > options.maximumBytes + ) { + throw new BombadilArtifactPolicyError(`${options.label} is not a bounded regular file`); + } + if ( + options.expected !== undefined + && ( + before.dev !== options.expected.device + || before.ino !== options.expected.inode + || size !== options.expected.size + ) + ) { + throw new BombadilArtifactPolicyError(`${options.label} changed after inventory`); + } + const bytes = Buffer.allocUnsafe(size); + let offset = 0; + while (offset < size) { + const read = await handle.read(bytes, offset, size - offset, offset); + if (read.bytesRead === 0) { + throw new BombadilArtifactPolicyError(`${options.label} changed while being read`); + } + offset += read.bytesRead; + } + const after = await handle.stat({ bigint: true }); + if (!sameBigIntFileMetadata(before, after)) { + throw new BombadilArtifactPolicyError(`${options.label} changed while being read`); + } + if ( + options.expected !== undefined + && sha256(bytes) !== options.expected.sha256 + ) { + throw new BombadilArtifactPolicyError(`${options.label} hash changed after inventory`); + } + return bytes; + }); + } catch (error) { + throw error instanceof BombadilArtifactPolicyError + ? error + : new BombadilArtifactPolicyError( + `${options.label} could not be read safely: ${renderUnknown(error)}`, + ); + } +} + +function decodeTraceLines(bytes: Uint8Array): readonly string[] { + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new Error("Bombadil trace is not valid UTF-8"); + } + const lines = text.split(/\r?\n/u); + if (lines.at(-1) === "") lines.pop(); + return lines; +} + +async function scanBombadilArtifactTree(options: { + readonly allowTransientEntryAbsence?: boolean; + readonly beforeDirectoryOpen?: (absolutePath: string) => Promise | void; + readonly beforeEntryInspect?: (absolutePath: string) => Promise | void; + readonly hashFiles: boolean; + readonly policy: ValidatedArtifactPolicy; + readonly root: string; + readonly rootMayBeAbsent?: boolean; +}): Promise { + let rootMetadata: BigIntStats | null; + try { + rootMetadata = await lstat(options.root, { bigint: true }); + } catch (error) { + if (!isRecord(error) || error.code !== "ENOENT") { + throw new BombadilArtifactPolicyError( + `Bombadil output root could not be inspected: ${renderUnknown(error)}`, + ); + } + rootMetadata = null; + } + if (rootMetadata === null) { + if (options.rootMayBeAbsent === true) { + return { + directories: Object.freeze([]), + entryCount: 0, + files: Object.freeze([]), + fileCount: 0, + inventorySha256: sha256(""), + totalBytes: 0, + }; + } + throw new BombadilArtifactPolicyError("Bombadil output directory does not exist"); + } + if (rootMetadata.isSymbolicLink() || !rootMetadata.isDirectory()) { + throw new BombadilArtifactPolicyError("Bombadil output root must be a non-symlink directory"); + } + const directories: string[] = []; + const files: ArtifactInventoryFile[] = []; + let entryCount = 0; + let totalBytes = 0; + const pending: Array<{ readonly absolutePath: string; readonly relativePath: string }> = [{ + absolutePath: options.root, + relativePath: "", + }]; + while (pending.length > 0) { + const current = pending.pop(); + if (current === undefined) continue; + await options.beforeDirectoryOpen?.(current.absolutePath); + const directory = await opendir(current.absolutePath).catch((error: unknown) => { + if ( + options.allowTransientEntryAbsence === true + && isRecord(error) + && error.code === "ENOENT" + ) { + throw error; + } + throw new BombadilArtifactPolicyError( + `Bombadil artifact directory could not be opened safely: ${renderUnknown(error)}`, + ); + }); + try { + await withClosedArtifactHandle(directory, async () => { + while (true) { + const entry = await directory.read(); + if (entry === null) break; + const relativePath = current.relativePath === "" + ? entry.name + : `${current.relativePath}/${entry.name}`; + validateArtifactRelativePath(relativePath, options.policy); + entryCount += 1; + if (entryCount > options.policy.maxEntries) { + throw new BombadilArtifactPolicyError("Bombadil artifact entry quota was exceeded"); + } + const absolutePath = join(current.absolutePath, entry.name); + await options.beforeEntryInspect?.(absolutePath); + const metadata = await lstat(absolutePath, { bigint: true }); + if (metadata.isSymbolicLink()) { + throw new BombadilArtifactPolicyError( + `Bombadil emitted a symbolic link at ${relativePath}`, + ); + } + if (metadata.isDirectory()) { + directories.push(relativePath); + pending.push({ absolutePath, relativePath }); + continue; + } + if (!metadata.isFile() || metadata.nlink !== 1n) { + throw new BombadilArtifactPolicyError( + `Bombadil emitted a non-regular or multiply-linked file at ${relativePath}`, + ); + } + if (!artifactOutputFileIsAllowed(relativePath)) { + throw new BombadilArtifactPolicyError( + `Bombadil emitted a file outside the artifact allowlist at ${relativePath}`, + ); + } + if (files.length + 1 > options.policy.maxFiles) { + throw new BombadilArtifactPolicyError("Bombadil artifact file quota was exceeded"); + } + const fileSize = Number(metadata.size); + if (!Number.isSafeInteger(fileSize) || fileSize > options.policy.maxFileBytes) { + throw new BombadilArtifactPolicyError( + `Bombadil artifact ${relativePath} exceeds the per-file byte quota`, + ); + } + totalBytes += fileSize; + if (!Number.isSafeInteger(totalBytes) || totalBytes > options.policy.maxTotalBytes) { + throw new BombadilArtifactPolicyError( + "Bombadil aggregate artifact byte quota was exceeded", + ); + } + files.push(options.hashFiles + ? await hashBoundRegularFile({ + expected: metadata, + path: absolutePath, + policy: options.policy, + relativePath, + }) + : { + device: 0n, + inode: 0n, + relativePath, + sha256: "", + size: fileSize, + }); + } + }); + } catch (error) { + if ( + options.allowTransientEntryAbsence === true + && isRecord(error) + && error.code === "ENOENT" + ) { + throw error; + } + throw error instanceof BombadilArtifactPolicyError + ? error + : new BombadilArtifactPolicyError( + `Bombadil artifact directory could not be inspected safely: ${renderUnknown(error)}`, + ); + } + } + let finalRootMetadata: BigIntStats; + try { + finalRootMetadata = await lstat(options.root, { bigint: true }); + } catch (error) { + throw new BombadilArtifactPolicyError( + `Bombadil output root could not be revalidated: ${renderUnknown(error)}`, + ); + } + if ( + !finalRootMetadata.isDirectory() + || finalRootMetadata.isSymbolicLink() + || finalRootMetadata.dev !== rootMetadata.dev + || finalRootMetadata.ino !== rootMetadata.ino + ) { + throw new BombadilArtifactPolicyError("Bombadil output root changed during inspection"); + } + directories.sort(compareCodeUnits); + files.sort((left, right) => compareCodeUnits(left.relativePath, right.relativePath)); + const inventorySha256 = sha256([ + ...directories.map((directory) => `D\0${directory}\n`), + ...files.map((file) => + `F\0${file.relativePath}\0${String(file.size)}\0${file.sha256}\n` + ), + ].join("")); + return { + directories: Object.freeze(directories), + entryCount, + files: Object.freeze(files), + fileCount: files.length, + inventorySha256, + totalBytes, + }; +} + +/** @internal Exercise transient versus authoritative artifact scans in package tests. */ +export async function inspectBombadilArtifactTreeForTest(options: { + readonly allowTransientEntryAbsence?: boolean; + readonly beforeDirectoryOpen?: (absolutePath: string) => Promise | void; + readonly beforeEntryInspect?: (absolutePath: string) => Promise | void; + readonly hashFiles: boolean; + readonly policy: DirectBombadilArtifactPolicy; + readonly root: string; +}): Promise { + await scanBombadilArtifactTree({ + ...options, + policy: validateArtifactPolicy(options.policy), + }); } -type ValidatedConfig = Omit< - DirectBombadilFuzzConfig, - "explorationPolicy" | "server" | "viewport" -> & { - readonly artifactRoot: string; - readonly baseUrl: string; - readonly bombadilExecutable: string; - readonly entryPath: `/${string}`; - readonly explorationPolicy: ValidatedExplorationPolicy | null; - readonly port: string; - readonly targetQuery: Readonly>; - readonly viewport: ValidatedViewport; - readonly server: DirectBombadilServerConfig & { - readonly readinessPath: `/${string}`; - readonly startupTimeoutMs: number; - }; -}; +async function ensureSafeChildDirectories( + root: string, + parts: readonly string[], +): Promise { + await requireSafeDirectory(root, "Bombadil upload staging root"); + let current = root; + for (const part of parts) { + if (!ARTIFACT_PATH_PART_PATTERN.test(part) || part.startsWith(".")) { + throw new BombadilArtifactPolicyError("Bombadil upload path contains an unsafe component"); + } + current = join(current, part); + try { + await mkdir(current, { mode: 0o700 }); + } catch (error) { + if (!isRecord(error) || error.code !== "EEXIST") throw error; + } + await requireSafeDirectory(current, "Bombadil upload directory"); + const resolved = await realpath(current); + if (!isWithin(root, resolved) || resolved !== current) { + throw new BombadilArtifactPolicyError("Bombadil upload directory escaped staging root"); + } + } + return current; +} -interface ValidatedViewport { - readonly deviceScaleFactor: number; - readonly height: number; - readonly width: number; +async function writeExclusiveBytes(path: string, bytes: Uint8Array): Promise { + const flags = fileSystemConstants.O_WRONLY + | fileSystemConstants.O_CREAT + | fileSystemConstants.O_EXCL + | fileSystemConstants.O_NOFOLLOW; + const handle = await open(path, flags, 0o600); + await withClosedArtifactHandle(handle, async () => { + let offset = 0; + while (offset < bytes.byteLength) { + const written = await handle.write(bytes, offset, bytes.byteLength - offset, offset); + if (written.bytesWritten === 0) throw new Error("Exclusive artifact write made no progress"); + offset += written.bytesWritten; + } + await handle.sync(); + }); } -interface ValidatedExplorationPolicy { - readonly minDistinctNamedSnapshotValues: Readonly>; - readonly minNamedSnapshotChangesAfterActionKind: Readonly>> - >>; - readonly minNamedSnapshotChangesAfterNonWait: Readonly>; - readonly minNonWaitActions: number; - readonly requireStableTargetUrl: boolean; - readonly requiredActionKinds: readonly DirectBombadilActionKind[]; - readonly requiredNamedSnapshots: readonly string[]; +async function writeExpectedJson( + root: string, + relativePath: string, + value: unknown, +): Promise { + const parts = relativePath.split("/"); + const fileName = parts.pop(); + if (fileName === undefined || !ARTIFACT_PATH_PART_PATTERN.test(fileName)) { + throw new BombadilArtifactPolicyError("Sanitized upload path is invalid"); + } + const directory = await ensureSafeChildDirectories(root, parts); + const bytes = Buffer.from(`${JSON.stringify(value, null, 2)}\n`, "utf8"); + await writeExclusiveBytes(join(directory, fileName), bytes); + return { + relativePath, + sha256: sha256(bytes), + size: bytes.byteLength, + }; } -function readOptionValue( - arguments_: readonly string[], - index: number, - option: string, -): { readonly index: number; readonly value: string } { - const value = arguments_[index + 1]; - if (value === undefined || value.startsWith("-")) { - throw new Error(`${option} requires a value`); +function expectedUploadDirectories( + files: readonly ExpectedUploadFile[], +): readonly string[] { + const directories = new Set(); + for (const file of files) { + const parts = file.relativePath.split("/"); + parts.pop(); + for (let index = 1; index <= parts.length; index += 1) { + directories.add(parts.slice(0, index).join("/")); + } } - return { index: index + 1, value }; + return Object.freeze([...directories].sort(compareCodeUnits)); } -function parseTimeLimit(value: string): number { - const match = /^([1-9][0-9]*)s$/u.exec(value); - if (match === null) { - throw new Error("--time-limit must be a whole number of seconds such as 20s"); +async function validateExpectedUploadTree( + root: string, + expectedInput: readonly ExpectedUploadFile[], +): Promise { + const expected = [...expectedInput].sort((left, right) => + compareCodeUnits(left.relativePath, right.relativePath) + ); + if (new Set(expected.map((file) => file.relativePath)).size !== expected.length) { + throw new BombadilArtifactPolicyError("Sanitized upload contains duplicate file paths"); } - const seconds = Number(match[1]); + const directories = expectedUploadDirectories(expected); + const maximumPathBytes = Math.max( + 1, + ...expected.map((file) => Buffer.byteLength(file.relativePath, "utf8")), + ); + const inventory = await scanBombadilArtifactTree({ + hashFiles: true, + policy: { + maxDepth: Math.max(1, ...expected.map((file) => file.relativePath.split("/").length)), + maxEntries: Math.max(1, expected.length + directories.length), + maxFileBytes: Math.max(1, ...expected.map((file) => file.size)), + maxFiles: Math.max(1, expected.length), + maxPathBytes: maximumPathBytes, + maxTotalBytes: Math.max(1, expected.reduce((total, file) => total + file.size, 0)), + }, + root, + }); if ( - !Number.isSafeInteger(seconds) - || seconds < MIN_TIME_LIMIT_SECONDS - || seconds > MAX_TIME_LIMIT_SECONDS + inventory.directories.length !== directories.length + || inventory.directories.some((directory, index) => directory !== directories[index]) + || inventory.files.length !== expected.length + || inventory.files.some((file, index) => { + const wanted = expected[index]; + return wanted === undefined + || file.relativePath !== wanted.relativePath + || file.sha256 !== wanted.sha256 + || file.size !== wanted.size; + }) ) { - throw new Error( - `--time-limit must be between ${String(MIN_TIME_LIMIT_SECONDS)}s and ${String(MAX_TIME_LIMIT_SECONDS)}s`, + throw new BombadilArtifactPolicyError( + "Sanitized upload tree differs from its exact expected inventory", ); } - return seconds; } -function bombadilNativeBinary(repositoryRoot: string): string { - let binary: string; - if (process.platform === "darwin" && process.arch === "arm64") { - binary = "bombadil-darwin-arm64"; - } else if (process.platform === "linux" && process.arch === "x64") { - binary = "bombadil-linux-x64"; - } else if (process.platform === "linux" && process.arch === "arm64") { - binary = "bombadil-linux-arm64"; - } else { - throw new Error(`Bombadil 0.7.2 does not support ${process.platform}-${process.arch}`); - } - return join( - repositoryRoot, - "node_modules", - "@antithesishq", - "bombadil", - "binaries", - binary, +async function copyVerifiedArtifactFile(options: { + readonly destinationRoot: string; + readonly file: ArtifactInventoryFile; + readonly sourceRoot: string; +}): Promise { + const parts = options.file.relativePath.split("/"); + const fileName = parts.pop(); + if (fileName === undefined) throw new BombadilArtifactPolicyError("Artifact copy path is empty"); + const destinationDirectory = await ensureSafeChildDirectories( + options.destinationRoot, + parts, ); + const destinationPath = join(destinationDirectory, fileName); + const sourcePath = join(options.sourceRoot, ...options.file.relativePath.split("/")); + const sourceFlags = fileSystemConstants.O_RDONLY + | fileSystemConstants.O_NOFOLLOW + | fileSystemConstants.O_NONBLOCK; + const destinationFlags = fileSystemConstants.O_WRONLY + | fileSystemConstants.O_CREAT + | fileSystemConstants.O_EXCL + | fileSystemConstants.O_NOFOLLOW; + const source = await open(sourcePath, sourceFlags); + let destination: Awaited> | null = null; + let copyFailure: unknown = null; + try { + const before = await source.stat({ bigint: true }); + if ( + !before.isFile() + || before.nlink !== 1n + || before.dev !== options.file.device + || before.ino !== options.file.inode + || Number(before.size) !== options.file.size + ) { + throw new BombadilArtifactPolicyError( + `Bombadil artifact ${options.file.relativePath} changed before private copy`, + ); + } + destination = await open(destinationPath, destinationFlags, 0o600); + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(64 * 1024); + let offset = 0; + while (offset < options.file.size) { + const read = await source.read( + buffer, + 0, + Math.min(buffer.length, options.file.size - offset), + offset, + ); + if (read.bytesRead === 0) { + throw new BombadilArtifactPolicyError( + `Bombadil artifact ${options.file.relativePath} changed during private copy`, + ); + } + hash.update(buffer.subarray(0, read.bytesRead)); + let writtenOffset = 0; + while (writtenOffset < read.bytesRead) { + const written = await destination.write( + buffer, + writtenOffset, + read.bytesRead - writtenOffset, + offset + writtenOffset, + ); + if (written.bytesWritten === 0) throw new Error("Private artifact copy made no progress"); + writtenOffset += written.bytesWritten; + } + offset += read.bytesRead; + } + await destination.sync(); + const after = await source.stat({ bigint: true }); + if ( + !sameBigIntFileMetadata(before, after) + || hash.digest("hex") !== options.file.sha256 + ) { + throw new BombadilArtifactPolicyError( + `Bombadil artifact ${options.file.relativePath} changed during private copy`, + ); + } + } catch (error) { + copyFailure = error; + await rm(destinationPath, { force: true }).catch(() => undefined); + } + let closeFailure: unknown = null; + try { + await closeBombadilArtifactCopyHandles(destination, source); + } catch (error) { + closeFailure = error; + } + if (copyFailure !== null) { + if (closeFailure !== null) { + throw new AggregateError( + [copyFailure, closeFailure], + "Bombadil artifact copy and descriptor cleanup both failed", + { cause: copyFailure }, + ); + } + throw copyFailure; + } + if (closeFailure !== null) throw closeFailure; } -function requireLocalRootHttpOrigin(value: string): string { - const baseUrl = normalizeRootHttpOrigin(value); - const url = new URL(baseUrl); - if (!canAutomaticallyStartLocalServer(baseUrl)) { - throw new Error("--base-url must use HTTP on 127.0.0.1 or localhost"); +/** @internal Close both descriptor-bound copy handles even when one close fails. */ +export async function closeBombadilArtifactCopyHandles( + destination: Readonly<{ close: () => Promise }> | null, + source: Readonly<{ close: () => Promise }>, +): Promise { + const failures: unknown[] = []; + if (destination !== null) { + try { + await destination.close(); + } catch (error) { + failures.push(error); + } } - if (url.port === "") { - throw new Error("--base-url must include an explicit local server port"); + try { + await source.close(); + } catch (error) { + failures.push(error); } - if (Number(url.port) < 1) { - throw new Error("--base-url port must be between 1 and 65535"); + if (failures.length === 1) throw failures[0]; + if (failures.length > 1) { + throw new AggregateError(failures, "Both Bombadil artifact copy descriptors failed to close"); } - return baseUrl; } -function hasControlCharacters(value: string): boolean { - for (const character of value) { - const code = character.charCodeAt(0); - if (code < 32 || code === 127) return true; +function emptyArtifactInventory(): ArtifactInventory { + return { + directories: Object.freeze([]), + entryCount: 0, + files: Object.freeze([]), + fileCount: 0, + inventorySha256: sha256(""), + totalBytes: 0, + }; +} + +function artifactFailureCode(error: unknown): DirectBombadilArtifactFailureCode { + if (error instanceof BombadilPersistenceError) return "persistence"; + if (error instanceof BombadilWriterSettlementError) return "writer-settlement"; + if (error instanceof BombadilArtifactPolicyError) return "artifact-policy"; + const message = renderUnknown(error); + if (message.includes("interrupted") || message.includes("SIGINT") || message.includes("SIGTERM")) { + return "interrupted"; } - return false; + if (message.includes("exploration policy")) return "exploration-policy"; + if (message.includes("trace") || message.includes("Direct contract")) return "trace-attestation"; + if (message.includes("server") || message.includes("reachable")) return "server"; + if (message.includes("Bombadil")) return "process"; + return "unknown"; } -function isRecord(value: unknown): value is Readonly> { - return typeof value === "object" && value !== null && !Array.isArray(value); +function failureAsError(error: unknown): Error { + return error instanceof Error ? error : new Error(renderUnknown(error)); } -function hasExactKeys( - value: Readonly>, - expected: ReadonlySet, -): boolean { - const keys = Object.keys(value); - return keys.length === expected.size && keys.every((key) => expected.has(key)); +function combinePersistenceFailure( + primary: unknown, + persistence: unknown, + message = "Bombadil persistence also failed", +): BombadilPersistenceError { + return new BombadilPersistenceError( + `${renderUnknown(primary)}; ${message}`, + [primary, persistence], + ); } -function compareCodeUnits(left: string, right: string): number { - if (left < right) return -1; - if (left > right) return 1; - return 0; +async function publishFailureAndThrow( + primary: unknown, + publish: () => Promise, +): Promise { + try { + await publish(); + } catch (persistence) { + throw combinePersistenceFailure( + primary, + persistence, + "sanitized Bombadil receipt publication also failed", + ); + } + throw failureAsError(primary); +} + +function createArtifactReceipt(options: { + readonly completedAt: Date; + readonly diagnosticsRetained: boolean; + readonly failureCode: DirectBombadilArtifactFailureCode | null; + readonly inventory: ArtifactInventory; + readonly policy: ValidatedArtifactPolicy; + readonly session: ArtifactUploadSession; + readonly status: "failed" | "passed" | "rejected"; +}): DirectBombadilArtifactReceipt { + return Object.freeze({ + schema: ARTIFACT_RECEIPT_SCHEMA, + completedAt: options.completedAt.toISOString(), + diagnosticsRetained: options.diagnosticsRetained, + failureCode: options.failureCode, + inventory: Object.freeze({ + entryCount: options.inventory.entryCount, + fileCount: options.inventory.fileCount, + inventorySha256: options.inventory.entryCount === 0 + ? null + : options.inventory.inventorySha256, + totalBytes: options.inventory.totalBytes, + }), + mode: options.session.mode, + policy: options.policy, + runId: options.session.runId, + status: options.status, + }); +} + +function createSanitizedRunSummary(options: { + readonly artifactName: string; + readonly attestation: DirectBombadilTraceAttestation | null; + readonly explorationSummary: DirectBombadilExplorationSummary | null; + readonly failureCode: DirectBombadilArtifactFailureCode | null; + readonly scenario: string; + readonly status: "failed" | "passed" | "rejected"; +}): DirectBombadilSanitizedRunSummary { + return Object.freeze({ + schema: ARTIFACT_SUMMARY_SCHEMA, + artifactName: options.artifactName, + scenario: options.scenario, + status: options.status, + failureCode: options.failureCode, + attestation: options.attestation === null + ? null + : Object.freeze({ + invalidObservationCount: options.attestation.invalidObservationCount, + observationCount: options.attestation.observationCount, + validObservationCount: options.attestation.validObservationCount, + }), + exploration: options.explorationSummary === null + ? null + : Object.freeze({ + actionCount: options.explorationSummary.actions.total, + nonWaitActionCount: options.explorationSummary.actions.nonWaitCount, + policySatisfied: options.explorationSummary.policy.satisfied, + traceBytes: options.explorationSummary.trace.bytes, + traceLineCount: options.explorationSummary.trace.lineCount, + traceSha256: options.explorationSummary.trace.sha256, + }), + }); +} + +async function resetUploadStaging(session: AtomicArtifactUploadSession): Promise { + await rm(session.stagingDirectory, { force: true, recursive: true }); + await createExclusiveDirectory(session.stagingDirectory, "Bombadil upload staging leaf"); +} + +async function withOwnedUploadStaging( + session: AtomicArtifactUploadSession, + operation: () => Promise, +): Promise { + await createExclusiveDirectory(session.stagingDirectory, "Bombadil upload staging leaf"); + try { + return await operation(); + } catch (error) { + try { + await rm(session.stagingDirectory, { force: true, recursive: true }); + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + "Bombadil upload staging operation and cleanup both failed", + { cause: error }, + ); + } + throw error; + } +} + +async function publishRunUpload(options: { + readonly abortSignal?: AbortSignal; + readonly artifactName: string; + readonly beforeCommitCheck?: (() => Promise | void) | undefined; + readonly attestation: DirectBombadilTraceAttestation | null; + readonly completedAt: Date; + readonly explorationSummary: DirectBombadilExplorationSummary | null; + readonly failure: unknown; + readonly failureCode?: DirectBombadilArtifactFailureCode; + readonly inventory: ArtifactInventory; + readonly interruptedSignal?: () => NodeJS.Signals | null; + readonly localOutputPath: string; + readonly policy: ValidatedArtifactPolicy; + readonly privateDiagnosticsAllowed: boolean; + readonly scenario: string; + readonly serverLog: string; + readonly processLog: string; + readonly session: ArtifactUploadSession; + readonly status: "failed" | "passed" | "rejected"; +}): Promise<{ + readonly failure: unknown; + readonly receipt: DirectBombadilArtifactReceipt; +}> { + let failure = options.failure; + let failureCode = failure === null + ? null + : options.failureCode ?? artifactFailureCode(failure); + let status = options.status; + const observeInterruption = (): boolean => { + if (failure !== null || options.abortSignal?.aborted !== true) return false; + const signal = options.interruptedSignal?.() ?? null; + failure = new Error( + signal === null + ? "Bombadil fuzzing was interrupted" + : `Bombadil fuzzing was interrupted by ${signal}`, + ); + failureCode = "interrupted"; + status = "failed"; + return true; + }; + observeInterruption(); + if (options.session.publication === "deferred" && options.session.mode !== "public-summary") { + throw new BombadilArtifactPolicyError( + "Bombadil matrices support public-summary uploads only", + ); + } + if (options.session.publication === "deferred") { + const receipt = createArtifactReceipt({ + completedAt: options.completedAt, + diagnosticsRetained: false, + failureCode, + inventory: options.inventory, + policy: options.policy, + session: options.session, + status, + }); + const summary = createSanitizedRunSummary({ + artifactName: options.artifactName, + attestation: options.attestation, + explorationSummary: options.explorationSummary, + failureCode, + scenario: options.scenario, + status, + }); + if (options.session.deferredPayload.value !== null) { + throw new BombadilArtifactPolicyError("Bombadil deferred upload state is invalid"); + } + options.session.deferredPayload.value = Object.freeze({ receipt, summary }); + return { failure, receipt }; + } + const session = options.session; + return await withOwnedUploadStaging(session, async () => { + const expectedFiles: ExpectedUploadFile[] = []; + let diagnosticsRetained = false; + if ( + session.mode === "private-vetted" + && options.privateDiagnosticsAllowed + && failureCode !== "interrupted" + ) { + try { + const diagnosticsRoot = await ensureSafeChildDirectories( + session.stagingDirectory, + ["diagnostics", "bombadil-output"], + ); + for (const file of options.inventory.files) { + await copyVerifiedArtifactFile({ + destinationRoot: diagnosticsRoot, + file, + sourceRoot: options.localOutputPath, + }); + expectedFiles.push({ + relativePath: `diagnostics/bombadil-output/${file.relativePath}`, + sha256: file.sha256, + size: file.size, + }); + } + const controlledLogs = await ensureSafeChildDirectories( + session.stagingDirectory, + ["diagnostics", "host"], + ); + const processLogBytes = Buffer.from(options.processLog, "utf8"); + const serverLogBytes = Buffer.from(options.serverLog, "utf8"); + await writeExclusiveBytes(join(controlledLogs, "bombadil.log"), processLogBytes); + await writeExclusiveBytes(join(controlledLogs, "server.log"), serverLogBytes); + expectedFiles.push( + { + relativePath: "diagnostics/host/bombadil.log", + sha256: sha256(processLogBytes), + size: processLogBytes.byteLength, + }, + { + relativePath: "diagnostics/host/server.log", + sha256: sha256(serverLogBytes), + size: serverLogBytes.byteLength, + }, + ); + diagnosticsRetained = true; + } catch (error) { + const persistence = new BombadilPersistenceError( + "Bombadil private diagnostics could not be persisted", + [error], + ); + failure = failure === null + ? persistence + : combinePersistenceFailure(failure, persistence); + failureCode = "persistence"; + status = "failed"; + await resetUploadStaging(session); + expectedFiles.length = 0; + } + } + const stageSanitizedPayload = async (): Promise => { + const receipt = createArtifactReceipt({ + completedAt: options.completedAt, + diagnosticsRetained, + failureCode, + inventory: options.inventory, + policy: options.policy, + session, + status, + }); + const summary = createSanitizedRunSummary({ + artifactName: options.artifactName, + attestation: options.attestation, + explorationSummary: options.explorationSummary, + failureCode, + scenario: options.scenario, + status, + }); + expectedFiles.push( + await writeExpectedJson(session.stagingDirectory, "summary.json", summary), + await writeExpectedJson(session.stagingDirectory, "receipt.json", receipt), + ); + await validateExpectedUploadTree(session.stagingDirectory, expectedFiles); + return receipt; + }; + let receipt = await stageSanitizedPayload(); + await options.beforeCommitCheck?.(); + await requireArtifactUploadLeafAbsent(session); + if (observeInterruption()) { + diagnosticsRetained = false; + await resetUploadStaging(session); + expectedFiles.length = 0; + receipt = await stageSanitizedPayload(); + await requireArtifactUploadLeafAbsent(session); + } + // Signals observed after this synchronous check belong to the caller after + // terminal publication has begun. The immutable rename remains uninterruptible. + await commitArtifactUploadSession(session); + return { failure, receipt }; + }); +} + +type MatrixCampaignTerminalStatus = DirectBombadilMatrixCampaignStatus; +type MatrixCampaignReceiptEntry = DirectBombadilMatrixCampaignReceiptEntry; + +interface MatrixSanitizedChild { + readonly campaignId: string; + readonly payload: SanitizedRunUploadPayload; +} + +async function publishMatrixUpload(options: { + readonly abortSignal?: AbortSignal; + readonly beforeCommitCheck?: (() => Promise | void) | undefined; + readonly campaigns: readonly MatrixCampaignReceiptEntry[]; + readonly children: readonly MatrixSanitizedChild[]; + readonly completedAt: Date; + readonly failure: unknown; + readonly failureCode?: DirectBombadilArtifactFailureCode; + readonly interruptedSignal?: () => NodeJS.Signals | null; + readonly omittedCampaignCount?: number; + readonly session: AtomicArtifactUploadSession; +}): Promise<{ readonly failure: unknown }> { + if (options.session.mode !== "public-summary") { + throw new BombadilArtifactPolicyError("Bombadil matrix upload session must be public-summary"); + } + let failure = options.failure; + let failureCode = failure === null + ? null + : options.failureCode ?? artifactFailureCode(failure); + let status: "failed" | "passed" = failure === null ? "passed" : "failed"; + const observeInterruption = (): boolean => { + if (failure !== null || options.abortSignal?.aborted !== true) return false; + const signal = options.interruptedSignal?.() ?? null; + failure = new Error( + signal === null + ? "Bombadil matrix was interrupted" + : `Bombadil matrix was interrupted by ${signal}`, + ); + failureCode = "interrupted"; + status = "failed"; + return true; + }; + observeInterruption(); + return await withOwnedUploadStaging(options.session, async () => { + const counts = new Map(); + for (const campaign of options.campaigns) { + counts.set(campaign.status, (counts.get(campaign.status) ?? 0) + 1); + } + const expectedFiles: ExpectedUploadFile[] = []; + const stageMatrixPayload = async (): Promise => { + const receipt: DirectBombadilMatrixReceipt = Object.freeze({ + schema: MATRIX_RECEIPT_SCHEMA, + completedAt: options.completedAt.toISOString(), + failureCode, + mode: options.session.mode, + runId: options.session.runId, + status, + omittedCampaignCount: options.omittedCampaignCount ?? 0, + campaigns: Object.freeze(options.campaigns.map((campaign) => Object.freeze(campaign))), + }); + const summary: DirectBombadilMatrixSummary = Object.freeze({ + schema: MATRIX_SUMMARY_SCHEMA, + failureCode, + status, + campaigns: Object.freeze({ + failed: counts.get("failed") ?? 0, + notRun: counts.get("not-run") ?? 0, + notSelected: counts.get("not-selected") ?? 0, + passed: counts.get("passed") ?? 0, + rejected: counts.get("rejected") ?? 0, + total: options.campaigns.length, + omitted: options.omittedCampaignCount ?? 0, + }), + }); + for (const child of options.children) { + expectedFiles.push( + await writeExpectedJson( + options.session.stagingDirectory, + `campaigns/${child.campaignId}/summary.json`, + child.payload.summary, + ), + await writeExpectedJson( + options.session.stagingDirectory, + `campaigns/${child.campaignId}/receipt.json`, + child.payload.receipt, + ), + ); + } + expectedFiles.push( + await writeExpectedJson(options.session.stagingDirectory, "summary.json", summary), + await writeExpectedJson(options.session.stagingDirectory, "receipt.json", receipt), + ); + await validateExpectedUploadTree(options.session.stagingDirectory, expectedFiles); + }; + await stageMatrixPayload(); + await options.beforeCommitCheck?.(); + await requireArtifactUploadLeafAbsent(options.session); + if (observeInterruption()) { + await resetUploadStaging(options.session); + expectedFiles.length = 0; + await stageMatrixPayload(); + await requireArtifactUploadLeafAbsent(options.session); + } + // The atomic rename is the matrix terminal-publication boundary. + await commitArtifactUploadSession(options.session); + return { failure }; + }); } function parseTraceDirectObservation(value: unknown): TraceDirectObservation { @@ -1120,24 +3415,27 @@ export async function attestDirectBombadilTrace(options: { readonly expectedScenario: string; readonly tracePath: string; }): Promise { - const metadata = await stat(options.tracePath).catch(() => null); - if (metadata === null || !metadata.isFile() || metadata.size === 0) { - throw new Error("Bombadil did not produce a nonempty trace.jsonl"); - } - if (metadata.size > TRACE_MAX_BYTES) { - throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_BYTES)} bytes`); - } + const traceBytes = await readBoundRegularFileBytes({ + label: "Bombadil trace.jsonl", + maximumBytes: TRACE_MAX_BYTES, + path: options.tracePath, + }); + return attestDirectBombadilTraceBytes({ ...options, traceBytes }); +} - const stream = createReadStream(options.tracePath, { encoding: "utf8" }); - const lines = createInterface({ input: stream, crlfDelay: Infinity }); +function attestDirectBombadilTraceBytes(options: { + readonly expectedRoute: string; + readonly expectedScenario: string; + readonly traceBytes: Uint8Array; +}): DirectBombadilTraceAttestation { + const lines = decodeTraceLines(options.traceBytes); let observationCount = 0; let invalidObservationCount = 0; let validObservationCount = 0; let initial: DirectBombadilTraceBinding | null = null; let final: ExactTraceDirectObservation | null = null; let finalWasInvalid = false; - try { - for await (const line of lines) { + for (const line of lines) { observationCount += 1; if (observationCount > TRACE_MAX_LINES) { throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); @@ -1189,11 +3487,7 @@ export async function attestDirectBombadilTrace(options: { } if (observation.violations.some((value) => value !== 0)) { throw new Error("Bombadil trace contains a nonzero Direct violation counter"); - } - } - } finally { - lines.close(); - stream.destroy(); + } } if (initial === null || final === null) { @@ -1240,13 +3534,19 @@ export async function summarizeDirectBombadilTrace(options: { readonly targetUrl: string; readonly tracePath: string; }): Promise { - const metadata = await stat(options.tracePath).catch(() => null); - if (metadata === null || !metadata.isFile() || metadata.size === 0) { - throw new Error("Bombadil did not produce a nonempty trace.jsonl"); - } - if (metadata.size > TRACE_MAX_BYTES) { - throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_BYTES)} bytes`); - } + const traceBytes = await readBoundRegularFileBytes({ + label: "Bombadil trace.jsonl", + maximumBytes: TRACE_MAX_BYTES, + path: options.tracePath, + }); + return summarizeDirectBombadilTraceBytes({ ...options, traceBytes }); +} + +function summarizeDirectBombadilTraceBytes(options: { + readonly explorationPolicy?: DirectBombadilExplorationPolicy; + readonly targetUrl: string; + readonly traceBytes: Uint8Array; +}): DirectBombadilExplorationSummary { let targetUrl: URL; try { targetUrl = new URL(options.targetUrl); @@ -1296,10 +3596,8 @@ export async function summarizeDirectBombadilTrace(options: { 0, TRACE_MAX_NAMED_SNAPSHOT_NAMES - strictDiagnosticSnapshotNames.size, ); - const stream = createReadStream(options.tracePath, { encoding: "utf8" }); - const lines = createInterface({ input: stream, crlfDelay: Infinity }); - try { - for await (const line of lines) { + const lines = decodeTraceLines(options.traceBytes); + for (const line of lines) { lineCount += 1; if (lineCount > TRACE_MAX_LINES) { throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); @@ -1440,10 +3738,6 @@ export async function summarizeDirectBombadilTrace(options: { entry.values.add(snapshot.valueSha256); } previousObservationWasExact = true; - } - } finally { - lines.close(); - stream.destroy(); } if (lineCount === 0) throw new Error("Bombadil did not produce a nonempty trace.jsonl"); @@ -1494,13 +3788,12 @@ export async function summarizeDirectBombadilTrace(options: { policyFailures.push("the browser did not remain on the exact target URL"); } } - const traceBytes = await readFile(options.tracePath); return Object.freeze({ schema: "direct.bombadil-exploration-summary/v2", trace: Object.freeze({ - bytes: metadata.size, + bytes: options.traceBytes.byteLength, lineCount, - sha256: sha256(traceBytes), + sha256: sha256(options.traceBytes), }), actions: Object.freeze({ byKind: sortedCountRecord(actionCounts), @@ -1915,7 +4208,7 @@ export function validateDirectBombadilFuzzConfig( if (!isAbsolute(config.repositoryRoot) || repositoryRoot !== config.repositoryRoot) { throw new Error("repositoryRoot must be an absolute normalized path"); } - if (!ARTIFACT_NAME_PATTERN.test(config.artifactName)) { + if (!isBoundedArtifactIdentifier(config.artifactName)) { throw new Error("artifactName must be a safe lowercase kebab identifier"); } if ( @@ -1926,8 +4219,7 @@ export function validateDirectBombadilFuzzConfig( throw new Error("label must contain 1-160 visible characters"); } if ( - config.scenario.length > 120 - || !SCENARIO_PATTERN.test(config.scenario) + !isBoundedScenarioIdentifier(config.scenario) ) { throw new Error("scenario must be a valid Direct scenario identifier"); } @@ -1977,6 +4269,7 @@ export function validateDirectBombadilFuzzConfig( const targetQuery = validateTargetQuery(config.targetQuery ?? {}); const viewport = validateViewport(config.viewport); const explorationPolicy = validateExplorationPolicy(config.explorationPolicy); + const artifactPolicy = validateArtifactPolicy(config.artifactPolicy); const startupTimeoutMs = config.server.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; if ( !Number.isSafeInteger(startupTimeoutMs) @@ -1992,6 +4285,7 @@ export function validateDirectBombadilFuzzConfig( const port = new URL(baseUrl).port; return { ...config, + artifactPolicy, repositoryRoot, specificationPath, baseUrl, @@ -2119,11 +4413,51 @@ function captureStream( function signalProcessGroup( process_: ReturnType, signal: "SIGKILL" | "SIGTERM", -): void { +): boolean { try { process.kill(-process_.pid, signal); - } catch { + return true; + } catch (error) { + if (!isRecord(error) || error.code !== "ESRCH") throw error; if (process_.exitCode === null) process_.kill(signal); + return false; + } +} + +function processGroupExists(processId: number): boolean { + try { + process.kill(-processId, 0); + return true; + } catch (error) { + if (isRecord(error) && error.code === "ESRCH") return false; + throw error; + } +} + +async function waitForProcessGroupExit( + processId: number, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + while (processGroupExists(processId)) { + if (Date.now() >= deadline) { + throw new Error(`Bombadil process group ${String(processId)} survived cleanup`); + } + await Bun.sleep(Math.min(25, Math.max(1, deadline - Date.now()))); + } +} + +async function waitForBombadilLeaderExit( + process_: ReturnType, + timeoutMs: number, +): Promise { + if (process_.exitCode !== null) return; + const exited = await Promise.race([ + process_.exited.then(() => true), + Bun.sleep(timeoutMs).then(() => false), + ]); + if (!exited && process_.exitCode === null) { + throw new Error(`Bombadil process ${String(process_.pid)} survived cleanup`); } } @@ -2132,25 +4466,82 @@ async function terminateProcessGroup( graceMs: number, ): Promise { signalProcessGroup(process_, "SIGTERM"); - await Bun.sleep(graceMs); + await Promise.race([ + process_.exited.then(() => undefined), + Bun.sleep(graceMs), + ]); // The group may still contain descendants after its leader exits on TERM. - signalProcessGroup(process_, "SIGKILL"); - await Promise.race([process_.exited.then(() => undefined), Bun.sleep(graceMs)]); + if (processGroupExists(process_.pid)) signalProcessGroup(process_, "SIGKILL"); + await waitForBombadilLeaderExit(process_, graceMs); + await waitForProcessGroupExit(process_.pid, graceMs); +} + +async function settleBombadilProcessGroup(options: { + readonly immediate: boolean; + readonly process: ReturnType; + readonly timeoutMs: number; +}): Promise { + try { + if (options.immediate) { + signalProcessGroup(options.process, "SIGKILL"); + await waitForBombadilLeaderExit(options.process, options.timeoutMs); + await waitForProcessGroupExit(options.process.pid, options.timeoutMs); + return; + } + await terminateProcessGroup(options.process, options.timeoutMs); + } catch (error) { + throw new BombadilWriterSettlementError( + `Bombadil process group ${String(options.process.pid)} did not settle safely`, + error, + ); + } +} + +async function monitorBombadilArtifactTree(options: { + readonly abortSignal: AbortSignal; + readonly outputPath: string; + readonly policy: ValidatedArtifactPolicy; +}): Promise { + while (!options.abortSignal.aborted) { + try { + await scanBombadilArtifactTree({ + allowTransientEntryAbsence: true, + hashFiles: false, + policy: options.policy, + root: options.outputPath, + rootMayBeAbsent: true, + }); + } catch (error) { + if (isRecord(error) && error.code === "ENOENT") { + // A live producer may atomically replace or remove an entry. The final + // stopped-process scan is authoritative; polling only bounds growth. + } else { + throw error instanceof BombadilArtifactPolicyError + ? error + : new BombadilArtifactPolicyError("Bombadil artifact monitor could not inspect output"); + } + } + await Bun.sleep(ARTIFACT_MONITOR_INTERVAL_MS); + } } export async function runBombadilNativeProcess( invocation: DirectBombadilInvocation, ): Promise { + const artifactPolicy = validateArtifactPolicy(invocation.artifactPolicy); + const childEnvironment = { ...process.env, NO_COLOR: "1" }; + delete childEnvironment[ARTIFACT_COORDINATION_ENVIRONMENT]; const process_ = Bun.spawn([...invocation.command], { cwd: invocation.cwd, detached: true, - env: { ...process.env, NO_COLOR: "1" }, + env: childEnvironment, stdin: "ignore", stdout: "pipe", stderr: "pipe", }); let timeout: ReturnType | undefined; let abortListener: (() => void) | undefined; + const monitorAbortController = new AbortController(); const timeoutPromise = new Promise<"timeout">((resolveTimeout) => { timeout = setTimeout(() => resolveTimeout("timeout"), invocation.wallClockTimeoutMs); }); @@ -2167,23 +4558,55 @@ export async function runBombadilNativeProcess( const stdoutCapture = captureStream(process_.stdout); const stderrCapture = captureStream(process_.stderr); const outputPromise = Promise.all([stdoutCapture.result, stderrCapture.result]); + const artifactMonitor = monitorBombadilArtifactTree({ + abortSignal: monitorAbortController.signal, + outputPath: invocation.outputPath, + policy: artifactPolicy, + }).then( + () => ({ kind: "monitor-stopped" as const }), + (error: unknown) => ({ kind: "artifact-policy" as const, error }), + ); const outcome = await Promise.race([ process_.exited.then((exitCode) => ({ kind: "exited" as const, exitCode })), timeoutPromise.then(() => ({ kind: "timeout" as const })), abortPromise.then(() => ({ kind: "aborted" as const })), + artifactMonitor, ]); + if (outcome.kind === "monitor-stopped") { + throw new BombadilArtifactPolicyError("Bombadil artifact monitor stopped unexpectedly"); + } const terminationGraceMs = invocation.terminationGraceMs ?? PROCESS_TERMINATION_GRACE_MS; - if (outcome.kind === "exited") { - // The native leader is done. Any member left in its group is stale and - // may otherwise keep inherited output pipes open indefinitely. - signalProcessGroup(process_, "SIGKILL"); - } else { - await terminateProcessGroup( - process_, - terminationGraceMs, - ); + try { + await settleBombadilProcessGroup({ + // Every terminal outcome is fail-closed: no writer receives a grace + // window in which it can keep growing or replacing artifact files. + immediate: true, + process: process_, + timeoutMs: terminationGraceMs, + }); + } catch (error) { + stdoutCapture.stop(); + stderrCapture.stop(); + throw error; } + let finalArtifactFailure: unknown = null; + try { + await scanBombadilArtifactTree({ + hashFiles: false, + policy: artifactPolicy, + root: invocation.outputPath, + rootMayBeAbsent: true, + }); + } catch (error) { + finalArtifactFailure = error instanceof BombadilArtifactPolicyError + ? error + : new BombadilArtifactPolicyError( + `Bombadil final artifact inventory could not be proven safe: ${renderUnknown(error)}`, + ); + } + monitorAbortController.abort(); + const finalMonitorOutcome = await artifactMonitor; const outputSettled = await Promise.race([ outputPromise.then( () => true, @@ -2196,6 +4619,16 @@ export async function runBombadilNativeProcess( stderrCapture.stop(); } const [stdout, stderr] = await outputPromise; + const artifactPolicyFailure = outcome.kind === "artifact-policy" + ? outcome.error + : finalMonitorOutcome.kind === "artifact-policy" + ? finalMonitorOutcome.error + : finalArtifactFailure; + if (artifactPolicyFailure !== null) { + throw artifactPolicyFailure instanceof BombadilArtifactPolicyError + ? artifactPolicyFailure + : new BombadilArtifactPolicyError("Bombadil artifact policy was violated"); + } return { exitCode: outcome.kind === "exited" ? outcome.exitCode : process_.exitCode ?? 137, stderr, @@ -2203,6 +4636,7 @@ export async function runBombadilNativeProcess( termination: outcome.kind === "exited" ? null : outcome.kind, }; } finally { + monitorAbortController.abort(); if (timeout !== undefined) clearTimeout(timeout); if (abortListener !== undefined) { invocation.abortSignal?.removeEventListener("abort", abortListener); @@ -2213,8 +4647,14 @@ export async function runBombadilNativeProcess( const defaultDependencies: DirectBombadilRunnerDependencies = { acquireServer: acquireVerificationServer, createAbortController: () => new AbortController(), + createRunId: randomUUID, now: () => new Date(), runBombadil: runBombadilNativeProcess, + signalController: { + forward: (signal) => process.kill(process.pid, signal), + once: (signal, listener) => process.once(signal, listener), + removeListener: (signal, listener) => process.removeListener(signal, listener), + }, serverOutputTimeoutMs: SERVER_OUTPUT_TIMEOUT_MS, spawnServer: spawnVerificationServer, stopServer: stopVerificationServer, @@ -2419,12 +4859,14 @@ function parseMatrixCampaignArgument(arguments_: readonly string[]): { function validateCampaignMatrix( campaigns: readonly DirectBombadilFuzzCampaign[], ): readonly DirectBombadilFuzzCampaign[] { - if (campaigns.length === 0 || campaigns.length > 32) { - throw new Error("Bombadil campaign matrix must contain 1-32 campaigns"); + if (campaigns.length === 0 || campaigns.length > MAX_MATRIX_CAMPAIGNS) { + throw new Error( + `Bombadil campaign matrix must contain 1-${String(MAX_MATRIX_CAMPAIGNS)} campaigns`, + ); } const ids = new Set(); for (const campaign of campaigns) { - if (!ARTIFACT_NAME_PATTERN.test(campaign.id) || ids.has(campaign.id)) { + if (!isBoundedArtifactIdentifier(campaign.id) || ids.has(campaign.id)) { throw new Error("Bombadil campaign IDs must be unique lowercase kebab identifiers"); } ids.add(campaign.id); @@ -2435,12 +4877,14 @@ function validateCampaignMatrix( /** Runs a bounded product-owned campaign matrix serially. */ export async function runDirectBombadilFuzzMatrix( campaignsInput: readonly DirectBombadilFuzzCampaign[], - arguments_: readonly string[] = process.argv.slice(2), + input: DirectBombadilMatrixRunInput = process.argv.slice(2), dependencyOverrides: Partial = {}, -): Promise { - const campaigns = validateCampaignMatrix(campaignsInput); - const parsed = parseMatrixCampaignArgument(arguments_); - if (parsed.help) { +): Promise { + const dependencies = { ...defaultDependencies, ...dependencyOverrides }; + const normalizedOptions = normalizeFuzzRunOptions(input); + if (normalizedOptions.arguments.some((argument) => argument === "--help" || argument === "-h")) { + const campaigns = validateCampaignMatrix(campaignsInput); + parseMatrixCampaignArgument(normalizedOptions.arguments); process.stdout.write(`${[ helpText(campaigns[0]?.config.baseUrl ?? ""), " --campaign Run one campaign; required with --replay", @@ -2449,36 +4893,236 @@ export async function runDirectBombadilFuzzMatrix( ].join("\n")}\n`); return { kind: "help" }; } - const selected = parsed.campaignId === null - ? campaigns - : campaigns.filter((campaign) => campaign.id === parsed.campaignId); - if (selected.length === 0) { - throw new Error(`Unknown Bombadil campaign ${parsed.campaignId ?? ""}`); + const matrixAbortController = dependencies.createAbortController?.() ?? new AbortController(); + let interruptedSignal: NodeJS.Signals | null = null; + const interrupt = (signal: NodeJS.Signals): void => { + interruptedSignal ??= signal; + matrixAbortController.abort(); + }; + const interruptSignals = ["SIGINT", "SIGTERM"] as const; + const processSignals = dependencies.signalController; + for (const signal of interruptSignals) processSignals.once(signal, interrupt); + const releaseSignalHandlers = (): void => { + for (const signal of interruptSignals) processSignals.removeListener(signal, interrupt); + }; + let invalidMatrixUploadMode: boolean; + let matrixPlan: DirectBombadilArtifactRunPlan; + let uploadSession: AtomicArtifactUploadSession; + try { + const firstRepositoryRoot = campaignsInput[0]?.config.repositoryRoot; + if (normalizedOptions.artifactRun === null && firstRepositoryRoot === undefined) { + throw new Error( + `Bombadil campaign matrix must contain 1-${String(MAX_MATRIX_CAMPAIGNS)} campaigns`, + ); + } + const requestedMatrixPlan = normalizedOptions.artifactRun ?? { + repositoryRoot: await realpath(resolve(firstRepositoryRoot ?? "")), + runId: dependencies.createRunId(), + uploadMode: "public-summary" as const, + }; + const requestedMatrixUploadMode = ( + requestedMatrixPlan as { readonly uploadMode?: unknown } + ).uploadMode ?? "public-summary"; + invalidMatrixUploadMode = requestedMatrixUploadMode !== "public-summary"; + matrixPlan = { + repositoryRoot: requestedMatrixPlan.repositoryRoot, + runId: requestedMatrixPlan.runId, + uploadMode: "public-summary", + }; + uploadSession = await prepareArtifactUploadSession(matrixPlan); + } catch (error) { + releaseSignalHandlers(); + const signalToForward = interruptedSignal as NodeJS.Signals | null; + if (signalToForward !== null) processSignals.forward(signalToForward); + throw error; } - if ( - parsed.campaignId === null - && parsed.arguments.some((argument) => - argument === "--replay" || argument.startsWith("--replay=") - ) - ) { - throw new Error("--replay requires exactly one --campaign in matrix mode"); - } - const results: Array<{ - readonly campaignId: string; - readonly result: Extract; - }> = []; - for (const campaign of selected) { - const result = await runDirectBombadilFuzz( - campaign.config, - parsed.arguments, - dependencyOverrides, - ); - if (result.kind !== "run") { - throw new Error("Bombadil campaign unexpectedly returned help during matrix execution"); + try { + let campaigns: readonly DirectBombadilFuzzCampaign[]; + let parsed: ReturnType; + let selected: readonly DirectBombadilFuzzCampaign[]; + try { + if (invalidMatrixUploadMode) { + throw new Error("Bombadil matrices support public-summary uploads only"); + } + campaigns = validateCampaignMatrix(campaignsInput); + parsed = parseMatrixCampaignArgument(normalizedOptions.arguments); + selected = parsed.campaignId === null + ? campaigns + : campaigns.filter((campaign) => campaign.id === parsed.campaignId); + if (selected.length === 0) { + throw new Error(`Unknown Bombadil campaign ${parsed.campaignId ?? ""}`); + } + if ( + parsed.campaignId === null + && parsed.arguments.some((argument) => + argument === "--replay" || argument.startsWith("--replay=") + ) + ) { + throw new Error("--replay requires exactly one --campaign in matrix mode"); + } + for (const campaign of selected) { + if (interruptedSignal !== null) throw new Error("Bombadil matrix was interrupted"); + const campaignArguments = parseDirectBombadilFuzzArguments( + parsed.arguments, + campaign.config.baseUrl, + ); + if (campaignArguments.kind !== "run") { + throw new Error("Bombadil matrix campaign unexpectedly entered help mode"); + } + const lexicalConfig = validateDirectBombadilFuzzConfig( + campaign.config, + campaignArguments.baseUrl, + ); + const resolvedPaths = await resolveDirectBombadilRealPaths( + lexicalConfig, + resolveReplayPath(lexicalConfig.repositoryRoot, campaignArguments.replayPath), + ); + if (resolvedPaths.config.repositoryRoot !== matrixPlan.repositoryRoot) { + throw new BombadilArtifactPolicyError( + "Every Bombadil matrix campaign must share artifactRun.repositoryRoot", + ); + } + } + } catch (error) { + const boundedCampaigns = campaignsInput.slice(0, MAX_MATRIX_CAMPAIGNS); + const entries = boundedCampaigns.map((campaign, index): MatrixCampaignReceiptEntry => ({ + campaignId: isBoundedArtifactIdentifier(campaign.id) ? campaign.id : null, + index, + receipt: null, + status: "rejected", + })); + await publishFailureAndThrow(error, async () => { + await publishMatrixUpload({ + abortSignal: matrixAbortController.signal, + beforeCommitCheck: dependencies.beforeArtifactCommit, + campaigns: entries, + children: [], + completedAt: dependencies.now(), + failure: error, + failureCode: interruptedSignal === null ? "configuration-rejected" : "interrupted", + interruptedSignal: () => interruptedSignal, + omittedCampaignCount: Math.max(0, campaignsInput.length - entries.length), + session: uploadSession, + }); + }); + } + + const results: Array<{ + readonly campaignId: string; + readonly result: Extract; + }> = []; + const entries: MatrixCampaignReceiptEntry[] = campaigns.map((campaign, index) => ({ + campaignId: campaign.id, + index, + receipt: null, + status: selected.includes(campaign) ? "not-run" : "not-selected", + })); + const children: MatrixSanitizedChild[] = []; + let executionFailure: unknown = null; + let executionFailureCode: DirectBombadilArtifactFailureCode | undefined; + for (const campaign of selected) { + if (interruptedSignal !== null) { + executionFailure = new Error("Bombadil matrix was interrupted"); + break; + } + const campaignIndex = campaigns.indexOf(campaign); + const deferredPayload = { value: null as SanitizedRunUploadPayload | null }; + const childSession: DeferredArtifactUploadSession = { + deferredPayload, + finalDirectory: join(uploadSession.finalDirectory, "campaigns", campaign.id), + mode: uploadSession.mode, + publication: "deferred", + receiptPath: join( + uploadSession.finalDirectory, + "campaigns", + campaign.id, + "receipt.json", + ), + runId: uploadSession.runId, + }; + try { + const result = await runDirectBombadilFuzzInternal( + campaign.config, + parsed.arguments, + dependencyOverrides, + { + abortSignal: matrixAbortController.signal, + forwardSignal: false, + interruptedSignal: () => interruptedSignal, + plan: matrixPlan, + session: childSession, + }, + ); + if (result.kind !== "run" || deferredPayload.value === null) { + throw new Error("Bombadil campaign did not finalize its sanitized receipt"); + } + children.push({ campaignId: campaign.id, payload: deferredPayload.value }); + results.push({ campaignId: campaign.id, result }); + entries[campaignIndex] = { + campaignId: campaign.id, + index: campaignIndex, + receipt: `campaigns/${campaign.id}/receipt.json`, + status: "passed", + }; + } catch (error) { + executionFailure = error; + const childPayload = deferredPayload.value; + if (childPayload !== null) { + children.push({ campaignId: campaign.id, payload: childPayload }); + executionFailureCode = childPayload.receipt.failureCode ?? undefined; + } + entries[campaignIndex] = { + campaignId: campaign.id, + index: campaignIndex, + receipt: childPayload === null + ? null + : `campaigns/${campaign.id}/receipt.json`, + status: childPayload?.receipt.status === "rejected" ? "rejected" : "failed", + }; + break; + } } - results.push({ campaignId: campaign.id, result }); + if (executionFailure === null && interruptedSignal !== null) { + executionFailure = new Error("Bombadil matrix was interrupted"); + executionFailureCode = "interrupted"; + } + if (executionFailure !== null) { + await publishFailureAndThrow(executionFailure, async () => { + await publishMatrixUpload({ + abortSignal: matrixAbortController.signal, + beforeCommitCheck: dependencies.beforeArtifactCommit, + campaigns: entries, + children, + completedAt: dependencies.now(), + failure: executionFailure, + ...(executionFailureCode === undefined ? {} : { failureCode: executionFailureCode }), + interruptedSignal: () => interruptedSignal, + session: uploadSession, + }); + }); + } + const published = await publishMatrixUpload({ + abortSignal: matrixAbortController.signal, + beforeCommitCheck: dependencies.beforeArtifactCommit, + campaigns: entries, + children, + completedAt: dependencies.now(), + failure: null, + interruptedSignal: () => interruptedSignal, + session: uploadSession, + }); + if (published.failure !== null) throw failureAsError(published.failure); + return { + kind: "matrix", + receiptPath: uploadSession.receiptPath, + results: Object.freeze(results), + uploadArtifactPath: uploadSession.finalDirectory, + }; + } finally { + releaseSignalHandlers(); + const signalToForward = interruptedSignal as NodeJS.Signals | null; + if (signalToForward !== null) processSignals.forward(signalToForward); } - return { kind: "matrix", results: Object.freeze(results) }; } function throwIfBombadilRunAborted(signal: AbortSignal): void { @@ -2495,37 +5139,155 @@ function terminateAbortedOwnedServer( } /** Runs one bounded diagnostic Bombadil campaign and always releases its server lease. */ -export async function runDirectBombadilFuzz( +async function runDirectBombadilFuzzInternal( config: DirectBombadilFuzzConfig, - arguments_: readonly string[] = process.argv.slice(2), + input: DirectBombadilFuzzRunInput = process.argv.slice(2), dependencyOverrides: Partial = {}, -): Promise { - const parsed = parseDirectBombadilFuzzArguments(arguments_, config.baseUrl); - if (parsed.kind === "help") { + preparedUpload?: Readonly<{ + readonly abortSignal?: AbortSignal; + readonly forwardSignal?: boolean; + readonly interruptedSignal?: () => NodeJS.Signals | null; + readonly plan: DirectBombadilArtifactRunPlan; + readonly session: ArtifactUploadSession; + }>, +): Promise { + const dependencies = { ...defaultDependencies, ...dependencyOverrides }; + const normalizedOptions = normalizeFuzzRunOptions(input); + if (normalizedOptions.arguments.some((argument) => argument === "--help" || argument === "-h")) { + parseDirectBombadilFuzzArguments(normalizedOptions.arguments, config.baseUrl); process.stdout.write(`${helpText(config.baseUrl)}\n`); return { kind: "help" }; } - - const lexicalConfig = validateDirectBombadilFuzzConfig(config, parsed.baseUrl); - const lexicalReplayPath = resolveReplayPath( - lexicalConfig.repositoryRoot, - parsed.replayPath, - ); - const resolvedPaths = await resolveDirectBombadilRealPaths( - lexicalConfig, - lexicalReplayPath, - ); - const validated = resolvedPaths.config; - const replayPath = resolvedPaths.replayPath; - const dependencies = { ...defaultDependencies, ...dependencyOverrides }; + const abortController = dependencies.createAbortController?.() ?? new AbortController(); + let interruptedSignal: NodeJS.Signals | null = null; + let ownedServer: ManagedVerificationServer | null = null; + const interrupt = (signal: NodeJS.Signals): void => { + interruptedSignal ??= signal; + abortController.abort(); + if (ownedServer?.exitCode() === null) ownedServer.terminate(); + }; + const interruptSignals = ["SIGINT", "SIGTERM"] as const; + // @types/bun augments Node's process events and has changed this overload + // across patch releases. Bind the stable signal subset used by this runner. + const processSignals = dependencies.signalController; + for (const signal of interruptSignals) processSignals.once(signal, interrupt); + const abortFromPreparedMatrix = (): void => { + interruptedSignal ??= preparedUpload?.interruptedSignal?.() ?? null; + abortController.abort(); + if (ownedServer?.exitCode() === null) ownedServer.terminate(); + }; + if (preparedUpload?.abortSignal !== undefined) { + if (preparedUpload.abortSignal.aborted) abortFromPreparedMatrix(); + else preparedUpload.abortSignal.addEventListener("abort", abortFromPreparedMatrix, { once: true }); + } + try { const generatedAt = dependencies.now(); - const artifactRun = await createArtifactRun({ - artifactRoot: validated.artifactRoot, - generatedAt: generatedAt.toISOString(), - }); + const artifactPlan = preparedUpload?.plan ?? normalizedOptions.artifactRun ?? { + repositoryRoot: await realpath(resolve(config.repositoryRoot)), + runId: dependencies.createRunId(), + uploadMode: "public-summary" as const, + }; + const uploadSession = preparedUpload?.session + ?? await prepareArtifactUploadSession(artifactPlan); + let parsed: Extract; + let validated: ValidatedConfig; + let replayPath: string | null; + try { + throwIfBombadilRunAborted(abortController.signal); + const parsedInput = parseDirectBombadilFuzzArguments( + normalizedOptions.arguments, + config.baseUrl, + ); + if (parsedInput.kind !== "run") { + throw new Error("Bombadil help was not handled before artifact allocation"); + } + parsed = parsedInput; + const lexicalConfig = validateDirectBombadilFuzzConfig(config, parsed.baseUrl); + const lexicalReplayPath = resolveReplayPath( + lexicalConfig.repositoryRoot, + parsed.replayPath, + ); + const resolvedPaths = await resolveDirectBombadilRealPaths( + lexicalConfig, + lexicalReplayPath, + ); + validated = resolvedPaths.config; + replayPath = resolvedPaths.replayPath; + throwIfBombadilRunAborted(abortController.signal); + if (validated.repositoryRoot !== resolve(artifactPlan.repositoryRoot)) { + throw new BombadilArtifactPolicyError( + "artifactRun.repositoryRoot must equal the campaign repositoryRoot", + ); + } + } catch (error) { + const policy = (() => { + try { + return validateArtifactPolicy(config.artifactPolicy); + } catch { + return validateArtifactPolicy(undefined); + } + })(); + await publishFailureAndThrow(error, async () => { + await publishRunUpload({ + abortSignal: abortController.signal, + artifactName: isBoundedArtifactIdentifier(config.artifactName) + ? config.artifactName + : "rejected", + beforeCommitCheck: dependencies.beforeArtifactCommit, + attestation: null, + completedAt: dependencies.now(), + explorationSummary: null, + failure: error, + failureCode: abortController.signal.aborted + ? "interrupted" + : "configuration-rejected", + inventory: emptyArtifactInventory(), + interruptedSignal: () => interruptedSignal, + localOutputPath: config.repositoryRoot, + policy, + privateDiagnosticsAllowed: false, + processLog: "", + scenario: isBoundedScenarioIdentifier(config.scenario) ? config.scenario : "rejected", + serverLog: "", + session: uploadSession, + status: abortController.signal.aborted ? "failed" : "rejected", + }); + }); + } + let artifactRun: Awaited>; + try { + throwIfBombadilRunAborted(abortController.signal); + artifactRun = await createBombadilArtifactRun({ + artifactName: validated.artifactName, + repositoryRoot: validated.repositoryRoot, + runId: dependencies.createRunId(), + }); + throwIfBombadilRunAborted(abortController.signal); + } catch (error) { + await publishFailureAndThrow(error, async () => { + await publishRunUpload({ + abortSignal: abortController.signal, + artifactName: validated.artifactName, + beforeCommitCheck: dependencies.beforeArtifactCommit, + attestation: null, + completedAt: dependencies.now(), + explorationSummary: null, + failure: error, + inventory: emptyArtifactInventory(), + interruptedSignal: () => interruptedSignal, + localOutputPath: validated.repositoryRoot, + policy: validated.artifactPolicy, + privateDiagnosticsAllowed: false, + processLog: "", + scenario: validated.scenario, + serverLog: "", + session: uploadSession, + status: "failed", + }); + }); + } const outputPath = join(artifactRun.runDirectory, "bombadil"); const tracePath = join(outputPath, "trace.jsonl"); - const abortController = dependencies.createAbortController?.() ?? new AbortController(); const invocation = createDirectBombadilInvocation({ baseUrl: validated.baseUrl, bombadilExecutable: validated.bombadilExecutable, @@ -2539,35 +5301,30 @@ export async function runDirectBombadilFuzz( timeLimitSeconds: parsed.timeLimitSeconds, viewport: validated.viewport, }); - const abortableInvocation = { ...invocation, abortSignal: abortController.signal }; + const abortableInvocation = { + ...invocation, + abortSignal: abortController.signal, + artifactPolicy: validated.artifactPolicy, + }; const serverCommand = validated.server.command.map((argument) => argument === "{port}" ? validated.port : argument ); let bombadilVersion: string | null = null; let lease: ServerLease | null = null; - let ownedServer: ManagedVerificationServer | null = null; let processResult: BombadilProcessResult | null = null; let attestation: DirectBombadilTraceAttestation | null = null; let attestationFailure: unknown = null; let explorationSummary: DirectBombadilExplorationSummary | null = null; let explorationSummaryFailure: unknown = null; + let artifactInventory = emptyArtifactInventory(); + let artifactInventoryVetted = false; let rawTracePath: string | null = null; let serverOutput = ""; let serverOutputFailure: unknown = null; let failure: unknown = null; - let interruptedSignal: NodeJS.Signals | null = null; - const interrupt = (signal: NodeJS.Signals): void => { - interruptedSignal ??= signal; - abortController.abort(); - if (ownedServer?.exitCode() === null) ownedServer.terminate(); - }; - const interruptSignals = ["SIGINT", "SIGTERM"] as const; - // @types/bun augments Node's process events and has changed this overload - // across patch releases. Bind the stable signal subset used by this runner. - const processSignals = process as unknown as ProcessSignalEmitter; - for (const signal of interruptSignals) processSignals.once(signal, interrupt); - try { + let writersSettled = true; + { try { await requireRegularFile(validated.bombadilExecutable, "The root Bombadil executable"); bombadilVersion = await readExactBombadilVersion(validated.repositoryRoot); @@ -2586,7 +5343,9 @@ export async function runDirectBombadilFuzz( ownedServer = dependencies.spawnServer({ command: serverCommand, cwd: validated.server.cwd, + detachedProcessGroup: true, ...(validated.server.env === undefined ? {} : { env: validated.server.env }), + omitEnvironment: [ARTIFACT_COORDINATION_ENVIRONMENT], }); terminateAbortedOwnedServer(abortController.signal, ownedServer); return ownedServer; @@ -2607,30 +5366,6 @@ export async function runDirectBombadilFuzz( } catch (error) { processFailure = error; } - const traceMetadata = await stat(tracePath).catch(() => null); - if (traceMetadata?.isFile() === true && traceMetadata.size > 0) { - rawTracePath = tracePath; - } - try { - attestation = await attestDirectBombadilTrace({ - expectedRoute: validated.expectedRoute, - expectedScenario: validated.scenario, - tracePath, - }); - } catch (error) { - attestationFailure = error; - } - try { - explorationSummary = await summarizeDirectBombadilTrace({ - ...(validated.explorationPolicy === null - ? {} - : { explorationPolicy: validated.explorationPolicy }), - targetUrl: invocation.targetUrl, - tracePath, - }); - } catch (error) { - explorationSummaryFailure = error; - } if (processFailure !== null) { throw processFailure instanceof Error ? processFailure @@ -2648,22 +5383,8 @@ export async function runDirectBombadilFuzz( if (processResult.exitCode !== 0) { throw new Error(`Bombadil exited with status ${String(processResult.exitCode)}`); } - if (attestationFailure !== null) { - throw attestationFailure instanceof Error - ? attestationFailure - : new Error(renderUnknown(attestationFailure)); - } - if (explorationSummaryFailure !== null) { - throw explorationSummaryFailure instanceof Error - ? explorationSummaryFailure - : new Error(renderUnknown(explorationSummaryFailure)); - } - if (explorationSummary?.policy.satisfied !== true) { - throw new Error( - `Bombadil exploration policy was not satisfied: ${explorationSummary?.policy.failures.join("; ") ?? "summary unavailable"}`, - ); - } } catch (error) { + if (error instanceof BombadilWriterSettlementError) writersSettled = false; failure = error; } @@ -2672,11 +5393,17 @@ export async function runDirectBombadilFuzz( try { await dependencies.stopServer(serverToStop); } catch (error) { - failure ??= error; + writersSettled = false; + failure = new BombadilWriterSettlementError( + "Bombadil server writers were not proven absent", + failure === null + ? error + : new AggregateError([failure, error], "Bombadil run and server cleanup both failed"), + ); } } const serverAfterRun = ownedServer as ManagedVerificationServer | null; - if (serverAfterRun !== null) { + if (serverAfterRun !== null && writersSettled) { try { serverOutput = await readServerOutputBounded( serverAfterRun, @@ -2687,87 +5414,236 @@ export async function runDirectBombadilFuzz( failure ??= error; } } - } finally { - for (const signal of interruptSignals) { - processSignals.removeListener(signal, interrupt); + if (writersSettled) { + try { + try { + artifactInventory = await scanBombadilArtifactTree({ + hashFiles: true, + policy: validated.artifactPolicy, + root: outputPath, + }); + } catch (error) { + throw error instanceof BombadilArtifactPolicyError + ? error + : new BombadilArtifactPolicyError( + `Bombadil artifact inventory could not be proven safe: ${renderUnknown(error)}`, + ); + } + artifactInventoryVetted = true; + const trace = artifactInventory.files.find((file) => file.relativePath === "trace.jsonl"); + if (trace === undefined || trace.size === 0) { + const missingTrace = new BombadilArtifactPolicyError( + "Bombadil did not produce a retained nonempty trace.jsonl", + ); + attestationFailure = missingTrace; + throw missingTrace; + } + rawTracePath = tracePath; + const traceBytes = await readBoundRegularFileBytes({ + expected: trace, + label: "Bombadil trace.jsonl", + maximumBytes: TRACE_MAX_BYTES, + path: tracePath, + }); + try { + attestation = attestDirectBombadilTraceBytes({ + expectedRoute: validated.expectedRoute, + expectedScenario: validated.scenario, + traceBytes, + }); + } catch (error) { + attestationFailure = error; + } + try { + explorationSummary = summarizeDirectBombadilTraceBytes({ + ...(validated.explorationPolicy === null + ? {} + : { explorationPolicy: validated.explorationPolicy }), + targetUrl: invocation.targetUrl, + traceBytes, + }); + } catch (error) { + explorationSummaryFailure = error; + } + if (attestationFailure !== null) { + throw attestationFailure instanceof Error + ? attestationFailure + : new Error(renderUnknown(attestationFailure)); + } + if (explorationSummaryFailure !== null) { + throw explorationSummaryFailure instanceof Error + ? explorationSummaryFailure + : new Error(renderUnknown(explorationSummaryFailure)); + } + if (explorationSummary?.policy.satisfied !== true) { + throw new Error( + `Bombadil exploration policy was not satisfied: ${explorationSummary?.policy.failures.join("; ") ?? "summary unavailable"}`, + ); + } + } catch (error) { + failure ??= error; + } + } else { + artifactInventory = emptyArtifactInventory(); + failure ??= new BombadilWriterSettlementError( + "Bombadil writers were not proven absent; artifact inspection was suppressed", + new Error("writer settlement unavailable"), + ); } } - const capturedSignal = interruptedSignal as NodeJS.Signals | null; - if (capturedSignal !== null && failure === null) { - failure = new Error(`Bombadil fuzzing was interrupted by ${capturedSignal}`); + const signalAfterRun = interruptedSignal as NodeJS.Signals | null; + if (signalAfterRun !== null && failure === null) { + failure = new Error(`Bombadil fuzzing was interrupted by ${signalAfterRun}`); } - const completedAt = dependencies.now(); - const status = failure === null ? "passed" : "failed"; const logPath = join(artifactRun.runDirectory, "bombadil.log"); const serverLogPath = join(artifactRun.runDirectory, "server.log"); const explorationSummaryPath = join( artifactRun.runDirectory, "exploration-summary.json", ); - const record = { - schema: ARTIFACT_SCHEMA, - evidenceClass: "diagnostic-fuzz", - artifactName: validated.artifactName, - label: validated.label, - status, - generatedAt: generatedAt.toISOString(), - completedAt: completedAt.toISOString(), - durationMs: Math.max(0, completedAt.getTime() - generatedAt.getTime()), - scenario: validated.scenario, - expectedRoute: validated.expectedRoute, - baseUrl: validated.baseUrl, - entryPath: validated.entryPath, - targetQuery: validated.targetQuery, - targetUrl: invocation.targetUrl, - viewport: validated.viewport, - explorationPolicy: validated.explorationPolicy, - specificationPath: validated.specificationPath, - replayPath, - timeLimitSeconds: replayPath === null ? parsed.timeLimitSeconds : null, - serverSource: lease?.source ?? null, - bombadil: { - version: bombadilVersion, - executable: validated.bombadilExecutable, - exitCode: processResult?.exitCode ?? null, - termination: processResult?.termination ?? null, - outputPath, - rawTracePath, - tracePath: attestation === null ? null : tracePath, - logPath, - }, - server: { - logPath: serverLogPath, - logPresent: serverOutput.length > 0, - outputFailure: serverOutputFailure === null ? null : renderUnknown(serverOutputFailure), - }, - attestation, - attestationFailure: attestationFailure === null ? null : renderUnknown(attestationFailure), - explorationSummary, - explorationSummaryPath: explorationSummary === null ? null : explorationSummaryPath, - explorationSummaryFailure: explorationSummaryFailure === null - ? null - : renderUnknown(explorationSummaryFailure), - initialDirect: attestation?.initial ?? null, - interruptedSignal: capturedSignal, - failure: failure === null ? null : renderUnknown(failure), - } as const; const log = [processResult?.stdout ?? "", processResult?.stderr ?? ""] .filter((part) => part.length > 0) .join("\n"); - try { - await writeFile(logPath, `${log}${log.length > 0 ? "\n" : ""}`, "utf8"); - await writeFile( - serverLogPath, - `${serverOutput}${serverOutput.length > 0 ? "\n" : ""}`, - "utf8", - ); - if (explorationSummary !== null) { - await writeJsonAtomically(explorationSummaryPath, explorationSummary); + try { + await writeExclusiveBytes( + logPath, + Buffer.from(`${log}${log.length > 0 ? "\n" : ""}`, "utf8"), + ); + await writeExclusiveBytes( + serverLogPath, + Buffer.from(`${serverOutput}${serverOutput.length > 0 ? "\n" : ""}`, "utf8"), + ); + if (explorationSummary !== null) { + await writeJsonAtomically(explorationSummaryPath, explorationSummary); + } + } catch (error) { + const persistence = new BombadilPersistenceError( + "Bombadil local diagnostic logs could not be persisted", + [error], + ); + failure = failure === null + ? persistence + : combinePersistenceFailure(failure, persistence); + } + + let completedAt = dependencies.now(); + const createRecord = (): unknown => ({ + schema: ARTIFACT_SCHEMA, + evidenceClass: "diagnostic-fuzz", + artifactName: validated.artifactName, + label: validated.label, + status: failure === null ? "passed" : "failed", + generatedAt: generatedAt.toISOString(), + completedAt: completedAt.toISOString(), + durationMs: Math.max(0, completedAt.getTime() - generatedAt.getTime()), + scenario: validated.scenario, + expectedRoute: validated.expectedRoute, + baseUrl: validated.baseUrl, + entryPath: validated.entryPath, + targetQuery: validated.targetQuery, + targetUrl: invocation.targetUrl, + viewport: validated.viewport, + artifactPolicy: validated.artifactPolicy, + artifactInventory: { + entryCount: artifactInventory.entryCount, + fileCount: artifactInventory.fileCount, + inventorySha256: artifactInventory.inventorySha256, + totalBytes: artifactInventory.totalBytes, + files: artifactInventory.files.map((file) => ({ + path: file.relativePath, + sha256: file.sha256, + size: file.size, + })), + }, + explorationPolicy: validated.explorationPolicy, + specificationPath: validated.specificationPath, + replayPath, + timeLimitSeconds: replayPath === null ? parsed.timeLimitSeconds : null, + serverSource: lease?.source ?? null, + bombadil: { + version: bombadilVersion, + executable: validated.bombadilExecutable, + exitCode: processResult?.exitCode ?? null, + termination: processResult?.termination ?? null, + outputPath, + rawTracePath, + tracePath: attestation === null ? null : tracePath, + logPath, + }, + server: { + logPath: serverLogPath, + logPresent: serverOutput.length > 0, + outputFailure: serverOutputFailure === null ? null : renderUnknown(serverOutputFailure), + }, + attestation, + attestationFailure: attestationFailure === null ? null : renderUnknown(attestationFailure), + explorationSummary, + explorationSummaryPath: explorationSummary === null ? null : explorationSummaryPath, + explorationSummaryFailure: explorationSummaryFailure === null + ? null + : renderUnknown(explorationSummaryFailure), + initialDirect: attestation?.initial ?? null, + interruptedSignal: interruptedSignal as NodeJS.Signals | null, + failure: failure === null ? null : renderUnknown(failure), + }); + const runRecordPath = join(artifactRun.runDirectory, "run.json"); + try { + await writeJsonAtomically(runRecordPath, createRecord()); + } catch (error) { + const persistence = new BombadilPersistenceError( + "Bombadil local run record could not be persisted", + [error], + ); + failure = failure === null + ? persistence + : combinePersistenceFailure(failure, persistence); + } + + const failureBeforeUpload = failure; + const signalBeforeUpload = interruptedSignal as NodeJS.Signals | null; + if (signalBeforeUpload !== null && failure === null) { + failure = new Error(`Bombadil fuzzing was interrupted by ${signalBeforeUpload}`); + } + let published: Awaited>; + try { + published = await publishRunUpload({ + abortSignal: abortController.signal, + artifactName: validated.artifactName, + beforeCommitCheck: dependencies.beforeArtifactCommit, + attestation, + completedAt, + explorationSummary, + failure, + inventory: artifactInventory, + interruptedSignal: () => interruptedSignal, + localOutputPath: outputPath, + policy: validated.artifactPolicy, + privateDiagnosticsAllowed: writersSettled && artifactInventoryVetted, + processLog: `${log}${log.length > 0 ? "\n" : ""}`, + scenario: validated.scenario, + serverLog: `${serverOutput}${serverOutput.length > 0 ? "\n" : ""}`, + session: uploadSession, + status: failure === null ? "passed" : "failed", + }); + } catch (persistence) { + if (failure === null) throw persistence; + throw combinePersistenceFailure( + failure, + persistence, + "sanitized Bombadil receipt publication also failed", + ); + } + failure = published.failure; + completedAt = dependencies.now(); + if (failure !== failureBeforeUpload) { + await writeJsonAtomically(runRecordPath, createRecord()).catch(() => undefined); } - await writeJsonAtomically(join(artifactRun.runDirectory, "run.json"), record); - await writeJsonAtomically(artifactRun.manifestPath, record); + // The rolling pointer is only a local convenience. The exclusive UUID leaf + // and its receipt are the authoritative upload identity. + await writeJsonAtomically(artifactRun.manifestPath, createRecord()).catch(() => undefined); + const status = failure === null ? "passed" : "failed"; const exploration = explorationSummary === null ? "exploration=unavailable" : [ @@ -2793,11 +5669,26 @@ export async function runDirectBombadilFuzz( kind: "run", artifactDirectory: artifactRun.runDirectory, manifestPath: artifactRun.manifestPath, + receiptPath: uploadSession.receiptPath, status: "passed", + uploadArtifactPath: uploadSession.finalDirectory, }; } finally { - if (capturedSignal !== null) { - process.kill(process.pid, capturedSignal); + preparedUpload?.abortSignal?.removeEventListener("abort", abortFromPreparedMatrix); + for (const signal of interruptSignals) { + processSignals.removeListener(signal, interrupt); + } + const signalToForward = interruptedSignal as NodeJS.Signals | null; + if (signalToForward !== null && preparedUpload?.forwardSignal !== false) { + processSignals.forward(signalToForward); } } } + +export async function runDirectBombadilFuzz( + config: DirectBombadilFuzzConfig, + input: DirectBombadilFuzzRunInput = process.argv.slice(2), + dependencyOverrides: Partial = {}, +): Promise { + return await runDirectBombadilFuzzInternal(config, input, dependencyOverrides); +} diff --git a/src/tooling/bombadil.ts b/src/tooling/bombadil.ts index ec72ce2..5fc4a69 100644 --- a/src/tooling/bombadil.ts +++ b/src/tooling/bombadil.ts @@ -1,14 +1,21 @@ import { attestDirectBombadilTrace as attestTrace, + parseDirectBombadilArtifactReceipt as parseArtifactReceipt, + parseDirectBombadilMatrixReceipt as parseMatrixReceipt, + parseDirectBombadilMatrixSummary as parseMatrixSummary, + parseDirectBombadilSanitizedRunSummary as parseRunSummary, + resolveDirectBombadilUploadLeaf as resolveUploadLeaf, runDirectBombadilFuzz as runFuzz, runDirectBombadilFuzzMatrix as runMatrix, summarizeDirectBombadilTrace as summarizeTrace, } from "./bombadil-runner.js"; import type { - DirectBombadilFuzzConfig, DirectBombadilFuzzCampaign, + DirectBombadilFuzzConfig, DirectBombadilFuzzMatrixResult, DirectBombadilFuzzResult, + DirectBombadilFuzzRunInput, + DirectBombadilMatrixRunInput, } from "./bombadil-runner.js"; /** Host-side exact attestation for one bounded Bombadil 0.7.2 JSONL trace. */ @@ -17,32 +24,66 @@ export const attestDirectBombadilTrace: typeof attestTrace = attestTrace; /** Derives bounded diagnostic navigation metadata without replacing the raw trace. */ export const summarizeDirectBombadilTrace: typeof summarizeTrace = summarizeTrace; +/** Parse, clone, and freeze a foreign sanitized Bombadil run receipt. */ +export const parseDirectBombadilArtifactReceipt: typeof parseArtifactReceipt = parseArtifactReceipt; + +/** Parse, clone, and freeze a foreign sanitized Bombadil run summary. */ +export const parseDirectBombadilSanitizedRunSummary: typeof parseRunSummary = parseRunSummary; + +/** Parse, clone, and freeze a foreign sanitized Bombadil matrix receipt. */ +export const parseDirectBombadilMatrixReceipt: typeof parseMatrixReceipt = parseMatrixReceipt; + +/** Parse, clone, and freeze a foreign sanitized Bombadil matrix summary. */ +export const parseDirectBombadilMatrixSummary: typeof parseMatrixSummary = parseMatrixSummary; + +/** Resolve the exact precomputed upload leaf used by failure-safe CI upload steps. */ +export const resolveDirectBombadilUploadLeaf: typeof resolveUploadLeaf = resolveUploadLeaf; + /** Runs one bounded local Bombadil campaign and preserves diagnostic artifacts. */ export function runDirectBombadilFuzz( config: DirectBombadilFuzzConfig, - arguments_?: readonly string[], + argumentsOrOptions?: DirectBombadilFuzzRunInput, ): Promise { - return arguments_ === undefined ? runFuzz(config) : runFuzz(config, arguments_); + return argumentsOrOptions === undefined + ? runFuzz(config) + : runFuzz(config, argumentsOrOptions); } /** Runs a bounded product campaign matrix serially and selects one for replay. */ export function runDirectBombadilFuzzMatrix( campaigns: readonly DirectBombadilFuzzCampaign[], - arguments_?: readonly string[], + argumentsOrOptions?: DirectBombadilMatrixRunInput, ): Promise { - return arguments_ === undefined ? runMatrix(campaigns) : runMatrix(campaigns, arguments_); + return argumentsOrOptions === undefined + ? runMatrix(campaigns) + : runMatrix(campaigns, argumentsOrOptions); } export type { DirectBombadilActionKind, + DirectBombadilArtifactFailureCode, + DirectBombadilArtifactParseError, + DirectBombadilArtifactPolicy, + DirectBombadilArtifactReceipt, + DirectBombadilArtifactRunPlan, DirectBombadilExplorationPolicy, DirectBombadilExplorationSummary, DirectBombadilFuzzCampaign, DirectBombadilFuzzConfig, DirectBombadilFuzzMatrixResult, DirectBombadilFuzzResult, + DirectBombadilFuzzRunInput, + DirectBombadilFuzzRunOptions, + DirectBombadilMatrixCampaignReceiptEntry, + DirectBombadilMatrixCampaignStatus, + DirectBombadilMatrixReceipt, + DirectBombadilMatrixRunInput, + DirectBombadilMatrixRunOptions, + DirectBombadilMatrixSummary, + DirectBombadilSanitizedRunSummary, DirectBombadilServerConfig, DirectBombadilTraceAttestation, DirectBombadilTraceBinding, + DirectBombadilUploadMode, DirectBombadilViewportConfig, } from "./bombadil-runner.js"; diff --git a/src/tooling/browser-verification.test.ts b/src/tooling/browser-verification.test.ts index 13e36e6..f31b29f 100644 --- a/src/tooling/browser-verification.test.ts +++ b/src/tooling/browser-verification.test.ts @@ -33,6 +33,7 @@ import { runVerificationCommand, serializeAgentBrowserLaunchArguments, serverIsReachable, + spawnVerificationServer, stopVerificationServer, tail, writeJsonAtomically, @@ -583,6 +584,49 @@ describe("Direct browser contract binding", () => { }); describe("server leases", () => { + test("omits coordination secrets from managed server environments", async () => { + const directory = await mkdtemp(join(tmpdir(), "direct-server-environment-")); + temporaryDirectories.push(directory); + const server = spawnVerificationServer({ + command: [ + process.execPath, + "-e", + "console.log(process.env.DIRECT_BOMBADIL_RUN_ID ?? 'absent')", + ], + cwd: directory, + env: { DIRECT_BOMBADIL_RUN_ID: "child-visible-secret" }, + omitEnvironment: ["DIRECT_BOMBADIL_RUN_ID"], + }); + await server.exited; + expect(await server.output).toBe("absent"); + }); + + test("stops descendants only for an explicitly detached owned server group", async () => { + const directory = await mkdtemp(join(tmpdir(), "direct-server-process-group-")); + temporaryDirectories.push(directory); + const childPidPath = join(directory, "child.pid"); + const source = [ + "const { spawn } = require('node:child_process');", + "const fs = require('node:fs');", + "const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });", + `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid));`, + "setInterval(() => {}, 1000);", + ].join(" "); + const server = spawnVerificationServer({ + command: [process.execPath, "-e", source], + cwd: directory, + detachedProcessGroup: true, + }); + for (let attempt = 0; attempt < 100 && !(await Bun.file(childPidPath).exists()); attempt += 1) { + await Bun.sleep(10); + } + expect(await Bun.file(childPidPath).exists()).toBeTrue(); + const childPid = Number.parseInt(await Bun.file(childPidPath).text(), 10); + await stopVerificationServer(server, 500); + expect(Number.isSafeInteger(childPid)).toBeTrue(); + expect(() => process.kill(childPid, 0)).toThrow(); + }); + test("bounds one-shot verification commands and reports their exact outcome", async () => { expect(await runVerificationCommand({ command: [process.execPath, "-e", "console.log('built')"], diff --git a/src/tooling/browser-verification.ts b/src/tooling/browser-verification.ts index 46ca7f0..c7bea0c 100644 --- a/src/tooling/browser-verification.ts +++ b/src/tooling/browser-verification.ts @@ -243,6 +243,7 @@ export function createDirectBrowserContractReader< export interface ManagedVerificationServer { readonly exited: Promise; readonly exitCode: () => number | null; + readonly killDescendants?: (timeoutMs: number) => void | Promise; readonly output: Promise; readonly terminate: () => void; readonly kill: () => void; @@ -680,15 +681,44 @@ async function collectStream(stream: ReadableStream, logLimit: numbe } } +function verificationProcessGroupExists(processId: number): boolean { + try { + process.kill(-processId, 0); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + throw error; + } +} + +async function waitForVerificationProcessGroupExit( + processId: number, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + while (verificationProcessGroupExists(processId)) { + if (Date.now() >= deadline) { + throw new Error(`verification server process group ${String(processId)} survived cleanup`); + } + await Bun.sleep(Math.min(25, Math.max(1, deadline - Date.now()))); + } +} + export function spawnVerificationServer(options: { readonly command: readonly string[]; readonly cwd: string; + readonly detachedProcessGroup?: boolean; readonly env?: Readonly>; readonly logLimit?: number; + readonly omitEnvironment?: readonly string[]; }): ManagedVerificationServer { + const detachedProcessGroup = options.detachedProcessGroup ?? false; + const environment = { ...process.env, ...options.env }; + for (const name of options.omitEnvironment ?? []) delete environment[name]; const process_ = Bun.spawn([...options.command], { cwd: options.cwd, - env: { ...process.env, ...options.env }, + detached: detachedProcessGroup, + env: environment, stdin: "ignore", stdout: "pipe", stderr: "pipe", @@ -699,12 +729,33 @@ export function spawnVerificationServer(options: { collectStream(process_.stderr, logLimit), ]).then(([stdout, stderr]) => tail(`${stdout}\n${stderr}`.trim(), logLimit)); + const signal = (value: "SIGKILL" | "SIGTERM"): void => { + if (detachedProcessGroup) { + try { + process.kill(-process_.pid, value); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + // The group may already be gone; fall back to the leader only while it lives. + } + } + if (process_.exitCode === null) process_.kill(value); + }; + return { exited: process_.exited, exitCode: () => process_.exitCode, + ...(detachedProcessGroup + ? { + killDescendants: async (timeoutMs: number): Promise => { + signal("SIGKILL"); + await waitForVerificationProcessGroupExit(process_.pid, timeoutMs); + }, + } + : {}), output, - terminate: () => process_.kill("SIGTERM"), - kill: () => process_.kill("SIGKILL"), + terminate: () => signal("SIGTERM"), + kill: () => signal("SIGKILL"), }; } @@ -797,6 +848,9 @@ async function stopVerificationServerWithOutput( ); } } + // A detached leader can exit before descendants close inherited output pipes. + // Reap only a process group that the verifier explicitly owns. + await server.killDescendants?.(stopTimeoutMs); const output = await settleWithin(server.output, stopTimeoutMs); if (!output.settled) { throw new Error( @@ -960,7 +1014,7 @@ export async function writeJsonAtomically(path: string, value: unknown): Promise await writeFile(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); await rename(temporaryPath, path); } catch (error) { - await rm(temporaryPath, { force: true }); + await rm(temporaryPath, { force: true }).catch(() => undefined); throw error; } } From b40294e009edec80353f19c3c5a10bf4ab15e6dc Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sat, 29 Aug 2026 11:41:18 -0400 Subject: [PATCH 22/25] fix: preserve immutable package recovery --- scripts/package-smoke.ts | 313 ++++++++++++++++------------ src/tooling/bombadil-runner.test.ts | 54 +++-- 2 files changed, 223 insertions(+), 144 deletions(-) diff --git a/scripts/package-smoke.ts b/scripts/package-smoke.ts index 5687d7a..a5c9831 100644 --- a/scripts/package-smoke.ts +++ b/scripts/package-smoke.ts @@ -328,6 +328,180 @@ function typeScriptConfig(options: { }, null, 2)}\n`; } +type BombadilFeatureProfile = "artifact-delivery" | "baseline" | "matrix"; + +function selectBombadilFeatureProfile(version: string): BombadilFeatureProfile { + if (Bun.semver.order(version, "0.7.8") >= 0) return "artifact-delivery"; + if (Bun.semver.order(version, "0.7.6") >= 0) return "matrix"; + return "baseline"; +} + +function bombadilToolingTypeChecks(profile: BombadilFeatureProfile): string { + if (profile === "artifact-delivery") { + return ` + type BombadilRunnerArity = Parameters["length"]; + type BombadilRunnerInput = Parameters[1]; + type BombadilMatrixInput = Parameters[1]; + const supportedBombadilRunnerArities: readonly BombadilRunnerArity[] = [1, 2]; + const supportedBombadilArguments = ["--time-limit=12s"] as const; + const supportedBombadilRunnerInput: BombadilRunnerInput = { + arguments: supportedBombadilArguments, + artifactRun: { + repositoryRoot: "/absolute/repository", + runId: "00000000-0000-4000-8000-000000000001", + uploadMode: "public-summary", + }, + }; + const supportedBombadilTupleInput: BombadilRunnerInput = supportedBombadilArguments; + const supportedBombadilMatrixInput: BombadilMatrixInput = { + arguments: supportedBombadilArguments, + artifactRun: { + repositoryRoot: "/absolute/repository", + runId: "00000000-0000-4000-8000-000000000002", + uploadMode: "public-summary", + }, + }; + const unsupportedPrivateBombadilMatrixInput: BombadilMatrixInput = { + artifactRun: { + repositoryRoot: "/absolute/repository", + runId: "00000000-0000-4000-8000-000000000003", + // @ts-expect-error Packaged matrix uploads are public-summary only. + uploadMode: "private-vetted", + }, + }; + // @ts-expect-error Public tooling does not expose dependency injection. + const unsupportedBombadilRunnerArity: BombadilRunnerArity = 3; + void [supportedBombadilMatrixInput, supportedBombadilRunnerArities, supportedBombadilRunnerInput, supportedBombadilTupleInput, unsupportedBombadilRunnerArity, unsupportedPrivateBombadilMatrixInput]; + `; + } + const matrixChecks = profile === "matrix" + ? ` + type BombadilMatrixInput = Parameters[1]; + const supportedBombadilMatrixInput: BombadilMatrixInput = supportedBombadilArguments; + void supportedBombadilMatrixInput; + ` + : ""; + return ` + type BombadilRunnerArity = Parameters["length"]; + type BombadilRunnerInput = Parameters[1]; + const supportedBombadilRunnerArities: readonly BombadilRunnerArity[] = [1, 2]; + const supportedBombadilArguments = ["--time-limit=12s"] as const; + const supportedBombadilRunnerInput: BombadilRunnerInput = supportedBombadilArguments; + // @ts-expect-error Public tooling does not expose dependency injection. + const unsupportedBombadilRunnerArity: BombadilRunnerArity = 3; + void [supportedBombadilRunnerArities, supportedBombadilRunnerInput, unsupportedBombadilRunnerArity]; + ${matrixChecks}`; +} + +function bombadilRuntimeImports(profile: BombadilFeatureProfile): string { + const importedNames = profile === "artifact-delivery" + ? [ + "parseDirectBombadilArtifactReceipt", + "parseDirectBombadilMatrixReceipt", + "parseDirectBombadilMatrixSummary", + "parseDirectBombadilSanitizedRunSummary", + "resolveDirectBombadilUploadLeaf", + "runDirectBombadilFuzz", + ] + : ["runDirectBombadilFuzz"]; + return `import {\n${importedNames.map((name) => ` ${name},`).join("\n")}\n } from "@hraness/direct/tooling/bombadil";`; +} + +function bombadilArtifactDeliverySmoke(profile: BombadilFeatureProfile): string { + if (profile !== "artifact-delivery") return ""; + return ` + const sha256 = "a".repeat(64); + const policy = { + maxDepth: 32, + maxEntries: 4096, + maxFileBytes: 67108864, + maxFiles: 2048, + maxPathBytes: 4096, + maxTotalBytes: 134217728, + }; + const runId = "00000000-0000-4000-8000-000000000001"; + const receipt = { + schema: "direct.bombadil-artifact-receipt/v1", + completedAt: "2026-08-29T00:00:00.000Z", + diagnosticsRetained: false, + failureCode: null, + inventory: { entryCount: 1, fileCount: 1, inventorySha256: sha256, totalBytes: 1 }, + mode: "public-summary", + policy, + runId, + status: "passed", + }; + const summary = { + schema: "direct.bombadil-upload-summary/v1", + artifactName: "package-smoke", + attestation: { invalidObservationCount: 0, observationCount: 1, validObservationCount: 1 }, + exploration: { + actionCount: 0, + nonWaitActionCount: 0, + policySatisfied: true, + traceBytes: 1, + traceLineCount: 1, + traceSha256: sha256, + }, + failureCode: null, + scenario: "package.ready", + status: "passed", + }; + const matrixReceipt = { + schema: "direct.bombadil-matrix-receipt/v1", + campaigns: [{ + campaignId: "package-smoke", + index: 0, + receipt: "campaigns/package-smoke/receipt.json", + status: "passed", + }], + completedAt: "2026-08-29T00:00:00.000Z", + failureCode: null, + mode: "public-summary", + omittedCampaignCount: 0, + runId, + status: "passed", + }; + const matrixSummary = { + schema: "direct.bombadil-matrix-summary/v1", + campaigns: { + failed: 0, + notRun: 0, + notSelected: 0, + omitted: 0, + passed: 1, + rejected: 0, + total: 1, + }, + failureCode: null, + status: "passed", + }; + if ( + !parseDirectBombadilArtifactReceipt(receipt).ok + || !parseDirectBombadilSanitizedRunSummary(summary).ok + || !parseDirectBombadilMatrixReceipt(matrixReceipt).ok + || !parseDirectBombadilMatrixSummary(matrixSummary).ok + ) { + throw new Error("Bombadil package evidence parsers rejected exact valid fixtures"); + } + if ( + parseDirectBombadilArtifactReceipt({ ...receipt, extra: true }).ok + || parseDirectBombadilMatrixReceipt({ ...matrixReceipt, schema: "wrong" }).ok + || parseDirectBombadilSanitizedRunSummary({ ...summary, failureCode: "unknown" }).ok + ) { + throw new Error("Bombadil package evidence parsers accepted malformed fixtures"); + } + const uploadLeaf = resolveDirectBombadilUploadLeaf({ + repositoryRoot: "/absolute/repository", + runId, + uploadMode: "public-summary", + }); + if (uploadLeaf !== "/absolute/repository/artifacts/direct-bombadil-upload/" + runId) { + throw new Error("Bombadil upload-leaf resolver returned an unexpected path"); + } + `; +} + const repository = process.cwd(); const packageManifest = await Bun.file(join(repository, "package.json")).json(); if ( @@ -338,6 +512,7 @@ if ( ) { throw new Error("package.json must declare a string version"); } +const bombadilFeatureProfile = selectBombadilFeatureProfile(packageManifest.version); const work = await mkdtemp(join(tmpdir(), "hraness-package-smoke-")); try { const packageInput = parsePackageInput(process.argv.slice(2), repository); @@ -386,41 +561,10 @@ try { `await Promise.all(${JSON.stringify(importSpecifiers)}.map((specifier) => import(specifier)))`, ], consumer); await writeFile(join(consumer, "runtime-index.ts"), typeImportSource(runtimeImportSpecifiers)); - await writeFile(join(consumer, "tooling-index.ts"), `${typeImportSource(toolingTypeImportSpecifiers)} - type BombadilRunnerArity = Parameters["length"]; - type BombadilRunnerInput = Parameters[1]; - type BombadilMatrixInput = Parameters[1]; - const supportedBombadilRunnerArities: readonly BombadilRunnerArity[] = [1, 2]; - const supportedBombadilArguments = ["--time-limit=12s"] as const; - const supportedBombadilRunnerInput: BombadilRunnerInput = { - arguments: supportedBombadilArguments, - artifactRun: { - repositoryRoot: "/absolute/repository", - runId: "00000000-0000-4000-8000-000000000001", - uploadMode: "public-summary", - }, - }; - const supportedBombadilTupleInput: BombadilRunnerInput = supportedBombadilArguments; - const supportedBombadilMatrixInput: BombadilMatrixInput = { - arguments: supportedBombadilArguments, - artifactRun: { - repositoryRoot: "/absolute/repository", - runId: "00000000-0000-4000-8000-000000000002", - uploadMode: "public-summary", - }, - }; - const unsupportedPrivateBombadilMatrixInput: BombadilMatrixInput = { - artifactRun: { - repositoryRoot: "/absolute/repository", - runId: "00000000-0000-4000-8000-000000000003", - // @ts-expect-error Packaged matrix uploads are public-summary only. - uploadMode: "private-vetted", - }, - }; - // @ts-expect-error Public tooling does not expose dependency injection. - const unsupportedBombadilRunnerArity: BombadilRunnerArity = 3; - void [supportedBombadilMatrixInput, supportedBombadilRunnerArities, supportedBombadilRunnerInput, supportedBombadilTupleInput, unsupportedBombadilRunnerArity, unsupportedPrivateBombadilMatrixInput]; - `); + await writeFile( + join(consumer, "tooling-index.ts"), + `${typeImportSource(toolingTypeImportSpecifiers)}${bombadilToolingTypeChecks(bombadilFeatureProfile)}`, + ); await writeFile(join(consumer, "tsconfig.bundler.json"), typeScriptConfig({ include: "runtime-index.ts", module: "Preserve", @@ -472,14 +616,7 @@ try { normalizeRootHttpOrigin, readDirectBrowserContract, } from "@hraness/direct/tooling/browser-verification"; - import { - parseDirectBombadilArtifactReceipt, - parseDirectBombadilMatrixReceipt, - parseDirectBombadilMatrixSummary, - parseDirectBombadilSanitizedRunSummary, - resolveDirectBombadilUploadLeaf, - runDirectBombadilFuzz, - } from "@hraness/direct/tooling/bombadil"; + ${bombadilRuntimeImports(bombadilFeatureProfile)} import { findForbiddenMarkers } from "@hraness/direct/tooling/bundle-boundary"; if (normalizeRootHttpOrigin("https://example.test/") !== "https://example.test") { @@ -498,95 +635,7 @@ try { if (typeof runDirectBombadilFuzz !== "function") { throw new Error("Bombadil host tooling runner is missing"); } - const sha256 = "a".repeat(64); - const policy = { - maxDepth: 32, - maxEntries: 4096, - maxFileBytes: 67108864, - maxFiles: 2048, - maxPathBytes: 4096, - maxTotalBytes: 134217728, - }; - const runId = "00000000-0000-4000-8000-000000000001"; - const receipt = { - schema: "direct.bombadil-artifact-receipt/v1", - completedAt: "2026-08-29T00:00:00.000Z", - diagnosticsRetained: false, - failureCode: null, - inventory: { entryCount: 1, fileCount: 1, inventorySha256: sha256, totalBytes: 1 }, - mode: "public-summary", - policy, - runId, - status: "passed", - }; - const summary = { - schema: "direct.bombadil-upload-summary/v1", - artifactName: "package-smoke", - attestation: { invalidObservationCount: 0, observationCount: 1, validObservationCount: 1 }, - exploration: { - actionCount: 0, - nonWaitActionCount: 0, - policySatisfied: true, - traceBytes: 1, - traceLineCount: 1, - traceSha256: sha256, - }, - failureCode: null, - scenario: "package.ready", - status: "passed", - }; - const matrixReceipt = { - schema: "direct.bombadil-matrix-receipt/v1", - campaigns: [{ - campaignId: "package-smoke", - index: 0, - receipt: "campaigns/package-smoke/receipt.json", - status: "passed", - }], - completedAt: "2026-08-29T00:00:00.000Z", - failureCode: null, - mode: "public-summary", - omittedCampaignCount: 0, - runId, - status: "passed", - }; - const matrixSummary = { - schema: "direct.bombadil-matrix-summary/v1", - campaigns: { - failed: 0, - notRun: 0, - notSelected: 0, - omitted: 0, - passed: 1, - rejected: 0, - total: 1, - }, - failureCode: null, - status: "passed", - }; - if ( - !parseDirectBombadilArtifactReceipt(receipt).ok - || !parseDirectBombadilSanitizedRunSummary(summary).ok - || !parseDirectBombadilMatrixReceipt(matrixReceipt).ok - || !parseDirectBombadilMatrixSummary(matrixSummary).ok - ) { - throw new Error("Bombadil package evidence parsers rejected exact valid fixtures"); - } - if ( - parseDirectBombadilArtifactReceipt({ ...receipt, extra: true }).ok - || parseDirectBombadilMatrixReceipt({ ...matrixReceipt, schema: "wrong" }).ok - || parseDirectBombadilSanitizedRunSummary({ ...summary, failureCode: "unknown" }).ok - ) { - throw new Error("Bombadil package evidence parsers accepted malformed fixtures"); - } - const uploadLeaf = resolveDirectBombadilUploadLeaf({ - repositoryRoot: "/absolute/repository", - runId, - uploadMode: "public-summary", - }); - if (uploadLeaf !== "/absolute/repository/artifacts/direct-bombadil-upload/" + runId) { - throw new Error("Bombadil upload-leaf resolver returned an unexpected path"); - } + ${bombadilArtifactDeliverySmoke(bombadilFeatureProfile)} type CampaignProperties = DirectBombadilProperties; void (undefined as unknown as CampaignProperties); `); diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index c7ea967..45926d8 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -43,6 +43,7 @@ import type { ServerLease, } from "./browser-verification.js"; +const ARTIFACT_IO_TEST_TIMEOUT_MS = 30_000; const temporaryDirectories: string[] = []; function artifactRunPlan< @@ -63,7 +64,7 @@ afterEach(async () => { await Promise.all(temporaryDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true }) )); -}); +}, ARTIFACT_IO_TEST_TIMEOUT_MS); function nativeBinaryName(): string { if (process.platform === "darwin" && process.arch === "arm64") { @@ -440,6 +441,7 @@ function dependencies(options: { readonly serverCommands: string[][]; } { const calls: string[] = []; + const signals = controllableSignals(); const serverCommands: string[][] = []; const server = fakeServer( calls, @@ -475,7 +477,9 @@ function dependencies(options: { `node_modules/@antithesishq/bombadil/binaries/${nativeBinaryName()}`, ); return (async () => { - if (options.noTrace !== true) { + if (options.noTrace === true) { + await mkdir(invocation.outputPath, { recursive: true }); + } else { await writeTrace( join(invocation.outputPath, "trace.jsonl"), options.observations ?? [absentObservation(), directObservation()], @@ -491,6 +495,7 @@ function dependencies(options: { }; })(); }, + signalController: signals.controller, ...(options.serverOutputTimeoutMs === undefined ? {} : { serverOutputTimeoutMs: options.serverOutputTimeoutMs }), @@ -786,12 +791,32 @@ describe("Direct Bombadil configuration and invocation", () => { const cwdLink = join(cwdFixture.repositoryRoot, "escaped-cwd"); await symlink(outside, cwdLink); const cwdRuntime = dependencies(); - expect((await rejection(runDirectBombadilFuzz({ + const cwdPlan = artifactRunPlan(cwdFixture.repositoryRoot, 43); + const publicationFailure = new Error("forced sanitized receipt publication failure"); + const cwdError = await rejection(runDirectBombadilFuzz({ ...cwdFixture.config, server: { ...cwdFixture.config.server, cwd: cwdLink }, - }, [], cwdRuntime.overrides))).message).toContain( + }, { arguments: [], artifactRun: cwdPlan }, { + ...cwdRuntime.overrides, + beforeArtifactCommit: () => { + throw publicationFailure; + }, + })); + expect(cwdError).toBeInstanceOf(AggregateError); + expect(cwdError.message).toContain( "server.cwd resolves outside repositoryRoot", ); + expect(cwdError.message).toContain( + "sanitized Bombadil receipt publication also failed", + ); + const persistenceErrors = (cwdError as AggregateError).errors; + expect(persistenceErrors).toHaveLength(2); + expect(persistenceErrors[0]).toBeInstanceOf(Error); + expect((persistenceErrors[0] as Error).message).toBe( + "server.cwd resolves outside repositoryRoot", + ); + expect(cwdError.cause).toBe(persistenceErrors[0]); + expect(persistenceErrors[1]).toBe(publicationFailure); expect(cwdRuntime.calls).toEqual([]); const replayFixture = await fixture(); @@ -804,7 +829,7 @@ describe("Direct Bombadil configuration and invocation", () => { replayRuntime.overrides, ))).message).toContain("--replay resolves outside repositoryRoot"); expect(replayRuntime.calls).toEqual([]); - }); + }, ARTIFACT_IO_TEST_TIMEOUT_MS); test("builds an argv-only native invocation with both Direct query bindings", () => { const invocation = createDirectBombadilInvocation({ @@ -940,7 +965,7 @@ describe("Direct Bombadil campaign matrix", () => { { id: "same", config }, { id: "same", config: { ...config, artifactName: "other" } }, ], []))).message).toContain("unique lowercase kebab"); - }); + }, ARTIFACT_IO_TEST_TIMEOUT_MS); test("publishes rejected and partially executed matrix terminal states", async () => { const { config, repositoryRoot } = await fixture(); @@ -1187,7 +1212,7 @@ describe("Direct Bombadil campaign matrix", () => { { campaignId: "secondary", status: "not-run" }, ], }); - }); + }, ARTIFACT_IO_TEST_TIMEOUT_MS); test("converts a parent-publication interruption and releases child abort listeners", async () => { const { config, repositoryRoot } = await fixture(); @@ -1228,7 +1253,7 @@ describe("Direct Bombadil campaign matrix", () => { campaigns: [{ status: "passed" }, { status: "passed" }], status: "failed", }); - }); + }, ARTIFACT_IO_TEST_TIMEOUT_MS); test("removes a failed matrix publication staging leaf", async () => { const { config, repositoryRoot } = await fixture(); @@ -1642,13 +1667,13 @@ describe("Direct Bombadil trace attestation", () => { expectedRoute: "/surface", expectedScenario: "surface.ready", tracePath, - }))).message).toContain("nonempty trace.jsonl"); + }))).message).toContain("not an openable regular file"); await writeFile(tracePath, "", "utf8"); expect((await rejection(attestDirectBombadilTrace({ expectedRoute: "/surface", expectedScenario: "surface.ready", tracePath, - }))).message).toContain("nonempty trace.jsonl"); + }))).message).toContain("not a bounded regular file"); }); }); @@ -2272,7 +2297,9 @@ describe("Direct Bombadil process lifecycle", () => { root: directory, })); expect(nestedFinal.name).toBe("BombadilArtifactPolicyError"); - expect(nestedFinal.message).toContain("could not be opened safely"); + expect(nestedFinal.message).toMatch( + /Bombadil artifact directory could not be (?:opened|inspected) safely:/u, + ); }); test("omits the upload coordination UUID from the native process environment", async () => { @@ -3180,7 +3207,10 @@ describe("Direct Bombadil run lifecycle", () => { arguments: [], artifactRun: plan, }, runtime.overrides)); - expect(error.message).toContain("inventory could not be proven safe"); + expect(error.name).toBe("BombadilArtifactPolicyError"); + expect(error.message).toContain( + "Bombadil artifact directory could not be inspected safely", + ); expect(JSON.parse(await readFile(join( resolveDirectBombadilUploadLeaf(plan), "receipt.json", From 2820be6d2980be53e62590b47f1cebae6aad0f58 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sat, 29 Aug 2026 12:42:53 -0400 Subject: [PATCH 23/25] fix: preserve Bombadil runner failure flow --- src/tooling/bombadil-runner.test.ts | 1 + src/tooling/bombadil-runner.ts | 45 ++++++++++++++++++----------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index 45926d8..9fd3905 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -938,6 +938,7 @@ describe("Direct Bombadil campaign matrix", () => { results: [{ campaignId: "secondary" }], }); expect(selectedRuntime.calls.filter((call) => call === "run-bombadil")).toHaveLength(1); + if (selected.kind !== "matrix") throw new Error("Expected a matrix result"); expect(JSON.parse(await readFile(selected.receiptPath, "utf8"))).toMatchObject({ campaigns: [ { campaignId: "primary", receipt: null, status: "not-selected" }, diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index 3135468..6986204 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -954,18 +954,20 @@ function normalizeFuzzRunOptions( artifactRun: null, }; } - if (!isRecord(input)) throw new Error("Bombadil run options must be an object or argument array"); - const keys = Object.keys(input); + const options: DirectBombadilFuzzRunOptions | DirectBombadilMatrixRunOptions = input; + const artifactRun = options.artifactRun; + if (!isRecord(options)) throw new Error("Bombadil run options must be an object or argument array"); + const keys = Object.keys(options); if (keys.some((key) => key !== "arguments" && key !== "artifactRun")) { throw new Error("Bombadil run options contain an unknown field"); } - const arguments_ = input.arguments ?? []; + const arguments_ = options.arguments ?? []; if (!isReadonlyStringArray(arguments_)) { throw new Error("Bombadil run options arguments must be a string array"); } return { arguments: Object.freeze([...arguments_]), - artifactRun: input.artifactRun ?? null, + artifactRun: artifactRun ?? null, }; } @@ -2716,7 +2718,8 @@ async function publishMatrixUpload(options: { readonly omittedCampaignCount?: number; readonly session: AtomicArtifactUploadSession; }): Promise<{ readonly failure: unknown }> { - if (options.session.mode !== "public-summary") { + const uploadMode = options.session.mode; + if (uploadMode !== "public-summary") { throw new BombadilArtifactPolicyError("Bombadil matrix upload session must be public-summary"); } let failure = options.failure; @@ -2748,7 +2751,7 @@ async function publishMatrixUpload(options: { schema: MATRIX_RECEIPT_SCHEMA, completedAt: options.completedAt.toISOString(), failureCode, - mode: options.session.mode, + mode: uploadMode, runId: options.session.runId, status, omittedCampaignCount: options.omittedCampaignCount ?? 0, @@ -4529,7 +4532,10 @@ export async function runBombadilNativeProcess( invocation: DirectBombadilInvocation, ): Promise { const artifactPolicy = validateArtifactPolicy(invocation.artifactPolicy); - const childEnvironment = { ...process.env, NO_COLOR: "1" }; + const childEnvironment: Record = { + ...process.env, + NO_COLOR: "1", + }; delete childEnvironment[ARTIFACT_COORDINATION_ENVIRONMENT]; const process_ = Bun.spawn([...invocation.command], { cwd: invocation.cwd, @@ -4619,14 +4625,19 @@ export async function runBombadilNativeProcess( stderrCapture.stop(); } const [stdout, stderr] = await outputPromise; - const artifactPolicyFailure = outcome.kind === "artifact-policy" - ? outcome.error - : finalMonitorOutcome.kind === "artifact-policy" + if (outcome.kind === "artifact-policy") { + throw outcome.error instanceof BombadilArtifactPolicyError + ? outcome.error + : new BombadilArtifactPolicyError("Bombadil artifact policy was violated"); + } + if (finalMonitorOutcome.kind === "artifact-policy") { + throw finalMonitorOutcome.error instanceof BombadilArtifactPolicyError ? finalMonitorOutcome.error - : finalArtifactFailure; - if (artifactPolicyFailure !== null) { - throw artifactPolicyFailure instanceof BombadilArtifactPolicyError - ? artifactPolicyFailure + : new BombadilArtifactPolicyError("Bombadil artifact policy was violated"); + } + if (finalArtifactFailure !== null) { + throw finalArtifactFailure instanceof BombadilArtifactPolicyError + ? finalArtifactFailure : new BombadilArtifactPolicyError("Bombadil artifact policy was violated"); } return { @@ -4991,7 +5002,7 @@ export async function runDirectBombadilFuzzMatrix( receipt: null, status: "rejected", })); - await publishFailureAndThrow(error, async () => { + return await publishFailureAndThrow(error, async () => { await publishMatrixUpload({ abortSignal: matrixAbortController.signal, beforeCommitCheck: dependencies.beforeArtifactCommit, @@ -5227,7 +5238,7 @@ async function runDirectBombadilFuzzInternal( return validateArtifactPolicy(undefined); } })(); - await publishFailureAndThrow(error, async () => { + return await publishFailureAndThrow(error, async () => { await publishRunUpload({ abortSignal: abortController.signal, artifactName: isBoundedArtifactIdentifier(config.artifactName) @@ -5264,7 +5275,7 @@ async function runDirectBombadilFuzzInternal( }); throwIfBombadilRunAborted(abortController.signal); } catch (error) { - await publishFailureAndThrow(error, async () => { + return await publishFailureAndThrow(error, async () => { await publishRunUpload({ abortSignal: abortController.signal, artifactName: validated.artifactName, From 591c7cf41a92a7317013711ebd9de8fc68885049 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sat, 29 Aug 2026 15:33:19 -0400 Subject: [PATCH 24/25] fix: harden Bombadil process lifecycle --- dist/tooling/bombadil.js | 2849 +++++++++++++++++--- dist/tooling/browser-verification-entry.js | 53 +- scripts/package-smoke.ts | 2 +- src/tooling/bombadil-runner.test.ts | 12 +- src/tooling/bombadil-runner.ts | 69 +- src/tooling/browser-verification.ts | 8 +- 6 files changed, 2509 insertions(+), 484 deletions(-) diff --git a/dist/tooling/bombadil.js b/dist/tooling/bombadil.js index 341bc26..13be32f 100644 --- a/dist/tooling/bombadil.js +++ b/dist/tooling/bombadil.js @@ -1,11 +1,20 @@ // @bun // src/tooling/bombadil-runner.ts -import { createReadStream } from "fs"; -import { readFile, realpath, stat, writeFile as writeFile2 } from "fs/promises"; -import { isAbsolute, join as join2, relative, resolve } from "path"; +import { constants as fileSystemConstants } from "fs"; +import { + lstat, + mkdir as mkdir2, + open, + opendir, + readFile, + realpath, + rename as rename2, + rm as rm2, + stat +} from "fs/promises"; +import { extname, isAbsolute, join as join2, relative, resolve } from "path"; import process2 from "process"; -import { createInterface } from "readline"; -import { createHash } from "crypto"; +import { createHash, randomUUID as randomUUID2 } from "crypto"; // src/core/result.ts function ok(value) { @@ -978,10 +987,33 @@ async function collectStream(stream, logLimit) { output = tail(`${output}${decoder.decode(chunk.value, { stream: true })}`, logLimit); } } +function verificationProcessGroupExists(processId) { + try { + process.kill(-processId, 0); + return true; + } catch (error) { + if (error.code === "ESRCH") + return false; + throw error; + } +} +async function waitForVerificationProcessGroupExit(processId, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (verificationProcessGroupExists(processId)) { + if (Date.now() >= deadline) { + throw new Error(`verification server process group ${String(processId)} survived cleanup`); + } + await Bun.sleep(Math.min(25, Math.max(1, deadline - Date.now()))); + } +} function spawnVerificationServer(options) { + const detachedProcessGroup = options.detachedProcessGroup ?? false; + const omittedEnvironment = new Set(options.omitEnvironment ?? []); + const environment = Object.fromEntries(Object.entries({ ...process.env, ...options.env }).filter(([name]) => !omittedEnvironment.has(name))); const process_ = Bun.spawn([...options.command], { cwd: options.cwd, - env: { ...process.env, ...options.env }, + detached: detachedProcessGroup, + env: environment, stdin: "ignore", stdout: "pipe", stderr: "pipe" @@ -992,12 +1024,31 @@ function spawnVerificationServer(options) { collectStream(process_.stderr, logLimit) ]).then(([stdout, stderr]) => tail(`${stdout} ${stderr}`.trim(), logLimit)); + const signal = (value) => { + if (detachedProcessGroup) { + try { + process.kill(-process_.pid, value); + return; + } catch (error) { + if (error.code !== "ESRCH") + throw error; + } + } + if (process_.exitCode === null) + process_.kill(value); + }; return { exited: process_.exited, exitCode: () => process_.exitCode, + ...detachedProcessGroup ? { + killDescendants: async (timeoutMs) => { + signal("SIGKILL"); + await waitForVerificationProcessGroupExit(process_.pid, timeoutMs); + } + } : {}, output, - terminate: () => process_.kill("SIGTERM"), - kill: () => process_.kill("SIGKILL") + terminate: () => signal("SIGTERM"), + kill: () => signal("SIGKILL") }; } async function settleWithin(promise, timeoutMs) { @@ -1037,6 +1088,7 @@ async function stopVerificationServerWithOutput(server, stopTimeoutMs = DEFAULT_ throw new Error(`verification server did not exit within ${stopTimeoutMs}ms after SIGKILL`); } } + await server.killDescendants?.(stopTimeoutMs); const output = await settleWithin(server.output, stopTimeoutMs); if (!output.settled) { throw new Error(`verification server output did not settle within ${stopTimeoutMs}ms after exit`); @@ -1124,19 +1176,6 @@ ${output2}`); throw new Error(output === "" ? timeoutMessage : `${timeoutMessage}: ${output}`); } -async function createArtifactRun(options) { - const generatedAt = options.generatedAt ?? new Date().toISOString(); - const processId = options.processId ?? process.pid; - const runId = `${generatedAt.replaceAll(/[^0-9A-Za-z]/gu, "-")}-${processId}`; - const runDirectory = join(options.artifactRoot, runId); - await mkdir(runDirectory, { recursive: true }); - return { - artifactRoot: options.artifactRoot, - generatedAt, - manifestPath: join(options.artifactRoot, "manifest.json"), - runDirectory - }; -} async function writeJsonAtomically(path, value) { const temporaryPath = join(dirname(path), `.${process.pid}-${randomUUID()}.tmp`); try { @@ -1144,7 +1183,9 @@ async function writeJsonAtomically(path, value) { `, "utf8"); await rename(temporaryPath, path); } catch (error) { - await rm(temporaryPath, { force: true }); + await rm(temporaryPath, { force: true }).catch(() => { + return; + }); throw error; } } @@ -1158,8 +1199,111 @@ var DEFAULT_STARTUP_TIMEOUT_MS = 60000; var MAX_STARTUP_TIMEOUT_MS = 120000; var LOG_LIMIT = 24000; var ARTIFACT_SCHEMA = "direct.bombadil-run/v1"; +var ARTIFACT_RECEIPT_SCHEMA = "direct.bombadil-artifact-receipt/v1"; +var ARTIFACT_SUMMARY_SCHEMA = "direct.bombadil-upload-summary/v1"; +var MATRIX_RECEIPT_SCHEMA = "direct.bombadil-matrix-receipt/v1"; +var MATRIX_SUMMARY_SCHEMA = "direct.bombadil-matrix-summary/v1"; +var ARTIFACT_FAILURE_CODES = new Set([ + "artifact-policy", + "configuration-rejected", + "exploration-policy", + "interrupted", + "persistence", + "process", + "server", + "trace-attestation", + "writer-settlement", + "unknown" +]); +var ARTIFACT_RECEIPT_KEYS = new Set([ + "completedAt", + "diagnosticsRetained", + "failureCode", + "inventory", + "mode", + "policy", + "runId", + "schema", + "status" +]); +var ARTIFACT_RECEIPT_INVENTORY_KEYS = new Set([ + "entryCount", + "fileCount", + "inventorySha256", + "totalBytes" +]); +var ARTIFACT_POLICY_RECEIPT_KEYS = new Set([ + "maxDepth", + "maxEntries", + "maxFileBytes", + "maxFiles", + "maxPathBytes", + "maxTotalBytes" +]); +var RUN_SUMMARY_KEYS = new Set([ + "artifactName", + "attestation", + "exploration", + "failureCode", + "scenario", + "schema", + "status" +]); +var RUN_SUMMARY_ATTESTATION_KEYS = new Set([ + "invalidObservationCount", + "observationCount", + "validObservationCount" +]); +var RUN_SUMMARY_EXPLORATION_KEYS = new Set([ + "actionCount", + "nonWaitActionCount", + "policySatisfied", + "traceBytes", + "traceLineCount", + "traceSha256" +]); +var MATRIX_RECEIPT_KEYS = new Set([ + "campaigns", + "completedAt", + "failureCode", + "mode", + "omittedCampaignCount", + "runId", + "schema", + "status" +]); +var MATRIX_CAMPAIGN_RECEIPT_KEYS = new Set([ + "campaignId", + "index", + "receipt", + "status" +]); +var MATRIX_SUMMARY_KEYS = new Set([ + "campaigns", + "failureCode", + "schema", + "status" +]); +var MATRIX_SUMMARY_CAMPAIGNS_KEYS = new Set([ + "failed", + "notRun", + "notSelected", + "omitted", + "passed", + "rejected", + "total" +]); +var SHA256_PATTERN = /^[0-9a-f]{64}$/u; +var ARTIFACT_EVIDENCE_JSON_LIMITS = Object.freeze({ + maxDepth: 8, + maxNodes: 2048, + maxStringBytes: 64 * 1024 +}); var SCENARIO_PATTERN = /^[a-z][a-z0-9]*(?:[._/-][a-z0-9]+)*$/u; var ARTIFACT_NAME_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u; +var MAX_ARTIFACT_IDENTIFIER_LENGTH = 80; +var MAX_MATRIX_CAMPAIGNS = 32; +var ARTIFACT_COORDINATION_ENVIRONMENT = "DIRECT_BOMBADIL_RUN_ID"; var ENVIRONMENT_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/u; var QUERY_PARAMETER_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_.-]*$/u; var PROTOTYPE_PROPERTY_NAMES = new Set(["__proto__", "constructor", "prototype"]); @@ -1178,6 +1322,31 @@ var REPLAY_WALL_CLOCK_TIMEOUT_MS = MAX_TIME_LIMIT_SECONDS * 1000 + RANDOM_RUN_OV var PROCESS_TERMINATION_GRACE_MS = 5000; var MIN_PROCESS_OUTPUT_DRAIN_MS = 500; var SERVER_OUTPUT_TIMEOUT_MS = 3000; +var ARTIFACT_MONITOR_INTERVAL_MS = 100; +var DEFAULT_ARTIFACT_MAX_ENTRIES = 4096; +var DEFAULT_ARTIFACT_MAX_FILES = 2048; +var DEFAULT_ARTIFACT_MAX_TOTAL_BYTES = 128 * 1024 * 1024; +var DEFAULT_ARTIFACT_MAX_FILE_BYTES = 64 * 1024 * 1024; +var DEFAULT_ARTIFACT_MAX_DEPTH = 32; +var DEFAULT_ARTIFACT_MAX_PATH_BYTES = 4096; +var MAX_ARTIFACT_ENTRIES = 16384; +var MAX_ARTIFACT_FILES = 8192; +var MAX_ARTIFACT_TOTAL_BYTES = 256 * 1024 * 1024; +var MAX_ARTIFACT_FILE_BYTES = 64 * 1024 * 1024; +var MAX_ARTIFACT_DEPTH = 64; +var MAX_ARTIFACT_PATH_BYTES = 4096; +var UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; +var ARTIFACT_PATH_PART_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u; +var PRIVATE_DIAGNOSTIC_EXTENSIONS = new Set([ + ".jpeg", + ".jpg", + ".json", + ".jsonl", + ".log", + ".png", + ".txt", + ".webp" +]); var DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; var TRACE_LINE_KEYS = new Set(["action", "snapshots", "state", "timestamp", "violations"]); var TRACE_SNAPSHOT_KEYS = new Set(["index", "name", "time", "value"]); @@ -1284,6 +1453,28 @@ var DIRECT_OBSERVATION_KEYS = new Set([ "violations", "violationsValid" ]); +var PROCESS_INTERRUPT_SIGNALS = ["SIGINT", "SIGTERM"]; + +class BombadilArtifactPolicyError extends Error { + constructor(message) { + super(message); + this.name = "BombadilArtifactPolicyError"; + } +} + +class BombadilWriterSettlementError extends Error { + constructor(message, cause) { + super(message, { cause }); + this.name = "BombadilWriterSettlementError"; + } +} + +class BombadilPersistenceError extends AggregateError { + constructor(message, errors) { + super(errors, message, { cause: errors[0] }); + this.name = "BombadilPersistenceError"; + } +} function readOptionValue(arguments_, index, option) { const value = arguments_[index + 1]; if (value === undefined || value.startsWith("-")) { @@ -1340,6 +1531,9 @@ function hasControlCharacters3(value) { function isRecord2(value) { return typeof value === "object" && value !== null && !Array.isArray(value); } +function isReadonlyStringArray(value) { + return Array.isArray(value) && value.every((entry) => typeof entry === "string"); +} function hasExactKeys(value, expected) { const keys = Object.keys(value); return keys.length === expected.size && keys.every((key) => expected.has(key)); @@ -1351,6 +1545,1269 @@ function compareCodeUnits(left, right) { return 1; return 0; } +function boundedArtifactInteger(options) { + const value = options.value ?? options.defaultValue; + if (!Number.isSafeInteger(value) || value < 1 || value > options.maximum) { + throw new Error(`${options.label} must be an integer between 1 and ${String(options.maximum)}`); + } + return value; +} +function validateArtifactPolicy(input) { + const value = input ?? {}; + return Object.freeze({ + maxDepth: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_DEPTH, + label: "artifactPolicy.maxDepth", + maximum: MAX_ARTIFACT_DEPTH, + value: value.maxDepth + }), + maxEntries: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_ENTRIES, + label: "artifactPolicy.maxEntries", + maximum: MAX_ARTIFACT_ENTRIES, + value: value.maxEntries + }), + maxFileBytes: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_FILE_BYTES, + label: "artifactPolicy.maxFileBytes", + maximum: MAX_ARTIFACT_FILE_BYTES, + value: value.maxFileBytes + }), + maxFiles: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_FILES, + label: "artifactPolicy.maxFiles", + maximum: MAX_ARTIFACT_FILES, + value: value.maxFiles + }), + maxPathBytes: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_PATH_BYTES, + label: "artifactPolicy.maxPathBytes", + maximum: MAX_ARTIFACT_PATH_BYTES, + value: value.maxPathBytes + }), + maxTotalBytes: boundedArtifactInteger({ + defaultValue: DEFAULT_ARTIFACT_MAX_TOTAL_BYTES, + label: "artifactPolicy.maxTotalBytes", + maximum: MAX_ARTIFACT_TOTAL_BYTES, + value: value.maxTotalBytes + }) + }); +} +function normalizeFuzzRunOptions(input) { + if (input === undefined || isReadonlyStringArray(input)) { + return { + arguments: Object.freeze([...input ?? []]), + artifactRun: null + }; + } + const options = input; + const artifactRun = options.artifactRun; + if (!isRecord2(options)) + throw new Error("Bombadil run options must be an object or argument array"); + const keys = Object.keys(options); + if (keys.some((key) => key !== "arguments" && key !== "artifactRun")) { + throw new Error("Bombadil run options contain an unknown field"); + } + const arguments_ = options.arguments ?? []; + if (!isReadonlyStringArray(arguments_)) { + throw new Error("Bombadil run options arguments must be a string array"); + } + return { + arguments: Object.freeze([...arguments_]), + artifactRun: artifactRun ?? null + }; +} +function validateArtifactRunPlan(input) { + const repositoryRoot = resolve(input.repositoryRoot); + if (!isAbsolute(input.repositoryRoot) || repositoryRoot !== input.repositoryRoot) { + throw new Error("artifactRun.repositoryRoot must be an absolute normalized path"); + } + if (!UUID_PATTERN.test(input.runId)) { + throw new Error("artifactRun.runId must be a lowercase RFC 4122 UUID"); + } + const uploadMode = input.uploadMode ?? "public-summary"; + if (uploadMode !== "public-summary" && uploadMode !== "private-vetted") { + throw new Error("artifactRun.uploadMode must be public-summary or private-vetted"); + } + return Object.freeze({ repositoryRoot, runId: input.runId, uploadMode }); +} +function isBoundedArtifactIdentifier(value) { + return value.length <= MAX_ARTIFACT_IDENTIFIER_LENGTH && ARTIFACT_NAME_PATTERN.test(value); +} +function isBoundedScenarioIdentifier(value) { + return value.length <= 120 && SCENARIO_PATTERN.test(value); +} +function requireEvidenceRecord(value, keys, label) { + if (!isRecord2(value) || !hasExactKeys(value, keys)) { + throw new Error(`${label} must contain exactly its documented fields`); + } + return value; +} +function requireEvidenceInteger(value, label, maximum = Number.MAX_SAFE_INTEGER) { + if (!Number.isSafeInteger(value) || value < 0 || value > maximum) { + throw new Error(`${label} must be a nonnegative safe integer no greater than ${String(maximum)}`); + } + return value; +} +function requireEvidencePositiveInteger(value, label, maximum) { + const parsed = requireEvidenceInteger(value, label, maximum); + if (parsed === 0) + throw new Error(`${label} must be greater than zero`); + return parsed; +} +function requireEvidenceSha256(value, label) { + if (typeof value !== "string" || !SHA256_PATTERN.test(value)) { + throw new Error(`${label} must be a lowercase SHA-256 digest`); + } + return value; +} +function requireEvidenceTimestamp(value, label) { + if (typeof value !== "string") + throw new Error(`${label} must be an ISO timestamp`); + const milliseconds = Date.parse(value); + if (!Number.isFinite(milliseconds) || new Date(milliseconds).toISOString() !== value) { + throw new Error(`${label} must be a canonical ISO timestamp`); + } + return value; +} +function parseEvidenceFailureCode(value, label) { + if (value === null) + return null; + if (typeof value !== "string" || !ARTIFACT_FAILURE_CODES.has(value)) { + throw new Error(`${label} is not a known Bombadil failure code`); + } + return value; +} +function requireEvidenceStatus(value, label) { + if (value !== "failed" && value !== "passed" && value !== "rejected") { + throw new Error(`${label} must be failed, passed, or rejected`); + } + return value; +} +function requireFailureStatusConsistency(status, failureCode, label) { + if (status === "passed" !== (failureCode === null)) { + throw new Error(`${label} status and failureCode are inconsistent`); + } + if (status === "rejected" && failureCode !== "configuration-rejected") { + throw new Error(`${label} rejected status requires configuration-rejected`); + } +} +function parseArtifactReceiptUnchecked(input) { + const value = requireEvidenceRecord(input, ARTIFACT_RECEIPT_KEYS, "Bombadil receipt"); + if (value.schema !== ARTIFACT_RECEIPT_SCHEMA) { + throw new Error("Bombadil receipt schema is unsupported"); + } + const completedAt = requireEvidenceTimestamp(value.completedAt, "Bombadil receipt completedAt"); + if (typeof value.diagnosticsRetained !== "boolean") { + throw new Error("Bombadil receipt diagnosticsRetained must be boolean"); + } + const failureCode = parseEvidenceFailureCode(value.failureCode, "Bombadil receipt failureCode"); + const status = requireEvidenceStatus(value.status, "Bombadil receipt status"); + requireFailureStatusConsistency(status, failureCode, "Bombadil receipt"); + if (value.mode !== "private-vetted" && value.mode !== "public-summary") { + throw new Error("Bombadil receipt mode is unsupported"); + } + if (value.diagnosticsRetained && value.mode !== "private-vetted") { + throw new Error("Public Bombadil receipts cannot retain diagnostics"); + } + if (typeof value.runId !== "string" || !UUID_PATTERN.test(value.runId)) { + throw new Error("Bombadil receipt runId must be a lowercase RFC 4122 UUID"); + } + const rawPolicy = requireEvidenceRecord(value.policy, ARTIFACT_POLICY_RECEIPT_KEYS, "Bombadil receipt policy"); + const policy = Object.freeze({ + maxDepth: requireEvidencePositiveInteger(rawPolicy.maxDepth, "Bombadil receipt policy.maxDepth", MAX_ARTIFACT_DEPTH), + maxEntries: requireEvidencePositiveInteger(rawPolicy.maxEntries, "Bombadil receipt policy.maxEntries", MAX_ARTIFACT_ENTRIES), + maxFileBytes: requireEvidencePositiveInteger(rawPolicy.maxFileBytes, "Bombadil receipt policy.maxFileBytes", MAX_ARTIFACT_FILE_BYTES), + maxFiles: requireEvidencePositiveInteger(rawPolicy.maxFiles, "Bombadil receipt policy.maxFiles", MAX_ARTIFACT_FILES), + maxPathBytes: requireEvidencePositiveInteger(rawPolicy.maxPathBytes, "Bombadil receipt policy.maxPathBytes", MAX_ARTIFACT_PATH_BYTES), + maxTotalBytes: requireEvidencePositiveInteger(rawPolicy.maxTotalBytes, "Bombadil receipt policy.maxTotalBytes", MAX_ARTIFACT_TOTAL_BYTES) + }); + const rawInventory = requireEvidenceRecord(value.inventory, ARTIFACT_RECEIPT_INVENTORY_KEYS, "Bombadil receipt inventory"); + const entryCount = requireEvidenceInteger(rawInventory.entryCount, "Bombadil receipt inventory.entryCount", policy.maxEntries); + const fileCount = requireEvidenceInteger(rawInventory.fileCount, "Bombadil receipt inventory.fileCount", policy.maxFiles); + const totalBytes = requireEvidenceInteger(rawInventory.totalBytes, "Bombadil receipt inventory.totalBytes", policy.maxTotalBytes); + if (fileCount > entryCount) { + throw new Error("Bombadil receipt inventory.fileCount cannot exceed entryCount"); + } + if (fileCount === 0 && totalBytes !== 0) { + throw new Error("Bombadil receipt inventory bytes require at least one file"); + } + const inventorySha256 = rawInventory.inventorySha256 === null ? null : requireEvidenceSha256(rawInventory.inventorySha256, "Bombadil receipt inventory.inventorySha256"); + if (entryCount === 0 && (fileCount !== 0 || totalBytes !== 0 || inventorySha256 !== null) || entryCount > 0 && inventorySha256 === null) { + throw new Error("Bombadil receipt empty-inventory fields are inconsistent"); + } + if (status === "passed" && (entryCount === 0 || fileCount === 0 || totalBytes === 0) || status === "passed" && value.mode === "private-vetted" && !value.diagnosticsRetained || failureCode === "interrupted" && value.diagnosticsRetained || failureCode === "configuration-rejected" && status !== "rejected" || failureCode === "writer-settlement" && (value.diagnosticsRetained || entryCount !== 0 || fileCount !== 0 || totalBytes !== 0) || status === "rejected" && (value.diagnosticsRetained || entryCount !== 0 || fileCount !== 0 || totalBytes !== 0)) { + throw new Error("Bombadil receipt terminal state and retained evidence are inconsistent"); + } + return Object.freeze({ + schema: ARTIFACT_RECEIPT_SCHEMA, + completedAt, + diagnosticsRetained: value.diagnosticsRetained, + failureCode, + inventory: Object.freeze({ entryCount, fileCount, inventorySha256, totalBytes }), + mode: value.mode, + policy, + runId: value.runId, + status + }); +} +function parseRunSummaryUnchecked(input) { + const value = requireEvidenceRecord(input, RUN_SUMMARY_KEYS, "Bombadil run summary"); + if (value.schema !== ARTIFACT_SUMMARY_SCHEMA) { + throw new Error("Bombadil run summary schema is unsupported"); + } + if (typeof value.artifactName !== "string" || !isBoundedArtifactIdentifier(value.artifactName)) { + throw new Error("Bombadil run summary artifactName is invalid"); + } + if (typeof value.scenario !== "string" || !isBoundedScenarioIdentifier(value.scenario)) { + throw new Error("Bombadil run summary scenario is invalid"); + } + const failureCode = parseEvidenceFailureCode(value.failureCode, "Bombadil run summary failureCode"); + const status = requireEvidenceStatus(value.status, "Bombadil run summary status"); + requireFailureStatusConsistency(status, failureCode, "Bombadil run summary"); + let attestation = null; + if (value.attestation !== null) { + const raw = requireEvidenceRecord(value.attestation, RUN_SUMMARY_ATTESTATION_KEYS, "Bombadil run summary attestation"); + const observationCount = requireEvidenceInteger(raw.observationCount, "Bombadil run summary attestation.observationCount", TRACE_MAX_LINES); + const invalidObservationCount = requireEvidenceInteger(raw.invalidObservationCount, "Bombadil run summary attestation.invalidObservationCount", observationCount); + const validObservationCount = requireEvidenceInteger(raw.validObservationCount, "Bombadil run summary attestation.validObservationCount", observationCount); + if (invalidObservationCount + validObservationCount !== observationCount) { + throw new Error("Bombadil run summary attestation counts do not reconcile"); + } + if (observationCount === 0 || validObservationCount === 0) { + throw new Error("Bombadil run summary attestation must contain a valid observation"); + } + attestation = Object.freeze({ + invalidObservationCount, + observationCount, + validObservationCount + }); + } + let exploration = null; + if (value.exploration !== null) { + const raw = requireEvidenceRecord(value.exploration, RUN_SUMMARY_EXPLORATION_KEYS, "Bombadil run summary exploration"); + const traceLineCount = requireEvidenceInteger(raw.traceLineCount, "Bombadil run summary exploration.traceLineCount", TRACE_MAX_LINES); + const actionCount = requireEvidenceInteger(raw.actionCount, "Bombadil run summary exploration.actionCount", traceLineCount); + const nonWaitActionCount = requireEvidenceInteger(raw.nonWaitActionCount, "Bombadil run summary exploration.nonWaitActionCount", actionCount); + if (typeof raw.policySatisfied !== "boolean") { + throw new Error("Bombadil run summary exploration.policySatisfied must be boolean"); + } + exploration = Object.freeze({ + actionCount, + nonWaitActionCount, + policySatisfied: raw.policySatisfied, + traceBytes: requireEvidenceInteger(raw.traceBytes, "Bombadil run summary exploration.traceBytes", TRACE_MAX_BYTES), + traceLineCount, + traceSha256: requireEvidenceSha256(raw.traceSha256, "Bombadil run summary exploration.traceSha256") + }); + if (exploration.traceBytes === 0 || exploration.traceLineCount === 0) { + throw new Error("Bombadil run summary exploration trace must be nonempty"); + } + } + if (status === "passed" && (attestation === null || attestation.observationCount === 0 || attestation.validObservationCount === 0 || exploration === null || !exploration.policySatisfied || attestation.observationCount !== exploration.traceLineCount)) { + throw new Error("A passed Bombadil run summary requires attested policy-satisfying evidence"); + } + if (attestation !== null && exploration !== null && attestation.observationCount !== exploration.traceLineCount) { + throw new Error("Bombadil run summary trace counts do not reconcile"); + } + if (status === "rejected" && (attestation !== null || exploration !== null)) { + throw new Error("A rejected Bombadil run summary cannot claim trace evidence"); + } + if (failureCode === "configuration-rejected" && status !== "rejected") { + throw new Error("A configuration-rejected Bombadil run summary must be rejected"); + } + if (failureCode === "writer-settlement" && (attestation !== null || exploration !== null)) { + throw new Error("A writer-settlement Bombadil run summary cannot claim trace evidence"); + } + return Object.freeze({ + schema: ARTIFACT_SUMMARY_SCHEMA, + artifactName: value.artifactName, + attestation, + exploration, + failureCode, + scenario: value.scenario, + status + }); +} +function parseMatrixReceiptUnchecked(input) { + const value = requireEvidenceRecord(input, MATRIX_RECEIPT_KEYS, "Bombadil matrix receipt"); + if (value.schema !== MATRIX_RECEIPT_SCHEMA || value.mode !== "public-summary") { + throw new Error("Bombadil matrix receipt schema or mode is unsupported"); + } + const completedAt = requireEvidenceTimestamp(value.completedAt, "Bombadil matrix receipt completedAt"); + const failureCode = parseEvidenceFailureCode(value.failureCode, "Bombadil matrix receipt failureCode"); + if (value.status !== "failed" && value.status !== "passed") { + throw new Error("Bombadil matrix receipt status must be failed or passed"); + } + requireFailureStatusConsistency(value.status, failureCode, "Bombadil matrix receipt"); + if (typeof value.runId !== "string" || !UUID_PATTERN.test(value.runId)) { + throw new Error("Bombadil matrix receipt runId must be a lowercase RFC 4122 UUID"); + } + if (!Array.isArray(value.campaigns) || value.campaigns.length > MAX_MATRIX_CAMPAIGNS) { + throw new Error("Bombadil matrix receipt campaigns exceed the bounded matrix size"); + } + const campaignIds = new Set; + const campaigns = value.campaigns.map((inputCampaign, index) => { + const campaign = requireEvidenceRecord(inputCampaign, MATRIX_CAMPAIGN_RECEIPT_KEYS, `Bombadil matrix receipt campaign ${String(index)}`); + if (campaign.index !== index) { + throw new Error("Bombadil matrix receipt campaign indices must be ordered and contiguous"); + } + const campaignId = campaign.campaignId; + if (campaignId !== null && (typeof campaignId !== "string" || !isBoundedArtifactIdentifier(campaignId) || campaignIds.has(campaignId))) { + throw new Error("Bombadil matrix receipt campaign IDs must be unique bounded identifiers"); + } + if (campaignId !== null) + campaignIds.add(campaignId); + if (campaign.status !== "failed" && campaign.status !== "not-run" && campaign.status !== "not-selected" && campaign.status !== "passed" && campaign.status !== "rejected") { + throw new Error("Bombadil matrix receipt campaign status is unsupported"); + } + const expectedReceipt = campaignId === null ? null : `campaigns/${campaignId}/receipt.json`; + if (campaign.receipt !== null && (typeof campaign.receipt !== "string" || campaign.receipt !== expectedReceipt)) { + throw new Error("Bombadil matrix child receipt path is not canonical"); + } + if ((campaign.status === "not-run" || campaign.status === "not-selected") && campaign.receipt !== null || campaign.status === "passed" && campaign.receipt !== expectedReceipt || campaignId === null && (campaign.status !== "rejected" || campaign.receipt !== null)) { + throw new Error("Bombadil matrix child terminal state is inconsistent"); + } + return Object.freeze({ + campaignId, + index, + receipt: campaign.receipt, + status: campaign.status + }); + }); + const omittedCampaignCount = requireEvidenceInteger(value.omittedCampaignCount, "Bombadil matrix receipt omittedCampaignCount"); + if (value.status === "passed" && (omittedCampaignCount !== 0 || !campaigns.some((campaign) => campaign.status === "passed") || campaigns.some((campaign) => campaign.status === "failed" || campaign.status === "not-run" || campaign.status === "rejected"))) { + throw new Error("A passed Bombadil matrix receipt has a nonterminal child"); + } + return Object.freeze({ + schema: MATRIX_RECEIPT_SCHEMA, + campaigns: Object.freeze(campaigns), + completedAt, + failureCode, + mode: "public-summary", + omittedCampaignCount, + runId: value.runId, + status: value.status + }); +} +function parseMatrixSummaryUnchecked(input) { + const value = requireEvidenceRecord(input, MATRIX_SUMMARY_KEYS, "Bombadil matrix summary"); + if (value.schema !== MATRIX_SUMMARY_SCHEMA) { + throw new Error("Bombadil matrix summary schema is unsupported"); + } + const failureCode = parseEvidenceFailureCode(value.failureCode, "Bombadil matrix summary failureCode"); + if (value.status !== "failed" && value.status !== "passed") { + throw new Error("Bombadil matrix summary status must be failed or passed"); + } + requireFailureStatusConsistency(value.status, failureCode, "Bombadil matrix summary"); + const rawCampaigns = requireEvidenceRecord(value.campaigns, MATRIX_SUMMARY_CAMPAIGNS_KEYS, "Bombadil matrix summary campaigns"); + const total = requireEvidenceInteger(rawCampaigns.total, "Bombadil matrix summary campaigns.total", MAX_MATRIX_CAMPAIGNS); + const campaigns = Object.freeze({ + failed: requireEvidenceInteger(rawCampaigns.failed, "Bombadil matrix summary failed", total), + notRun: requireEvidenceInteger(rawCampaigns.notRun, "Bombadil matrix summary notRun", total), + notSelected: requireEvidenceInteger(rawCampaigns.notSelected, "Bombadil matrix summary notSelected", total), + omitted: requireEvidenceInteger(rawCampaigns.omitted, "Bombadil matrix summary omitted"), + passed: requireEvidenceInteger(rawCampaigns.passed, "Bombadil matrix summary passed", total), + rejected: requireEvidenceInteger(rawCampaigns.rejected, "Bombadil matrix summary rejected", total), + total + }); + if (campaigns.failed + campaigns.notRun + campaigns.notSelected + campaigns.passed + campaigns.rejected !== campaigns.total) { + throw new Error("Bombadil matrix summary campaign counts do not reconcile"); + } + if (value.status === "passed" && (campaigns.failed !== 0 || campaigns.notRun !== 0 || campaigns.rejected !== 0 || campaigns.omitted !== 0 || campaigns.passed === 0)) { + throw new Error("A passed Bombadil matrix summary contains unsuccessful campaigns"); + } + return Object.freeze({ + schema: MATRIX_SUMMARY_SCHEMA, + campaigns, + failureCode, + status: value.status + }); +} +function artifactEvidenceError(error) { + return Object.freeze({ + code: "invalid-bombadil-artifact-evidence", + message: renderUnknown(error) + }); +} +function cloneArtifactEvidence(input) { + const parsed = parseJsonValue(input, ARTIFACT_EVIDENCE_JSON_LIMITS); + if (!parsed.ok) { + throw new Error(`Bombadil artifact evidence is not bounded inert JSON: ${parsed.error.message}`); + } + return parsed.value; +} +function parseDirectBombadilArtifactReceipt(input) { + try { + return ok(parseArtifactReceiptUnchecked(cloneArtifactEvidence(input))); + } catch (error) { + return err(artifactEvidenceError(error)); + } +} +function parseDirectBombadilSanitizedRunSummary(input) { + try { + return ok(parseRunSummaryUnchecked(cloneArtifactEvidence(input))); + } catch (error) { + return err(artifactEvidenceError(error)); + } +} +function parseDirectBombadilMatrixReceipt(input) { + try { + return ok(parseMatrixReceiptUnchecked(cloneArtifactEvidence(input))); + } catch (error) { + return err(artifactEvidenceError(error)); + } +} +function parseDirectBombadilMatrixSummary(input) { + try { + return ok(parseMatrixSummaryUnchecked(cloneArtifactEvidence(input))); + } catch (error) { + return err(artifactEvidenceError(error)); + } +} +function resolveDirectBombadilUploadLeaf(input) { + const plan = validateArtifactRunPlan(input); + return join2(plan.repositoryRoot, "artifacts", "direct-bombadil-upload", plan.runId); +} +async function requireSafeDirectory(path, label) { + let metadata; + try { + metadata = await lstat(path); + } catch { + throw new BombadilArtifactPolicyError(`${label} does not exist`); + } + if (metadata.isSymbolicLink() || !metadata.isDirectory()) { + throw new BombadilArtifactPolicyError(`${label} must be a non-symlink directory`); + } +} +async function ensureSafeDirectoryChain(repositoryRoot, parts) { + await requireSafeDirectory(repositoryRoot, "repositoryRoot"); + let current = repositoryRoot; + for (const part of parts) { + if (!ARTIFACT_PATH_PART_PATTERN.test(part) || part === "." || part === "..") { + throw new BombadilArtifactPolicyError("Artifact directory contains an unsafe path component"); + } + current = join2(current, part); + try { + await mkdir2(current, { mode: 448 }); + } catch (error) { + if (!isRecord2(error) || error.code !== "EEXIST") + throw error; + } + await requireSafeDirectory(current, `Artifact directory ${part}`); + const resolved = await realpath(current); + if (!isWithin(repositoryRoot, resolved) || resolved !== current) { + throw new BombadilArtifactPolicyError("Artifact directory escaped repositoryRoot"); + } + } + return current; +} +async function createExclusiveDirectory(path, label) { + try { + await mkdir2(path, { mode: 448 }); + } catch (error) { + if (isRecord2(error) && error.code === "EEXIST") { + throw new BombadilArtifactPolicyError(`${label} already exists`); + } + throw error; + } + await requireSafeDirectory(path, label); +} +async function createBombadilArtifactRun(options) { + if (!UUID_PATTERN.test(options.runId)) { + throw new BombadilArtifactPolicyError("Bombadil raw artifact run ID must be a UUID"); + } + const artifactRoot = await ensureSafeDirectoryChain(options.repositoryRoot, [ + "artifacts", + "direct-bombadil", + options.artifactName + ]); + const runDirectory = join2(artifactRoot, options.runId); + await createExclusiveDirectory(runDirectory, "Bombadil artifact run leaf"); + return { + artifactRoot, + manifestPath: join2(artifactRoot, "manifest.json"), + runDirectory + }; +} +async function prepareArtifactUploadSession(planInput) { + const plan = validateArtifactRunPlan(planInput); + let repositoryRoot; + try { + repositoryRoot = await realpath(plan.repositoryRoot); + } catch (error) { + if (!isRecord2(error) || error.code !== "ENOENT") { + throw new BombadilArtifactPolicyError(`artifactRun.repositoryRoot could not be proven safe: ${renderUnknown(error)}`); + } + repositoryRoot = null; + } + if (repositoryRoot === null || repositoryRoot !== plan.repositoryRoot) { + throw new BombadilArtifactPolicyError("artifactRun.repositoryRoot must resolve to its exact configured directory"); + } + const root = await ensureSafeDirectoryChain(repositoryRoot, [ + "artifacts", + "direct-bombadil-upload" + ]); + const finalDirectory = join2(root, plan.runId); + let finalMetadata; + try { + finalMetadata = await lstat(finalDirectory); + } catch (error) { + if (!isRecord2(error) || error.code !== "ENOENT") { + throw new BombadilArtifactPolicyError(`Bombadil upload run leaf could not be inspected: ${renderUnknown(error)}`); + } + finalMetadata = null; + } + if (finalMetadata !== null) { + throw new BombadilArtifactPolicyError("Bombadil upload run leaf already exists"); + } + const stagingDirectory = join2(root, `.staging-${plan.runId}`); + return { + finalDirectory, + mode: plan.uploadMode, + publication: "atomic-leaf", + receiptPath: join2(finalDirectory, "receipt.json"), + runId: plan.runId, + stagingDirectory + }; +} +async function requireArtifactUploadLeafAbsent(session2) { + let existing; + try { + existing = await lstat(session2.finalDirectory); + } catch (error) { + if (!isRecord2(error) || error.code !== "ENOENT") { + throw new BombadilArtifactPolicyError(`Bombadil upload run leaf could not be inspected: ${renderUnknown(error)}`); + } + existing = null; + } + if (existing !== null) { + throw new BombadilArtifactPolicyError("Bombadil upload run leaf appeared before publication"); + } +} +async function commitArtifactUploadSession(session2) { + await rename2(session2.stagingDirectory, session2.finalDirectory); +} +function validateArtifactRelativePath(relativePath, policy) { + const parts = relativePath.split("/"); + if (relativePath.length === 0 || relativePath.includes("\\") || Buffer.byteLength(relativePath, "utf8") > policy.maxPathBytes || parts.length > policy.maxDepth || parts.some((part) => part === "" || part === "." || part === ".." || part.startsWith(".") || !ARTIFACT_PATH_PART_PATTERN.test(part))) { + throw new BombadilArtifactPolicyError(`Bombadil emitted unsafe artifact path ${relativePath}`); + } + return parts; +} +function artifactOutputFileIsAllowed(relativePath) { + return relativePath === "trace.jsonl" || PRIVATE_DIAGNOSTIC_EXTENSIONS.has(extname(relativePath).toLowerCase()); +} +function sameBigIntFileMetadata(left, right) { + return left.dev === right.dev && left.ino === right.ino && left.size === right.size && left.ctimeNs === right.ctimeNs && left.mtimeNs === right.mtimeNs; +} +async function withClosedArtifactHandle(handle, operation) { + let value; + let operationFailure = null; + try { + value = await operation(); + } catch (error) { + operationFailure = error; + } + let closeFailure = null; + try { + await handle.close(); + } catch (error) { + closeFailure = error; + } + if (operationFailure !== null) { + if (closeFailure !== null) { + throw new AggregateError([operationFailure, closeFailure], "Bombadil artifact operation and descriptor cleanup both failed", { cause: operationFailure }); + } + throw operationFailure; + } + if (closeFailure !== null) + throw closeFailure; + return value; +} +async function hashBoundRegularFile(options) { + const flags = fileSystemConstants.O_RDONLY | fileSystemConstants.O_NOFOLLOW | fileSystemConstants.O_NONBLOCK; + const handle = await open(options.path, flags); + return await withClosedArtifactHandle(handle, async () => { + const before = await handle.stat({ bigint: true }); + if (!before.isFile() || before.nlink !== 1n || !options.expected.isFile() || options.expected.nlink !== 1n || !sameBigIntFileMetadata(before, options.expected)) { + throw new BombadilArtifactPolicyError(`Bombadil artifact ${options.relativePath} changed identity before inspection`); + } + const size = Number(before.size); + if (!Number.isSafeInteger(size) || size > options.policy.maxFileBytes) { + throw new BombadilArtifactPolicyError(`Bombadil artifact ${options.relativePath} exceeds the per-file byte quota`); + } + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(64 * 1024); + let offset = 0; + while (offset < size) { + const length = Math.min(buffer.length, size - offset); + const read = await handle.read(buffer, 0, length, offset); + if (read.bytesRead === 0) { + throw new BombadilArtifactPolicyError(`Bombadil artifact ${options.relativePath} changed while inspected`); + } + hash.update(buffer.subarray(0, read.bytesRead)); + offset += read.bytesRead; + } + const after = await handle.stat({ bigint: true }); + if (!sameBigIntFileMetadata(before, after)) { + throw new BombadilArtifactPolicyError(`Bombadil artifact ${options.relativePath} changed while inspected`); + } + return { + device: before.dev, + inode: before.ino, + relativePath: options.relativePath, + sha256: hash.digest("hex"), + size + }; + }); +} +async function readBoundRegularFileBytes(options) { + const flags = fileSystemConstants.O_RDONLY | fileSystemConstants.O_NOFOLLOW | fileSystemConstants.O_NONBLOCK; + let handle; + try { + handle = await open(options.path, flags); + } catch { + throw new BombadilArtifactPolicyError(`${options.label} is not an openable regular file`); + } + try { + return await withClosedArtifactHandle(handle, async () => { + const before = await handle.stat({ bigint: true }); + const size = Number(before.size); + if (!before.isFile() || before.nlink !== 1n || !Number.isSafeInteger(size) || size < 1 || size > options.maximumBytes) { + throw new BombadilArtifactPolicyError(`${options.label} is not a bounded regular file`); + } + if (options.expected !== undefined && (before.dev !== options.expected.device || before.ino !== options.expected.inode || size !== options.expected.size)) { + throw new BombadilArtifactPolicyError(`${options.label} changed after inventory`); + } + const bytes = Buffer.allocUnsafe(size); + let offset = 0; + while (offset < size) { + const read = await handle.read(bytes, offset, size - offset, offset); + if (read.bytesRead === 0) { + throw new BombadilArtifactPolicyError(`${options.label} changed while being read`); + } + offset += read.bytesRead; + } + const after = await handle.stat({ bigint: true }); + if (!sameBigIntFileMetadata(before, after)) { + throw new BombadilArtifactPolicyError(`${options.label} changed while being read`); + } + if (options.expected !== undefined && sha256(bytes) !== options.expected.sha256) { + throw new BombadilArtifactPolicyError(`${options.label} hash changed after inventory`); + } + return bytes; + }); + } catch (error) { + throw error instanceof BombadilArtifactPolicyError ? error : new BombadilArtifactPolicyError(`${options.label} could not be read safely: ${renderUnknown(error)}`); + } +} +function decodeTraceLines(bytes) { + let text; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new Error("Bombadil trace is not valid UTF-8"); + } + const lines = text.split(/\r?\n/u); + if (lines.at(-1) === "") + lines.pop(); + return lines; +} +async function scanBombadilArtifactTree(options) { + let rootMetadata; + try { + rootMetadata = await lstat(options.root, { bigint: true }); + } catch (error) { + if (!isRecord2(error) || error.code !== "ENOENT") { + throw new BombadilArtifactPolicyError(`Bombadil output root could not be inspected: ${renderUnknown(error)}`); + } + rootMetadata = null; + } + if (rootMetadata === null) { + if (options.rootMayBeAbsent === true) { + return { + directories: Object.freeze([]), + entryCount: 0, + files: Object.freeze([]), + fileCount: 0, + inventorySha256: sha256(""), + totalBytes: 0 + }; + } + throw new BombadilArtifactPolicyError("Bombadil output directory does not exist"); + } + if (rootMetadata.isSymbolicLink() || !rootMetadata.isDirectory()) { + throw new BombadilArtifactPolicyError("Bombadil output root must be a non-symlink directory"); + } + const directories = []; + const files = []; + let entryCount = 0; + let totalBytes = 0; + const pending = [{ + absolutePath: options.root, + relativePath: "" + }]; + while (pending.length > 0) { + const current = pending.pop(); + if (current === undefined) + continue; + await options.beforeDirectoryOpen?.(current.absolutePath); + const directory = await opendir(current.absolutePath).catch((error) => { + if (options.allowTransientEntryAbsence === true && isRecord2(error) && error.code === "ENOENT") { + throw error; + } + throw new BombadilArtifactPolicyError(`Bombadil artifact directory could not be opened safely: ${renderUnknown(error)}`); + }); + try { + await withClosedArtifactHandle(directory, async () => { + while (true) { + const entry = await directory.read(); + if (entry === null) + break; + const relativePath = current.relativePath === "" ? entry.name : `${current.relativePath}/${entry.name}`; + validateArtifactRelativePath(relativePath, options.policy); + entryCount += 1; + if (entryCount > options.policy.maxEntries) { + throw new BombadilArtifactPolicyError("Bombadil artifact entry quota was exceeded"); + } + const absolutePath = join2(current.absolutePath, entry.name); + await options.beforeEntryInspect?.(absolutePath); + const metadata = await lstat(absolutePath, { bigint: true }); + if (metadata.isSymbolicLink()) { + throw new BombadilArtifactPolicyError(`Bombadil emitted a symbolic link at ${relativePath}`); + } + if (metadata.isDirectory()) { + directories.push(relativePath); + pending.push({ absolutePath, relativePath }); + continue; + } + if (!metadata.isFile() || metadata.nlink !== 1n) { + throw new BombadilArtifactPolicyError(`Bombadil emitted a non-regular or multiply-linked file at ${relativePath}`); + } + if (!artifactOutputFileIsAllowed(relativePath)) { + throw new BombadilArtifactPolicyError(`Bombadil emitted a file outside the artifact allowlist at ${relativePath}`); + } + if (files.length + 1 > options.policy.maxFiles) { + throw new BombadilArtifactPolicyError("Bombadil artifact file quota was exceeded"); + } + const fileSize = Number(metadata.size); + if (!Number.isSafeInteger(fileSize) || fileSize > options.policy.maxFileBytes) { + throw new BombadilArtifactPolicyError(`Bombadil artifact ${relativePath} exceeds the per-file byte quota`); + } + totalBytes += fileSize; + if (!Number.isSafeInteger(totalBytes) || totalBytes > options.policy.maxTotalBytes) { + throw new BombadilArtifactPolicyError("Bombadil aggregate artifact byte quota was exceeded"); + } + files.push(options.hashFiles ? await hashBoundRegularFile({ + expected: metadata, + path: absolutePath, + policy: options.policy, + relativePath + }) : { + device: 0n, + inode: 0n, + relativePath, + sha256: "", + size: fileSize + }); + } + }); + } catch (error) { + if (options.allowTransientEntryAbsence === true && isRecord2(error) && error.code === "ENOENT") { + throw error; + } + throw error instanceof BombadilArtifactPolicyError ? error : new BombadilArtifactPolicyError(`Bombadil artifact directory could not be inspected safely: ${renderUnknown(error)}`); + } + } + let finalRootMetadata; + try { + finalRootMetadata = await lstat(options.root, { bigint: true }); + } catch (error) { + throw new BombadilArtifactPolicyError(`Bombadil output root could not be revalidated: ${renderUnknown(error)}`); + } + if (!finalRootMetadata.isDirectory() || finalRootMetadata.isSymbolicLink() || finalRootMetadata.dev !== rootMetadata.dev || finalRootMetadata.ino !== rootMetadata.ino) { + throw new BombadilArtifactPolicyError("Bombadil output root changed during inspection"); + } + directories.sort(compareCodeUnits); + files.sort((left, right) => compareCodeUnits(left.relativePath, right.relativePath)); + const inventorySha256 = sha256([ + ...directories.map((directory) => `D\x00${directory} +`), + ...files.map((file) => `F\x00${file.relativePath}\x00${String(file.size)}\x00${file.sha256} +`) + ].join("")); + return { + directories: Object.freeze(directories), + entryCount, + files: Object.freeze(files), + fileCount: files.length, + inventorySha256, + totalBytes + }; +} +async function ensureSafeChildDirectories(root, parts) { + await requireSafeDirectory(root, "Bombadil upload staging root"); + let current = root; + for (const part of parts) { + if (!ARTIFACT_PATH_PART_PATTERN.test(part) || part.startsWith(".")) { + throw new BombadilArtifactPolicyError("Bombadil upload path contains an unsafe component"); + } + current = join2(current, part); + try { + await mkdir2(current, { mode: 448 }); + } catch (error) { + if (!isRecord2(error) || error.code !== "EEXIST") + throw error; + } + await requireSafeDirectory(current, "Bombadil upload directory"); + const resolved = await realpath(current); + if (!isWithin(root, resolved) || resolved !== current) { + throw new BombadilArtifactPolicyError("Bombadil upload directory escaped staging root"); + } + } + return current; +} +async function writeExclusiveBytes(path, bytes) { + const flags = fileSystemConstants.O_WRONLY | fileSystemConstants.O_CREAT | fileSystemConstants.O_EXCL | fileSystemConstants.O_NOFOLLOW; + const handle = await open(path, flags, 384); + await withClosedArtifactHandle(handle, async () => { + let offset = 0; + while (offset < bytes.byteLength) { + const written = await handle.write(bytes, offset, bytes.byteLength - offset, offset); + if (written.bytesWritten === 0) + throw new Error("Exclusive artifact write made no progress"); + offset += written.bytesWritten; + } + await handle.sync(); + }); +} +async function writeExpectedJson(root, relativePath, value) { + const parts = relativePath.split("/"); + const fileName = parts.pop(); + if (fileName === undefined || !ARTIFACT_PATH_PART_PATTERN.test(fileName)) { + throw new BombadilArtifactPolicyError("Sanitized upload path is invalid"); + } + const directory = await ensureSafeChildDirectories(root, parts); + const bytes = Buffer.from(`${JSON.stringify(value, null, 2)} +`, "utf8"); + await writeExclusiveBytes(join2(directory, fileName), bytes); + return { + relativePath, + sha256: sha256(bytes), + size: bytes.byteLength + }; +} +function expectedUploadDirectories(files) { + const directories = new Set; + for (const file of files) { + const parts = file.relativePath.split("/"); + parts.pop(); + for (let index = 1;index <= parts.length; index += 1) { + directories.add(parts.slice(0, index).join("/")); + } + } + return Object.freeze([...directories].sort(compareCodeUnits)); +} +async function validateExpectedUploadTree(root, expectedInput) { + const expected = [...expectedInput].sort((left, right) => compareCodeUnits(left.relativePath, right.relativePath)); + if (new Set(expected.map((file) => file.relativePath)).size !== expected.length) { + throw new BombadilArtifactPolicyError("Sanitized upload contains duplicate file paths"); + } + const directories = expectedUploadDirectories(expected); + const maximumPathBytes = Math.max(1, ...expected.map((file) => Buffer.byteLength(file.relativePath, "utf8"))); + const inventory = await scanBombadilArtifactTree({ + hashFiles: true, + policy: { + maxDepth: Math.max(1, ...expected.map((file) => file.relativePath.split("/").length)), + maxEntries: Math.max(1, expected.length + directories.length), + maxFileBytes: Math.max(1, ...expected.map((file) => file.size)), + maxFiles: Math.max(1, expected.length), + maxPathBytes: maximumPathBytes, + maxTotalBytes: Math.max(1, expected.reduce((total, file) => total + file.size, 0)) + }, + root + }); + if (inventory.directories.length !== directories.length || inventory.directories.some((directory, index) => directory !== directories[index]) || inventory.files.length !== expected.length || inventory.files.some((file, index) => { + const wanted = expected[index]; + return wanted === undefined || file.relativePath !== wanted.relativePath || file.sha256 !== wanted.sha256 || file.size !== wanted.size; + })) { + throw new BombadilArtifactPolicyError("Sanitized upload tree differs from its exact expected inventory"); + } +} +async function copyVerifiedArtifactFile(options) { + const parts = options.file.relativePath.split("/"); + const fileName = parts.pop(); + if (fileName === undefined) + throw new BombadilArtifactPolicyError("Artifact copy path is empty"); + const destinationDirectory = await ensureSafeChildDirectories(options.destinationRoot, parts); + const destinationPath = join2(destinationDirectory, fileName); + const sourcePath = join2(options.sourceRoot, ...options.file.relativePath.split("/")); + const sourceFlags = fileSystemConstants.O_RDONLY | fileSystemConstants.O_NOFOLLOW | fileSystemConstants.O_NONBLOCK; + const destinationFlags = fileSystemConstants.O_WRONLY | fileSystemConstants.O_CREAT | fileSystemConstants.O_EXCL | fileSystemConstants.O_NOFOLLOW; + const source = await open(sourcePath, sourceFlags); + let destination = null; + let copyFailure = null; + try { + const before = await source.stat({ bigint: true }); + if (!before.isFile() || before.nlink !== 1n || before.dev !== options.file.device || before.ino !== options.file.inode || Number(before.size) !== options.file.size) { + throw new BombadilArtifactPolicyError(`Bombadil artifact ${options.file.relativePath} changed before private copy`); + } + destination = await open(destinationPath, destinationFlags, 384); + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(64 * 1024); + let offset = 0; + while (offset < options.file.size) { + const read = await source.read(buffer, 0, Math.min(buffer.length, options.file.size - offset), offset); + if (read.bytesRead === 0) { + throw new BombadilArtifactPolicyError(`Bombadil artifact ${options.file.relativePath} changed during private copy`); + } + hash.update(buffer.subarray(0, read.bytesRead)); + let writtenOffset = 0; + while (writtenOffset < read.bytesRead) { + const written = await destination.write(buffer, writtenOffset, read.bytesRead - writtenOffset, offset + writtenOffset); + if (written.bytesWritten === 0) + throw new Error("Private artifact copy made no progress"); + writtenOffset += written.bytesWritten; + } + offset += read.bytesRead; + } + await destination.sync(); + const after = await source.stat({ bigint: true }); + if (!sameBigIntFileMetadata(before, after) || hash.digest("hex") !== options.file.sha256) { + throw new BombadilArtifactPolicyError(`Bombadil artifact ${options.file.relativePath} changed during private copy`); + } + } catch (error) { + copyFailure = error; + await rm2(destinationPath, { force: true }).catch(() => { + return; + }); + } + let closeFailure = null; + try { + await closeBombadilArtifactCopyHandles(destination, source); + } catch (error) { + closeFailure = error; + } + if (copyFailure !== null) { + if (closeFailure !== null) { + throw new AggregateError([copyFailure, closeFailure], "Bombadil artifact copy and descriptor cleanup both failed", { cause: copyFailure }); + } + throw copyFailure; + } + if (closeFailure !== null) + throw closeFailure; +} +async function closeBombadilArtifactCopyHandles(destination, source) { + const failures = []; + if (destination !== null) { + try { + await destination.close(); + } catch (error) { + failures.push(error); + } + } + try { + await source.close(); + } catch (error) { + failures.push(error); + } + if (failures.length === 1) + throw failures[0]; + if (failures.length > 1) { + throw new AggregateError(failures, "Both Bombadil artifact copy descriptors failed to close"); + } +} +function emptyArtifactInventory() { + return { + directories: Object.freeze([]), + entryCount: 0, + files: Object.freeze([]), + fileCount: 0, + inventorySha256: sha256(""), + totalBytes: 0 + }; +} +function artifactFailureCode(error) { + if (error instanceof BombadilPersistenceError) + return "persistence"; + if (error instanceof BombadilWriterSettlementError) + return "writer-settlement"; + if (error instanceof BombadilArtifactPolicyError) + return "artifact-policy"; + const message = renderUnknown(error); + if (message.includes("interrupted") || message.includes("SIGINT") || message.includes("SIGTERM")) { + return "interrupted"; + } + if (message.includes("exploration policy")) + return "exploration-policy"; + if (message.includes("trace") || message.includes("Direct contract")) + return "trace-attestation"; + if (message.includes("server") || message.includes("reachable")) + return "server"; + if (message.includes("Bombadil")) + return "process"; + return "unknown"; +} +function failureAsError(error) { + return error instanceof Error ? error : new Error(renderUnknown(error)); +} +function combinePersistenceFailure(primary, persistence, message = "Bombadil persistence also failed") { + return new BombadilPersistenceError(`${renderUnknown(primary)}; ${message}`, [primary, persistence]); +} +async function publishFailureAndThrow(primary, publish) { + try { + await publish(); + } catch (persistence) { + throw combinePersistenceFailure(primary, persistence, "sanitized Bombadil receipt publication also failed"); + } + throw failureAsError(primary); +} +function createArtifactReceipt(options) { + return Object.freeze({ + schema: ARTIFACT_RECEIPT_SCHEMA, + completedAt: options.completedAt.toISOString(), + diagnosticsRetained: options.diagnosticsRetained, + failureCode: options.failureCode, + inventory: Object.freeze({ + entryCount: options.inventory.entryCount, + fileCount: options.inventory.fileCount, + inventorySha256: options.inventory.entryCount === 0 ? null : options.inventory.inventorySha256, + totalBytes: options.inventory.totalBytes + }), + mode: options.session.mode, + policy: options.policy, + runId: options.session.runId, + status: options.status + }); +} +function createSanitizedRunSummary(options) { + return Object.freeze({ + schema: ARTIFACT_SUMMARY_SCHEMA, + artifactName: options.artifactName, + scenario: options.scenario, + status: options.status, + failureCode: options.failureCode, + attestation: options.attestation === null ? null : Object.freeze({ + invalidObservationCount: options.attestation.invalidObservationCount, + observationCount: options.attestation.observationCount, + validObservationCount: options.attestation.validObservationCount + }), + exploration: options.explorationSummary === null ? null : Object.freeze({ + actionCount: options.explorationSummary.actions.total, + nonWaitActionCount: options.explorationSummary.actions.nonWaitCount, + policySatisfied: options.explorationSummary.policy.satisfied, + traceBytes: options.explorationSummary.trace.bytes, + traceLineCount: options.explorationSummary.trace.lineCount, + traceSha256: options.explorationSummary.trace.sha256 + }) + }); +} +async function resetUploadStaging(session2) { + await rm2(session2.stagingDirectory, { force: true, recursive: true }); + await createExclusiveDirectory(session2.stagingDirectory, "Bombadil upload staging leaf"); +} +async function withOwnedUploadStaging(session2, operation) { + await createExclusiveDirectory(session2.stagingDirectory, "Bombadil upload staging leaf"); + try { + return await operation(); + } catch (error) { + try { + await rm2(session2.stagingDirectory, { force: true, recursive: true }); + } catch (cleanupError) { + throw new AggregateError([error, cleanupError], "Bombadil upload staging operation and cleanup both failed", { cause: error }); + } + throw error; + } +} +async function publishRunUpload(options) { + let failure = options.failure; + let failureCode = failure === null ? null : options.failureCode ?? artifactFailureCode(failure); + let status = options.status; + const observeInterruption = () => { + if (failure !== null || options.abortSignal?.aborted !== true) + return false; + const signal = options.interruptedSignal?.() ?? null; + failure = new Error(signal === null ? "Bombadil fuzzing was interrupted" : `Bombadil fuzzing was interrupted by ${signal}`); + failureCode = "interrupted"; + status = "failed"; + return true; + }; + observeInterruption(); + if (options.session.publication === "deferred" && options.session.mode !== "public-summary") { + throw new BombadilArtifactPolicyError("Bombadil matrices support public-summary uploads only"); + } + if (options.session.publication === "deferred") { + const receipt = createArtifactReceipt({ + completedAt: options.completedAt, + diagnosticsRetained: false, + failureCode, + inventory: options.inventory, + policy: options.policy, + session: options.session, + status + }); + const summary = createSanitizedRunSummary({ + artifactName: options.artifactName, + attestation: options.attestation, + explorationSummary: options.explorationSummary, + failureCode, + scenario: options.scenario, + status + }); + if (options.session.deferredPayload.value !== null) { + throw new BombadilArtifactPolicyError("Bombadil deferred upload state is invalid"); + } + options.session.deferredPayload.value = Object.freeze({ receipt, summary }); + return { failure, receipt }; + } + const session2 = options.session; + return await withOwnedUploadStaging(session2, async () => { + const expectedFiles = []; + let diagnosticsRetained = false; + if (session2.mode === "private-vetted" && options.privateDiagnosticsAllowed && failureCode !== "interrupted") { + try { + const diagnosticsRoot = await ensureSafeChildDirectories(session2.stagingDirectory, ["diagnostics", "bombadil-output"]); + for (const file of options.inventory.files) { + await copyVerifiedArtifactFile({ + destinationRoot: diagnosticsRoot, + file, + sourceRoot: options.localOutputPath + }); + expectedFiles.push({ + relativePath: `diagnostics/bombadil-output/${file.relativePath}`, + sha256: file.sha256, + size: file.size + }); + } + const controlledLogs = await ensureSafeChildDirectories(session2.stagingDirectory, ["diagnostics", "host"]); + const processLogBytes = Buffer.from(options.processLog, "utf8"); + const serverLogBytes = Buffer.from(options.serverLog, "utf8"); + await writeExclusiveBytes(join2(controlledLogs, "bombadil.log"), processLogBytes); + await writeExclusiveBytes(join2(controlledLogs, "server.log"), serverLogBytes); + expectedFiles.push({ + relativePath: "diagnostics/host/bombadil.log", + sha256: sha256(processLogBytes), + size: processLogBytes.byteLength + }, { + relativePath: "diagnostics/host/server.log", + sha256: sha256(serverLogBytes), + size: serverLogBytes.byteLength + }); + diagnosticsRetained = true; + } catch (error) { + const persistence = new BombadilPersistenceError("Bombadil private diagnostics could not be persisted", [error]); + failure = failure === null ? persistence : combinePersistenceFailure(failure, persistence); + failureCode = "persistence"; + status = "failed"; + await resetUploadStaging(session2); + expectedFiles.length = 0; + } + } + const stageSanitizedPayload = async () => { + const receipt2 = createArtifactReceipt({ + completedAt: options.completedAt, + diagnosticsRetained, + failureCode, + inventory: options.inventory, + policy: options.policy, + session: session2, + status + }); + const summary = createSanitizedRunSummary({ + artifactName: options.artifactName, + attestation: options.attestation, + explorationSummary: options.explorationSummary, + failureCode, + scenario: options.scenario, + status + }); + expectedFiles.push(await writeExpectedJson(session2.stagingDirectory, "summary.json", summary), await writeExpectedJson(session2.stagingDirectory, "receipt.json", receipt2)); + await validateExpectedUploadTree(session2.stagingDirectory, expectedFiles); + return receipt2; + }; + let receipt = await stageSanitizedPayload(); + await options.beforeCommitCheck?.(); + await requireArtifactUploadLeafAbsent(session2); + if (observeInterruption()) { + diagnosticsRetained = false; + await resetUploadStaging(session2); + expectedFiles.length = 0; + receipt = await stageSanitizedPayload(); + await requireArtifactUploadLeafAbsent(session2); + } + await commitArtifactUploadSession(session2); + return { failure, receipt }; + }); +} +async function publishMatrixUpload(options) { + const uploadMode = options.session.mode; + if (uploadMode !== "public-summary") { + throw new BombadilArtifactPolicyError("Bombadil matrix upload session must be public-summary"); + } + let failure = options.failure; + let failureCode = failure === null ? null : options.failureCode ?? artifactFailureCode(failure); + let status = failure === null ? "passed" : "failed"; + const observeInterruption = () => { + if (failure !== null || options.abortSignal?.aborted !== true) + return false; + const signal = options.interruptedSignal?.() ?? null; + failure = new Error(signal === null ? "Bombadil matrix was interrupted" : `Bombadil matrix was interrupted by ${signal}`); + failureCode = "interrupted"; + status = "failed"; + return true; + }; + observeInterruption(); + return await withOwnedUploadStaging(options.session, async () => { + const counts = new Map; + for (const campaign of options.campaigns) { + counts.set(campaign.status, (counts.get(campaign.status) ?? 0) + 1); + } + const expectedFiles = []; + const stageMatrixPayload = async () => { + const receipt = Object.freeze({ + schema: MATRIX_RECEIPT_SCHEMA, + completedAt: options.completedAt.toISOString(), + failureCode, + mode: uploadMode, + runId: options.session.runId, + status, + omittedCampaignCount: options.omittedCampaignCount ?? 0, + campaigns: Object.freeze(options.campaigns.map((campaign) => Object.freeze(campaign))) + }); + const summary = Object.freeze({ + schema: MATRIX_SUMMARY_SCHEMA, + failureCode, + status, + campaigns: Object.freeze({ + failed: counts.get("failed") ?? 0, + notRun: counts.get("not-run") ?? 0, + notSelected: counts.get("not-selected") ?? 0, + passed: counts.get("passed") ?? 0, + rejected: counts.get("rejected") ?? 0, + total: options.campaigns.length, + omitted: options.omittedCampaignCount ?? 0 + }) + }); + for (const child of options.children) { + expectedFiles.push(await writeExpectedJson(options.session.stagingDirectory, `campaigns/${child.campaignId}/summary.json`, child.payload.summary), await writeExpectedJson(options.session.stagingDirectory, `campaigns/${child.campaignId}/receipt.json`, child.payload.receipt)); + } + expectedFiles.push(await writeExpectedJson(options.session.stagingDirectory, "summary.json", summary), await writeExpectedJson(options.session.stagingDirectory, "receipt.json", receipt)); + await validateExpectedUploadTree(options.session.stagingDirectory, expectedFiles); + }; + await stageMatrixPayload(); + await options.beforeCommitCheck?.(); + await requireArtifactUploadLeafAbsent(options.session); + if (observeInterruption()) { + await resetUploadStaging(options.session); + expectedFiles.length = 0; + await stageMatrixPayload(); + await requireArtifactUploadLeafAbsent(options.session); + } + await commitArtifactUploadSession(options.session); + return { failure }; + }); +} function parseTraceDirectObservation(value) { if (!isRecord2(value) || !hasExactKeys(value, DIRECT_OBSERVATION_KEYS)) { throw new Error("Bombadil trace has an invalid named direct observation"); @@ -1727,71 +3184,66 @@ function parseTraceLine(line, lineNumber, strictDiagnosticSnapshotNames) { }; } async function attestDirectBombadilTrace(options) { - const metadata = await stat(options.tracePath).catch(() => null); - if (metadata === null || !metadata.isFile() || metadata.size === 0) { - throw new Error("Bombadil did not produce a nonempty trace.jsonl"); - } - if (metadata.size > TRACE_MAX_BYTES) { - throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_BYTES)} bytes`); - } - const stream = createReadStream(options.tracePath, { encoding: "utf8" }); - const lines = createInterface({ input: stream, crlfDelay: Infinity }); - let observationCount = 0; - let invalidObservationCount = 0; - let validObservationCount = 0; - let initial = null; - let final = null; - let finalWasInvalid = false; - try { - for await (const line of lines) { - observationCount += 1; - if (observationCount > TRACE_MAX_LINES) { - throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); - } - if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { - throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); - } - const observation = parseDirectTraceLine(line, observationCount); - const exact = exactTraceDirectObservation(observation); - if (exact === null) { - if (initial !== null) { - throw new Error("Bombadil trace lost the Direct bridge after exact activation"); - } - invalidObservationCount += 1; - finalWasInvalid = true; - continue; - } - validObservationCount += 1; - final = exact; - finalWasInvalid = false; - if (initial === null) { - if (exact.source !== "scenario" || exact.scenario !== options.expectedScenario || exact.route !== options.expectedRoute) { - throw new Error("Bombadil trace first valid Direct activation does not match the requested scenario and route"); - } - initial = { - activationHash: exact.activationHash, - catalogHash: exact.catalogHash, - route: exact.route, - scenario: exact.scenario, - source: exact.source - }; - } - if (exact.source !== "scenario") { - throw new Error("Bombadil trace left scenario activation during the run"); - } - if (exact.scenario !== initial.scenario || exact.route !== initial.route || exact.activationHash !== initial.activationHash) { - throw new Error("Bombadil trace Direct activation changed during the run"); - } - if (exact.catalogHash !== initial.catalogHash) { - throw new Error("Bombadil trace Direct catalog changed during the run"); + const traceBytes = await readBoundRegularFileBytes({ + label: "Bombadil trace.jsonl", + maximumBytes: TRACE_MAX_BYTES, + path: options.tracePath + }); + return attestDirectBombadilTraceBytes({ ...options, traceBytes }); +} +function attestDirectBombadilTraceBytes(options) { + const lines = decodeTraceLines(options.traceBytes); + let observationCount = 0; + let invalidObservationCount = 0; + let validObservationCount = 0; + let initial = null; + let final = null; + let finalWasInvalid = false; + for (const line of lines) { + observationCount += 1; + if (observationCount > TRACE_MAX_LINES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); + } + if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { + throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); + } + const observation = parseDirectTraceLine(line, observationCount); + const exact = exactTraceDirectObservation(observation); + if (exact === null) { + if (initial !== null) { + throw new Error("Bombadil trace lost the Direct bridge after exact activation"); } - if (observation.violations.some((value) => value !== 0)) { - throw new Error("Bombadil trace contains a nonzero Direct violation counter"); + invalidObservationCount += 1; + finalWasInvalid = true; + continue; + } + validObservationCount += 1; + final = exact; + finalWasInvalid = false; + if (initial === null) { + if (exact.source !== "scenario" || exact.scenario !== options.expectedScenario || exact.route !== options.expectedRoute) { + throw new Error("Bombadil trace first valid Direct activation does not match the requested scenario and route"); } + initial = { + activationHash: exact.activationHash, + catalogHash: exact.catalogHash, + route: exact.route, + scenario: exact.scenario, + source: exact.source + }; + } + if (exact.source !== "scenario") { + throw new Error("Bombadil trace left scenario activation during the run"); + } + if (exact.scenario !== initial.scenario || exact.route !== initial.route || exact.activationHash !== initial.activationHash) { + throw new Error("Bombadil trace Direct activation changed during the run"); + } + if (exact.catalogHash !== initial.catalogHash) { + throw new Error("Bombadil trace Direct catalog changed during the run"); + } + if (observation.violations.some((value) => value !== 0)) { + throw new Error("Bombadil trace contains a nonzero Direct violation counter"); } - } finally { - lines.close(); - stream.destroy(); } if (initial === null || final === null) { throw new Error("Bombadil trace never reached a valid Direct contract"); @@ -1823,13 +3275,14 @@ function sortedCountRecord(values) { return Object.freeze(Object.fromEntries([...values.entries()].sort(([left], [right]) => compareCodeUnits(left, right)))); } async function summarizeDirectBombadilTrace(options) { - const metadata = await stat(options.tracePath).catch(() => null); - if (metadata === null || !metadata.isFile() || metadata.size === 0) { - throw new Error("Bombadil did not produce a nonempty trace.jsonl"); - } - if (metadata.size > TRACE_MAX_BYTES) { - throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_BYTES)} bytes`); - } + const traceBytes = await readBoundRegularFileBytes({ + label: "Bombadil trace.jsonl", + maximumBytes: TRACE_MAX_BYTES, + path: options.tracePath + }); + return summarizeDirectBombadilTraceBytes({ ...options, traceBytes }); +} +function summarizeDirectBombadilTraceBytes(options) { let targetUrl; try { targetUrl = new URL(options.targetUrl); @@ -1869,118 +3322,112 @@ async function summarizeDirectBombadilTrace(options) { let stableTarget = true; let trackedUnrelatedSnapshotNameCount = 0; const unrelatedSnapshotNameLimit = Math.max(0, TRACE_MAX_NAMED_SNAPSHOT_NAMES - strictDiagnosticSnapshotNames.size); - const stream = createReadStream(options.tracePath, { encoding: "utf8" }); - const lines = createInterface({ input: stream, crlfDelay: Infinity }); - try { - for await (const line of lines) { - lineCount += 1; - if (lineCount > TRACE_MAX_LINES) { - throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); - } - if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { - throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); - } - const parsed = parseTraceLine(line, lineCount, strictDiagnosticSnapshotNames); - const rawRelativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; - rawUrlFingerprints.add(sha256(rawRelativeUrl)); - if (rawUrlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { - throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct raw URL fingerprints`); - } - if (parsed.state.currentHash !== null) { - rawNonNullHashCount += 1; - rawTransitionHashes.add(String(parsed.state.currentHash)); - } - for (const name of parsed.propertyViolationNames) { - if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { - throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`); - } - propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); - } - for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { - resources[outputName] = Math.max(resources[outputName], parsed.state.resources[sourceName]); - } - const currentObservationIsExact = exactTraceDirectObservation(parsed.directObservation) !== null; - if (!currentObservationIsExact) { - previousObservationWasExact = false; - continue; + const lines = decodeTraceLines(options.traceBytes); + for (const line of lines) { + lineCount += 1; + if (lineCount > TRACE_MAX_LINES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); + } + if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { + throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); + } + const parsed = parseTraceLine(line, lineCount, strictDiagnosticSnapshotNames); + const rawRelativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + rawUrlFingerprints.add(sha256(rawRelativeUrl)); + if (rawUrlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct raw URL fingerprints`); + } + if (parsed.state.currentHash !== null) { + rawNonNullHashCount += 1; + rawTransitionHashes.add(String(parsed.state.currentHash)); + } + for (const name of parsed.propertyViolationNames) { + if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`); } - policyObservationCount += 1; - const actionFollowsExactObservation = previousObservationWasExact; - const recordedActionKind = actionFollowsExactObservation ? parsed.action?.kind ?? null : null; - if (actionFollowsExactObservation && parsed.action !== null) { - totalActions += 1; - actionCounts.set(parsed.action.kind, (actionCounts.get(parsed.action.kind) ?? 0) + 1); - if (parsed.action.kind === "Wait") { - waitStreak += 1; - maxWaitStreak = Math.max(maxWaitStreak, waitStreak); - } else { - nonWaitCount += 1; - waitStreak = 0; - } - if (parsed.action.targetTag !== null) { - if (!targetTags.has(parsed.action.targetTag) && targetTags.size >= 128) { - throw new Error("Bombadil trace exceeds 128 distinct action target tags"); - } - targetTags.set(parsed.action.targetTag, (targetTags.get(parsed.action.targetTag) ?? 0) + 1); - } - } else if (actionFollowsExactObservation) { + propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); + } + for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { + resources[outputName] = Math.max(resources[outputName], parsed.state.resources[sourceName]); + } + const currentObservationIsExact = exactTraceDirectObservation(parsed.directObservation) !== null; + if (!currentObservationIsExact) { + previousObservationWasExact = false; + continue; + } + policyObservationCount += 1; + const actionFollowsExactObservation = previousObservationWasExact; + const recordedActionKind = actionFollowsExactObservation ? parsed.action?.kind ?? null : null; + if (actionFollowsExactObservation && parsed.action !== null) { + totalActions += 1; + actionCounts.set(parsed.action.kind, (actionCounts.get(parsed.action.kind) ?? 0) + 1); + if (parsed.action.kind === "Wait") { + waitStreak += 1; + maxWaitStreak = Math.max(maxWaitStreak, waitStreak); + } else { + nonWaitCount += 1; waitStreak = 0; } - const relativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; - urlFingerprints.add(sha256(relativeUrl)); - if (urlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { - throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct URL fingerprints`); - } - stableTarget &&= parsed.state.url.href === targetUrl.href; - if (parsed.state.currentHash !== null) { - nonNullHashCount += 1; - transitionHashes.add(String(parsed.state.currentHash)); - } - for (const snapshot of parsed.namedSnapshots) { - let entry = snapshots.get(snapshot.name); - if (entry === undefined) { - const isStrictSnapshot = snapshot.name === "direct" || strictDiagnosticSnapshotNames.has(snapshot.name); - if (!isStrictSnapshot && trackedUnrelatedSnapshotNameCount >= unrelatedSnapshotNameLimit) { - continue; - } - if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { - throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`); - } - entry = { - changeAfterActionKind: new Map, - changeAfterNonWaitCount: 0, - lastObservationIndex: null, - lastValueSha256: null, - observationCount: 0, - values: new Set - }; - snapshots.set(snapshot.name, entry); - if (!isStrictSnapshot) - trackedUnrelatedSnapshotNameCount += 1; + if (parsed.action.targetTag !== null) { + if (!targetTags.has(parsed.action.targetTag) && targetTags.size >= 128) { + throw new Error("Bombadil trace exceeds 128 distinct action target tags"); } - if (!entry.values.has(snapshot.valueSha256) && entry.values.size >= TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { - if (snapshot.name === "direct" || strictDiagnosticSnapshotNames.has(snapshot.name)) { - throw new Error(`Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`); - } + targetTags.set(parsed.action.targetTag, (targetTags.get(parsed.action.targetTag) ?? 0) + 1); + } + } else if (actionFollowsExactObservation) { + waitStreak = 0; + } + const relativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + urlFingerprints.add(sha256(relativeUrl)); + if (urlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct URL fingerprints`); + } + stableTarget &&= parsed.state.url.href === targetUrl.href; + if (parsed.state.currentHash !== null) { + nonNullHashCount += 1; + transitionHashes.add(String(parsed.state.currentHash)); + } + for (const snapshot of parsed.namedSnapshots) { + let entry = snapshots.get(snapshot.name); + if (entry === undefined) { + const isStrictSnapshot = snapshot.name === "direct" || strictDiagnosticSnapshotNames.has(snapshot.name); + if (!isStrictSnapshot && trackedUnrelatedSnapshotNameCount >= unrelatedSnapshotNameLimit) { continue; } - const changedAfterRecordedAction = recordedActionKind !== null && entry.lastObservationIndex === policyObservationCount - 1 && entry.lastValueSha256 !== null && entry.lastValueSha256 !== snapshot.valueSha256; - if (changedAfterRecordedAction) { - entry.changeAfterActionKind.set(recordedActionKind, (entry.changeAfterActionKind.get(recordedActionKind) ?? 0) + 1); + if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`); } - if (changedAfterRecordedAction && recordedActionKind !== "Wait") { - entry.changeAfterNonWaitCount += 1; + entry = { + changeAfterActionKind: new Map, + changeAfterNonWaitCount: 0, + lastObservationIndex: null, + lastValueSha256: null, + observationCount: 0, + values: new Set + }; + snapshots.set(snapshot.name, entry); + if (!isStrictSnapshot) + trackedUnrelatedSnapshotNameCount += 1; + } + if (!entry.values.has(snapshot.valueSha256) && entry.values.size >= TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { + if (snapshot.name === "direct" || strictDiagnosticSnapshotNames.has(snapshot.name)) { + throw new Error(`Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`); } - entry.lastObservationIndex = policyObservationCount; - entry.lastValueSha256 = snapshot.valueSha256; - entry.observationCount += 1; - entry.values.add(snapshot.valueSha256); + continue; } - previousObservationWasExact = true; + const changedAfterRecordedAction = recordedActionKind !== null && entry.lastObservationIndex === policyObservationCount - 1 && entry.lastValueSha256 !== null && entry.lastValueSha256 !== snapshot.valueSha256; + if (changedAfterRecordedAction) { + entry.changeAfterActionKind.set(recordedActionKind, (entry.changeAfterActionKind.get(recordedActionKind) ?? 0) + 1); + } + if (changedAfterRecordedAction && recordedActionKind !== "Wait") { + entry.changeAfterNonWaitCount += 1; + } + entry.lastObservationIndex = policyObservationCount; + entry.lastValueSha256 = snapshot.valueSha256; + entry.observationCount += 1; + entry.values.add(snapshot.valueSha256); } - } finally { - lines.close(); - stream.destroy(); + previousObservationWasExact = true; } if (lineCount === 0) throw new Error("Bombadil did not produce a nonempty trace.jsonl"); @@ -2020,13 +3467,12 @@ async function summarizeDirectBombadilTrace(options) { policyFailures.push("the browser did not remain on the exact target URL"); } } - const traceBytes = await readFile(options.tracePath); return Object.freeze({ schema: "direct.bombadil-exploration-summary/v2", trace: Object.freeze({ - bytes: metadata.size, + bytes: options.traceBytes.byteLength, lineCount, - sha256: sha256(traceBytes) + sha256: sha256(options.traceBytes) }), actions: Object.freeze({ byKind: sortedCountRecord(actionCounts), @@ -2328,13 +3774,13 @@ function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { if (!isAbsolute(config.repositoryRoot) || repositoryRoot !== config.repositoryRoot) { throw new Error("repositoryRoot must be an absolute normalized path"); } - if (!ARTIFACT_NAME_PATTERN.test(config.artifactName)) { + if (!isBoundedArtifactIdentifier(config.artifactName)) { throw new Error("artifactName must be a safe lowercase kebab identifier"); } if (config.label.trim().length === 0 || config.label.length > 160 || hasControlCharacters3(config.label)) { throw new Error("label must contain 1-160 visible characters"); } - if (config.scenario.length > 120 || !SCENARIO_PATTERN.test(config.scenario)) { + if (!isBoundedScenarioIdentifier(config.scenario)) { throw new Error("scenario must be a valid Direct scenario identifier"); } if (config.expectedRoute.trim().length === 0 || config.expectedRoute.length > 256 || hasControlCharacters3(config.expectedRoute)) { @@ -2377,6 +3823,7 @@ function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { const targetQuery = validateTargetQuery(config.targetQuery ?? {}); const viewport = validateViewport(config.viewport); const explorationPolicy = validateExplorationPolicy(config.explorationPolicy); + const artifactPolicy = validateArtifactPolicy(config.artifactPolicy); const startupTimeoutMs = config.server.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; if (!Number.isSafeInteger(startupTimeoutMs) || startupTimeoutMs < 1000 || startupTimeoutMs > MAX_STARTUP_TIMEOUT_MS) { throw new Error(`server.startupTimeoutMs must be an integer between 1000 and ${String(MAX_STARTUP_TIMEOUT_MS)}`); @@ -2385,6 +3832,7 @@ function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { const port = new URL(baseUrl).port; return { ...config, + artifactPolicy, repositoryRoot, specificationPath, baseUrl, @@ -2481,30 +3929,106 @@ function captureStream(stream, maximumLength = LOG_LIMIT) { function signalProcessGroup(process_, signal) { try { process2.kill(-process_.pid, signal); - } catch { + return true; + } catch (error) { + if (!isRecord2(error) || error.code !== "ESRCH") + throw error; if (process_.exitCode === null) process_.kill(signal); + return false; + } +} +function processGroupExists(processId) { + try { + process2.kill(-processId, 0); + return true; + } catch (error) { + if (isRecord2(error) && error.code === "ESRCH") + return false; + throw error; + } +} +async function waitForProcessGroupExit(processId, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (processGroupExists(processId)) { + if (Date.now() >= deadline) { + throw new Error(`Bombadil process group ${String(processId)} survived cleanup`); + } + await Bun.sleep(Math.min(25, Math.max(1, deadline - Date.now()))); + } +} +async function waitForBombadilLeaderExit(process_, timeoutMs) { + if (process_.exitCode !== null) + return; + const exited = await Promise.race([ + process_.exited.then(() => true), + Bun.sleep(timeoutMs).then(() => false) + ]); + if (!exited && process_.exitCode === null) { + throw new Error(`Bombadil process ${String(process_.pid)} survived cleanup`); } } async function terminateProcessGroup(process_, graceMs) { signalProcessGroup(process_, "SIGTERM"); - await Bun.sleep(graceMs); - signalProcessGroup(process_, "SIGKILL"); - await Promise.race([process_.exited.then(() => { - return; - }), Bun.sleep(graceMs)]); + await Promise.race([ + process_.exited.then(() => { + return; + }), + Bun.sleep(graceMs) + ]); + if (processGroupExists(process_.pid)) + signalProcessGroup(process_, "SIGKILL"); + await waitForBombadilLeaderExit(process_, graceMs); + await waitForProcessGroupExit(process_.pid, graceMs); +} +async function settleBombadilProcessGroup(options) { + try { + if (options.immediate) { + signalProcessGroup(options.process, "SIGKILL"); + await waitForBombadilLeaderExit(options.process, options.timeoutMs); + await waitForProcessGroupExit(options.process.pid, options.timeoutMs); + return; + } + await terminateProcessGroup(options.process, options.timeoutMs); + } catch (error) { + throw new BombadilWriterSettlementError(`Bombadil process group ${String(options.process.pid)} did not settle safely`, error); + } +} +async function monitorBombadilArtifactTree(options) { + while (!options.abortSignal.aborted) { + try { + await scanBombadilArtifactTree({ + allowTransientEntryAbsence: true, + hashFiles: false, + policy: options.policy, + root: options.outputPath, + rootMayBeAbsent: true + }); + } catch (error) { + if (isRecord2(error) && error.code === "ENOENT") {} else { + throw error instanceof BombadilArtifactPolicyError ? error : new BombadilArtifactPolicyError("Bombadil artifact monitor could not inspect output"); + } + } + await Bun.sleep(ARTIFACT_MONITOR_INTERVAL_MS); + } } async function runBombadilNativeProcess(invocation) { + const artifactPolicy = validateArtifactPolicy(invocation.artifactPolicy); + const childEnvironment = Object.fromEntries(Object.entries({ + ...process2.env, + NO_COLOR: "1" + }).filter(([name]) => name !== ARTIFACT_COORDINATION_ENVIRONMENT)); const process_ = Bun.spawn([...invocation.command], { cwd: invocation.cwd, detached: true, - env: { ...process2.env, NO_COLOR: "1" }, + env: childEnvironment, stdin: "ignore", stdout: "pipe", stderr: "pipe" }); let timeout; let abortListener; + const monitorAbortController = new AbortController; const timeoutPromise = new Promise((resolveTimeout) => { timeout = setTimeout(() => resolveTimeout("timeout"), invocation.wallClockTimeoutMs); }); @@ -2522,17 +4046,45 @@ async function runBombadilNativeProcess(invocation) { const stdoutCapture = captureStream(process_.stdout); const stderrCapture = captureStream(process_.stderr); const outputPromise = Promise.all([stdoutCapture.result, stderrCapture.result]); + const artifactMonitor = monitorBombadilArtifactTree({ + abortSignal: monitorAbortController.signal, + outputPath: invocation.outputPath, + policy: artifactPolicy + }).then(() => ({ kind: "monitor-stopped" }), (error) => ({ kind: "artifact-policy", error })); const outcome = await Promise.race([ process_.exited.then((exitCode) => ({ kind: "exited", exitCode })), timeoutPromise.then(() => ({ kind: "timeout" })), - abortPromise.then(() => ({ kind: "aborted" })) + abortPromise.then(() => ({ kind: "aborted" })), + artifactMonitor ]); + if (outcome.kind === "monitor-stopped") { + throw new BombadilArtifactPolicyError("Bombadil artifact monitor stopped unexpectedly"); + } const terminationGraceMs = invocation.terminationGraceMs ?? PROCESS_TERMINATION_GRACE_MS; - if (outcome.kind === "exited") { - signalProcessGroup(process_, "SIGKILL"); - } else { - await terminateProcessGroup(process_, terminationGraceMs); + try { + await settleBombadilProcessGroup({ + immediate: true, + process: process_, + timeoutMs: terminationGraceMs + }); + } catch (error) { + stdoutCapture.stop(); + stderrCapture.stop(); + throw error; + } + let finalArtifactFailure = null; + try { + await scanBombadilArtifactTree({ + hashFiles: false, + policy: artifactPolicy, + root: invocation.outputPath, + rootMayBeAbsent: true + }); + } catch (error) { + finalArtifactFailure = error instanceof BombadilArtifactPolicyError ? error : new BombadilArtifactPolicyError(`Bombadil final artifact inventory could not be proven safe: ${renderUnknown(error)}`); } + monitorAbortController.abort(); + const finalMonitorOutcome = await artifactMonitor; const outputSettled = await Promise.race([ outputPromise.then(() => true, () => true), Bun.sleep(Math.max(terminationGraceMs, MIN_PROCESS_OUTPUT_DRAIN_MS)).then(() => false) @@ -2542,6 +4094,15 @@ async function runBombadilNativeProcess(invocation) { stderrCapture.stop(); } const [stdout, stderr] = await outputPromise; + if (outcome.kind === "artifact-policy") { + throw outcome.error instanceof BombadilArtifactPolicyError ? outcome.error : new BombadilArtifactPolicyError("Bombadil artifact policy was violated"); + } + if (finalMonitorOutcome.kind === "artifact-policy") { + throw finalMonitorOutcome.error instanceof BombadilArtifactPolicyError ? finalMonitorOutcome.error : new BombadilArtifactPolicyError("Bombadil artifact policy was violated"); + } + if (finalArtifactFailure !== null) { + throw finalArtifactFailure instanceof BombadilArtifactPolicyError ? finalArtifactFailure : new BombadilArtifactPolicyError("Bombadil artifact policy was violated"); + } return { exitCode: outcome.kind === "exited" ? outcome.exitCode : process_.exitCode ?? 137, stderr, @@ -2549,6 +4110,7 @@ async function runBombadilNativeProcess(invocation) { termination: outcome.kind === "exited" ? null : outcome.kind }; } finally { + monitorAbortController.abort(); if (timeout !== undefined) clearTimeout(timeout); if (abortListener !== undefined) { @@ -2556,11 +4118,18 @@ async function runBombadilNativeProcess(invocation) { } } } +var processEvents = process2; var defaultDependencies = { acquireServer: acquireVerificationServer, createAbortController: () => new AbortController, + createRunId: randomUUID2, now: () => new Date, runBombadil: runBombadilNativeProcess, + signalController: { + forward: (signal) => process2.kill(process2.pid, signal), + once: (signal, listener) => processEvents.once(signal, listener), + removeListener: (signal, listener) => processEvents.removeListener(signal, listener) + }, serverOutputTimeoutMs: SERVER_OUTPUT_TIMEOUT_MS, spawnServer: spawnVerificationServer, stopServer: stopVerificationServer @@ -2718,22 +4287,24 @@ function parseMatrixCampaignArgument(arguments_) { return { arguments: Object.freeze(forwarded), campaignId, help }; } function validateCampaignMatrix(campaigns) { - if (campaigns.length === 0 || campaigns.length > 32) { - throw new Error("Bombadil campaign matrix must contain 1-32 campaigns"); + if (campaigns.length === 0 || campaigns.length > MAX_MATRIX_CAMPAIGNS) { + throw new Error(`Bombadil campaign matrix must contain 1-${String(MAX_MATRIX_CAMPAIGNS)} campaigns`); } const ids = new Set; for (const campaign of campaigns) { - if (!ARTIFACT_NAME_PATTERN.test(campaign.id) || ids.has(campaign.id)) { + if (!isBoundedArtifactIdentifier(campaign.id) || ids.has(campaign.id)) { throw new Error("Bombadil campaign IDs must be unique lowercase kebab identifiers"); } ids.add(campaign.id); } return campaigns; } -async function runDirectBombadilFuzzMatrix(campaignsInput, arguments_ = process2.argv.slice(2), dependencyOverrides = {}) { - const campaigns = validateCampaignMatrix(campaignsInput); - const parsed = parseMatrixCampaignArgument(arguments_); - if (parsed.help) { +async function runDirectBombadilFuzzMatrix(campaignsInput, input = process2.argv.slice(2), dependencyOverrides = {}) { + const dependencies = { ...defaultDependencies, ...dependencyOverrides }; + const normalizedOptions = normalizeFuzzRunOptions(input); + if (normalizedOptions.arguments.some((argument) => argument === "--help" || argument === "-h")) { + const campaigns = validateCampaignMatrix(campaignsInput); + parseMatrixCampaignArgument(normalizedOptions.arguments); process2.stdout.write(`${[ helpText(campaigns[0]?.config.baseUrl ?? ""), " --campaign Run one campaign; required with --replay", @@ -2744,22 +4315,204 @@ async function runDirectBombadilFuzzMatrix(campaignsInput, arguments_ = process2 `); return { kind: "help" }; } - const selected = parsed.campaignId === null ? campaigns : campaigns.filter((campaign) => campaign.id === parsed.campaignId); - if (selected.length === 0) { - throw new Error(`Unknown Bombadil campaign ${parsed.campaignId ?? ""}`); - } - if (parsed.campaignId === null && parsed.arguments.some((argument) => argument === "--replay" || argument.startsWith("--replay="))) { - throw new Error("--replay requires exactly one --campaign in matrix mode"); + const matrixAbortController = dependencies.createAbortController?.() ?? new AbortController; + let interruptedSignal = null; + const interrupt = (signal) => { + interruptedSignal ??= signal; + matrixAbortController.abort(); + }; + const processSignals = dependencies.signalController; + for (const signal of PROCESS_INTERRUPT_SIGNALS) + processSignals.once(signal, interrupt); + const releaseSignalHandlers = () => { + for (const signal of PROCESS_INTERRUPT_SIGNALS) { + processSignals.removeListener(signal, interrupt); + } + }; + let invalidMatrixUploadMode; + let matrixPlan; + let uploadSession; + try { + const firstRepositoryRoot = campaignsInput[0]?.config.repositoryRoot; + if (normalizedOptions.artifactRun === null && firstRepositoryRoot === undefined) { + throw new Error(`Bombadil campaign matrix must contain 1-${String(MAX_MATRIX_CAMPAIGNS)} campaigns`); + } + const requestedMatrixPlan = normalizedOptions.artifactRun ?? { + repositoryRoot: await realpath(resolve(firstRepositoryRoot ?? "")), + runId: dependencies.createRunId(), + uploadMode: "public-summary" + }; + const requestedMatrixUploadMode = requestedMatrixPlan.uploadMode ?? "public-summary"; + invalidMatrixUploadMode = requestedMatrixUploadMode !== "public-summary"; + matrixPlan = { + repositoryRoot: requestedMatrixPlan.repositoryRoot, + runId: requestedMatrixPlan.runId, + uploadMode: "public-summary" + }; + uploadSession = await prepareArtifactUploadSession(matrixPlan); + } catch (error) { + releaseSignalHandlers(); + const signalToForward = interruptedSignal; + if (signalToForward !== null) + processSignals.forward(signalToForward); + throw error; } - const results = []; - for (const campaign of selected) { - const result = await runDirectBombadilFuzz(campaign.config, parsed.arguments, dependencyOverrides); - if (result.kind !== "run") { - throw new Error("Bombadil campaign unexpectedly returned help during matrix execution"); + try { + let campaigns; + let parsed; + let selected; + try { + if (invalidMatrixUploadMode) { + throw new Error("Bombadil matrices support public-summary uploads only"); + } + campaigns = validateCampaignMatrix(campaignsInput); + parsed = parseMatrixCampaignArgument(normalizedOptions.arguments); + selected = parsed.campaignId === null ? campaigns : campaigns.filter((campaign) => campaign.id === parsed.campaignId); + if (selected.length === 0) { + throw new Error(`Unknown Bombadil campaign ${parsed.campaignId ?? ""}`); + } + if (parsed.campaignId === null && parsed.arguments.some((argument) => argument === "--replay" || argument.startsWith("--replay="))) { + throw new Error("--replay requires exactly one --campaign in matrix mode"); + } + for (const campaign of selected) { + if (interruptedSignal !== null) + throw new Error("Bombadil matrix was interrupted"); + const campaignArguments = parseDirectBombadilFuzzArguments(parsed.arguments, campaign.config.baseUrl); + if (campaignArguments.kind !== "run") { + throw new Error("Bombadil matrix campaign unexpectedly entered help mode"); + } + const lexicalConfig = validateDirectBombadilFuzzConfig(campaign.config, campaignArguments.baseUrl); + const resolvedPaths = await resolveDirectBombadilRealPaths(lexicalConfig, resolveReplayPath(lexicalConfig.repositoryRoot, campaignArguments.replayPath)); + if (resolvedPaths.config.repositoryRoot !== matrixPlan.repositoryRoot) { + throw new BombadilArtifactPolicyError("Every Bombadil matrix campaign must share artifactRun.repositoryRoot"); + } + } + } catch (error) { + const boundedCampaigns = campaignsInput.slice(0, MAX_MATRIX_CAMPAIGNS); + const entries2 = boundedCampaigns.map((campaign, index) => ({ + campaignId: isBoundedArtifactIdentifier(campaign.id) ? campaign.id : null, + index, + receipt: null, + status: "rejected" + })); + return await publishFailureAndThrow(error, async () => { + await publishMatrixUpload({ + abortSignal: matrixAbortController.signal, + beforeCommitCheck: dependencies.beforeArtifactCommit, + campaigns: entries2, + children: [], + completedAt: dependencies.now(), + failure: error, + failureCode: interruptedSignal === null ? "configuration-rejected" : "interrupted", + interruptedSignal: () => interruptedSignal, + omittedCampaignCount: Math.max(0, campaignsInput.length - entries2.length), + session: uploadSession + }); + }); + } + const results = []; + const entries = campaigns.map((campaign, index) => ({ + campaignId: campaign.id, + index, + receipt: null, + status: selected.includes(campaign) ? "not-run" : "not-selected" + })); + const children = []; + let executionFailure = null; + let executionFailureCode; + for (const campaign of selected) { + if (interruptedSignal !== null) { + executionFailure = new Error("Bombadil matrix was interrupted"); + break; + } + const campaignIndex = campaigns.indexOf(campaign); + const deferredPayload = { value: null }; + const childSession = { + deferredPayload, + finalDirectory: join2(uploadSession.finalDirectory, "campaigns", campaign.id), + mode: uploadSession.mode, + publication: "deferred", + receiptPath: join2(uploadSession.finalDirectory, "campaigns", campaign.id, "receipt.json"), + runId: uploadSession.runId + }; + try { + const result = await runDirectBombadilFuzzInternal(campaign.config, parsed.arguments, dependencyOverrides, { + abortSignal: matrixAbortController.signal, + forwardSignal: false, + interruptedSignal: () => interruptedSignal, + plan: matrixPlan, + session: childSession + }); + if (result.kind !== "run" || deferredPayload.value === null) { + throw new Error("Bombadil campaign did not finalize its sanitized receipt"); + } + children.push({ campaignId: campaign.id, payload: deferredPayload.value }); + results.push({ campaignId: campaign.id, result }); + entries[campaignIndex] = { + campaignId: campaign.id, + index: campaignIndex, + receipt: `campaigns/${campaign.id}/receipt.json`, + status: "passed" + }; + } catch (error) { + executionFailure = error; + const childPayload = deferredPayload.value; + if (childPayload !== null) { + children.push({ campaignId: campaign.id, payload: childPayload }); + executionFailureCode = childPayload.receipt.failureCode ?? undefined; + } + entries[campaignIndex] = { + campaignId: campaign.id, + index: campaignIndex, + receipt: childPayload === null ? null : `campaigns/${campaign.id}/receipt.json`, + status: childPayload?.receipt.status === "rejected" ? "rejected" : "failed" + }; + break; + } + } + if (executionFailure === null && interruptedSignal !== null) { + executionFailure = new Error("Bombadil matrix was interrupted"); + executionFailureCode = "interrupted"; + } + if (executionFailure !== null) { + await publishFailureAndThrow(executionFailure, async () => { + await publishMatrixUpload({ + abortSignal: matrixAbortController.signal, + beforeCommitCheck: dependencies.beforeArtifactCommit, + campaigns: entries, + children, + completedAt: dependencies.now(), + failure: executionFailure, + ...executionFailureCode === undefined ? {} : { failureCode: executionFailureCode }, + interruptedSignal: () => interruptedSignal, + session: uploadSession + }); + }); } - results.push({ campaignId: campaign.id, result }); + const published = await publishMatrixUpload({ + abortSignal: matrixAbortController.signal, + beforeCommitCheck: dependencies.beforeArtifactCommit, + campaigns: entries, + children, + completedAt: dependencies.now(), + failure: null, + interruptedSignal: () => interruptedSignal, + session: uploadSession + }); + if (published.failure !== null) + throw failureAsError(published.failure); + return { + kind: "matrix", + receiptPath: uploadSession.receiptPath, + results: Object.freeze(results), + uploadArtifactPath: uploadSession.finalDirectory + }; + } finally { + releaseSignalHandlers(); + const signalToForward = interruptedSignal; + if (signalToForward !== null) + processSignals.forward(signalToForward); } - return { kind: "matrix", results: Object.freeze(results) }; } function throwIfBombadilRunAborted(signal) { if (signal.aborted) @@ -2772,242 +4525,434 @@ function terminateAbortedOwnedServer(signal, server) { server.terminate(); throwIfBombadilRunAborted(signal); } -async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2), dependencyOverrides = {}) { - const parsed = parseDirectBombadilFuzzArguments(arguments_, config.baseUrl); - if (parsed.kind === "help") { +async function runDirectBombadilFuzzInternal(config, input = process2.argv.slice(2), dependencyOverrides = {}, preparedUpload) { + const dependencies = { ...defaultDependencies, ...dependencyOverrides }; + const normalizedOptions = normalizeFuzzRunOptions(input); + if (normalizedOptions.arguments.some((argument) => argument === "--help" || argument === "-h")) { + parseDirectBombadilFuzzArguments(normalizedOptions.arguments, config.baseUrl); process2.stdout.write(`${helpText(config.baseUrl)} `); return { kind: "help" }; } - const lexicalConfig = validateDirectBombadilFuzzConfig(config, parsed.baseUrl); - const lexicalReplayPath = resolveReplayPath(lexicalConfig.repositoryRoot, parsed.replayPath); - const resolvedPaths = await resolveDirectBombadilRealPaths(lexicalConfig, lexicalReplayPath); - const validated = resolvedPaths.config; - const replayPath = resolvedPaths.replayPath; - const dependencies = { ...defaultDependencies, ...dependencyOverrides }; - const generatedAt = dependencies.now(); - const artifactRun = await createArtifactRun({ - artifactRoot: validated.artifactRoot, - generatedAt: generatedAt.toISOString() - }); - const outputPath = join2(artifactRun.runDirectory, "bombadil"); - const tracePath = join2(outputPath, "trace.jsonl"); const abortController = dependencies.createAbortController?.() ?? new AbortController; - const invocation = createDirectBombadilInvocation({ - baseUrl: validated.baseUrl, - bombadilExecutable: validated.bombadilExecutable, - entryPath: validated.entryPath, - outputPath, - replayPath, - repositoryRoot: validated.repositoryRoot, - scenario: validated.scenario, - specificationPath: validated.specificationPath, - targetQuery: validated.targetQuery, - timeLimitSeconds: parsed.timeLimitSeconds, - viewport: validated.viewport - }); - const abortableInvocation = { ...invocation, abortSignal: abortController.signal }; - const serverCommand = validated.server.command.map((argument) => argument === "{port}" ? validated.port : argument); - let bombadilVersion = null; - let lease = null; - let ownedServer = null; - let processResult = null; - let attestation = null; - let attestationFailure = null; - let explorationSummary = null; - let explorationSummaryFailure = null; - let rawTracePath = null; - let serverOutput = ""; - let serverOutputFailure = null; - let failure = null; let interruptedSignal = null; + let ownedServer = null; const interrupt = (signal) => { interruptedSignal ??= signal; abortController.abort(); if (ownedServer?.exitCode() === null) ownedServer.terminate(); }; - const interruptSignals = ["SIGINT", "SIGTERM"]; - const processSignals = process2; - for (const signal of interruptSignals) + const processSignals = dependencies.signalController; + for (const signal of PROCESS_INTERRUPT_SIGNALS) processSignals.once(signal, interrupt); + const abortFromPreparedMatrix = () => { + interruptedSignal ??= preparedUpload?.interruptedSignal?.() ?? null; + abortController.abort(); + if (ownedServer?.exitCode() === null) + ownedServer.terminate(); + }; + if (preparedUpload?.abortSignal !== undefined) { + if (preparedUpload.abortSignal.aborted) + abortFromPreparedMatrix(); + else + preparedUpload.abortSignal.addEventListener("abort", abortFromPreparedMatrix, { once: true }); + } try { + const generatedAt = dependencies.now(); + const artifactPlan = preparedUpload?.plan ?? normalizedOptions.artifactRun ?? { + repositoryRoot: await realpath(resolve(config.repositoryRoot)), + runId: dependencies.createRunId(), + uploadMode: "public-summary" + }; + const uploadSession = preparedUpload?.session ?? await prepareArtifactUploadSession(artifactPlan); + let parsed; + let validated; + let replayPath; try { - await requireRegularFile(validated.bombadilExecutable, "The root Bombadil executable"); - bombadilVersion = await readExactBombadilVersion(validated.repositoryRoot); throwIfBombadilRunAborted(abortController.signal); - try { - lease = await dependencies.acquireServer({ - abortSignal: abortController.signal, - baseUrl: validated.baseUrl, - label: validated.label, - readinessPath: validated.server.readinessPath, - reuseExistingLocalServer: false, - startupTimeoutMs: validated.server.startupTimeoutMs, - startServer: () => { - throwIfBombadilRunAborted(abortController.signal); - ownedServer = dependencies.spawnServer({ - command: serverCommand, - cwd: validated.server.cwd, - ...validated.server.env === undefined ? {} : { env: validated.server.env } - }); - terminateAbortedOwnedServer(abortController.signal, ownedServer); - return ownedServer; - } - }); - } catch (error) { - if (abortController.signal.aborted) - throwIfBombadilRunAborted(abortController.signal); - throw error; + const parsedInput = parseDirectBombadilFuzzArguments(normalizedOptions.arguments, config.baseUrl); + if (parsedInput.kind !== "run") { + throw new Error("Bombadil help was not handled before artifact allocation"); } - if (abortController.signal.aborted) { - const acquiredOwnedServer = ownedServer; - if (acquiredOwnedServer?.exitCode() === null) - acquiredOwnedServer.terminate(); - throwIfBombadilRunAborted(abortController.signal); - } - let processFailure = null; - try { - processResult = await dependencies.runBombadil(abortableInvocation); - } catch (error) { - processFailure = error; - } - const traceMetadata = await stat(tracePath).catch(() => null); - if (traceMetadata?.isFile() === true && traceMetadata.size > 0) { - rawTracePath = tracePath; + parsed = parsedInput; + const lexicalConfig = validateDirectBombadilFuzzConfig(config, parsed.baseUrl); + const lexicalReplayPath = resolveReplayPath(lexicalConfig.repositoryRoot, parsed.replayPath); + const resolvedPaths = await resolveDirectBombadilRealPaths(lexicalConfig, lexicalReplayPath); + validated = resolvedPaths.config; + replayPath = resolvedPaths.replayPath; + throwIfBombadilRunAborted(abortController.signal); + if (validated.repositoryRoot !== resolve(artifactPlan.repositoryRoot)) { + throw new BombadilArtifactPolicyError("artifactRun.repositoryRoot must equal the campaign repositoryRoot"); } - try { - attestation = await attestDirectBombadilTrace({ - expectedRoute: validated.expectedRoute, - expectedScenario: validated.scenario, - tracePath + } catch (error) { + const policy = (() => { + try { + return validateArtifactPolicy(config.artifactPolicy); + } catch { + return validateArtifactPolicy(undefined); + } + })(); + return await publishFailureAndThrow(error, async () => { + await publishRunUpload({ + abortSignal: abortController.signal, + artifactName: isBoundedArtifactIdentifier(config.artifactName) ? config.artifactName : "rejected", + beforeCommitCheck: dependencies.beforeArtifactCommit, + attestation: null, + completedAt: dependencies.now(), + explorationSummary: null, + failure: error, + failureCode: abortController.signal.aborted ? "interrupted" : "configuration-rejected", + inventory: emptyArtifactInventory(), + interruptedSignal: () => interruptedSignal, + localOutputPath: config.repositoryRoot, + policy, + privateDiagnosticsAllowed: false, + processLog: "", + scenario: isBoundedScenarioIdentifier(config.scenario) ? config.scenario : "rejected", + serverLog: "", + session: uploadSession, + status: abortController.signal.aborted ? "failed" : "rejected" }); - } catch (error) { - attestationFailure = error; - } - try { - explorationSummary = await summarizeDirectBombadilTrace({ - ...validated.explorationPolicy === null ? {} : { explorationPolicy: validated.explorationPolicy }, - targetUrl: invocation.targetUrl, - tracePath + }); + } + let artifactRun; + try { + throwIfBombadilRunAborted(abortController.signal); + artifactRun = await createBombadilArtifactRun({ + artifactName: validated.artifactName, + repositoryRoot: validated.repositoryRoot, + runId: dependencies.createRunId() + }); + throwIfBombadilRunAborted(abortController.signal); + } catch (error) { + return await publishFailureAndThrow(error, async () => { + await publishRunUpload({ + abortSignal: abortController.signal, + artifactName: validated.artifactName, + beforeCommitCheck: dependencies.beforeArtifactCommit, + attestation: null, + completedAt: dependencies.now(), + explorationSummary: null, + failure: error, + inventory: emptyArtifactInventory(), + interruptedSignal: () => interruptedSignal, + localOutputPath: validated.repositoryRoot, + policy: validated.artifactPolicy, + privateDiagnosticsAllowed: false, + processLog: "", + scenario: validated.scenario, + serverLog: "", + session: uploadSession, + status: "failed" }); + }); + } + const outputPath = join2(artifactRun.runDirectory, "bombadil"); + const tracePath = join2(outputPath, "trace.jsonl"); + const invocation = createDirectBombadilInvocation({ + baseUrl: validated.baseUrl, + bombadilExecutable: validated.bombadilExecutable, + entryPath: validated.entryPath, + outputPath, + replayPath, + repositoryRoot: validated.repositoryRoot, + scenario: validated.scenario, + specificationPath: validated.specificationPath, + targetQuery: validated.targetQuery, + timeLimitSeconds: parsed.timeLimitSeconds, + viewport: validated.viewport + }); + const abortableInvocation = { + ...invocation, + abortSignal: abortController.signal, + artifactPolicy: validated.artifactPolicy + }; + const serverCommand = validated.server.command.map((argument) => argument === "{port}" ? validated.port : argument); + let bombadilVersion = null; + let lease = null; + let processResult = null; + let attestation = null; + let attestationFailure = null; + let explorationSummary = null; + let explorationSummaryFailure = null; + let artifactInventory = emptyArtifactInventory(); + let artifactInventoryVetted = false; + let rawTracePath = null; + let serverOutput = ""; + let serverOutputFailure = null; + let failure = null; + let writersSettled = true; + { + try { + await requireRegularFile(validated.bombadilExecutable, "The root Bombadil executable"); + bombadilVersion = await readExactBombadilVersion(validated.repositoryRoot); + throwIfBombadilRunAborted(abortController.signal); + try { + lease = await dependencies.acquireServer({ + abortSignal: abortController.signal, + baseUrl: validated.baseUrl, + label: validated.label, + readinessPath: validated.server.readinessPath, + reuseExistingLocalServer: false, + startupTimeoutMs: validated.server.startupTimeoutMs, + startServer: () => { + throwIfBombadilRunAborted(abortController.signal); + ownedServer = dependencies.spawnServer({ + command: serverCommand, + cwd: validated.server.cwd, + detachedProcessGroup: true, + ...validated.server.env === undefined ? {} : { env: validated.server.env }, + omitEnvironment: [ARTIFACT_COORDINATION_ENVIRONMENT] + }); + terminateAbortedOwnedServer(abortController.signal, ownedServer); + return ownedServer; + } + }); + } catch (error) { + if (abortController.signal.aborted) + throwIfBombadilRunAborted(abortController.signal); + throw error; + } + if (abortController.signal.aborted) { + const acquiredOwnedServer = ownedServer; + if (acquiredOwnedServer?.exitCode() === null) + acquiredOwnedServer.terminate(); + throwIfBombadilRunAborted(abortController.signal); + } + let processFailure = null; + try { + processResult = await dependencies.runBombadil(abortableInvocation); + } catch (error) { + processFailure = error; + } + if (processFailure !== null) { + throw processFailure instanceof Error ? processFailure : new Error(renderUnknown(processFailure)); + } + if (processResult === null) + throw new Error("Bombadil did not return a process result"); + if (processResult.termination === "timeout") { + throw new Error(`Bombadil exceeded its ${String(invocation.wallClockTimeoutMs)}ms wall-clock limit`); + } + if (processResult.termination === "aborted") { + throw new Error("Bombadil process was interrupted"); + } + if (processResult.exitCode !== 0) { + throw new Error(`Bombadil exited with status ${String(processResult.exitCode)}`); + } } catch (error) { - explorationSummaryFailure = error; - } - if (processFailure !== null) { - throw processFailure instanceof Error ? processFailure : new Error(renderUnknown(processFailure)); - } - if (processResult === null) - throw new Error("Bombadil did not return a process result"); - if (processResult.termination === "timeout") { - throw new Error(`Bombadil exceeded its ${String(invocation.wallClockTimeoutMs)}ms wall-clock limit`); - } - if (processResult.termination === "aborted") { - throw new Error("Bombadil process was interrupted"); + if (error instanceof BombadilWriterSettlementError) + writersSettled = false; + failure = error; } - if (processResult.exitCode !== 0) { - throw new Error(`Bombadil exited with status ${String(processResult.exitCode)}`); + const serverToStop = lease?.source === "started" ? lease.server : ownedServer; + if (serverToStop !== null) { + try { + await dependencies.stopServer(serverToStop); + } catch (error) { + writersSettled = false; + failure = new BombadilWriterSettlementError("Bombadil server writers were not proven absent", failure === null ? error : new AggregateError([failure, error], "Bombadil run and server cleanup both failed")); + } } - if (attestationFailure !== null) { - throw attestationFailure instanceof Error ? attestationFailure : new Error(renderUnknown(attestationFailure)); + const serverAfterRun = ownedServer; + if (serverAfterRun !== null && writersSettled) { + try { + serverOutput = await readServerOutputBounded(serverAfterRun, dependencies.serverOutputTimeoutMs); + } catch (error) { + serverOutputFailure = error; + failure ??= error; + } } - if (explorationSummaryFailure !== null) { - throw explorationSummaryFailure instanceof Error ? explorationSummaryFailure : new Error(renderUnknown(explorationSummaryFailure)); + if (writersSettled) { + try { + try { + artifactInventory = await scanBombadilArtifactTree({ + hashFiles: true, + policy: validated.artifactPolicy, + root: outputPath + }); + } catch (error) { + throw error instanceof BombadilArtifactPolicyError ? error : new BombadilArtifactPolicyError(`Bombadil artifact inventory could not be proven safe: ${renderUnknown(error)}`); + } + artifactInventoryVetted = true; + const trace = artifactInventory.files.find((file) => file.relativePath === "trace.jsonl"); + if (trace === undefined || trace.size === 0) { + const missingTrace = new BombadilArtifactPolicyError("Bombadil did not produce a retained nonempty trace.jsonl"); + attestationFailure = missingTrace; + throw missingTrace; + } + rawTracePath = tracePath; + const traceBytes = await readBoundRegularFileBytes({ + expected: trace, + label: "Bombadil trace.jsonl", + maximumBytes: TRACE_MAX_BYTES, + path: tracePath + }); + try { + attestation = attestDirectBombadilTraceBytes({ + expectedRoute: validated.expectedRoute, + expectedScenario: validated.scenario, + traceBytes + }); + } catch (error) { + attestationFailure = error; + } + try { + explorationSummary = summarizeDirectBombadilTraceBytes({ + ...validated.explorationPolicy === null ? {} : { explorationPolicy: validated.explorationPolicy }, + targetUrl: invocation.targetUrl, + traceBytes + }); + } catch (error) { + explorationSummaryFailure = error; + } + if (attestationFailure !== null) { + throw attestationFailure instanceof Error ? attestationFailure : new Error(renderUnknown(attestationFailure)); + } + if (explorationSummaryFailure !== null) { + throw explorationSummaryFailure instanceof Error ? explorationSummaryFailure : new Error(renderUnknown(explorationSummaryFailure)); + } + if (explorationSummary?.policy.satisfied !== true) { + throw new Error(`Bombadil exploration policy was not satisfied: ${explorationSummary?.policy.failures.join("; ") ?? "summary unavailable"}`); + } + } catch (error) { + failure ??= error; + } + } else { + artifactInventory = emptyArtifactInventory(); + failure ??= new BombadilWriterSettlementError("Bombadil writers were not proven absent; artifact inspection was suppressed", new Error("writer settlement unavailable")); } - if (explorationSummary?.policy.satisfied !== true) { - throw new Error(`Bombadil exploration policy was not satisfied: ${explorationSummary?.policy.failures.join("; ") ?? "summary unavailable"}`); + } + const signalAfterRun = interruptedSignal; + if (signalAfterRun !== null && failure === null) { + failure = new Error(`Bombadil fuzzing was interrupted by ${signalAfterRun}`); + } + const logPath = join2(artifactRun.runDirectory, "bombadil.log"); + const serverLogPath = join2(artifactRun.runDirectory, "server.log"); + const explorationSummaryPath = join2(artifactRun.runDirectory, "exploration-summary.json"); + const log = [processResult?.stdout ?? "", processResult?.stderr ?? ""].filter((part) => part.length > 0).join(` +`); + try { + await writeExclusiveBytes(logPath, Buffer.from(`${log}${log.length > 0 ? ` +` : ""}`, "utf8")); + await writeExclusiveBytes(serverLogPath, Buffer.from(`${serverOutput}${serverOutput.length > 0 ? ` +` : ""}`, "utf8")); + if (explorationSummary !== null) { + await writeJsonAtomically(explorationSummaryPath, explorationSummary); } } catch (error) { - failure = error; + const persistence = new BombadilPersistenceError("Bombadil local diagnostic logs could not be persisted", [error]); + failure = failure === null ? persistence : combinePersistenceFailure(failure, persistence); } - const serverToStop = lease?.source === "started" ? lease.server : ownedServer; - if (serverToStop !== null) { - try { - await dependencies.stopServer(serverToStop); - } catch (error) { - failure ??= error; - } + let completedAt = dependencies.now(); + const createRecord = () => ({ + schema: ARTIFACT_SCHEMA, + evidenceClass: "diagnostic-fuzz", + artifactName: validated.artifactName, + label: validated.label, + status: failure === null ? "passed" : "failed", + generatedAt: generatedAt.toISOString(), + completedAt: completedAt.toISOString(), + durationMs: Math.max(0, completedAt.getTime() - generatedAt.getTime()), + scenario: validated.scenario, + expectedRoute: validated.expectedRoute, + baseUrl: validated.baseUrl, + entryPath: validated.entryPath, + targetQuery: validated.targetQuery, + targetUrl: invocation.targetUrl, + viewport: validated.viewport, + artifactPolicy: validated.artifactPolicy, + artifactInventory: { + entryCount: artifactInventory.entryCount, + fileCount: artifactInventory.fileCount, + inventorySha256: artifactInventory.inventorySha256, + totalBytes: artifactInventory.totalBytes, + files: artifactInventory.files.map((file) => ({ + path: file.relativePath, + sha256: file.sha256, + size: file.size + })) + }, + explorationPolicy: validated.explorationPolicy, + specificationPath: validated.specificationPath, + replayPath, + timeLimitSeconds: replayPath === null ? parsed.timeLimitSeconds : null, + serverSource: lease?.source ?? null, + bombadil: { + version: bombadilVersion, + executable: validated.bombadilExecutable, + exitCode: processResult?.exitCode ?? null, + termination: processResult?.termination ?? null, + outputPath, + rawTracePath, + tracePath: attestation === null ? null : tracePath, + logPath + }, + server: { + logPath: serverLogPath, + logPresent: serverOutput.length > 0, + outputFailure: serverOutputFailure === null ? null : renderUnknown(serverOutputFailure) + }, + attestation, + attestationFailure: attestationFailure === null ? null : renderUnknown(attestationFailure), + explorationSummary, + explorationSummaryPath: explorationSummary === null ? null : explorationSummaryPath, + explorationSummaryFailure: explorationSummaryFailure === null ? null : renderUnknown(explorationSummaryFailure), + initialDirect: attestation?.initial ?? null, + interruptedSignal, + failure: failure === null ? null : renderUnknown(failure) + }); + const runRecordPath = join2(artifactRun.runDirectory, "run.json"); + try { + await writeJsonAtomically(runRecordPath, createRecord()); + } catch (error) { + const persistence = new BombadilPersistenceError("Bombadil local run record could not be persisted", [error]); + failure = failure === null ? persistence : combinePersistenceFailure(failure, persistence); } - const serverAfterRun = ownedServer; - if (serverAfterRun !== null) { - try { - serverOutput = await readServerOutputBounded(serverAfterRun, dependencies.serverOutputTimeoutMs); - } catch (error) { - serverOutputFailure = error; - failure ??= error; - } + const failureBeforeUpload = failure; + const signalBeforeUpload = interruptedSignal; + if (signalBeforeUpload !== null && failure === null) { + failure = new Error(`Bombadil fuzzing was interrupted by ${signalBeforeUpload}`); } - } finally { - for (const signal of interruptSignals) { - processSignals.removeListener(signal, interrupt); + let published; + try { + published = await publishRunUpload({ + abortSignal: abortController.signal, + artifactName: validated.artifactName, + beforeCommitCheck: dependencies.beforeArtifactCommit, + attestation, + completedAt, + explorationSummary, + failure, + inventory: artifactInventory, + interruptedSignal: () => interruptedSignal, + localOutputPath: outputPath, + policy: validated.artifactPolicy, + privateDiagnosticsAllowed: writersSettled && artifactInventoryVetted, + processLog: `${log}${log.length > 0 ? ` +` : ""}`, + scenario: validated.scenario, + serverLog: `${serverOutput}${serverOutput.length > 0 ? ` +` : ""}`, + session: uploadSession, + status: failure === null ? "passed" : "failed" + }); + } catch (persistence) { + if (failure === null) + throw persistence; + throw combinePersistenceFailure(failure, persistence, "sanitized Bombadil receipt publication also failed"); } - } - const capturedSignal = interruptedSignal; - if (capturedSignal !== null && failure === null) { - failure = new Error(`Bombadil fuzzing was interrupted by ${capturedSignal}`); - } - const completedAt = dependencies.now(); - const status = failure === null ? "passed" : "failed"; - const logPath = join2(artifactRun.runDirectory, "bombadil.log"); - const serverLogPath = join2(artifactRun.runDirectory, "server.log"); - const explorationSummaryPath = join2(artifactRun.runDirectory, "exploration-summary.json"); - const record = { - schema: ARTIFACT_SCHEMA, - evidenceClass: "diagnostic-fuzz", - artifactName: validated.artifactName, - label: validated.label, - status, - generatedAt: generatedAt.toISOString(), - completedAt: completedAt.toISOString(), - durationMs: Math.max(0, completedAt.getTime() - generatedAt.getTime()), - scenario: validated.scenario, - expectedRoute: validated.expectedRoute, - baseUrl: validated.baseUrl, - entryPath: validated.entryPath, - targetQuery: validated.targetQuery, - targetUrl: invocation.targetUrl, - viewport: validated.viewport, - explorationPolicy: validated.explorationPolicy, - specificationPath: validated.specificationPath, - replayPath, - timeLimitSeconds: replayPath === null ? parsed.timeLimitSeconds : null, - serverSource: lease?.source ?? null, - bombadil: { - version: bombadilVersion, - executable: validated.bombadilExecutable, - exitCode: processResult?.exitCode ?? null, - termination: processResult?.termination ?? null, - outputPath, - rawTracePath, - tracePath: attestation === null ? null : tracePath, - logPath - }, - server: { - logPath: serverLogPath, - logPresent: serverOutput.length > 0, - outputFailure: serverOutputFailure === null ? null : renderUnknown(serverOutputFailure) - }, - attestation, - attestationFailure: attestationFailure === null ? null : renderUnknown(attestationFailure), - explorationSummary, - explorationSummaryPath: explorationSummary === null ? null : explorationSummaryPath, - explorationSummaryFailure: explorationSummaryFailure === null ? null : renderUnknown(explorationSummaryFailure), - initialDirect: attestation?.initial ?? null, - interruptedSignal: capturedSignal, - failure: failure === null ? null : renderUnknown(failure) - }; - const log = [processResult?.stdout ?? "", processResult?.stderr ?? ""].filter((part) => part.length > 0).join(` -`); - try { - await writeFile2(logPath, `${log}${log.length > 0 ? ` -` : ""}`, "utf8"); - await writeFile2(serverLogPath, `${serverOutput}${serverOutput.length > 0 ? ` -` : ""}`, "utf8"); - if (explorationSummary !== null) { - await writeJsonAtomically(explorationSummaryPath, explorationSummary); - } - await writeJsonAtomically(join2(artifactRun.runDirectory, "run.json"), record); - await writeJsonAtomically(artifactRun.manifestPath, record); + failure = published.failure; + completedAt = dependencies.now(); + if (failure !== failureBeforeUpload) { + await writeJsonAtomically(runRecordPath, createRecord()).catch(() => { + return; + }); + } + await writeJsonAtomically(artifactRun.manifestPath, createRecord()).catch(() => { + return; + }); + const status = failure === null ? "passed" : "failed"; const exploration = explorationSummary === null ? "exploration=unavailable" : [ `nonWait=${String(explorationSummary.actions.nonWaitCount)}`, `maxWaitStreak=${String(explorationSummary.actions.maxWaitStreak)}`, @@ -3029,27 +4974,47 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) kind: "run", artifactDirectory: artifactRun.runDirectory, manifestPath: artifactRun.manifestPath, - status: "passed" + receiptPath: uploadSession.receiptPath, + status: "passed", + uploadArtifactPath: uploadSession.finalDirectory }; } finally { - if (capturedSignal !== null) { - process2.kill(process2.pid, capturedSignal); + preparedUpload?.abortSignal?.removeEventListener("abort", abortFromPreparedMatrix); + for (const signal of PROCESS_INTERRUPT_SIGNALS) { + processSignals.removeListener(signal, interrupt); + } + const signalToForward = interruptedSignal; + if (signalToForward !== null && preparedUpload?.forwardSignal !== false) { + processSignals.forward(signalToForward); } } } +async function runDirectBombadilFuzz(config, input = process2.argv.slice(2), dependencyOverrides = {}) { + return await runDirectBombadilFuzzInternal(config, input, dependencyOverrides); +} // src/tooling/bombadil.ts var attestDirectBombadilTrace2 = attestDirectBombadilTrace; var summarizeDirectBombadilTrace2 = summarizeDirectBombadilTrace; -function runDirectBombadilFuzz2(config, arguments_) { - return arguments_ === undefined ? runDirectBombadilFuzz(config) : runDirectBombadilFuzz(config, arguments_); +var parseDirectBombadilArtifactReceipt2 = parseDirectBombadilArtifactReceipt; +var parseDirectBombadilSanitizedRunSummary2 = parseDirectBombadilSanitizedRunSummary; +var parseDirectBombadilMatrixReceipt2 = parseDirectBombadilMatrixReceipt; +var parseDirectBombadilMatrixSummary2 = parseDirectBombadilMatrixSummary; +var resolveDirectBombadilUploadLeaf2 = resolveDirectBombadilUploadLeaf; +function runDirectBombadilFuzz2(config, argumentsOrOptions) { + return argumentsOrOptions === undefined ? runDirectBombadilFuzz(config) : runDirectBombadilFuzz(config, argumentsOrOptions); } -function runDirectBombadilFuzzMatrix2(campaigns, arguments_) { - return arguments_ === undefined ? runDirectBombadilFuzzMatrix(campaigns) : runDirectBombadilFuzzMatrix(campaigns, arguments_); +function runDirectBombadilFuzzMatrix2(campaigns, argumentsOrOptions) { + return argumentsOrOptions === undefined ? runDirectBombadilFuzzMatrix(campaigns) : runDirectBombadilFuzzMatrix(campaigns, argumentsOrOptions); } export { summarizeDirectBombadilTrace2 as summarizeDirectBombadilTrace, runDirectBombadilFuzzMatrix2 as runDirectBombadilFuzzMatrix, runDirectBombadilFuzz2 as runDirectBombadilFuzz, + resolveDirectBombadilUploadLeaf2 as resolveDirectBombadilUploadLeaf, + parseDirectBombadilSanitizedRunSummary2 as parseDirectBombadilSanitizedRunSummary, + parseDirectBombadilMatrixSummary2 as parseDirectBombadilMatrixSummary, + parseDirectBombadilMatrixReceipt2 as parseDirectBombadilMatrixReceipt, + parseDirectBombadilArtifactReceipt2 as parseDirectBombadilArtifactReceipt, attestDirectBombadilTrace2 as attestDirectBombadilTrace }; diff --git a/dist/tooling/browser-verification-entry.js b/dist/tooling/browser-verification-entry.js index 77fa758..7c34731 100644 --- a/dist/tooling/browser-verification-entry.js +++ b/dist/tooling/browser-verification-entry.js @@ -1293,10 +1293,33 @@ async function collectStream(stream, logLimit) { output = tail(`${output}${decoder.decode(chunk.value, { stream: true })}`, logLimit); } } +function verificationProcessGroupExists(processId) { + try { + process.kill(-processId, 0); + return true; + } catch (error) { + if (error.code === "ESRCH") + return false; + throw error; + } +} +async function waitForVerificationProcessGroupExit(processId, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (verificationProcessGroupExists(processId)) { + if (Date.now() >= deadline) { + throw new Error(`verification server process group ${String(processId)} survived cleanup`); + } + await Bun.sleep(Math.min(25, Math.max(1, deadline - Date.now()))); + } +} function spawnVerificationServer(options) { + const detachedProcessGroup = options.detachedProcessGroup ?? false; + const omittedEnvironment = new Set(options.omitEnvironment ?? []); + const environment = Object.fromEntries(Object.entries({ ...process.env, ...options.env }).filter(([name]) => !omittedEnvironment.has(name))); const process_ = Bun.spawn([...options.command], { cwd: options.cwd, - env: { ...process.env, ...options.env }, + detached: detachedProcessGroup, + env: environment, stdin: "ignore", stdout: "pipe", stderr: "pipe" @@ -1307,12 +1330,31 @@ function spawnVerificationServer(options) { collectStream(process_.stderr, logLimit) ]).then(([stdout, stderr]) => tail(`${stdout} ${stderr}`.trim(), logLimit)); + const signal = (value) => { + if (detachedProcessGroup) { + try { + process.kill(-process_.pid, value); + return; + } catch (error) { + if (error.code !== "ESRCH") + throw error; + } + } + if (process_.exitCode === null) + process_.kill(value); + }; return { exited: process_.exited, exitCode: () => process_.exitCode, + ...detachedProcessGroup ? { + killDescendants: async (timeoutMs) => { + signal("SIGKILL"); + await waitForVerificationProcessGroupExit(process_.pid, timeoutMs); + } + } : {}, output, - terminate: () => process_.kill("SIGTERM"), - kill: () => process_.kill("SIGKILL") + terminate: () => signal("SIGTERM"), + kill: () => signal("SIGKILL") }; } async function runVerificationCommand(options) { @@ -1384,6 +1426,7 @@ async function stopVerificationServerWithOutput(server, stopTimeoutMs = DEFAULT_ throw new Error(`verification server did not exit within ${stopTimeoutMs}ms after SIGKILL`); } } + await server.killDescendants?.(stopTimeoutMs); const output = await settleWithin(server.output, stopTimeoutMs); if (!output.settled) { throw new Error(`verification server output did not settle within ${stopTimeoutMs}ms after exit`); @@ -1491,7 +1534,9 @@ async function writeJsonAtomically(path, value) { `, "utf8"); await rename(temporaryPath, path); } catch (error) { - await rm(temporaryPath, { force: true }); + await rm(temporaryPath, { force: true }).catch(() => { + return; + }); throw error; } } diff --git a/scripts/package-smoke.ts b/scripts/package-smoke.ts index a5c9831..a3708fe 100644 --- a/scripts/package-smoke.ts +++ b/scripts/package-smoke.ts @@ -361,11 +361,11 @@ function bombadilToolingTypeChecks(profile: BombadilFeatureProfile): string { uploadMode: "public-summary", }, }; + // @ts-expect-error Packaged matrix uploads are public-summary only. const unsupportedPrivateBombadilMatrixInput: BombadilMatrixInput = { artifactRun: { repositoryRoot: "/absolute/repository", runId: "00000000-0000-4000-8000-000000000003", - // @ts-expect-error Packaged matrix uploads are public-summary only. uploadMode: "private-vetted", }, }; diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index 9fd3905..f82b3c4 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -389,14 +389,18 @@ async function rejection(promise: Promise): Promise { throw new Error("Expected the operation to reject"); } +type ControllableSignal = Parameters< + DirectBombadilRunnerDependencies["signalController"]["forward"] +>[0]; + function controllableSignals(): { readonly controller: DirectBombadilRunnerDependencies["signalController"]; - readonly emit: (signal: NodeJS.Signals) => void; - readonly forwarded: NodeJS.Signals[]; + readonly emit: (signal: ControllableSignal) => void; + readonly forwarded: ControllableSignal[]; readonly listenerCount: () => number; } { - const listeners = new Map void>>(); - const forwarded: NodeJS.Signals[] = []; + const listeners = new Map void>>(); + const forwarded: ControllableSignal[] = []; return { controller: { forward: (signal) => { diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index 6986204..40f1185 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -1,3 +1,4 @@ +import { EventEmitter } from "node:events"; import { constants as fileSystemConstants, type BigIntStats } from "node:fs"; import { lstat, @@ -664,19 +665,22 @@ export interface DirectBombadilRunnerDependencies { readonly stopServer: typeof stopVerificationServer; } +const PROCESS_INTERRUPT_SIGNALS = ["SIGINT", "SIGTERM"] as const; +type ProcessInterruptSignal = (typeof PROCESS_INTERRUPT_SIGNALS)[number]; + interface ProcessSignalEmitter { readonly once: ( - signal: NodeJS.Signals, - listener: (signal: NodeJS.Signals) => void, + signal: ProcessInterruptSignal, + listener: (signal: ProcessInterruptSignal) => void, ) => unknown; readonly removeListener: ( - signal: NodeJS.Signals, - listener: (signal: NodeJS.Signals) => void, + signal: ProcessInterruptSignal, + listener: (signal: ProcessInterruptSignal) => void, ) => unknown; } interface ProcessSignalController extends ProcessSignalEmitter { - readonly forward: (signal: NodeJS.Signals) => void; + readonly forward: (signal: ProcessInterruptSignal) => void; } type ValidatedConfig = Omit< @@ -2535,7 +2539,7 @@ async function publishRunUpload(options: { readonly failure: unknown; readonly failureCode?: DirectBombadilArtifactFailureCode; readonly inventory: ArtifactInventory; - readonly interruptedSignal?: () => NodeJS.Signals | null; + readonly interruptedSignal?: () => ProcessInterruptSignal | null; readonly localOutputPath: string; readonly policy: ValidatedArtifactPolicy; readonly privateDiagnosticsAllowed: boolean; @@ -2714,7 +2718,7 @@ async function publishMatrixUpload(options: { readonly completedAt: Date; readonly failure: unknown; readonly failureCode?: DirectBombadilArtifactFailureCode; - readonly interruptedSignal?: () => NodeJS.Signals | null; + readonly interruptedSignal?: () => ProcessInterruptSignal | null; readonly omittedCampaignCount?: number; readonly session: AtomicArtifactUploadSession; }): Promise<{ readonly failure: unknown }> { @@ -4532,11 +4536,12 @@ export async function runBombadilNativeProcess( invocation: DirectBombadilInvocation, ): Promise { const artifactPolicy = validateArtifactPolicy(invocation.artifactPolicy); - const childEnvironment: Record = { - ...process.env, - NO_COLOR: "1", - }; - delete childEnvironment[ARTIFACT_COORDINATION_ENVIRONMENT]; + const childEnvironment: Record = Object.fromEntries( + Object.entries({ + ...process.env, + NO_COLOR: "1", + }).filter(([name]) => name !== ARTIFACT_COORDINATION_ENVIRONMENT), + ); const process_ = Bun.spawn([...invocation.command], { cwd: invocation.cwd, detached: true, @@ -4655,6 +4660,8 @@ export async function runBombadilNativeProcess( } } +const processEvents: EventEmitter = process; + const defaultDependencies: DirectBombadilRunnerDependencies = { acquireServer: acquireVerificationServer, createAbortController: () => new AbortController(), @@ -4663,8 +4670,8 @@ const defaultDependencies: DirectBombadilRunnerDependencies = { runBombadil: runBombadilNativeProcess, signalController: { forward: (signal) => process.kill(process.pid, signal), - once: (signal, listener) => process.once(signal, listener), - removeListener: (signal, listener) => process.removeListener(signal, listener), + once: (signal, listener) => processEvents.once(signal, listener), + removeListener: (signal, listener) => processEvents.removeListener(signal, listener), }, serverOutputTimeoutMs: SERVER_OUTPUT_TIMEOUT_MS, spawnServer: spawnVerificationServer, @@ -4905,16 +4912,17 @@ export async function runDirectBombadilFuzzMatrix( return { kind: "help" }; } const matrixAbortController = dependencies.createAbortController?.() ?? new AbortController(); - let interruptedSignal: NodeJS.Signals | null = null; - const interrupt = (signal: NodeJS.Signals): void => { + let interruptedSignal: ProcessInterruptSignal | null = null; + const interrupt = (signal: ProcessInterruptSignal): void => { interruptedSignal ??= signal; matrixAbortController.abort(); }; - const interruptSignals = ["SIGINT", "SIGTERM"] as const; const processSignals = dependencies.signalController; - for (const signal of interruptSignals) processSignals.once(signal, interrupt); + for (const signal of PROCESS_INTERRUPT_SIGNALS) processSignals.once(signal, interrupt); const releaseSignalHandlers = (): void => { - for (const signal of interruptSignals) processSignals.removeListener(signal, interrupt); + for (const signal of PROCESS_INTERRUPT_SIGNALS) { + processSignals.removeListener(signal, interrupt); + } }; let invalidMatrixUploadMode: boolean; let matrixPlan: DirectBombadilArtifactRunPlan; @@ -4943,7 +4951,7 @@ export async function runDirectBombadilFuzzMatrix( uploadSession = await prepareArtifactUploadSession(matrixPlan); } catch (error) { releaseSignalHandlers(); - const signalToForward = interruptedSignal as NodeJS.Signals | null; + const signalToForward = interruptedSignal as ProcessInterruptSignal | null; if (signalToForward !== null) processSignals.forward(signalToForward); throw error; } @@ -5131,7 +5139,7 @@ export async function runDirectBombadilFuzzMatrix( }; } finally { releaseSignalHandlers(); - const signalToForward = interruptedSignal as NodeJS.Signals | null; + const signalToForward = interruptedSignal as ProcessInterruptSignal | null; if (signalToForward !== null) processSignals.forward(signalToForward); } } @@ -5157,7 +5165,7 @@ async function runDirectBombadilFuzzInternal( preparedUpload?: Readonly<{ readonly abortSignal?: AbortSignal; readonly forwardSignal?: boolean; - readonly interruptedSignal?: () => NodeJS.Signals | null; + readonly interruptedSignal?: () => ProcessInterruptSignal | null; readonly plan: DirectBombadilArtifactRunPlan; readonly session: ArtifactUploadSession; }>, @@ -5170,18 +5178,17 @@ async function runDirectBombadilFuzzInternal( return { kind: "help" }; } const abortController = dependencies.createAbortController?.() ?? new AbortController(); - let interruptedSignal: NodeJS.Signals | null = null; + let interruptedSignal: ProcessInterruptSignal | null = null; let ownedServer: ManagedVerificationServer | null = null; - const interrupt = (signal: NodeJS.Signals): void => { + const interrupt = (signal: ProcessInterruptSignal): void => { interruptedSignal ??= signal; abortController.abort(); if (ownedServer?.exitCode() === null) ownedServer.terminate(); }; - const interruptSignals = ["SIGINT", "SIGTERM"] as const; // @types/bun augments Node's process events and has changed this overload // across patch releases. Bind the stable signal subset used by this runner. const processSignals = dependencies.signalController; - for (const signal of interruptSignals) processSignals.once(signal, interrupt); + for (const signal of PROCESS_INTERRUPT_SIGNALS) processSignals.once(signal, interrupt); const abortFromPreparedMatrix = (): void => { interruptedSignal ??= preparedUpload?.interruptedSignal?.() ?? null; abortController.abort(); @@ -5502,7 +5509,7 @@ async function runDirectBombadilFuzzInternal( ); } } - const signalAfterRun = interruptedSignal as NodeJS.Signals | null; + const signalAfterRun = interruptedSignal as ProcessInterruptSignal | null; if (signalAfterRun !== null && failure === null) { failure = new Error(`Bombadil fuzzing was interrupted by ${signalAfterRun}`); } @@ -5595,7 +5602,7 @@ async function runDirectBombadilFuzzInternal( ? null : renderUnknown(explorationSummaryFailure), initialDirect: attestation?.initial ?? null, - interruptedSignal: interruptedSignal as NodeJS.Signals | null, + interruptedSignal: interruptedSignal as ProcessInterruptSignal | null, failure: failure === null ? null : renderUnknown(failure), }); const runRecordPath = join(artifactRun.runDirectory, "run.json"); @@ -5612,7 +5619,7 @@ async function runDirectBombadilFuzzInternal( } const failureBeforeUpload = failure; - const signalBeforeUpload = interruptedSignal as NodeJS.Signals | null; + const signalBeforeUpload = interruptedSignal as ProcessInterruptSignal | null; if (signalBeforeUpload !== null && failure === null) { failure = new Error(`Bombadil fuzzing was interrupted by ${signalBeforeUpload}`); } @@ -5686,10 +5693,10 @@ async function runDirectBombadilFuzzInternal( }; } finally { preparedUpload?.abortSignal?.removeEventListener("abort", abortFromPreparedMatrix); - for (const signal of interruptSignals) { + for (const signal of PROCESS_INTERRUPT_SIGNALS) { processSignals.removeListener(signal, interrupt); } - const signalToForward = interruptedSignal as NodeJS.Signals | null; + const signalToForward = interruptedSignal as ProcessInterruptSignal | null; if (signalToForward !== null && preparedUpload?.forwardSignal !== false) { processSignals.forward(signalToForward); } diff --git a/src/tooling/browser-verification.ts b/src/tooling/browser-verification.ts index c7bea0c..8a6e12f 100644 --- a/src/tooling/browser-verification.ts +++ b/src/tooling/browser-verification.ts @@ -713,8 +713,12 @@ export function spawnVerificationServer(options: { readonly omitEnvironment?: readonly string[]; }): ManagedVerificationServer { const detachedProcessGroup = options.detachedProcessGroup ?? false; - const environment = { ...process.env, ...options.env }; - for (const name of options.omitEnvironment ?? []) delete environment[name]; + const omittedEnvironment = new Set(options.omitEnvironment ?? []); + const environment: Record = Object.fromEntries( + Object.entries({ ...process.env, ...options.env }).filter( + ([name]) => !omittedEnvironment.has(name), + ), + ); const process_ = Bun.spawn([...options.command], { cwd: options.cwd, detached: detachedProcessGroup, From 1a5431ce5c88d3a276c9506305ab41e2f6bafb61 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Sat, 29 Aug 2026 20:43:06 -0400 Subject: [PATCH 25/25] fix: harden Bombadil process settlement --- dist/tooling/bombadil.js | 54 +++---- dist/tooling/browser-verification-entry.js | 2 + scripts/npm-publish-workflow.test.ts | 168 +++++++++++--------- scripts/package-smoke.ts | 2 +- src/tooling/bombadil-runner.test.ts | 165 +++++++++++++++++++- src/tooling/bombadil-runner.ts | 63 +++----- src/tooling/browser-verification.test.ts | 169 +++++++++++++++++++-- src/tooling/browser-verification.ts | 2 + 8 files changed, 472 insertions(+), 153 deletions(-) diff --git a/dist/tooling/bombadil.js b/dist/tooling/bombadil.js index 13be32f..c697614 100644 --- a/dist/tooling/bombadil.js +++ b/dist/tooling/bombadil.js @@ -994,6 +994,8 @@ function verificationProcessGroupExists(processId) { } catch (error) { if (error.code === "ESRCH") return false; + if (error.code === "EPERM") + return true; throw error; } } @@ -3929,28 +3931,29 @@ function captureStream(stream, maximumLength = LOG_LIMIT) { function signalProcessGroup(process_, signal) { try { process2.kill(-process_.pid, signal); - return true; + return; } catch (error) { if (!isRecord2(error) || error.code !== "ESRCH") throw error; if (process_.exitCode === null) process_.kill(signal); - return false; } } -function processGroupExists(processId) { +function processGroupMayExist(processId) { try { process2.kill(-processId, 0); return true; } catch (error) { if (isRecord2(error) && error.code === "ESRCH") return false; + if (isRecord2(error) && error.code === "EPERM") + return true; throw error; } } async function waitForProcessGroupExit(processId, timeoutMs) { const deadline = Date.now() + timeoutMs; - while (processGroupExists(processId)) { + while (processGroupMayExist(processId)) { if (Date.now() >= deadline) { throw new Error(`Bombadil process group ${String(processId)} survived cleanup`); } @@ -3968,28 +3971,11 @@ async function waitForBombadilLeaderExit(process_, timeoutMs) { throw new Error(`Bombadil process ${String(process_.pid)} survived cleanup`); } } -async function terminateProcessGroup(process_, graceMs) { - signalProcessGroup(process_, "SIGTERM"); - await Promise.race([ - process_.exited.then(() => { - return; - }), - Bun.sleep(graceMs) - ]); - if (processGroupExists(process_.pid)) - signalProcessGroup(process_, "SIGKILL"); - await waitForBombadilLeaderExit(process_, graceMs); - await waitForProcessGroupExit(process_.pid, graceMs); -} async function settleBombadilProcessGroup(options) { try { - if (options.immediate) { - signalProcessGroup(options.process, "SIGKILL"); - await waitForBombadilLeaderExit(options.process, options.timeoutMs); - await waitForProcessGroupExit(options.process.pid, options.timeoutMs); - return; - } - await terminateProcessGroup(options.process, options.timeoutMs); + signalProcessGroup(options.process, "SIGKILL"); + await waitForBombadilLeaderExit(options.process, options.timeoutMs); + await waitForProcessGroupExit(options.process.pid, options.timeoutMs); } catch (error) { throw new BombadilWriterSettlementError(`Bombadil process group ${String(options.process.pid)} did not settle safely`, error); } @@ -4063,7 +4049,6 @@ async function runBombadilNativeProcess(invocation) { const terminationGraceMs = invocation.terminationGraceMs ?? PROCESS_TERMINATION_GRACE_MS; try { await settleBombadilProcessGroup({ - immediate: true, process: process_, timeoutMs: terminationGraceMs }); @@ -4389,12 +4374,19 @@ async function runDirectBombadilFuzzMatrix(campaignsInput, input = process2.argv } } catch (error) { const boundedCampaigns = campaignsInput.slice(0, MAX_MATRIX_CAMPAIGNS); - const entries2 = boundedCampaigns.map((campaign, index) => ({ - campaignId: isBoundedArtifactIdentifier(campaign.id) ? campaign.id : null, - index, - receipt: null, - status: "rejected" - })); + const retainedCampaignIds = new Set; + const entries2 = boundedCampaigns.map((campaign, index) => { + const boundedCampaignId = isBoundedArtifactIdentifier(campaign.id) ? campaign.id : null; + const campaignId = boundedCampaignId !== null && !retainedCampaignIds.has(boundedCampaignId) ? boundedCampaignId : null; + if (campaignId !== null) + retainedCampaignIds.add(campaignId); + return { + campaignId, + index, + receipt: null, + status: "rejected" + }; + }); return await publishFailureAndThrow(error, async () => { await publishMatrixUpload({ abortSignal: matrixAbortController.signal, diff --git a/dist/tooling/browser-verification-entry.js b/dist/tooling/browser-verification-entry.js index 7c34731..17d938f 100644 --- a/dist/tooling/browser-verification-entry.js +++ b/dist/tooling/browser-verification-entry.js @@ -1300,6 +1300,8 @@ function verificationProcessGroupExists(processId) { } catch (error) { if (error.code === "ESRCH") return false; + if (error.code === "EPERM") + return true; throw error; } } diff --git a/scripts/npm-publish-workflow.test.ts b/scripts/npm-publish-workflow.test.ts index 681f2a4..28b2797 100644 --- a/scripts/npm-publish-workflow.test.ts +++ b/scripts/npm-publish-workflow.test.ts @@ -25,7 +25,26 @@ const publishingGuideUrl = new URL("../docs/publishing.md", import.meta.url); const agentGuideUrl = new URL("../AGENTS.md", import.meta.url); const npmRegistry = "https://registry.npmjs.org"; const repository = fileURLToPath(new URL("../", import.meta.url)); -const firstPublicSourceCommit = "c6aa5a49c531b45216e3fb043b6e0ab8a392c13d"; +const historicalRecoverySources = [ + { + commit: "c6aa5a49c531b45216e3fb043b6e0ab8a392c13d", + expectedFileCount: 56, + expectedUnpackedBytes: 697_651, + version: "0.7.5", + }, + { + commit: "3f7c821ffaff1d28ccbde1c635d95f584c1af875", + version: "0.7.6", + }, + { + commit: "8953550e298df061e9b9f4081aced158e497b906", + version: "0.7.7", + }, + { + commit: "13e5fa5d4628706d113252420b57579090363ffc", + version: "0.7.8", + }, +] as const; function workflowStepScript(workflow: string, name: string): string { const stepMarker = ` - name: ${name}\n`; @@ -792,73 +811,82 @@ describe("canonical npm package identity", () => { } }); - test("current tools prepare and smoke the exact v0.7.5 source without tagged helpers", async () => { - const work = await mkdtemp(join(tmpdir(), "direct-release-recovery-test-")); - try { - const sourceArchive = join(work, "v0.7.5-source.tar"); - const sourceTree = join(work, "source"); - const packageOutput = join(work, "package"); - await mkdir(sourceTree); - await run([ - "git", - "cat-file", - "-e", - `${firstPublicSourceCommit}^{commit}`, - ], repository); - await run([ - "git", - "archive", - "--format=tar", - `--output=${sourceArchive}`, - firstPublicSourceCommit, - ], repository); - await run(["tar", "-xf", sourceArchive, "-C", sourceTree], repository); - - const manifest = JSON.parse(await readFile(join(sourceTree, "package.json"), "utf8")) as { - readonly name?: unknown; - readonly scripts?: Readonly>; - readonly version?: unknown; - }; - expect(manifest.name).toBe("@hraness/direct"); - expect(manifest.version).toBe("0.7.5"); - expect(manifest.scripts?.prepack).toBe("bun run check"); - - await rm(join(sourceTree, "scripts"), { recursive: true }); - expect(await readdir(sourceTree)).not.toContain("node_modules"); - await run([ - process.execPath, - "--no-env-file", - "--config=/dev/null", - "run", - fileURLToPath(packagePreparationUrl), - packageOutput, - ], sourceTree); - - const filename = "hraness-direct-0.7.5.tgz"; - expect(new Set(await readdir(packageOutput))).toEqual(new Set([ - filename, - "npm-pack.json", - ])); - const inventory = await inspectPackageArtifact(join(packageOutput, filename)); - expect(inventory.fileCount).toBe(56); - expect(inventory.unpackedBytes).toBe(697_651); - - await run([ - process.execPath, - "--no-env-file", - "--config=/dev/null", - "run", - fileURLToPath(packageSmokeUrl), - "--archive", - join(packageOutput, filename), - "--pack-json", - join(packageOutput, "npm-pack.json"), - ], sourceTree); - const finalSourceEntries = await readdir(sourceTree); - expect(finalSourceEntries).not.toContain("scripts"); - expect(finalSourceEntries).not.toContain("node_modules"); - } finally { - await rm(work, { force: true, recursive: true }); - } - }, 180_000); + for (const release of historicalRecoverySources) test( + `current tools prepare and smoke exact v${release.version} source without tagged helpers`, + async () => { + const work = await mkdtemp(join(tmpdir(), "direct-release-recovery-test-")); + try { + const sourceArchive = join(work, `v${release.version}-source.tar`); + const sourceTree = join(work, "source"); + const packageOutput = join(work, "package"); + await mkdir(sourceTree); + await run([ + "git", + "cat-file", + "-e", + `${release.commit}^{commit}`, + ], repository); + await run([ + "git", + "archive", + "--format=tar", + `--output=${sourceArchive}`, + release.commit, + ], repository); + await run(["tar", "-xf", sourceArchive, "-C", sourceTree], repository); + + const manifest = JSON.parse(await readFile(join(sourceTree, "package.json"), "utf8")) as { + readonly name?: unknown; + readonly scripts?: Readonly>; + readonly version?: unknown; + }; + expect(manifest.name).toBe("@hraness/direct"); + expect(manifest.version).toBe(release.version); + expect(manifest.scripts?.prepack).toBe("bun run check"); + + await rm(join(sourceTree, "scripts"), { recursive: true }); + expect(await readdir(sourceTree)).not.toContain("node_modules"); + await run([ + process.execPath, + "--no-env-file", + "--config=/dev/null", + "run", + fileURLToPath(packagePreparationUrl), + packageOutput, + ], sourceTree); + + const filename = `hraness-direct-${release.version}.tgz`; + expect(new Set(await readdir(packageOutput))).toEqual(new Set([ + filename, + "npm-pack.json", + ])); + const inventory = await inspectPackageArtifact(join(packageOutput, filename)); + if ("expectedFileCount" in release) { + expect(inventory.fileCount).toBe(release.expectedFileCount); + expect(inventory.unpackedBytes).toBe(release.expectedUnpackedBytes); + } else { + expect(inventory.fileCount).toBeGreaterThan(0); + expect(inventory.unpackedBytes).toBeGreaterThan(0); + } + + await run([ + process.execPath, + "--no-env-file", + "--config=/dev/null", + "run", + fileURLToPath(packageSmokeUrl), + "--archive", + join(packageOutput, filename), + "--pack-json", + join(packageOutput, "npm-pack.json"), + ], sourceTree); + const finalSourceEntries = await readdir(sourceTree); + expect(finalSourceEntries).not.toContain("scripts"); + expect(finalSourceEntries).not.toContain("node_modules"); + } finally { + await rm(work, { force: true, recursive: true }); + } + }, + 180_000, + ); }); diff --git a/scripts/package-smoke.ts b/scripts/package-smoke.ts index 2dddb6b..72917bf 100644 --- a/scripts/package-smoke.ts +++ b/scripts/package-smoke.ts @@ -332,7 +332,7 @@ type BombadilFeatureProfile = "artifact-delivery" | "baseline" | "matrix"; function selectBombadilFeatureProfile(version: string): BombadilFeatureProfile { if (Bun.semver.order(version, "0.7.9") >= 0) return "artifact-delivery"; - if (Bun.semver.order(version, "0.7.6") >= 0) return "matrix"; + if (Bun.semver.order(version, "0.7.7") >= 0) return "matrix"; return "baseline"; } diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index f82b3c4..ca05f6d 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -389,6 +389,47 @@ async function rejection(promise: Promise): Promise { throw new Error("Expected the operation to reject"); } +type ProcessKill = ( + processId: number, + signal?: NodeJS.Signals | number, +) => boolean; + +async function withProcessKillAdapter( + createAdapter: (kill: ProcessKill) => ProcessKill, + operation: () => Promise, +): Promise { + const descriptor = Object.getOwnPropertyDescriptor(process, "kill"); + if (descriptor === undefined) throw new Error("process.kill descriptor is unavailable"); + const originalKill = process.kill.bind(process); + const kill: ProcessKill = (processId, signal) => originalKill(processId, signal); + Object.defineProperty(process, "kill", { + ...descriptor, + value: createAdapter(kill), + }); + try { + return await operation(); + } finally { + Object.defineProperty(process, "kill", descriptor); + } +} + +async function waitForMissingProcessGroup(processGroupId: number): Promise { + const deadline = Date.now() + 1_000; + for (;;) { + try { + process.kill(-processGroupId, 0); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ESRCH") return; + if (code !== "EPERM") throw error; + } + if (Date.now() >= deadline) { + throw new Error(`Process group ${String(processGroupId)} survived its test cleanup`); + } + await Bun.sleep(10); + } +} + type ControllableSignal = Parameters< DirectBombadilRunnerDependencies["signalController"]["forward"] >[0]; @@ -979,18 +1020,23 @@ describe("Direct Bombadil campaign matrix", () => { { id: "same", config }, { id: "same", config: { ...config, artifactName: "other" } }, ], { arguments: [], artifactRun: duplicatePlan })); - const duplicateReceipt = JSON.parse(await readFile(join( + const duplicateReceipt = parseDirectBombadilMatrixReceipt(JSON.parse(await readFile(join( repositoryRoot, "artifacts", "direct-bombadil-upload", duplicatePlan.runId, "receipt.json", - ), "utf8")) as Record; - expect(duplicateReceipt).toMatchObject({ + ), "utf8"))); + expect(duplicateReceipt.ok).toBeTrue(); + if (!duplicateReceipt.ok) throw new Error("Expected a parser-valid rejected matrix receipt"); + expect(duplicateReceipt.value).toMatchObject({ schema: "direct.bombadil-matrix-receipt/v1", failureCode: "configuration-rejected", status: "failed", - campaigns: [{ status: "rejected" }, { status: "rejected" }], + campaigns: [ + { campaignId: "same", index: 0, receipt: null, status: "rejected" }, + { campaignId: null, index: 1, receipt: null, status: "rejected" }, + ], }); const runtime = dependencies(); @@ -2416,6 +2462,117 @@ describe("Direct Bombadil process lifecycle", () => { expect(Date.now() - startedAt).toBeLessThan(3_500); }, 10_000); + test("retries a transient EPERM process-group probe without signaling again", async () => { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-probe-eperm-")); + temporaryDirectories.push(directory); + const childSource = [ + "process.on('SIGTERM', () => {});", + "setTimeout(() => process.exit(0), 5000);", + "setInterval(() => {}, 1000);", + ].join(" "); + const leaderSource = [ + "const { spawn } = require('node:child_process');", + `const child = spawn(process.execPath, ['-e', ${JSON.stringify(childSource)}], { stdio: ['ignore', 'inherit', 'inherit'] });`, + "child.unref();", + "console.log('normal leader output');", + ].join(" "); + let processGroupId: number | null = null; + let processGroupKills = 0; + let processGroupProbes = 0; + const result = await withProcessKillAdapter( + (kill) => (processId, signal) => { + if (processId < 0 && signal === "SIGKILL") { + processGroupId = -processId; + processGroupKills += 1; + } + if ( + processGroupId !== null + && processId === -processGroupId + && signal === 0 + ) { + processGroupProbes += 1; + if (processGroupProbes === 1) { + throw Object.assign(new Error("synthetic transient process-group probe"), { + code: "EPERM", + }); + } + } + return kill(processId, signal); + }, + async () => await runBombadilNativeProcess({ + command: [process.execPath, "-e", leaderSource], + cwd: directory, + outputPath: directory, + targetUrl: "http://127.0.0.1:4919/", + terminationGraceMs: 100, + wallClockTimeoutMs: 5_000, + }), + ); + expect(result).toMatchObject({ exitCode: 0, termination: null }); + expect(result.stdout).toContain("normal leader output"); + expect(processGroupKills).toBe(1); + expect(processGroupProbes).toBeGreaterThanOrEqual(2); + }, 10_000); + + test("fails closed after a persistent EPERM process-group probe", async () => { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-probe-eperm-timeout-")); + temporaryDirectories.push(directory); + const processIdPath = join(directory, "process.pid"); + const controller = new AbortController(); + let processGroupId: number | null = null; + let processGroupKills = 0; + let processGroupProbes = 0; + const failure = await withProcessKillAdapter( + (kill) => (processId, signal) => { + if (processId < 0 && signal === "SIGKILL") { + processGroupId = -processId; + processGroupKills += 1; + } + if ( + processGroupId !== null + && processId === -processGroupId + && signal === 0 + ) { + processGroupProbes += 1; + throw Object.assign(new Error("synthetic persistent process-group probe"), { + code: "EPERM", + }); + } + return kill(processId, signal); + }, + async () => { + const running = runBombadilNativeProcess({ + abortSignal: controller.signal, + command: [ + process.execPath, + "-e", + `require("node:fs").writeFileSync(${JSON.stringify(processIdPath)}, String(process.pid)); setTimeout(() => process.exit(0), 5000); setInterval(() => {}, 1000);`, + ], + cwd: directory, + outputPath: directory, + targetUrl: "http://127.0.0.1:4919/", + terminationGraceMs: 50, + wallClockTimeoutMs: 5_000, + }); + for (let attempt = 0; attempt < 100 && !(await Bun.file(processIdPath).exists()); attempt += 1) { + await Bun.sleep(10); + } + expect(await Bun.file(processIdPath).exists()).toBeTrue(); + controller.abort(); + return await rejection(running); + }, + ); + expect(failure.name).toBe("BombadilWriterSettlementError"); + expect(failure.message).toContain("did not settle safely"); + expect(processGroupKills).toBe(1); + expect(processGroupProbes).toBeGreaterThanOrEqual(2); + const settledProcessGroupId = processGroupId; + if (settledProcessGroupId === null) { + throw new Error("Expected an owned process-group signal"); + } + await waitForMissingProcessGroup(settledProcessGroupId); + }, 10_000); + test("kills an uncooperative native child after the outer wall-clock limit", async () => { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-timeout-")); temporaryDirectories.push(directory); diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index 40f1185..21108a7 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -4419,24 +4419,26 @@ function captureStream( function signalProcessGroup( process_: ReturnType, - signal: "SIGKILL" | "SIGTERM", -): boolean { + signal: "SIGKILL", +): void { try { process.kill(-process_.pid, signal); - return true; + return; } catch (error) { if (!isRecord(error) || error.code !== "ESRCH") throw error; if (process_.exitCode === null) process_.kill(signal); - return false; } } -function processGroupExists(processId: number): boolean { +function processGroupMayExist(processId: number): boolean { try { process.kill(-processId, 0); return true; } catch (error) { if (isRecord(error) && error.code === "ESRCH") return false; + // EPERM does not prove absence. Keep polling the already-killed group; + // settlement must never authorize a second signal from a failed probe. + if (isRecord(error) && error.code === "EPERM") return true; throw error; } } @@ -4446,7 +4448,7 @@ async function waitForProcessGroupExit( timeoutMs: number, ): Promise { const deadline = Date.now() + timeoutMs; - while (processGroupExists(processId)) { + while (processGroupMayExist(processId)) { if (Date.now() >= deadline) { throw new Error(`Bombadil process group ${String(processId)} survived cleanup`); } @@ -4468,34 +4470,14 @@ async function waitForBombadilLeaderExit( } } -async function terminateProcessGroup( - process_: ReturnType, - graceMs: number, -): Promise { - signalProcessGroup(process_, "SIGTERM"); - await Promise.race([ - process_.exited.then(() => undefined), - Bun.sleep(graceMs), - ]); - // The group may still contain descendants after its leader exits on TERM. - if (processGroupExists(process_.pid)) signalProcessGroup(process_, "SIGKILL"); - await waitForBombadilLeaderExit(process_, graceMs); - await waitForProcessGroupExit(process_.pid, graceMs); -} - async function settleBombadilProcessGroup(options: { - readonly immediate: boolean; readonly process: ReturnType; readonly timeoutMs: number; }): Promise { try { - if (options.immediate) { - signalProcessGroup(options.process, "SIGKILL"); - await waitForBombadilLeaderExit(options.process, options.timeoutMs); - await waitForProcessGroupExit(options.process.pid, options.timeoutMs); - return; - } - await terminateProcessGroup(options.process, options.timeoutMs); + signalProcessGroup(options.process, "SIGKILL"); + await waitForBombadilLeaderExit(options.process, options.timeoutMs); + await waitForProcessGroupExit(options.process.pid, options.timeoutMs); } catch (error) { throw new BombadilWriterSettlementError( `Bombadil process group ${String(options.process.pid)} did not settle safely`, @@ -4590,9 +4572,6 @@ export async function runBombadilNativeProcess( ?? PROCESS_TERMINATION_GRACE_MS; try { await settleBombadilProcessGroup({ - // Every terminal outcome is fail-closed: no writer receives a grace - // window in which it can keep growing or replacing artifact files. - immediate: true, process: process_, timeoutMs: terminationGraceMs, }); @@ -5004,12 +4983,20 @@ export async function runDirectBombadilFuzzMatrix( } } catch (error) { const boundedCampaigns = campaignsInput.slice(0, MAX_MATRIX_CAMPAIGNS); - const entries = boundedCampaigns.map((campaign, index): MatrixCampaignReceiptEntry => ({ - campaignId: isBoundedArtifactIdentifier(campaign.id) ? campaign.id : null, - index, - receipt: null, - status: "rejected", - })); + const retainedCampaignIds = new Set(); + const entries = boundedCampaigns.map((campaign, index): MatrixCampaignReceiptEntry => { + const boundedCampaignId = isBoundedArtifactIdentifier(campaign.id) ? campaign.id : null; + const campaignId = boundedCampaignId !== null && !retainedCampaignIds.has(boundedCampaignId) + ? boundedCampaignId + : null; + if (campaignId !== null) retainedCampaignIds.add(campaignId); + return { + campaignId, + index, + receipt: null, + status: "rejected", + }; + }); return await publishFailureAndThrow(error, async () => { await publishMatrixUpload({ abortSignal: matrixAbortController.signal, diff --git a/src/tooling/browser-verification.test.ts b/src/tooling/browser-verification.test.ts index f31b29f..9356939 100644 --- a/src/tooling/browser-verification.test.ts +++ b/src/tooling/browser-verification.test.ts @@ -155,6 +155,57 @@ async function rejection(promise: Promise): Promise { throw new Error("Expected the operation to reject."); } +type ProcessKill = ( + processId: number, + signal?: NodeJS.Signals | number, +) => boolean; + +async function withProcessKillAdapter( + createAdapter: (kill: ProcessKill) => ProcessKill, + operation: () => Promise, +): Promise { + const descriptor = Object.getOwnPropertyDescriptor(process, "kill"); + if (descriptor === undefined) throw new Error("process.kill descriptor is unavailable"); + const originalKill = process.kill.bind(process); + const kill: ProcessKill = (processId, signal) => originalKill(processId, signal); + Object.defineProperty(process, "kill", { + ...descriptor, + value: createAdapter(kill), + }); + try { + return await operation(); + } finally { + Object.defineProperty(process, "kill", descriptor); + } +} + +async function waitForMissingProcess(processId: number): Promise { + const deadline = Date.now() + 1_000; + for (;;) { + try { + process.kill(processId, 0); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ESRCH") return; + if (code !== "EPERM") throw error; + } + if (Date.now() >= deadline) { + throw new Error(`Process ${String(processId)} survived its test cleanup`); + } + await Bun.sleep(10); + } +} + +async function forceCleanupProcess(processId: number): Promise { + try { + process.kill(processId, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return; + throw error; + } + await waitForMissingProcess(processId); +} + describe("browser verification targets", () => { test("normalizes only credential-free HTTP server roots", () => { expect(normalizeRootHttpOrigin("https://example.test/")).toBe("https://example.test"); @@ -608,23 +659,123 @@ describe("server leases", () => { const source = [ "const { spawn } = require('node:child_process');", "const fs = require('node:fs');", - "const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });", + "const child = spawn(process.execPath, ['-e', `process.on('SIGTERM', () => {}); setTimeout(() => process.exit(0), 5000); setInterval(() => {}, 1000);`], { stdio: 'ignore' });", + "child.unref();", + `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid));`, + ].join(" "); + const server = spawnVerificationServer({ + command: [process.execPath, "-e", source], + cwd: directory, + detachedProcessGroup: true, + }); + let childPid: number | null = null; + let childMissing = false; + let stopped = false; + let processGroupId: number | null = null; + let processGroupKills = 0; + let processGroupProbes = 0; + try { + for (let attempt = 0; attempt < 100 && !(await Bun.file(childPidPath).exists()); attempt += 1) { + await Bun.sleep(10); + } + expect(await Bun.file(childPidPath).exists()).toBeTrue(); + childPid = Number.parseInt(await Bun.file(childPidPath).text(), 10); + await server.exited; + await withProcessKillAdapter( + (kill) => (processId, signal) => { + if (processId < 0 && signal === "SIGKILL") { + processGroupId = -processId; + processGroupKills += 1; + } + if ( + processGroupId !== null + && processId === -processGroupId + && signal === 0 + ) { + processGroupProbes += 1; + if (processGroupProbes === 1) { + throw Object.assign(new Error("synthetic transient process-group probe"), { + code: "EPERM", + }); + } + } + return kill(processId, signal); + }, + async () => await stopVerificationServer(server, 500), + ); + stopped = true; + expect(processGroupKills).toBe(1); + expect(processGroupProbes).toBeGreaterThanOrEqual(2); + expect(Number.isSafeInteger(childPid)).toBeTrue(); + await waitForMissingProcess(childPid); + childMissing = true; + } finally { + if (!stopped) await stopVerificationServer(server, 500); + if (childPid !== null && !childMissing) await forceCleanupProcess(childPid); + } + }); + + test("fails closed after persistent EPERM while stopping detached descendants", async () => { + const directory = await mkdtemp(join(tmpdir(), "direct-server-process-group-eperm-")); + temporaryDirectories.push(directory); + const childPidPath = join(directory, "child.pid"); + const source = [ + "const { spawn } = require('node:child_process');", + "const fs = require('node:fs');", + "const child = spawn(process.execPath, ['-e', `process.on('SIGTERM', () => {}); setTimeout(() => process.exit(0), 5000); setInterval(() => {}, 1000);`], { stdio: 'ignore' });", + "child.unref();", `fs.writeFileSync(${JSON.stringify(childPidPath)}, String(child.pid));`, - "setInterval(() => {}, 1000);", ].join(" "); const server = spawnVerificationServer({ command: [process.execPath, "-e", source], cwd: directory, detachedProcessGroup: true, }); - for (let attempt = 0; attempt < 100 && !(await Bun.file(childPidPath).exists()); attempt += 1) { - await Bun.sleep(10); + let childPid: number | null = null; + let childMissing = false; + try { + for (let attempt = 0; attempt < 100 && !(await Bun.file(childPidPath).exists()); attempt += 1) { + await Bun.sleep(10); + } + expect(await Bun.file(childPidPath).exists()).toBeTrue(); + childPid = Number.parseInt(await Bun.file(childPidPath).text(), 10); + await server.exited; + let processGroupId: number | null = null; + let processGroupKills = 0; + let processGroupProbes = 0; + const failure = await withProcessKillAdapter( + (kill) => (processId, signal) => { + if (processId < 0 && signal === "SIGKILL") { + processGroupId = -processId; + processGroupKills += 1; + } + if ( + processGroupId !== null + && processId === -processGroupId + && signal === 0 + ) { + processGroupProbes += 1; + throw Object.assign(new Error("synthetic persistent process-group probe"), { + code: "EPERM", + }); + } + return kill(processId, signal); + }, + async () => await rejection(stopVerificationServer(server, 50)), + ); + expect(failure.message).toContain("survived cleanup"); + expect(processGroupKills).toBe(1); + expect(processGroupProbes).toBeGreaterThanOrEqual(2); + expect(Number.isSafeInteger(childPid)).toBeTrue(); + await waitForMissingProcess(childPid); + childMissing = true; + } finally { + if (childPid === null) { + await stopVerificationServer(server, 500); + } else if (!childMissing) { + await forceCleanupProcess(childPid); + } } - expect(await Bun.file(childPidPath).exists()).toBeTrue(); - const childPid = Number.parseInt(await Bun.file(childPidPath).text(), 10); - await stopVerificationServer(server, 500); - expect(Number.isSafeInteger(childPid)).toBeTrue(); - expect(() => process.kill(childPid, 0)).toThrow(); }); test("bounds one-shot verification commands and reports their exact outcome", async () => { diff --git a/src/tooling/browser-verification.ts b/src/tooling/browser-verification.ts index 8a6e12f..85bce85 100644 --- a/src/tooling/browser-verification.ts +++ b/src/tooling/browser-verification.ts @@ -687,6 +687,8 @@ function verificationProcessGroupExists(processId: number): boolean { return true; } catch (error) { if ((error as NodeJS.ErrnoException).code === "ESRCH") return false; + // EPERM leaves group absence unproven, so poll until ESRCH or timeout. + if ((error as NodeJS.ErrnoException).code === "EPERM") return true; throw error; } }