From ee58332fb901d2786f3859ebedde81679faa2497 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:20:33 -0400 Subject: [PATCH 01/20] feat: deepen Bombadil exploration diagnostics --- README.md | 20 +- docs/verification.md | 143 +++- src/exports.test.ts | 4 + src/tooling/bombadil-campaign.test.ts | 101 ++- src/tooling/bombadil-campaign.ts | 120 ++++ src/tooling/bombadil-runner.test.ts | 326 ++++++++- src/tooling/bombadil-runner.ts | 937 +++++++++++++++++++++++++- src/tooling/bombadil.ts | 21 + 8 files changed, 1652 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index e31c3e6..ee5cecd 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ A quiet probe means the declared deterministic work settled. It does not prove t | `@hraness/direct/web` | Atomic browser installation, with low-level bridge and firewall escape hatches | Browser only | | `@hraness/direct/tooling/browser-verification` | Protocol-bound bridge reads, bounded agent-browser commands, local server leases, and artifact writes | Bun 1.3.14 with Node APIs | | `@hraness/direct/tooling/bombadil-campaign` | Direct property and conservative action factories for a Bombadil specification | Bombadil 0.7.2 specification compiler | -| `@hraness/direct/tooling/bombadil` | Local server ownership, native Bombadil lifecycle, trace attestation, replay, and diagnostic artifacts | Bun 1.3.14 with Node APIs | +| `@hraness/direct/tooling/bombadil` | Local server ownership, native Bombadil lifecycle, serial campaign matrices, trace attestation and summaries, replay, and diagnostic artifacts | Bun 1.3.14 with Node APIs | | `@hraness/direct/tooling/bundle-boundary` | Deterministic emitted-file scans and exact versioned-wire evidence | Bun 1.3.14 with Node APIs | The tooling subpaths are development-only. They are built separately from the @@ -272,7 +272,23 @@ path, and any additional safe actions. Call `runDirectBombadilFuzz` from `@hraness/direct/tooling/bombadil` in a small Bun wrapper. The runner accepts only an explicit local HTTP origin, starts an argv-only server command, invokes the exact native 0.7.2 binary, attests the bounded trace with Direct's canonical -parsers, writes pass or failure artifacts, and releases its owned processes. +parsers, writes pass or failure artifacts plus a compact exploration summary, +and releases its owned processes. Use `runDirectBombadilFuzzMatrix` when a +product owns several scenarios; it runs them serially and requires one exact +campaign selector for replay. + +Keep liveness formulas time-bounded. Prefer guarded product actions with +explicit weights over unrestricted browser actions, and name small JSON +snapshots that expose semantic state without retaining page content. Run short +12–30 second campaigns while editing and longer 60–300 second matrices in a +scheduled diagnostic lane. Inspect and replay a retained failing trace, then +promote the smallest readable failure to a deterministic product regression. +When a campaign must exercise an interaction, require a named product value to +change after a non-Wait action so bootstrap and idle transitions do not satisfy +the exploration policy. +The raw trace remains authoritative and may contain screenshots, URLs, typed +text, accessible labels, and local paths; treat it as potentially sensitive. +Summary counts and hashes help triage exploration but are not Direct coverage. See [Verification](./docs/verification.md#run-a-bounded-bombadil-campaign) for the complete configuration and proof limits. diff --git a/docs/verification.md b/docs/verification.md index 52c3e58..c4ae45a 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -237,6 +237,11 @@ deterministic checks. This path does not replace product-owned semantic or accessibility assertions. It also does not prove a substituted service, adapter, browser host, operating system, or device. +Bombadil is an experimental 0.x tool. Review the official +[manual](https://antithesishq.github.io/bombadil/) when changing its pinned +version, specification, actions, properties, or CLI invocation; minor releases +may change interfaces or trace details. + Install the one supported release directly in the consumer. Direct declares it as an exact optional peer so products that do not use fuzzing do not install a browser tool: @@ -253,20 +258,36 @@ TypeScript source. Import it only from a Bombadil specification; use the built Create a specification such as `direct/bombadil-campaign.ts`: ```ts +import { always, eventually } from "@antithesishq/bombadil"; import { createDirectBombadilActions, + createDirectBombadilNamedSnapshot, createDirectBombadilProperties, + createDirectBombadilResourceLeakProperty, } from "@hraness/direct/tooling/bombadil-campaign"; export * from "@antithesishq/bombadil/browser/defaults/properties"; const direct = createDirectBombadilProperties(); +const phase = createDirectBombadilNamedSnapshot({ + fallback: "unavailable", + name: "todos.phase", + read: ({ window }) => Reflect.get(window, "__todosPhase"), +}); export const direct_safe_actions = createDirectBombadilActions(); export const direct_exact_contract = direct.exactContract; export const direct_stable_catalog = direct.stableCatalog; export const direct_no_declared_violations = direct.noDeclaredViolations; export const direct_eventual_quiescence = direct.eventualQuiescence; +export const todos_phase_is_known = always( + eventually(() => phase.current !== "unavailable").within(5, "seconds"), +); +export const no_dom_node_leak = createDirectBombadilResourceLeakProperty({ + metric: "dom_nodes", + growthLimit: 500, + windowMillis: 10_000, +}); ``` The Direct action generator deliberately excludes reload, history traversal, @@ -275,6 +296,31 @@ Enter key. It retains ordinary buttons, text input, scrolling, and an always-eligible low-weight wait. This preserves the post-handshake contract within one document. Add product actions only when their navigation and form effects are understood, and keep product-specific assertions in the campaign. +Guard domain actions on the state that makes them valid, then weight valuable +state-changing actions above Wait. Do not increase throughput by admitting +reload, navigation, submission, destructive controls, or arbitrary generated +input. A generated action sequence is useful only while it preserves the +scenario boundary and exercises behavior the product can interpret. + +`createDirectBombadilNamedSnapshot` gives product properties and post-run +diagnostics a small semantic signal. It requires a safe bounded name, clones no +more than two million JSON characters, and fails closed to the explicit +fallback when a page getter throws or returns unsuitable data. Extract state, +not page content or credentials. Named values are represented only by canonical +SHA-256 hashes in the exploration summary; the authoritative raw trace still +contains the original values. + +Bombadil's 0.7.2 manual documents a sliding-window resource property at +`@antithesishq/bombadil/browser/extras/resources`, but the published npm +package omits that subpath from its `exports` map. Use Direct's +`createDirectBombadilResourceLeakProperty` implementation until a reviewed +Bombadil release exports the official helper. Add a tuned property when +repeated product actions allocate DOM nodes, listeners, layout objects, or +heap. Prefer DOM-node or listener thresholds when they express the defect +because heap samples move with garbage collection. Measure a normal run in +Inspect before setting a growth limit, and keep the window longer than ordinary +rendering bursts. The summary's resource high-water marks help choose and +review those thresholds; a maximum alone does not prove or disprove a leak. Create a Bun wrapper such as `direct/fuzz-browser.ts`: @@ -297,6 +343,14 @@ await runDirectBombadilFuzz({ repositoryRoot, scenario: "todos.populated", specificationPath: resolve(directRoot, "bombadil-campaign.ts"), + viewport: { width: 1_024, height: 768, deviceScaleFactor: 2 }, + explorationPolicy: { + minNonWaitActions: 1, + requiredNamedSnapshots: ["direct", "todos.phase"], + minDistinctNamedSnapshotValues: { "todos.phase": 2 }, + minNamedSnapshotChangesAfterNonWait: { "todos.phase": 1 }, + requireStableTargetUrl: true, + }, server: { command: [ process.execPath, @@ -327,6 +381,26 @@ the runner rejects Direct's reserved scenario and fixture keys there because it binds the requested scenario itself. The repository root, campaign path, server working directory, and replay trace are resolved canonically before use; a symlink that escapes the configured repository is rejected. +`viewport` is optional and defaults to Bombadil 0.7.2's 1024×768 viewport at a +device scale factor of 2. The runner always passes the validated exact values +to random and replay invocations and records them in `run.json`. +Assign one reviewed viewport to each existing scenario campaign. Alternate a +stable wide and narrow viewport across the matrix instead of duplicating every +scenario at every size; add a second size only when the scenario owns a +responsive behavior that needs separate exploration. + +`explorationPolicy` is optional. When present, it can require a minimum count +of non-Wait actions, particular action kinds and named snapshots, minimum +distinct hashed values for named snapshots, named-value changes observed after +a non-Wait action, and an exact stable target URL. The change requirement keeps +bootstrap or Wait-only transitions from satisfying a product-interaction +threshold. The trace records the last action with the resulting state, so this +count identifies a named value transition associated with an active product +action. It does not prove causality. The policy detects a campaign that passed +its properties without doing the intended exploration. It is a diagnostic +sufficiency check, not a coverage claim. Start with observed stable behavior +and raise thresholds only when the action generator makes them reliably +reachable. Run random exploration for 12 to 300 seconds. The default is 20 seconds: @@ -334,19 +408,54 @@ Run random exploration for 12 to 300 seconds. The default is 20 seconds: bun direct/fuzz-browser.ts --time-limit 20s ``` +Use 12–30 seconds in the edit loop. A scheduled diagnostic lane can run each +campaign for 60–300 seconds, serially, with an outer job timeout and retained +failure artifacts. Random exploration should supplement the deterministic +required gate rather than make an otherwise healthy pull request depend on one +particular random path. + +For multiple product scenarios, pass a bounded matrix to the shared runner: + +```ts +import { runDirectBombadilFuzzMatrix } from "@hraness/direct/tooling/bombadil"; + +await runDirectBombadilFuzzMatrix([ + { id: "populated", config: populatedCampaign }, + { id: "empty", config: emptyCampaign }, +], process.argv.slice(2)); +``` + +Without `--campaign`, the matrix runs every unique campaign serially. Select +one for focused work with `--campaign empty`. Replay is intentionally rejected +without that selector so a trace cannot be applied to the wrong scenario. + Use `--base-url` to select another local root origin. Use `--replay` with a repository-local `.jsonl` trace instead of `--time-limit` to reproduce a prior run. The native runner supports Bombadil 0.7.2 on Apple silicon macOS and x64 or arm64 Linux. Unsupported platform and architecture pairs fail before a server starts. +Bombadil's browser Inspect UI is the fastest way to examine the actions, +screenshots, resource timeline, snapshots, and violations in a retained run: + +```sh +bunx bombadil browser inspect artifacts/direct-bombadil/todos//bombadil +bun direct/fuzz-browser.ts --replay artifacts/direct-bombadil/todos//bombadil/trace.jsonl +``` + +Use the same campaign, viewport, specification, scenario, application tree, +and server configuration for replay. Bombadil rejects a replay that diverges, +so reproduction is strong debugging evidence but not guaranteed after the +product changes. + The browser formulas require an exact scenario contract, a nonempty catalog, zero declared violations, and quiescence to recur within ten seconds from every -observed state. Bombadil's temporal engine cannot decide an unbounded `always` -inside a bounded `eventually`, so continuous identity and catalog binding stay -in the host attestation below. A formula result and Bombadil exit status are not -sufficient evidence because a short or incomplete trace could otherwise pass -vacuously. +observed state. Keep every liveness obligation bounded to a product latency +budget. A finite run cannot decide an unbounded future obligation, and nesting +unbounded `always` inside bounded `eventually` does not make it decidable. +Continuous identity and catalog binding therefore stay in the host attestation +below. A formula result and Bombadil exit status are not sufficient evidence +because a short or incomplete trace could otherwise pass vacuously. After every random run or replay, the host runner streams the bounded 0.7.2 JSONL trace from foreign input and requires one named `direct` observation per @@ -369,12 +478,30 @@ configured local server is always stopped through the shared browser-verification lease helpers, and its output drain remains bounded even when cleanup itself fails. -Each attempt writes `run.json`, `bombadil.log`, and `server.log` below +Each attempt writes `run.json`, `exploration-summary.json`, `bombadil.log`, and `server.log` below `artifacts/direct-bombadil///`, including failures. The rolling `manifest.json` points to the latest record. `rawTracePath` reports a regular nonempty trace even if attestation fails; `tracePath` is present only -after exact attestation. Keep these diagnostic artifacts out of source control -unless the consumer explicitly reviews them as fixtures. +after exact attestation. The summary strictly parses the 0.7.2 envelopes and +records the raw trace SHA-256, action-kind and safe target-tag counts, non-Wait +count and longest Wait streak, origin-relative URL fingerprints, non-null +transition-hash cardinality, canonical named-snapshot value hashes, property +violation names and counts, named-value changes after non-Wait actions, and +browser resource high-water marks. It excludes +typed text, accessible names, snapshot values, URLs, screenshots, and absolute +paths. These diagnostics describe what Bombadil happened to explore. They do +not measure code, state, interaction, or Direct catalog coverage, and the raw +trace remains authoritative. + +Keep all generated artifacts out of source control by default. Upload a failed +scheduled run to access-controlled CI storage with a bounded retention period; +the raw trace can contain screenshots, query values, typed text, accessibility +labels, extracted values, and local paths. Preserve it long enough to inspect +and replay. Once the defect is understood, add the smallest deterministic +regression at the owning parser, reducer, port, component, semantic browser, or +Direct scenario boundary. Verify that regression fails before the fix and +passes after it. Retain a reviewed trace fixture only when replay itself adds +durable value; otherwise remove the sensitive trace after promotion. ## Report coverage without promotion diff --git a/src/exports.test.ts b/src/exports.test.ts index 5d42c43..6672cd7 100644 --- a/src/exports.test.ts +++ b/src/exports.test.ts @@ -61,12 +61,16 @@ describe("public package exports", () => { expect(Object.keys(bombadil).toSorted()).toEqual([ "attestDirectBombadilTrace", "runDirectBombadilFuzz", + "runDirectBombadilFuzzMatrix", + "summarizeDirectBombadilTrace", ]); expect(typeof browserVerification.createAgentBrowser).toBe("function"); expect(typeof browserVerification.createDirectBrowserContractReader).toBe("function"); expect(typeof browserVerification.readDirectBrowserContract).toBe("function"); expect(typeof bombadil.runDirectBombadilFuzz).toBe("function"); expect(typeof bombadil.attestDirectBombadilTrace).toBe("function"); + expect(typeof bombadil.runDirectBombadilFuzzMatrix).toBe("function"); + expect(typeof bombadil.summarizeDirectBombadilTrace).toBe("function"); expect(typeof bundleBoundary.checkBundleBoundary).toBe("function"); expect(typeof bundleBoundary.findForbiddenMarkers).toBe("function"); expect("createAgentBrowser" in root).toBeFalse(); diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 22db343..63ac1cb 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -13,7 +13,10 @@ interface FakeCell { current: unknown; name: string | null; readonly named: (name: string) => FakeCell; - readonly read: (state: { readonly window: unknown }) => unknown; + readonly read: (state: { + readonly resources?: Readonly>; + readonly window: unknown; + }) => unknown; } interface FakeActionGenerator { @@ -68,7 +71,9 @@ void mock.module("@antithesishq/bombadil/browser/defaults/actions", () => ({ const { createDirectBombadilActions, + createDirectBombadilNamedSnapshot, createDirectBombadilProperties, + createDirectBombadilResourceLeakProperty, readDirectBombadilObservation, } = await import("./bombadil-campaign.js"); @@ -229,6 +234,100 @@ describe("Direct Bombadil observation", () => { }); }); +describe("Direct Bombadil named snapshots", () => { + test("names bounded JSON and fails closed around hostile or oversized page values", () => { + const snapshot = createDirectBombadilNamedSnapshot({ + fallback: { status: "unavailable" }, + name: "product.phase", + read: (state) => Reflect.get(state.window, "phase"), + }) as unknown as FakeCell; + expect(snapshot.name).toBe("product.phase"); + expect(snapshot.read({ window: { phase: { status: "ready" } } })).toEqual({ + status: "ready", + }); + + const hostile = {}; + Object.defineProperty(hostile, "phase", { + get: () => { + throw new Error("hostile getter"); + }, + }); + expect(snapshot.read({ window: hostile })).toEqual({ status: "unavailable" }); + expect(snapshot.read({ + window: { phase: "x".repeat(2_000_001) }, + })).toEqual({ status: "unavailable" }); + }); + + test("rejects unsafe names and non-JSON fallbacks before registering an extractor", () => { + expect(() => createDirectBombadilNamedSnapshot({ + fallback: null, + name: "unsafe name", + read: () => null, + })).toThrow("safe 1-128 character identifier"); + expect(() => createDirectBombadilNamedSnapshot({ + fallback: undefined as never, + name: "safe", + read: () => null, + })).toThrow("fallback must be bounded JSON"); + }); +}); + +describe("Direct Bombadil resource properties", () => { + const resources = (timestamp: number, domNodes: number) => ({ + documents: 1, + dom_nodes: domNodes, + js_event_listeners: 2, + js_heap_total: 2_000, + js_heap_used: 1_000, + layout_objects: 10, + script_duration: 0, + task_duration: 0, + thread_time: 0, + timestamp, + }); + + test("detects excessive growth across a bounded sliding resource window", () => { + const property = createDirectBombadilResourceLeakProperty({ + growthLimit: 10, + metric: "dom_nodes", + windowMillis: 1_000, + }) as unknown as FakeFormula; + const cell = cells.at(-1); + if (cell === undefined) throw new Error("resource extractor was not registered"); + + cell.current = cell.read({ resources: resources(1, 100), window: {} }); + expect(evaluate(property.body)).toBeTrue(); + cell.current = cell.read({ resources: resources(1.5, 105), window: {} }); + expect(evaluate(property.body)).toBeTrue(); + cell.current = cell.read({ resources: resources(1.75, 120), window: {} }); + expect(evaluate(property.body)).toBeFalse(); + }); + + test("rejects unknown fields, metrics, and unsafe numeric bounds", () => { + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: 1, + metric: "dom_nodes", + windowMillis: 1_000, + extra: true, + } as never)).toThrow("must contain metric"); + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: 1, + metric: "documents" as never, + windowMillis: 1_000, + })).toThrow("metric is unsupported"); + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: Number.POSITIVE_INFINITY, + metric: "dom_nodes", + windowMillis: 1_000, + })).toThrow("positive finite safe number"); + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: 1, + metric: "dom_nodes", + windowMillis: 300_001, + })).toThrow("between 1 and 300000"); + }); +}); + function clickAction(overrides: Readonly> = {}): unknown { return { Click: { diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index 5a932d4..5650e12 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -9,6 +9,7 @@ import { extract, weighted, type ActionGenerator, + type Cell, type Formula, type JSON as BombadilJson, type Tree, @@ -29,6 +30,16 @@ const DIRECT_PROBE_SCHEMA = "direct.probe/v1"; const MAX_RAW_CONTRACT_CHARACTERS = 2_000_000; const BRIDGE_KEYS = new Set(["manifest", "reset", "schema", "snapshot"]); const UNSAFE_CLICK_INPUT_TYPES = new Set(["image", "reset", "submit"]); +const SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +const RESOURCE_LEAK_OPTION_KEYS = new Set(["growthLimit", "metric", "windowMillis"]); +const RESOURCE_METRICS = [ + "dom_nodes", + "js_event_listeners", + "js_heap_total", + "js_heap_used", + "layout_objects", +] as const; +const RESOURCE_METRIC_SET = new Set(RESOURCE_METRICS); export interface DirectBombadilObservation { readonly [key: string | number | symbol]: BombadilJson; @@ -54,6 +65,14 @@ export interface DirectBombadilProperties { readonly eventualQuiescence: Formula; } +export type DirectBombadilResourceMetric = (typeof RESOURCE_METRICS)[number]; + +export interface DirectBombadilResourceLeakOptions { + readonly growthLimit: number; + readonly metric: DirectBombadilResourceMetric; + readonly windowMillis: number; +} + function safeClickAction(action: ActionTemplate): boolean { if (typeof action !== "object" || action === null) return false; const candidate = "Click" in action @@ -126,6 +145,72 @@ function hasExactKeys( return keys.length === expected.size && keys.every((key) => expected.has(key)); } +/** + * Builds Bombadil's documented sliding-window resource growth invariant from + * the public browser state because 0.7.2 omits its extras module from exports. + */ +export function createDirectBombadilResourceLeakProperty( + options: DirectBombadilResourceLeakOptions, +): Formula { + if (!isRecord(options) || !hasExactKeys(options, RESOURCE_LEAK_OPTION_KEYS)) { + throw new Error("Bombadil resource leak options must contain metric, growthLimit, and windowMillis"); + } + if (typeof options.metric !== "string" || !RESOURCE_METRIC_SET.has(options.metric)) { + throw new Error("Bombadil resource leak metric is unsupported"); + } + if ( + typeof options.growthLimit !== "number" + || !Number.isFinite(options.growthLimit) + || options.growthLimit <= 0 + || options.growthLimit > Number.MAX_SAFE_INTEGER + ) { + throw new Error("Bombadil resource leak growthLimit must be a positive finite safe number"); + } + if ( + typeof options.windowMillis !== "number" + || !Number.isSafeInteger(options.windowMillis) + || options.windowMillis < 1 + || options.windowMillis > 300_000 + ) { + throw new Error("Bombadil resource leak windowMillis must be an integer between 1 and 300000"); + } + const metric = options.metric; + const samples: Array<{ readonly timestamp: number; readonly value: number }> = []; + let previousTimestamp = -1; + const window = extract((state) => { + const timestamp = state.resources.timestamp * 1_000; + const value = state.resources[metric]; + if ( + !Number.isFinite(timestamp) + || timestamp < 0 + || timestamp < previousTimestamp + || !Number.isFinite(value) + || value < 0 + ) { + return { baseline: 0, valid: false, value: 0 }; + } + previousTimestamp = timestamp; + samples.push({ timestamp, value }); + const cutoff = timestamp - options.windowMillis; + while (samples.length > 2 && (samples[1]?.timestamp ?? Number.POSITIVE_INFINITY) <= cutoff) { + samples.shift(); + } + return { + baseline: samples[0]?.value ?? value, + valid: true, + value, + }; + }); + return always(() => + window.current.valid + && window.current.value - window.current.baseline <= options.growthLimit + ); +} + function invalidObservation(bridgePresent = false): DirectBombadilObservation { return { activationHash: "", @@ -150,6 +235,41 @@ function boundedJsonClone(value: unknown): BombadilJson | null { return JSON.parse(source) as BombadilJson; } +function optionalBoundedJsonClone(value: unknown): BombadilJson | undefined { + const source = JSON.stringify(value); + if (source === undefined || source.length > MAX_RAW_CONTRACT_CHARACTERS) return undefined; + return JSON.parse(source) as BombadilJson; +} + +/** + * Creates a named, bounded JSON extractor that fails closed to an explicit + * fallback when a page getter throws or returns non-JSON or oversized data. + */ +export function createDirectBombadilNamedSnapshot(options: { + readonly fallback: T; + readonly name: string; + readonly read: (state: BombadilBrowserState) => unknown; +}): Cell { + if ( + options.name.length === 0 + || options.name.length > 128 + || !SNAPSHOT_NAME_PATTERN.test(options.name) + ) { + throw new Error("Bombadil snapshot name must be a safe 1-128 character identifier"); + } + const fallback = optionalBoundedJsonClone(options.fallback); + if (fallback === undefined) { + throw new Error("Bombadil snapshot fallback must be bounded JSON"); + } + return extract((state) => { + try { + return (optionalBoundedJsonClone(options.read(state)) ?? fallback) as T; + } catch { + return fallback as T; + } + }).named(options.name); +} + function readNonNegativeCounters(value: unknown): { readonly valid: boolean; readonly values: number[]; diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index a3e0709..a164b38 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -11,6 +11,8 @@ import { parseDirectBombadilFuzzArguments, runBombadilNativeProcess, runDirectBombadilFuzz, + runDirectBombadilFuzzMatrix, + summarizeDirectBombadilTrace, validateDirectBombadilFuzzConfig, type DirectBombadilFuzzConfig, type DirectBombadilInvocation, @@ -187,13 +189,49 @@ function absentObservation(): Record { }; } -function traceLine(observation: unknown, timestamp: number): string { +interface TraceLineOptions { + readonly action?: unknown; + readonly namedSnapshots?: readonly { readonly name: string; readonly value: unknown }[]; + readonly url?: string; + readonly violations?: readonly unknown[]; +} + +function traceLine( + observation: unknown, + timestamp: number, + options: TraceLineOptions = {}, +): string { return JSON.stringify({ timestamp, - action: null, - state: {}, - snapshots: [{ index: 0, name: "direct", value: observation, time: timestamp }], - violations: [], + action: options.action ?? null, + state: { + url: options.url ?? "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + hash_previous: timestamp === 1 ? null : timestamp - 1, + hash_current: timestamp, + screenshot: `/tmp/${String(timestamp)}.png`, + resources: { + js_heap_used: timestamp * 10, + js_heap_total: timestamp * 20, + dom_nodes: timestamp, + documents: 1, + js_event_listeners: timestamp * 2, + layout_objects: timestamp * 3, + timestamp, + thread_time: timestamp / 10, + task_duration: timestamp / 20, + script_duration: timestamp / 30, + }, + }, + snapshots: [ + { index: 0, name: "direct", value: observation, time: timestamp }, + ...(options.namedSnapshots ?? []).map((snapshot, index) => ({ + index: index + 1, + name: snapshot.name, + value: snapshot.value, + time: timestamp, + })), + ], + violations: options.violations ?? [], }); } @@ -392,6 +430,11 @@ describe("Direct Bombadil configuration and invocation", () => { expect(validated.entryPath).toBe("/"); expect(validated.targetQuery).toEqual({}); expect(validated.server.readinessPath).toBe("/ready"); + expect(validated.viewport).toEqual({ + deviceScaleFactor: 2, + height: 768, + width: 1_024, + }); expect(validated.bombadilExecutable).toEndWith( `node_modules/@antithesishq/bombadil/binaries/${nativeBinaryName()}`, ); @@ -435,6 +478,26 @@ describe("Direct Bombadil configuration and invocation", () => { ...config, server: { ...config.server, readinessPath: "//external.test" }, })).toThrow("origin-relative"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + viewport: { width: 0 }, + })).toThrow("viewport.width"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + viewport: { deviceScaleFactor: Number.POSITIVE_INFINITY }, + })).toThrow("deviceScaleFactor"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { requiredActionKinds: ["Wait", "Wait"] }, + })).toThrow("duplicate kind"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { minDistinctNamedSnapshotValues: { "unsafe name": 2 } }, + })).toThrow("safe bounded snapshot name"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { minNamedSnapshotChangesAfterNonWait: { phase: 10_001 } }, + })).toThrow("between 1 and 10000"); }); test("rejects specification, server cwd, and replay symlinks that escape the repository", async () => { @@ -493,6 +556,7 @@ describe("Direct Bombadil configuration and invocation", () => { specificationPath: "/repo/spec.ts", targetQuery: { workbench: "frame" }, timeLimitSeconds: 20, + viewport: { deviceScaleFactor: 1.25, height: 720, width: 1_280 }, }); expect(invocation.targetUrl).toBe( "http://127.0.0.1:5184/direct/?__direct_scenario=surface.ready&workbench=frame", @@ -508,6 +572,12 @@ describe("Direct Bombadil configuration and invocation", () => { "/repo/artifacts/run/bombadil", "--headless", "--instrument-javascript=", + "--width", + "1280", + "--height", + "720", + "--device-scale-factor", + "1.25", "--exit-on-violation", "--time-limit", "20s", @@ -526,12 +596,65 @@ describe("Direct Bombadil configuration and invocation", () => { timeLimitSeconds: 20, }); expect(invocation.command).toContain("--reproduce"); + expect(invocation.command).toEqual(expect.arrayContaining([ + "--width", "1024", "--height", "768", "--device-scale-factor", "2", + ])); expect(invocation.command).not.toContain("--time-limit"); expect(invocation.command).not.toContain("--exit-on-violation"); expect(invocation.wallClockTimeoutMs).toBe(330_000); }); }); +describe("Direct Bombadil campaign matrix", () => { + test("runs unique bounded campaigns serially and selects exactly one", async () => { + const { config } = await fixture(); + const campaigns = [{ id: "primary", config }, { + id: "secondary", + config: { ...config, artifactName: "fixture-secondary" }, + }] as const; + const allRuntime = dependencies(); + const all = await runDirectBombadilFuzzMatrix( + campaigns, + ["--time-limit=12s"], + allRuntime.overrides, + ); + expect(all).toMatchObject({ + kind: "matrix", + results: [{ campaignId: "primary" }, { campaignId: "secondary" }], + }); + expect(allRuntime.calls.filter((call) => call === "run-bombadil")).toHaveLength(2); + + const selectedRuntime = dependencies(); + const selected = await runDirectBombadilFuzzMatrix( + campaigns, + ["--campaign=secondary", "--time-limit=12s"], + selectedRuntime.overrides, + ); + expect(selected).toMatchObject({ + kind: "matrix", + results: [{ campaignId: "secondary" }], + }); + expect(selectedRuntime.calls.filter((call) => call === "run-bombadil")).toHaveLength(1); + }); + + test("rejects ambiguous replay, duplicate IDs, and unknown selection", async () => { + const { config } = await fixture(); + const campaigns = [{ id: "primary", config }] as const; + expect((await rejection(runDirectBombadilFuzzMatrix( + campaigns, + ["--replay=artifacts/trace.jsonl"], + ))).message).toContain("requires exactly one --campaign"); + expect((await rejection(runDirectBombadilFuzzMatrix( + campaigns, + ["--campaign=missing"], + ))).message).toContain("Unknown Bombadil campaign"); + expect((await rejection(runDirectBombadilFuzzMatrix([ + { id: "same", config }, + { id: "same", config: { ...config, artifactName: "other" } }, + ], []))).message).toContain("unique lowercase kebab"); + }); +}); + describe("Direct Bombadil trace attestation", () => { async function attest(observations: readonly unknown[]) { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-trace-")); @@ -657,6 +780,165 @@ describe("Direct Bombadil trace attestation", () => { }); }); +describe("Direct Bombadil exploration summary", () => { + async function summaryTrace(lines: readonly string[]): Promise { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-summary-")); + temporaryDirectories.push(directory); + const tracePath = join(directory, "trace.jsonl"); + await writeFile(tracePath, `${lines.join("\n")}\n`, "utf8"); + return tracePath; + } + + test("derives deterministic bounded diagnostics without retaining typed text or labels", async () => { + const observation = directObservation(); + const tracePath = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: { b: 2, a: 1 } }], + }), + traceLine(observation, 2, { + action: { + Click: { + fingerprint: { + accessible_name: "sensitive button label", + tag: "button", + }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: { a: 2 } }], + }), + traceLine(observation, 3, { + action: "Wait", + namedSnapshots: [{ name: "phase", value: { a: 2 } }], + }), + traceLine(observation, 4, { + action: { TypeText: { delay_millis: 0, text: "sensitive typed text" } }, + namedSnapshots: [{ name: "phase", value: { a: 2 } }], + violations: [{ + name: "noConsoleErrors", + violation: { False: { condition: "sensitive formula source" } }, + }], + }), + ]); + const options = { + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + minNonWaitActions: 2, + requireStableTargetUrl: true, + requiredActionKinds: ["Click", "TypeText"] as const, + requiredNamedSnapshots: ["phase"], + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }; + const first = await summarizeDirectBombadilTrace(options); + const second = await summarizeDirectBombadilTrace(options); + expect(second).toEqual(first); + expect(first).toMatchObject({ + schema: "direct.bombadil-exploration-summary/v1", + actions: { + byKind: { Click: 1, TypeText: 1, Wait: 1 }, + maxWaitStreak: 1, + nonWaitCount: 2, + targetTags: { button: 1 }, + total: 3, + }, + urls: { + distinctFingerprintCount: 1, + observationCount: 4, + stableTarget: true, + }, + transitions: { distinctNonNullHashCount: 4, nonNullHashCount: 4 }, + propertyViolations: { byName: { noConsoleErrors: 1 }, total: 1 }, + resourceHighWaterMarks: { + domNodes: 4, + jsHeapUsedBytes: 40, + }, + policy: { configured: true, failures: [], satisfied: true }, + }); + expect(first.namedSnapshots.find((entry) => entry.name === "phase")).toMatchObject({ + changeAfterNonWaitCount: 1, + distinctValueCount: 2, + observationCount: 4, + }); + expect(first.trace.sha256).toHaveLength(64); + const serialized = JSON.stringify(first); + expect(serialized).not.toContain("sensitive button label"); + expect(serialized).not.toContain("sensitive typed text"); + expect(serialized).not.toContain("sensitive formula source"); + expect(serialized).not.toContain("127.0.0.1"); + expect(serialized).not.toContain("/tmp/"); + }); + + test("reports strict policy misses while keeping the diagnostic summary", async () => { + const tracePath = await summaryTrace([traceLine(directObservation(), 1)]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNonWaitActions: 1, + requireStableTargetUrl: true, + requiredActionKinds: ["Click"], + requiredNamedSnapshots: ["phase"], + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.policy.satisfied).toBeFalse(); + expect(summary.policy.failures).toHaveLength(4); + }); + + test("does not credit bootstrap or Wait-only state changes to product actions", async () => { + const observation = directObservation(); + const tracePath = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: "loading" }], + }), + traceLine(observation, 2, { + action: "Wait", + namedSnapshots: [{ name: "phase", value: "ready" }], + }), + ]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.find((entry) => entry.name === "phase")).toMatchObject({ + changeAfterNonWaitCount: 0, + distinctValueCount: 2, + }); + expect(summary.policy).toMatchObject({ + failures: [expect.stringContaining("post-non-Wait change minimum")], + satisfied: false, + }); + }); + + test("rejects hostile envelopes, action targets, and excessively deep snapshot JSON", async () => { + const observation = directObservation(); + const badTarget = await summaryTrace([traceLine(observation, 1, { + action: { Click: { fingerprint: { tag: "unsafe tag" } } }, + })]); + expect((await rejection(summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: badTarget, + }))).message).toContain("invalid action target tag"); + + let deep: unknown = null; + for (let index = 0; index < 70; index += 1) deep = [deep]; + const deepTrace = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: [{ name: "deep", value: deep }], + })]); + expect((await rejection(summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: deepTrace, + }))).message).toContain("exceeds JSON depth"); + }); +}); + describe("Direct Bombadil process lifecycle", () => { test("cleans descendants and inherited pipes after a normal leader exit", async () => { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-normal-exit-")); @@ -790,6 +1072,11 @@ describe("Direct Bombadil run lifecycle", () => { }, initialDirect: { source: "scenario", scenario: "surface.ready", route: "/surface" }, server: { logPresent: true }, + explorationSummary: { + schema: "direct.bombadil-exploration-summary/v1", + policy: { configured: false, satisfied: true }, + }, + viewport: { deviceScaleFactor: 2, height: 768, width: 1_024 }, }); const bombadil = record(manifest.bombadil, "bombadil"); expect(bombadil.version).toBe("0.7.2"); @@ -800,6 +1087,35 @@ describe("Direct Bombadil run lifecycle", () => { expect(await readFile(String(bombadil.logPath), "utf8")).toContain("bombadil stdout"); const server = record(manifest.server, "server"); expect(await readFile(String(server.logPath), "utf8")).toContain("server output"); + const summaryPath = String(manifest.explorationSummaryPath); + expect(JSON.parse(await readFile(summaryPath, "utf8"))).toMatchObject({ + schema: "direct.bombadil-exploration-summary/v1", + trace: { lineCount: 2 }, + }); + }); + + test("fails a configured exploration policy while retaining the derived sidecar", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies(); + expect((await rejection(runDirectBombadilFuzz({ + ...config, + explorationPolicy: { + minNonWaitActions: 1, + requiredActionKinds: ["Click"], + }, + }, [], runtime.overrides))).message).toContain("exploration policy was not satisfied"); + const manifest = JSON.parse(await readFile( + join(repositoryRoot, "artifacts", "direct-bombadil", "fixture-product", "manifest.json"), + "utf8", + )) as Record; + expect(manifest).toMatchObject({ + status: "failed", + explorationSummary: { + policy: { configured: true, satisfied: false }, + }, + }); + expect(await readFile(String(manifest.explorationSummaryPath), "utf8")) + .toContain("direct.bombadil-exploration-summary/v1"); }); test("retains an attested failure artifact and server log for a nonzero exit", async () => { diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index 61cf990..f2a2ef2 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -3,6 +3,7 @@ import { readFile, realpath, stat, writeFile } from "node:fs/promises"; import { isAbsolute, join, relative, resolve } from "node:path"; import process from "node:process"; import { createInterface } from "node:readline"; +import { createHash } from "node:crypto"; import { parseDirectProbeSnapshot, @@ -44,6 +45,12 @@ const TRACE_MAX_BYTES = 64 * 1024 * 1024; const TRACE_MAX_LINE_BYTES = 16 * 1024 * 1024; const TRACE_MAX_LINES = 10_000; const TRACE_MAX_SNAPSHOTS_PER_LINE = 4_096; +const TRACE_MAX_NAMED_SNAPSHOT_NAMES = 128; +const TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME = 1_024; +const TRACE_MAX_DISTINCT_URLS = 1_024; +const TRACE_MAX_PROPERTY_NAMES = 128; +const TRACE_MAX_CANONICAL_SNAPSHOT_BYTES = 2 * 1024 * 1024; +const TRACE_MAX_JSON_DEPTH = 64; const RANDOM_RUN_OVERHEAD_MS = 30_000; const REPLAY_WALL_CLOCK_TIMEOUT_MS = MAX_TIME_LIMIT_SECONDS * 1_000 + RANDOM_RUN_OVERHEAD_MS; const PROCESS_TERMINATION_GRACE_MS = 5_000; @@ -52,6 +59,66 @@ const SERVER_OUTPUT_TIMEOUT_MS = 3_000; const DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; const TRACE_LINE_KEYS = new Set(["action", "snapshots", "state", "timestamp", "violations"]); const TRACE_SNAPSHOT_KEYS = new Set(["index", "name", "time", "value"]); +const TRACE_STATE_KEYS = new Set([ + "hash_current", + "hash_previous", + "resources", + "screenshot", + "url", +]); +const TRACE_RESOURCE_KEYS = new Set([ + "documents", + "dom_nodes", + "js_event_listeners", + "js_heap_total", + "js_heap_used", + "layout_objects", + "script_duration", + "task_duration", + "thread_time", + "timestamp", +]); +const TRACE_VIOLATION_KEYS = new Set(["name", "violation"]); +const VIEWPORT_KEYS = new Set(["deviceScaleFactor", "height", "width"]); +const EXPLORATION_POLICY_KEYS = new Set([ + "minDistinctNamedSnapshotValues", + "minNamedSnapshotChangesAfterNonWait", + "minNonWaitActions", + "requireStableTargetUrl", + "requiredActionKinds", + "requiredNamedSnapshots", +]); +const DEFAULT_VIEWPORT_WIDTH = 1_024; +const DEFAULT_VIEWPORT_HEIGHT = 768; +const DEFAULT_DEVICE_SCALE_FACTOR = 2; +const SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +const TARGET_TAG_PATTERN = /^[a-z][a-z0-9-]*$/u; +const ACTION_KINDS = [ + "Back", + "Click", + "DoubleClick", + "Forward", + "MouseDrag", + "PressKey", + "Reload", + "ScrollDown", + "ScrollUp", + "SetFileInputFiles", + "SetViewport", + "TypeText", + "Wait", +] as const; +const ACTION_KIND_SET = new Set(ACTION_KINDS); +const UNIT_ACTION_KINDS = new Set([ + "Back", + "Forward", + "Reload", + "Wait", +]); +const TARGETED_ACTION_KINDS = new Set([ + "Click", + "DoubleClick", +]); const DIRECT_OBSERVATION_KEYS = new Set([ "activationHash", "activeRoute", @@ -76,6 +143,76 @@ export interface DirectBombadilServerConfig { readonly startupTimeoutMs?: number; } +export type DirectBombadilActionKind = (typeof ACTION_KINDS)[number]; + +export interface DirectBombadilViewportConfig { + readonly deviceScaleFactor?: number; + readonly height?: number; + readonly width?: number; +} + +export interface DirectBombadilExplorationPolicy { + readonly minDistinctNamedSnapshotValues?: Readonly>; + readonly minNamedSnapshotChangesAfterNonWait?: Readonly>; + readonly minNonWaitActions?: number; + readonly requireStableTargetUrl?: boolean; + readonly requiredActionKinds?: readonly DirectBombadilActionKind[]; + readonly requiredNamedSnapshots?: readonly string[]; +} + +export interface DirectBombadilExplorationSummary { + readonly schema: "direct.bombadil-exploration-summary/v1"; + readonly trace: { + readonly bytes: number; + readonly lineCount: number; + readonly sha256: string; + }; + readonly actions: { + readonly byKind: Readonly>>; + readonly maxWaitStreak: number; + readonly nonWaitCount: number; + readonly targetTags: Readonly>; + readonly total: number; + }; + readonly urls: { + readonly distinctFingerprintCount: number; + readonly fingerprintSha256: readonly string[]; + readonly observationCount: number; + readonly stableTarget: boolean; + }; + readonly transitions: { + readonly distinctNonNullHashCount: number; + readonly nonNullHashCount: number; + }; + readonly namedSnapshots: readonly { + readonly changeAfterNonWaitCount: number; + readonly distinctValueCount: number; + readonly distinctValueSha256: readonly string[]; + readonly name: string; + readonly observationCount: number; + }[]; + readonly propertyViolations: { + readonly byName: Readonly>; + readonly total: number; + }; + readonly resourceHighWaterMarks: { + readonly documents: number; + readonly domNodes: number; + readonly jsEventListeners: number; + readonly jsHeapTotalBytes: number; + readonly jsHeapUsedBytes: number; + readonly layoutObjects: number; + readonly scriptDurationSeconds: number; + readonly taskDurationSeconds: number; + readonly threadTimeSeconds: number; + }; + readonly policy: { + readonly configured: boolean; + readonly failures: readonly string[]; + readonly satisfied: boolean; + }; +} + export interface DirectBombadilFuzzConfig { readonly artifactName: string; readonly baseUrl: string; @@ -86,6 +223,8 @@ export interface DirectBombadilFuzzConfig { readonly scenario: string; readonly specificationPath: string; readonly targetQuery?: Readonly>; + readonly explorationPolicy?: DirectBombadilExplorationPolicy; + readonly viewport?: DirectBombadilViewportConfig; readonly server: DirectBombadilServerConfig; } @@ -107,6 +246,21 @@ export type DirectBombadilFuzzResult = readonly status: "passed"; }; +export interface DirectBombadilFuzzCampaign { + readonly config: DirectBombadilFuzzConfig; + readonly id: string; +} + +export type DirectBombadilFuzzMatrixResult = + | { readonly kind: "help" } + | { + readonly kind: "matrix"; + readonly results: readonly { + readonly campaignId: string; + readonly result: Extract; + }[]; + }; + export interface DirectBombadilInvocation { readonly abortSignal?: AbortSignal; readonly command: readonly string[]; @@ -184,17 +338,37 @@ interface ProcessSignalEmitter { ) => unknown; } -interface ValidatedConfig extends DirectBombadilFuzzConfig { +type ValidatedConfig = Omit< + DirectBombadilFuzzConfig, + "explorationPolicy" | "server" | "viewport" +> & { readonly artifactRoot: string; readonly baseUrl: string; readonly bombadilExecutable: string; readonly entryPath: `/${string}`; + readonly explorationPolicy: ValidatedExplorationPolicy | null; readonly port: string; readonly targetQuery: Readonly>; + readonly viewport: ValidatedViewport; readonly server: DirectBombadilServerConfig & { readonly readinessPath: `/${string}`; readonly startupTimeoutMs: number; }; +}; + +interface ValidatedViewport { + readonly deviceScaleFactor: number; + readonly height: number; + readonly width: number; +} + +interface ValidatedExplorationPolicy { + readonly minDistinctNamedSnapshotValues: Readonly>; + readonly minNamedSnapshotChangesAfterNonWait: Readonly>; + readonly minNonWaitActions: number; + readonly requireStableTargetUrl: boolean; + readonly requiredActionKinds: readonly DirectBombadilActionKind[]; + readonly requiredNamedSnapshots: readonly string[]; } function readOptionValue( @@ -427,7 +601,177 @@ function exactTraceDirectObservation( }; } -function parseTraceLine(line: string, lineNumber: number): TraceDirectObservation { +interface ParsedTraceAction { + readonly kind: DirectBombadilActionKind; + readonly targetTag: string | null; +} + +interface ParsedTraceState { + readonly currentHash: number | null; + readonly resources: Readonly>; + readonly url: URL; +} + +interface ParsedTraceLine { + readonly action: ParsedTraceAction | null; + readonly directObservation: TraceDirectObservation; + readonly namedSnapshots: readonly { + readonly name: string; + readonly valueSha256: string; + }[]; + readonly propertyViolationNames: readonly string[]; + readonly state: ParsedTraceState; +} + +const RESOURCE_FIELD_MAP = { + documents: "documents", + dom_nodes: "domNodes", + js_event_listeners: "jsEventListeners", + js_heap_total: "jsHeapTotalBytes", + js_heap_used: "jsHeapUsedBytes", + layout_objects: "layoutObjects", + script_duration: "scriptDurationSeconds", + task_duration: "taskDurationSeconds", + thread_time: "threadTimeSeconds", +} as const; + +function canonicalJson(value: unknown, depth = 0): string { + if (depth > TRACE_MAX_JSON_DEPTH) { + throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(TRACE_MAX_JSON_DEPTH)}`); + } + if (value === null || typeof value === "boolean" || typeof value === "string") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("Bombadil named snapshot has a non-finite number"); + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((entry) => canonicalJson(entry, depth + 1)).join(",")}]`; + } + if (!isRecord(value)) throw new Error("Bombadil named snapshot is not JSON"); + const entries = Object.keys(value).sort().map((key) => + `${JSON.stringify(key)}:${canonicalJson(value[key], depth + 1)}` + ); + return `{${entries.join(",")}}`; +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +function namedSnapshotValueSha256(value: unknown): string { + const canonical = canonicalJson(value); + if (Buffer.byteLength(canonical, "utf8") > TRACE_MAX_CANONICAL_SNAPSHOT_BYTES) { + throw new Error( + `Bombadil named snapshot exceeds ${String(TRACE_MAX_CANONICAL_SNAPSHOT_BYTES)} canonical bytes`, + ); + } + return sha256(canonical); +} + +function parseTraceAction(value: unknown, lineNumber: number): ParsedTraceAction | null { + if (value === null) return null; + if (typeof value === "string") { + if (!ACTION_KIND_SET.has(value) || !UNIT_ACTION_KINDS.has(value as DirectBombadilActionKind)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + } + return { kind: value as DirectBombadilActionKind, targetTag: null }; + } + if (!isRecord(value) || Object.keys(value).length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + } + const kind = Object.keys(value)[0]; + if ( + kind === undefined + || !ACTION_KIND_SET.has(kind) + || UNIT_ACTION_KINDS.has(kind as DirectBombadilActionKind) + || !isRecord(value[kind]) + ) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + } + const actionKind = kind as DirectBombadilActionKind; + let targetTag: string | null = null; + if (TARGETED_ACTION_KINDS.has(actionKind)) { + const fingerprint = value[kind].fingerprint; + if (!isRecord(fingerprint)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + const tag = fingerprint.tag; + if ( + typeof tag !== "string" + || tag.length === 0 + || tag.length > 64 + || !TARGET_TAG_PATTERN.test(tag) + ) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target tag`); + } + targetTag = tag; + } + return { kind: actionKind, targetTag }; +} + +function parseNonNegativeFiniteNumber( + value: unknown, + lineNumber: number, + field: string, +): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + return value; +} + +function parseTraceState(value: unknown, lineNumber: number): ParsedTraceState { + if (!isRecord(value) || !hasExactKeys(value, TRACE_STATE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser state`); + } + if (typeof value.url !== "string" || value.url.length === 0 || value.url.length > 8_192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + let url: URL; + try { + url = new URL(value.url); + } catch { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL protocol`); + } + if (typeof value.screenshot !== "string" || value.screenshot.length > 8_192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid screenshot path`); + } + for (const field of ["hash_previous", "hash_current"] as const) { + const hash = value[field]; + if (hash !== null && ( + typeof hash !== "number" + || !Number.isFinite(hash) + || !Number.isInteger(hash) + || hash < 0 + )) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + } + if (!isRecord(value.resources) || !hasExactKeys(value.resources, TRACE_RESOURCE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid browser resources`); + } + const resources: Record = {}; + for (const field of Object.keys(RESOURCE_FIELD_MAP) as (keyof typeof RESOURCE_FIELD_MAP)[]) { + resources[field] = parseNonNegativeFiniteNumber( + value.resources[field], + lineNumber, + `resources.${field}`, + ); + } + parseNonNegativeFiniteNumber(value.resources.timestamp, lineNumber, "resources.timestamp"); + return { + currentHash: value.hash_current as number | null, + resources: resources as Readonly>, + url, + }; +} + +function parseTraceLine(line: string, lineNumber: number): ParsedTraceLine { let input: unknown; try { input = JSON.parse(line) as unknown; @@ -447,7 +791,38 @@ function parseTraceLine(line: string, lineNumber: number): TraceDirectObservatio ) { throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid state fields`); } + const state = parseTraceState(input.state, lineNumber); + const action = parseTraceAction(input.action, lineNumber); const snapshots = input.snapshots as unknown[]; + const namedSnapshots: Array<{ readonly name: string; readonly valueSha256: string }> = []; + const namedSnapshotNames = new Set(); + for (const snapshotValue of snapshots) { + if ( + !isRecord(snapshotValue) + || !hasExactKeys(snapshotValue, TRACE_SNAPSHOT_KEYS) + || !Number.isSafeInteger(snapshotValue.index) + || !Number.isSafeInteger(snapshotValue.time) + || (snapshotValue.index as number) < 0 + || (snapshotValue.time as number) < 0 + || (snapshotValue.name !== null && typeof snapshotValue.name !== "string") + ) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid snapshot`); + } + if (snapshotValue.name !== null) { + const name = validateSnapshotName( + snapshotValue.name, + `Bombadil trace line ${String(lineNumber)} snapshot name`, + ); + if (namedSnapshotNames.has(name)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} repeats named snapshot ${name}`); + } + namedSnapshotNames.add(name); + namedSnapshots.push({ + name, + valueSha256: namedSnapshotValueSha256(snapshotValue.value), + }); + } + } const directSnapshots = snapshots.filter((snapshot): snapshot is Readonly> => isRecord(snapshot) && snapshot.name === "direct" ); @@ -465,7 +840,26 @@ function parseTraceLine(line: string, lineNumber: number): TraceDirectObservatio ) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); } - return parseTraceDirectObservation(snapshot.value); + const propertyViolationNames: string[] = []; + for (const violation of input.violations) { + if (!isRecord(violation) || !hasExactKeys(violation, TRACE_VIOLATION_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + propertyViolationNames.push(validateSnapshotName( + violation.name, + `Bombadil trace line ${String(lineNumber)} property violation name`, + )); + if (!isRecord(violation.violation) || Object.keys(violation.violation).length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + } + return { + action, + directObservation: parseTraceDirectObservation(snapshot.value), + namedSnapshots, + propertyViolationNames, + state, + }; } /** Exact post-run proof over Bombadil 0.7.2's bounded JSONL trace. */ @@ -499,7 +893,7 @@ export async function attestDirectBombadilTrace(options: { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); } - const observation = parseTraceLine(line, observationCount); + const observation = parseTraceLine(line, observationCount).directObservation; const exact = exactTraceDirectObservation(observation); if (exact === null) { if (initial !== null) { @@ -577,6 +971,248 @@ export async function attestDirectBombadilTrace(options: { }; } +function sortedCountRecord( + values: ReadonlyMap, +): Readonly>> { + return Object.freeze(Object.fromEntries( + [...values.entries()].sort(([left], [right]) => left.localeCompare(right)), + ) as Partial>); +} + +/** + * Derives bounded diagnostic exploration metadata from the raw Bombadil trace. + * The hashes and counts are navigation aids, not Direct coverage evidence. + */ +export async function summarizeDirectBombadilTrace(options: { + readonly explorationPolicy?: DirectBombadilExplorationPolicy; + readonly targetUrl: string; + readonly tracePath: string; +}): Promise { + const metadata = await stat(options.tracePath).catch(() => null); + if (metadata === null || !metadata.isFile() || metadata.size === 0) { + throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + } + if (metadata.size > TRACE_MAX_BYTES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_BYTES)} bytes`); + } + let targetUrl: URL; + try { + targetUrl = new URL(options.targetUrl); + } catch { + throw new Error("targetUrl must be an absolute URL"); + } + const policy = validateExplorationPolicy(options.explorationPolicy); + const actionCounts = new Map(); + const targetTags = new Map(); + const urlFingerprints = new Set(); + const transitionHashes = new Set(); + const snapshots = new Map; + }>(); + const propertyViolations = new Map(); + const resources = { + documents: 0, + domNodes: 0, + jsEventListeners: 0, + jsHeapTotalBytes: 0, + jsHeapUsedBytes: 0, + layoutObjects: 0, + scriptDurationSeconds: 0, + taskDurationSeconds: 0, + threadTimeSeconds: 0, + }; + let lineCount = 0; + let totalActions = 0; + let nonWaitCount = 0; + let waitStreak = 0; + let maxWaitStreak = 0; + let nonNullHashCount = 0; + let stableTarget = true; + const stream = createReadStream(options.tracePath, { encoding: "utf8" }); + const lines = createInterface({ input: stream, crlfDelay: Infinity }); + try { + for await (const line of lines) { + lineCount += 1; + if (lineCount > TRACE_MAX_LINES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); + } + if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { + throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); + } + const parsed = parseTraceLine(line, lineCount); + if (parsed.action !== null) { + totalActions += 1; + actionCounts.set(parsed.action.kind, (actionCounts.get(parsed.action.kind) ?? 0) + 1); + if (parsed.action.kind === "Wait") { + waitStreak += 1; + maxWaitStreak = Math.max(maxWaitStreak, waitStreak); + } else { + nonWaitCount += 1; + waitStreak = 0; + } + if (parsed.action.targetTag !== null) { + if (!targetTags.has(parsed.action.targetTag) && targetTags.size >= 128) { + throw new Error("Bombadil trace exceeds 128 distinct action target tags"); + } + targetTags.set( + parsed.action.targetTag, + (targetTags.get(parsed.action.targetTag) ?? 0) + 1, + ); + } + } else { + waitStreak = 0; + } + + const relativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + urlFingerprints.add(sha256(relativeUrl)); + if (urlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct URL fingerprints`, + ); + } + stableTarget &&= parsed.state.url.href === targetUrl.href; + if (parsed.state.currentHash !== null) { + nonNullHashCount += 1; + transitionHashes.add(String(parsed.state.currentHash)); + } + + for (const snapshot of parsed.namedSnapshots) { + let entry = snapshots.get(snapshot.name); + if (entry === undefined) { + if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`, + ); + } + entry = { + changeAfterNonWaitCount: 0, + lastValueSha256: null, + observationCount: 0, + values: new Set(), + }; + snapshots.set(snapshot.name, entry); + } + if ( + parsed.action !== null + && parsed.action.kind !== "Wait" + && entry.lastValueSha256 !== null + && entry.lastValueSha256 !== snapshot.valueSha256 + ) { + entry.changeAfterNonWaitCount += 1; + } + entry.lastValueSha256 = snapshot.valueSha256; + entry.observationCount += 1; + entry.values.add(snapshot.valueSha256); + if (entry.values.size > TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { + throw new Error( + `Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`, + ); + } + } + for (const name of parsed.propertyViolationNames) { + if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`, + ); + } + propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); + } + for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { + resources[outputName] = Math.max( + resources[outputName], + parsed.state.resources[sourceName as keyof typeof RESOURCE_FIELD_MAP], + ); + } + } + } finally { + lines.close(); + stream.destroy(); + } + if (lineCount === 0) throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + + const policyFailures: string[] = []; + if (policy !== null) { + if (nonWaitCount < policy.minNonWaitActions) { + policyFailures.push("minimum non-Wait action count was not reached"); + } + for (const kind of policy.requiredActionKinds) { + if ((actionCounts.get(kind) ?? 0) === 0) { + policyFailures.push(`required action kind ${kind} was not observed`); + } + } + for (const name of policy.requiredNamedSnapshots) { + if (!snapshots.has(name)) { + policyFailures.push(`required named snapshot ${name} was not observed`); + } + } + for (const [name, minimum] of Object.entries(policy.minDistinctNamedSnapshotValues)) { + if ((snapshots.get(name)?.values.size ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its distinct-value minimum`); + } + } + for (const [name, minimum] of Object.entries( + policy.minNamedSnapshotChangesAfterNonWait, + )) { + if ((snapshots.get(name)?.changeAfterNonWaitCount ?? 0) < minimum) { + policyFailures.push( + `named snapshot ${name} did not reach its post-non-Wait change minimum`, + ); + } + } + if (policy.requireStableTargetUrl && !stableTarget) { + policyFailures.push("the browser did not remain on the exact target URL"); + } + } + const traceBytes = await readFile(options.tracePath); + return Object.freeze({ + schema: "direct.bombadil-exploration-summary/v1", + trace: Object.freeze({ + bytes: metadata.size, + lineCount, + sha256: sha256(traceBytes), + }), + actions: Object.freeze({ + byKind: sortedCountRecord(actionCounts), + maxWaitStreak, + nonWaitCount, + targetTags: sortedCountRecord(targetTags) as Readonly>, + total: totalActions, + }), + urls: Object.freeze({ + distinctFingerprintCount: urlFingerprints.size, + fingerprintSha256: Object.freeze([...urlFingerprints].sort()), + observationCount: lineCount, + stableTarget, + }), + transitions: Object.freeze({ + distinctNonNullHashCount: transitionHashes.size, + nonNullHashCount, + }), + namedSnapshots: Object.freeze([...snapshots.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, entry]) => Object.freeze({ + changeAfterNonWaitCount: entry.changeAfterNonWaitCount, + distinctValueCount: entry.values.size, + distinctValueSha256: Object.freeze([...entry.values].sort()), + name, + observationCount: entry.observationCount, + }))), + propertyViolations: Object.freeze({ + byName: sortedCountRecord(propertyViolations) as Readonly>, + total: [...propertyViolations.values()].reduce((total, value) => total + value, 0), + }), + resourceHighWaterMarks: Object.freeze(resources), + policy: Object.freeze({ + configured: policy !== null, + failures: Object.freeze(policyFailures), + satisfied: policyFailures.length === 0, + }), + }); +} + export function parseDirectBombadilFuzzArguments( arguments_: readonly string[], defaultBaseUrl: string, @@ -714,6 +1350,155 @@ function validateTargetQuery(value: unknown): Readonly> { return Object.freeze(validated); } +function validateSnapshotName(value: unknown, label: string): string { + if ( + typeof value !== "string" + || value.length === 0 + || value.length > 128 + || !SNAPSHOT_NAME_PATTERN.test(value) + || PROTOTYPE_PROPERTY_NAMES.has(value) + || hasControlCharacters(value) + ) { + throw new Error(`${label} must be a safe bounded snapshot name`); + } + return value; +} + +function validateViewport(value: unknown): ValidatedViewport { + if (value === undefined) { + return Object.freeze({ + deviceScaleFactor: DEFAULT_DEVICE_SCALE_FACTOR, + height: DEFAULT_VIEWPORT_HEIGHT, + width: DEFAULT_VIEWPORT_WIDTH, + }); + } + if (!isRecord(value) || !Object.keys(value).every((key) => VIEWPORT_KEYS.has(key))) { + throw new Error("viewport must contain only width, height, and deviceScaleFactor"); + } + const width = value.width ?? DEFAULT_VIEWPORT_WIDTH; + const height = value.height ?? DEFAULT_VIEWPORT_HEIGHT; + const deviceScaleFactor = value.deviceScaleFactor ?? DEFAULT_DEVICE_SCALE_FACTOR; + for (const [name, dimension] of [["width", width], ["height", height]] as const) { + if ( + typeof dimension !== "number" + || !Number.isSafeInteger(dimension) + || dimension < 1 + || dimension > 65_535 + ) { + throw new Error(`viewport.${name} must be an integer between 1 and 65535`); + } + } + if ( + typeof deviceScaleFactor !== "number" + || !Number.isFinite(deviceScaleFactor) + || deviceScaleFactor < 0.1 + || deviceScaleFactor > 10 + ) { + throw new Error("viewport.deviceScaleFactor must be a finite number between 0.1 and 10"); + } + return Object.freeze({ deviceScaleFactor, height, width }); +} + +function validateSnapshotMinimumMap(options: { + readonly label: string; + readonly maximum: number; + readonly value: unknown; +}): Readonly> { + if (!isRecord(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated: Record = {}; + for (const [rawName, minimum] of Object.entries(options.value).sort(([left], [right]) => + left.localeCompare(right) + )) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if ( + typeof minimum !== "number" + || !Number.isSafeInteger(minimum) + || minimum < 1 + || minimum > options.maximum + ) { + throw new Error( + `${options.label} ${name} must be an integer between 1 and ${String(options.maximum)}`, + ); + } + validated[name] = minimum; + } + return Object.freeze(validated); +} + +function validateExplorationPolicy( + value: unknown, +): ValidatedExplorationPolicy | null { + if (value === undefined) return null; + if ( + !isRecord(value) + || !Object.keys(value).every((key) => EXPLORATION_POLICY_KEYS.has(key)) + ) { + throw new Error("explorationPolicy contains an unknown field"); + } + const minNonWaitActions = value.minNonWaitActions ?? 0; + if ( + typeof minNonWaitActions !== "number" + || !Number.isSafeInteger(minNonWaitActions) + || minNonWaitActions < 0 + || minNonWaitActions > TRACE_MAX_LINES + ) { + throw new Error( + `explorationPolicy.minNonWaitActions must be an integer between 0 and ${String(TRACE_MAX_LINES)}`, + ); + } + const requiredActionKindsInput = value.requiredActionKinds ?? []; + if (!Array.isArray(requiredActionKindsInput) || requiredActionKindsInput.length > ACTION_KINDS.length) { + throw new Error("explorationPolicy.requiredActionKinds must be a bounded array"); + } + const requiredActionKinds = [...requiredActionKindsInput]; + if ( + !requiredActionKinds.every((kind): kind is DirectBombadilActionKind => + typeof kind === "string" && ACTION_KIND_SET.has(kind) + ) + || new Set(requiredActionKinds).size !== requiredActionKinds.length + ) { + throw new Error("explorationPolicy.requiredActionKinds contains an unknown or duplicate kind"); + } + requiredActionKinds.sort(); + + const requiredNamedSnapshotsInput = value.requiredNamedSnapshots ?? []; + if (!Array.isArray(requiredNamedSnapshotsInput) || requiredNamedSnapshotsInput.length > 32) { + throw new Error("explorationPolicy.requiredNamedSnapshots must be a bounded array"); + } + const requiredNamedSnapshots = requiredNamedSnapshotsInput.map((name) => + validateSnapshotName(name, "explorationPolicy.requiredNamedSnapshots entry") + ); + if (new Set(requiredNamedSnapshots).size !== requiredNamedSnapshots.length) { + throw new Error("explorationPolicy.requiredNamedSnapshots contains a duplicate name"); + } + requiredNamedSnapshots.sort(); + + const minDistinctNamedSnapshotValues = validateSnapshotMinimumMap({ + label: "explorationPolicy.minDistinctNamedSnapshotValues", + maximum: TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME, + value: value.minDistinctNamedSnapshotValues ?? {}, + }); + const minNamedSnapshotChangesAfterNonWait = validateSnapshotMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterNonWait", + maximum: TRACE_MAX_LINES, + value: value.minNamedSnapshotChangesAfterNonWait ?? {}, + }); + const requireStableTargetUrl = value.requireStableTargetUrl ?? false; + if (typeof requireStableTargetUrl !== "boolean") { + throw new Error("explorationPolicy.requireStableTargetUrl must be a boolean"); + } + return Object.freeze({ + minDistinctNamedSnapshotValues, + minNamedSnapshotChangesAfterNonWait, + minNonWaitActions, + requireStableTargetUrl, + requiredActionKinds: Object.freeze(requiredActionKinds), + requiredNamedSnapshots: Object.freeze(requiredNamedSnapshots), + }); +} + export function validateDirectBombadilFuzzConfig( config: DirectBombadilFuzzConfig, baseUrlOverride?: string, @@ -782,6 +1567,8 @@ export function validateDirectBombadilFuzzConfig( const entryPath = config.entryPath ?? "/"; validateEntryPath(entryPath); const targetQuery = validateTargetQuery(config.targetQuery ?? {}); + const viewport = validateViewport(config.viewport); + const explorationPolicy = validateExplorationPolicy(config.explorationPolicy); const startupTimeoutMs = config.server.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; if ( !Number.isSafeInteger(startupTimeoutMs) @@ -803,8 +1590,10 @@ export function validateDirectBombadilFuzzConfig( artifactRoot: join(repositoryRoot, "artifacts", "direct-bombadil", config.artifactName), bombadilExecutable: bombadilNativeBinary(repositoryRoot), entryPath, + explorationPolicy, port, targetQuery, + viewport, server: { ...config.server, cwd: serverCwd, @@ -834,7 +1623,9 @@ export function createDirectBombadilInvocation(options: { readonly specificationPath: string; readonly targetQuery?: Readonly>; readonly timeLimitSeconds: number; + readonly viewport?: DirectBombadilViewportConfig; }): DirectBombadilInvocation { + const viewport = validateViewport(options.viewport); const target = new URL(options.entryPath ?? "/", `${options.baseUrl}/`); target.searchParams.set(SCENARIO_QUERY_KEY, options.scenario); for (const [name, value] of Object.entries(options.targetQuery ?? {})) { @@ -850,6 +1641,12 @@ export function createDirectBombadilInvocation(options: { options.outputPath, "--headless", "--instrument-javascript=", + "--width", + String(viewport.width), + "--height", + String(viewport.height), + "--device-scale-factor", + String(viewport.deviceScaleFactor), ]; if (options.replayPath === null) { command.push( @@ -1181,6 +1978,100 @@ function helpText(defaultBaseUrl: string): string { ].join("\n"); } +function parseMatrixCampaignArgument(arguments_: readonly string[]): { + readonly arguments: readonly string[]; + readonly campaignId: string | null; + readonly help: boolean; +} { + const forwarded: string[] = []; + let campaignId: string | null = null; + let help = false; + for (let index = 0; index < arguments_.length; index += 1) { + const argument = arguments_[index]; + if (argument === undefined) continue; + if (argument === "--help" || argument === "-h") help = true; + if (argument === "--campaign" || argument.startsWith("--campaign=")) { + if (campaignId !== null) throw new Error("--campaign may be provided only once"); + if (argument === "--campaign") { + const next = readOptionValue(arguments_, index, "--campaign"); + campaignId = next.value; + index = next.index; + } else { + campaignId = argument.slice("--campaign=".length); + } + if (campaignId.length === 0) throw new Error("--campaign requires a value"); + continue; + } + forwarded.push(argument); + } + return { arguments: Object.freeze(forwarded), campaignId, help }; +} + +function validateCampaignMatrix( + campaigns: readonly DirectBombadilFuzzCampaign[], +): readonly DirectBombadilFuzzCampaign[] { + if (campaigns.length === 0 || campaigns.length > 32) { + throw new Error("Bombadil campaign matrix must contain 1-32 campaigns"); + } + const ids = new Set(); + for (const campaign of campaigns) { + if (!ARTIFACT_NAME_PATTERN.test(campaign.id) || ids.has(campaign.id)) { + throw new Error("Bombadil campaign IDs must be unique lowercase kebab identifiers"); + } + ids.add(campaign.id); + } + return campaigns; +} + +/** Runs a bounded product-owned campaign matrix serially. */ +export async function runDirectBombadilFuzzMatrix( + campaignsInput: readonly DirectBombadilFuzzCampaign[], + arguments_: readonly string[] = process.argv.slice(2), + dependencyOverrides: Partial = {}, +): Promise { + const campaigns = validateCampaignMatrix(campaignsInput); + const parsed = parseMatrixCampaignArgument(arguments_); + if (parsed.help) { + process.stdout.write(`${[ + helpText(campaigns[0]?.config.baseUrl ?? ""), + " --campaign Run one campaign; required with --replay", + "", + `Campaigns: ${campaigns.map((campaign) => campaign.id).join(", ")}`, + ].join("\n")}\n`); + return { kind: "help" }; + } + const selected = parsed.campaignId === null + ? campaigns + : campaigns.filter((campaign) => campaign.id === parsed.campaignId); + if (selected.length === 0) { + throw new Error(`Unknown Bombadil campaign ${parsed.campaignId ?? ""}`); + } + if ( + parsed.campaignId === null + && parsed.arguments.some((argument) => + argument === "--replay" || argument.startsWith("--replay=") + ) + ) { + throw new Error("--replay requires exactly one --campaign in matrix mode"); + } + const results: Array<{ + readonly campaignId: string; + readonly result: Extract; + }> = []; + for (const campaign of selected) { + const result = await runDirectBombadilFuzz( + campaign.config, + parsed.arguments, + dependencyOverrides, + ); + if (result.kind !== "run") { + throw new Error("Bombadil campaign unexpectedly returned help during matrix execution"); + } + results.push({ campaignId: campaign.id, result }); + } + return { kind: "matrix", results: Object.freeze(results) }; +} + /** Runs one bounded diagnostic Bombadil campaign and always releases its server lease. */ export async function runDirectBombadilFuzz( config: DirectBombadilFuzzConfig, @@ -1224,6 +2115,7 @@ export async function runDirectBombadilFuzz( specificationPath: validated.specificationPath, targetQuery: validated.targetQuery, timeLimitSeconds: parsed.timeLimitSeconds, + viewport: validated.viewport, }); const abortableInvocation = { ...invocation, abortSignal: abortController.signal }; const serverCommand = validated.server.command.map((argument) => @@ -1236,6 +2128,8 @@ export async function runDirectBombadilFuzz( let processResult: BombadilProcessResult | null = null; let attestation: DirectBombadilTraceAttestation | null = null; let attestationFailure: unknown = null; + let explorationSummary: DirectBombadilExplorationSummary | null = null; + let explorationSummaryFailure: unknown = null; let rawTracePath: string | null = null; let serverOutput = ""; let serverOutputFailure: unknown = null; @@ -1291,6 +2185,17 @@ export async function runDirectBombadilFuzz( } catch (error) { attestationFailure = error; } + try { + explorationSummary = await summarizeDirectBombadilTrace({ + ...(validated.explorationPolicy === null + ? {} + : { explorationPolicy: validated.explorationPolicy }), + targetUrl: invocation.targetUrl, + tracePath, + }); + } catch (error) { + explorationSummaryFailure = error; + } if (processFailure !== null) { throw processFailure instanceof Error ? processFailure @@ -1313,6 +2218,16 @@ export async function runDirectBombadilFuzz( ? attestationFailure : new Error(renderUnknown(attestationFailure)); } + if (explorationSummaryFailure !== null) { + throw explorationSummaryFailure instanceof Error + ? explorationSummaryFailure + : new Error(renderUnknown(explorationSummaryFailure)); + } + if (explorationSummary?.policy.satisfied !== true) { + throw new Error( + `Bombadil exploration policy was not satisfied: ${explorationSummary?.policy.failures.join("; ") ?? "summary unavailable"}`, + ); + } } catch (error) { failure = error; } @@ -1351,6 +2266,10 @@ export async function runDirectBombadilFuzz( const status = failure === null ? "passed" : "failed"; const logPath = join(artifactRun.runDirectory, "bombadil.log"); const serverLogPath = join(artifactRun.runDirectory, "server.log"); + const explorationSummaryPath = join( + artifactRun.runDirectory, + "exploration-summary.json", + ); const record = { schema: ARTIFACT_SCHEMA, evidenceClass: "diagnostic-fuzz", @@ -1366,6 +2285,8 @@ export async function runDirectBombadilFuzz( entryPath: validated.entryPath, targetQuery: validated.targetQuery, targetUrl: invocation.targetUrl, + viewport: validated.viewport, + explorationPolicy: validated.explorationPolicy, specificationPath: validated.specificationPath, replayPath, timeLimitSeconds: replayPath === null ? parsed.timeLimitSeconds : null, @@ -1387,6 +2308,11 @@ export async function runDirectBombadilFuzz( }, attestation, attestationFailure: attestationFailure === null ? null : renderUnknown(attestationFailure), + explorationSummary, + explorationSummaryPath: explorationSummary === null ? null : explorationSummaryPath, + explorationSummaryFailure: explorationSummaryFailure === null + ? null + : renderUnknown(explorationSummaryFailure), initialDirect: attestation?.initial ?? null, interruptedSignal: capturedSignal, failure: failure === null ? null : renderUnknown(failure), @@ -1401,6 +2327,9 @@ export async function runDirectBombadilFuzz( `${serverOutput}${serverOutput.length > 0 ? "\n" : ""}`, "utf8", ); + if (explorationSummary !== null) { + await writeJsonAtomically(explorationSummaryPath, explorationSummary); + } await writeJsonAtomically(join(artifactRun.runDirectory, "run.json"), record); await writeJsonAtomically(artifactRun.manifestPath, record); diff --git a/src/tooling/bombadil.ts b/src/tooling/bombadil.ts index 96f2b64..ec72ce2 100644 --- a/src/tooling/bombadil.ts +++ b/src/tooling/bombadil.ts @@ -1,15 +1,22 @@ import { attestDirectBombadilTrace as attestTrace, runDirectBombadilFuzz as runFuzz, + runDirectBombadilFuzzMatrix as runMatrix, + summarizeDirectBombadilTrace as summarizeTrace, } from "./bombadil-runner.js"; import type { DirectBombadilFuzzConfig, + DirectBombadilFuzzCampaign, + DirectBombadilFuzzMatrixResult, DirectBombadilFuzzResult, } from "./bombadil-runner.js"; /** Host-side exact attestation for one bounded Bombadil 0.7.2 JSONL trace. */ export const attestDirectBombadilTrace: typeof attestTrace = attestTrace; +/** Derives bounded diagnostic navigation metadata without replacing the raw trace. */ +export const summarizeDirectBombadilTrace: typeof summarizeTrace = summarizeTrace; + /** Runs one bounded local Bombadil campaign and preserves diagnostic artifacts. */ export function runDirectBombadilFuzz( config: DirectBombadilFuzzConfig, @@ -18,10 +25,24 @@ export function runDirectBombadilFuzz( return arguments_ === undefined ? runFuzz(config) : runFuzz(config, arguments_); } +/** Runs a bounded product campaign matrix serially and selects one for replay. */ +export function runDirectBombadilFuzzMatrix( + campaigns: readonly DirectBombadilFuzzCampaign[], + arguments_?: readonly string[], +): Promise { + return arguments_ === undefined ? runMatrix(campaigns) : runMatrix(campaigns, arguments_); +} + export type { + DirectBombadilActionKind, + DirectBombadilExplorationPolicy, + DirectBombadilExplorationSummary, + DirectBombadilFuzzCampaign, DirectBombadilFuzzConfig, + DirectBombadilFuzzMatrixResult, DirectBombadilFuzzResult, DirectBombadilServerConfig, DirectBombadilTraceAttestation, DirectBombadilTraceBinding, + DirectBombadilViewportConfig, } from "./bombadil-runner.js"; From b17332687e79aa53b17d266303ce2a0fa072343d Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:23:09 -0400 Subject: [PATCH 02/20] release: prepare Direct 0.7.6 --- README.md | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ee5cecd..47d4845 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Copy this prompt into Codex, Claude Code, or another coding agent: ```text Use $direct to install hraness/direct from -the npm registry at the exact 0.7.5 version. Follow the repository README, add +the npm registry at the exact 0.7.6 version. Follow the repository README, add `@hraness/direct` to devDependencies only, and verify that the production dependency graph excludes Direct. Do not add a fixture composition until I ask. @@ -66,7 +66,7 @@ Pin the public npm package to an exact immutable version: ```json { "devDependencies": { - "@hraness/direct": "0.7.5" + "@hraness/direct": "0.7.6" } } ``` diff --git a/package.json b/package.json index b993123..74e3254 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hraness/direct", - "version": "0.7.5", + "version": "0.7.6", "description": "A general harness for repeatable app states.", "license": "MIT", "type": "module", From 19852e4b84a462c084b5e5ca3b2bc949ac121b53 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:31:45 -0400 Subject: [PATCH 03/20] feat: report Bombadil exploration sufficiency --- src/tooling/bombadil-runner.ts | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index f2a2ef2..8be7621 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -2333,7 +2333,22 @@ export async function runDirectBombadilFuzz( await writeJsonAtomically(join(artifactRun.runDirectory, "run.json"), record); await writeJsonAtomically(artifactRun.manifestPath, record); - const summary = `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}; artifacts: ${artifactRun.runDirectory}; log: ${logPath}`; + const exploration = explorationSummary === null + ? "exploration=unavailable" + : [ + `nonWait=${String(explorationSummary.actions.nonWaitCount)}`, + `maxWaitStreak=${String(explorationSummary.actions.maxWaitStreak)}`, + `namedChanges=${explorationSummary.namedSnapshots + .map((snapshot) => `${snapshot.name}:${String(snapshot.changeAfterNonWaitCount)}`) + .join(",") || "none"}`, + `policy=${explorationSummary.policy.satisfied ? "satisfied" : "failed"}`, + ].join("; "); + const summary = [ + `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}`, + exploration, + `artifacts: ${artifactRun.runDirectory}`, + `log: ${logPath}`, + ].join("; "); (status === "passed" ? process.stdout : process.stderr).write(`${summary}\n`); if (failure !== null) { From 2347b0c9414029376ba3e33a7646d101a0a8fa60 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:35:02 -0400 Subject: [PATCH 04/20] fix: exclude destructive generic Bombadil clicks --- src/tooling/bombadil-campaign.test.ts | 6 ++++++ src/tooling/bombadil-campaign.ts | 23 +++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 63ac1cb..56a6b25 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -365,6 +365,11 @@ describe("Direct Bombadil actions", () => { [2, { value: clickAction({ textContent: "ReSeT" }) }], [2, { value: clickAction({ accessibleName: "RESET", textContent: "Again" }) }], [2, { value: clickAction({ accessibleName: "", textContent: "Reset" }) }], + [2, { value: clickAction({ textContent: "Delete track" }) }], + [2, { value: clickAction({ accessibleName: "Close editor", textContent: "Done" }) }], + [2, { value: clickAction({ textContent: "Sign-out" }) }], + [2, { value: clickAction({ textContent: "Remove from playlist" }) }], + [2, { value: clickAction({ textContent: "Unclear status" }) }], [1, { branches: [ [9, { value: clickAction({ tag: "input", inputType: "text" }) }], @@ -383,6 +388,7 @@ describe("Direct Bombadil actions", () => { expect(filtered?.generate()).toEqual({ branches: [ [7, { value: clickAction() }], + [2, { value: clickAction({ textContent: "Unclear status" }) }], [1, { branches: [ [9, { value: clickAction({ tag: "input", inputType: "text" }) }], diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index 5650e12..abb31b3 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -30,6 +30,18 @@ const DIRECT_PROBE_SCHEMA = "direct.probe/v1"; const MAX_RAW_CONTRACT_CHARACTERS = 2_000_000; const BRIDGE_KEYS = new Set(["manifest", "reset", "schema", "snapshot"]); const UNSAFE_CLICK_INPUT_TYPES = new Set(["image", "reset", "submit"]); +const UNSAFE_CLICK_LABEL_PHRASES = [ + "clear", + "close", + "delete", + "discard", + "erase", + "log out", + "remove", + "reset", + "sign out", + "unlink", +] as const; const SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; const RESOURCE_LEAK_OPTION_KEYS = new Set(["growthLimit", "metric", "windowMillis"]); const RESOURCE_METRICS = [ @@ -86,11 +98,18 @@ function safeClickAction(action: ActionTemplate): boolean { const inputType = fingerprint.inputType?.toLowerCase() ?? ""; const labels = [fingerprint.accessibleName, fingerprint.textContent] .filter((label): label is string => label !== null) - .map((label) => label.trim().toLowerCase()); + .map((label) => label + .trim() + .toLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, " ")); + const hasUnsafeLabel = labels.some((label) => { + const padded = ` ${label} `; + return UNSAFE_CLICK_LABEL_PHRASES.some((phrase) => padded.includes(` ${phrase} `)); + }); return fingerprint.href === null && tag !== "a" && fingerprint.role?.toLowerCase() !== "link" - && !labels.includes("reset") + && !hasUnsafeLabel && !UNSAFE_CLICK_INPUT_TYPES.has(inputType) && (tag !== "button" || inputType === "button"); } From 83f4fa9305da4af1d053e174dfae3df15a40ef14 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:40:23 -0400 Subject: [PATCH 05/20] docs: define conservative Bombadil click boundaries --- docs/verification.md | 10 ++++++---- src/tooling/bombadil-campaign.test.ts | 2 +- src/tooling/bombadil-campaign.ts | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/verification.md b/docs/verification.md index c4ae45a..5a6f696 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -291,11 +291,13 @@ export const no_dom_node_leak = createDirectBombadilResourceLeakProperty({ ``` The Direct action generator deliberately excludes reload, history traversal, -visible links, anchors, href targets, form submission, reset controls, and the -Enter key. It retains ordinary buttons, text input, scrolling, and an +visible links, anchors, href targets, form submission, reset controls, +destructive labels such as delete, remove, clear, discard, unlink, and close, +and the Enter key. It retains ordinary buttons, text input, scrolling, and an always-eligible low-weight wait. This preserves the post-handshake contract -within one document. Add product actions only when their navigation and form -effects are understood, and keep product-specific assertions in the campaign. +within one document. Add product actions only when their navigation, form, and +destructive effects are understood, and keep product-specific assertions in +the campaign. Guard domain actions on the state that makes them valid, then weight valuable state-changing actions above Wait. Do not increase throughput by admitting reload, navigation, submission, destructive controls, or arbitrary generated diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 56a6b25..cd77767 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -351,7 +351,7 @@ function clickAction(overrides: Readonly> = {}): u } describe("Direct Bombadil actions", () => { - test("prunes nested visible navigation and submission clicks", () => { + test("prunes nested navigation, submission, and destructive clicks", () => { fakeClicks.generate = () => ({ branches: [ [7, { value: clickAction() }], diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index abb31b3..a9eb154 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -138,7 +138,8 @@ function pruneActionTree( /** * Builds a browser action generator without reload/history actions or visible - * navigation and submission click targets, keeping Direct continuously bound. + * navigation, submission, reset, or destructive click targets, keeping Direct + * continuously bound. */ export function createDirectBombadilActions(): ActionGenerator { const safeClicks = actions(() => pruneActionTree(clicks.generate(), safeClickAction) ?? []); From 40f53bc3182fbf13a1732f8003866ab84fbf5d09 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:49:26 -0400 Subject: [PATCH 06/20] docs: separate responsive and interaction evidence --- README.md | 4 ++++ docs/verification.md | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 47d4845..db5e358 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,10 @@ promote the smallest readable failure to a deterministic product regression. When a campaign must exercise an interaction, require a named product value to change after a non-Wait action so bootstrap and idle transitions do not satisfy the exploration policy. +If the full product snapshot includes viewport dimensions, put that requirement +on a separate interaction snapshot without viewport fields and require an +opposite-size `SetViewport` independently. Latch the first ready product state +for initial-world properties so later actions cannot repair a bad initial state. The raw trace remains authoritative and may contain screenshots, URLs, typed text, accessible labels, and local paths; treat it as potentially sensitive. Summary counts and hashes help triage exploration but are not Direct coverage. diff --git a/docs/verification.md b/docs/verification.md index 5a6f696..f39983f 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -404,6 +404,15 @@ sufficiency check, not a coverage claim. Start with observed stable behavior and raise thresholds only when the action generator makes them reliably reachable. +Do not let responsive evidence stand in for product interaction. When a full +product snapshot includes viewport dimensions, name a second compact +interaction snapshot that excludes them and put the distinct-value and +post-action-change requirements on that snapshot. Require `SetViewport` +separately, and generate only a width or height different from the current +viewport. Likewise, latch the first product-ready observation for initial-world +properties so a later generated action cannot repair an incorrect initial +state before the bounded formula completes. + Run random exploration for 12 to 300 seconds. The default is 20 seconds: ```sh From 2977c6241582b33bd38a8c82eb4a8e0fe3936420 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 11:59:05 -0400 Subject: [PATCH 07/20] fix: preserve validated viewport number types --- src/tooling/bombadil-runner.test.ts | 8 ++++++++ src/tooling/bombadil-runner.ts | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index a164b38..82e9595 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -435,6 +435,14 @@ describe("Direct Bombadil configuration and invocation", () => { height: 768, width: 1_024, }); + expect(validateDirectBombadilFuzzConfig({ + ...config, + viewport: { deviceScaleFactor: 1.5, height: 720, width: 1_280 }, + }).viewport).toEqual({ + deviceScaleFactor: 1.5, + height: 720, + width: 1_280, + }); expect(validated.bombadilExecutable).toEndWith( `node_modules/@antithesishq/bombadil/binaries/${nativeBinaryName()}`, ); diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index 8be7621..a76deaf 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -1375,19 +1375,20 @@ function validateViewport(value: unknown): ValidatedViewport { if (!isRecord(value) || !Object.keys(value).every((key) => VIEWPORT_KEYS.has(key))) { throw new Error("viewport must contain only width, height, and deviceScaleFactor"); } - const width = value.width ?? DEFAULT_VIEWPORT_WIDTH; - const height = value.height ?? DEFAULT_VIEWPORT_HEIGHT; - const deviceScaleFactor = value.deviceScaleFactor ?? DEFAULT_DEVICE_SCALE_FACTOR; - for (const [name, dimension] of [["width", width], ["height", height]] as const) { + const validateDimension = (name: "height" | "width", input: unknown): number => { if ( - typeof dimension !== "number" - || !Number.isSafeInteger(dimension) - || dimension < 1 - || dimension > 65_535 + typeof input !== "number" + || !Number.isSafeInteger(input) + || input < 1 + || input > 65_535 ) { throw new Error(`viewport.${name} must be an integer between 1 and 65535`); } - } + return input; + }; + const width = validateDimension("width", value.width ?? DEFAULT_VIEWPORT_WIDTH); + const height = validateDimension("height", value.height ?? DEFAULT_VIEWPORT_HEIGHT); + const deviceScaleFactor = value.deviceScaleFactor ?? DEFAULT_DEVICE_SCALE_FACTOR; if ( typeof deviceScaleFactor !== "number" || !Number.isFinite(deviceScaleFactor) From 9cdf1e9cd9bf716f670654fd4f7fe68b643ef8bd Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 12:03:10 -0400 Subject: [PATCH 08/20] chore: review Direct package growth --- scripts/package-artifact.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/package-artifact.ts b/scripts/package-artifact.ts index c8b667f..08f043c 100644 --- a/scripts/package-artifact.ts +++ b/scripts/package-artifact.ts @@ -10,7 +10,7 @@ const packageBudget = Object.freeze({ entryCount: { min: 50, max: 120 }, fileCount: { min: 50, max: 60 }, packedBytes: { min: 140_000, max: 180_000 }, - unpackedBytes: { min: 650_000, max: 750_000 }, + unpackedBytes: { min: 650_000, max: 800_000 }, }); const requiredPaths = Object.freeze([ From 7e27258a875c3587874156f2f2be922709e97fb7 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 12:05:17 -0400 Subject: [PATCH 09/20] docs: publish Direct 0.7.6 install pins --- README.md | 4 ++-- skills/direct/references/install.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index db5e358..747025c 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ state with predictable local stand-ins. it does not click through the browser or test the systems it replaces. ```sh -bun add --dev @hraness/direct@0.7.5 +bun add --dev @hraness/direct@0.7.6 # or -npm install --save-dev @hraness/direct@0.7.5 +npm install --save-dev @hraness/direct@0.7.6 ``` [overview](https://hraness.com/direct) diff --git a/skills/direct/references/install.md b/skills/direct/references/install.md index 90c38b9..6d240e2 100644 --- a/skills/direct/references/install.md +++ b/skills/direct/references/install.md @@ -21,9 +21,9 @@ global `direct` CLI. For a new installation, pin the reviewed public release: ```sh -bun add --dev @hraness/direct@0.7.5 +bun add --dev @hraness/direct@0.7.6 # or, in an npm project -npm install --save-dev @hraness/direct@0.7.5 +npm install --save-dev @hraness/direct@0.7.6 ``` The equivalent manifest entry is: @@ -31,7 +31,7 @@ The equivalent manifest entry is: ```json { "devDependencies": { - "@hraness/direct": "0.7.5" + "@hraness/direct": "0.7.6" } } ``` From a15ce86eac6192c0e4d80ccec04c0114f743abe0 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 12:50:47 -0400 Subject: [PATCH 10/20] fix: make Bombadil evidence fail closed --- README.md | 6 + docs/verification.md | 37 +++-- src/tooling/bombadil-campaign.test.ts | 214 +++++++++++++++++++------- src/tooling/bombadil-campaign.ts | 168 +++++++++++++++----- src/tooling/bombadil-runner.test.ts | 137 +++++++++++++++++ src/tooling/bombadil-runner.ts | 47 ++++-- 6 files changed, 490 insertions(+), 119 deletions(-) diff --git a/README.md b/README.md index 747025c..b461193 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,7 @@ export * from "@antithesishq/bombadil/browser/defaults/properties"; const direct = createDirectBombadilProperties(); export const direct_safe_actions = createDirectBombadilActions(); +export const direct_startup_contract = direct.startupContract; export const direct_exact_contract = direct.exactContract; export const direct_stable_catalog = direct.stableCatalog; export const direct_no_declared_violations = direct.noDeclaredViolations; @@ -277,6 +278,11 @@ and releases its owned processes. Use `runDirectBombadilFuzzMatrix` when a product owns several scenarios; it runs them serially and requires one exact campaign selector for replay. +Startup is the only repairable contract phase. It must reach one exact Direct +observation within ten seconds. From that sample onward, activation identity, +route, scenario, catalog, and zero declared violations are immediate safety +invariants; only quiescence remains bounded liveness. + Keep liveness formulas time-bounded. Prefer guarded product actions with explicit weights over unrestricted browser actions, and name small JSON snapshots that expose semantic state without retaining page content. Run short diff --git a/docs/verification.md b/docs/verification.md index f39983f..cb765f9 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -272,10 +272,12 @@ const direct = createDirectBombadilProperties(); const phase = createDirectBombadilNamedSnapshot({ fallback: "unavailable", name: "todos.phase", + parse: (value) => typeof value === "string" ? value : null, read: ({ window }) => Reflect.get(window, "__todosPhase"), }); export const direct_safe_actions = createDirectBombadilActions(); +export const direct_startup_contract = direct.startupContract; export const direct_exact_contract = direct.exactContract; export const direct_stable_catalog = direct.stableCatalog; export const direct_no_declared_violations = direct.noDeclaredViolations; @@ -293,7 +295,9 @@ export const no_dom_node_leak = createDirectBombadilResourceLeakProperty({ The Direct action generator deliberately excludes reload, history traversal, visible links, anchors, href targets, form submission, reset controls, destructive labels such as delete, remove, clear, discard, unlink, and close, -and the Enter key. It retains ordinary buttons, text input, scrolling, and an +all labels because their fingerprints do not expose the associated control's +submit, reset, or button type, and the Enter key. It retains ordinary buttons, +text input, scrolling, and an always-eligible low-weight wait. This preserves the post-handshake contract within one document. Add product actions only when their navigation, form, and destructive effects are understood, and keep product-specific assertions in @@ -305,12 +309,13 @@ input. A generated action sequence is useful only while it preserves the scenario boundary and exercises behavior the product can interpret. `createDirectBombadilNamedSnapshot` gives product properties and post-run -diagnostics a small semantic signal. It requires a safe bounded name, clones no -more than two million JSON characters, and fails closed to the explicit -fallback when a page getter throws or returns unsuitable data. Extract state, -not page content or credentials. Named values are represented only by canonical -SHA-256 hashes in the exploration summary; the authoritative raw trace still -contains the original values. +diagnostics a small semantic signal. It requires a safe bounded name distinct +from `direct` and JavaScript prototype names, an explicit product parser, at +most 64 JSON levels, and at most 2 MiB of UTF-8 JSON. It fails closed to the +parsed fallback when a page getter throws or returns unsuitable data. Extract +state, not page content or credentials. Named values are represented only by +canonical SHA-256 hashes in the exploration summary; the authoritative raw +trace still contains the original values. Bombadil's 0.7.2 manual documents a sliding-window resource property at `@antithesishq/bombadil/browser/extras/resources`, but the published npm @@ -459,14 +464,16 @@ and server configuration for replay. Bombadil rejects a replay that diverges, so reproduction is strong debugging evidence but not guaranteed after the product changes. -The browser formulas require an exact scenario contract, a nonempty catalog, -zero declared violations, and quiescence to recur within ten seconds from every -observed state. Keep every liveness obligation bounded to a product latency -budget. A finite run cannot decide an unbounded future obligation, and nesting -unbounded `always` inside bounded `eventually` does not make it decidable. -Continuous identity and catalog binding therefore stay in the host attestation -below. A formula result and Bombadil exit status are not sufficient evidence -because a short or incomplete trace could otherwise pass vacuously. +The startup formula requires an exact scenario contract within ten seconds. +After that first exact sample, the browser formulas require continuous +activation identity, catalog identity, and zero declared violations. Only +quiescence may recover within a bounded ten-second liveness window. Keep every +liveness obligation bounded to a product latency budget. A finite run cannot +decide an unbounded future obligation, and nesting unbounded `always` inside +bounded `eventually` does not make it decidable. The host attestation below +rechecks the full trace independently. A formula result and Bombadil exit status +are not sufficient evidence because a short or incomplete trace could otherwise +pass vacuously. After every random run or replay, the host runner streams the bounded 0.7.2 JSONL trace from foreign input and requires one named `direct` observation per diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index cd77767..86c94cd 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -1,7 +1,13 @@ import { describe, expect, mock, test } from "bun:test"; +import type { JSON as BombadilJson } from "@antithesishq/bombadil"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { defineDirect } from "@hraness/direct"; import { createDirectSession } from "@hraness/direct/testing"; +import { summarizeDirectBombadilTrace } from "./bombadil-runner.js"; + interface FakeFormula { readonly body: unknown; readonly kind: "always" | "eventually"; @@ -239,6 +245,13 @@ describe("Direct Bombadil named snapshots", () => { const snapshot = createDirectBombadilNamedSnapshot({ fallback: { status: "unavailable" }, name: "product.phase", + parse: (value) => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return null; + } + const status = Reflect.get(value, "status"); + return typeof status === "string" ? { status } : null; + }, read: (state) => Reflect.get(state.window, "phase"), }) as unknown as FakeCell; expect(snapshot.name).toBe("product.phase"); @@ -254,21 +267,115 @@ describe("Direct Bombadil named snapshots", () => { }); expect(snapshot.read({ window: hostile })).toEqual({ status: "unavailable" }); expect(snapshot.read({ - window: { phase: "x".repeat(2_000_001) }, + window: { phase: { status: "é".repeat(1_100_000) } }, })).toEqual({ status: "unavailable" }); + expect(snapshot.read({ window: { phase: null } })).toEqual({ + status: "unavailable", + }); + }); + + test("produces a named value accepted by the host summary contract", async () => { + const snapshot = createDirectBombadilNamedSnapshot({ + fallback: { status: "unavailable" }, + name: "product.compat", + parse: (value) => { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return null; + } + const status = Reflect.get(value, "status"); + return typeof status === "string" ? { status } : null; + }, + read: (state) => Reflect.get(state.window, "phase"), + }) as unknown as FakeCell; + const value = snapshot.read({ window: { phase: { status: "ready" } } }); + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-helper-summary-")); + const tracePath = join(directory, "trace.jsonl"); + try { + await writeFile(tracePath, `${JSON.stringify({ + action: null, + snapshots: [ + { + index: 0, + name: "direct", + time: 1, + value: readDirectBombadilObservation(contractFixture()), + }, + { index: 1, name: snapshot.name, time: 1, value }, + ], + state: { + hash_current: 1, + hash_previous: null, + resources: { + documents: 1, + dom_nodes: 1, + js_event_listeners: 1, + js_heap_total: 1, + js_heap_used: 1, + layout_objects: 1, + script_duration: 0, + task_duration: 0, + thread_time: 0, + timestamp: 1, + }, + screenshot: "1.png", + url: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + }, + timestamp: 1, + violations: [], + })}\n`, "utf8"); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.find(({ name }) => name === snapshot.name)) + .toMatchObject({ distinctValueCount: 1, observationCount: 1 }); + } finally { + await rm(directory, { force: true, recursive: true }); + } + }); + + test("matches the host name, UTF-8 size, and JSON depth boundary", () => { + for (const name of [ + "direct", + "__proto__", + "constructor", + "prototype", + "unsafe name", + ]) { + expect(() => createDirectBombadilNamedSnapshot({ + fallback: null, + name, + parse: (value) => value, + read: () => null, + })).toThrow("safe, unreserved"); + } + + const snapshot = createDirectBombadilNamedSnapshot({ + fallback: null, + name: "safe", + parse: (value) => value, + read: (state) => Reflect.get(state.window, "phase"), + }) as unknown as FakeCell; + let atLimit: BombadilJson = null; + for (let index = 0; index < 64; index += 1) atLimit = [atLimit]; + expect(snapshot.read({ window: { phase: atLimit } })).toEqual(atLimit); + const beyondLimit: BombadilJson = [atLimit]; + expect(snapshot.read({ window: { phase: beyondLimit } })).toBeNull(); }); - test("rejects unsafe names and non-JSON fallbacks before registering an extractor", () => { + test("rejects non-JSON or parser-invalid fallbacks before registering an extractor", () => { expect(() => createDirectBombadilNamedSnapshot({ fallback: null, - name: "unsafe name", + name: "safe", + parse: () => null, read: () => null, - })).toThrow("safe 1-128 character identifier"); - expect(() => createDirectBombadilNamedSnapshot({ + })).toThrow("accepted by parse"); + expect(() => createDirectBombadilNamedSnapshot({ fallback: undefined as never, name: "safe", + parse: (value) => value, read: () => null, - })).toThrow("fallback must be bounded JSON"); + })).toThrow("fallback must be bounded JSON accepted by parse"); }); }); @@ -374,6 +481,7 @@ describe("Direct Bombadil actions", () => { branches: [ [9, { value: clickAction({ tag: "input", inputType: "text" }) }], [8, { value: clickAction({ tag: "A" }) }], + [7, { value: clickAction({ tag: "label", textContent: "Continue" }) }], ], }], ], @@ -430,25 +538,38 @@ describe("Direct Bombadil actions", () => { }); describe("Direct Bombadil formulas", () => { - test("returns four recurring bounded health formulas", () => { + test("splits bounded startup from strict recurring health", () => { const properties = createDirectBombadilProperties() as unknown as Record; const cell = cells.at(-1); if (cell === undefined) throw new Error("Expected one Direct extractor cell"); expect(cell.name).toBe("direct"); - cell.current = readDirectBombadilObservation(contractFixture()); + const sample = (window: unknown): void => { + cell.current = cell.read({ window }); + }; + sample({}); expect(Object.keys(properties).sort()).toEqual([ "eventualQuiescence", "exactContract", "noDeclaredViolations", "stableCatalog", + "startupContract", ]); - const exactContract = properties.exactContract; - expect(exactContract?.kind).toBe("always"); - const exactEventually = requireFormula(exactContract?.body as FakeFormula); - expect(exactEventually.kind).toBe("eventually"); - expect(exactEventually.milliseconds).toBe(10_000); - expect(evaluate(exactEventually.body)).toBeTrue(); + expect(properties.startupContract?.kind).toBe("eventually"); + expect(properties.startupContract?.milliseconds).toBe(10_000); + expect(evaluate(properties.startupContract?.body)).toBeFalse(); + expect(properties.exactContract?.kind).toBe("always"); + expect(properties.stableCatalog?.kind).toBe("always"); + expect(properties.noDeclaredViolations?.kind).toBe("always"); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); + + sample(contractFixture()); + expect(evaluate(properties.startupContract?.body)).toBeTrue(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); const initial = cell.current as Readonly>; for (const [key, value] of [ ["activeScenario", ""], @@ -457,29 +578,18 @@ describe("Direct Bombadil formulas", () => { ["activeSource", "fixture"], ] as const) { cell.current = { ...initial, [key]: value }; - expect(evaluate(exactEventually.body), key).toBeFalse(); + expect(evaluate(properties.exactContract?.body), key).toBeFalse(); } cell.current = initial; - const noDeclaredViolations = properties.noDeclaredViolations; - expect(noDeclaredViolations?.kind).toBe("always"); - const violationsEventually = requireFormula( - noDeclaredViolations?.body as FakeFormula, - ); - expect(violationsEventually.kind).toBe("eventually"); - expect(violationsEventually.milliseconds).toBe(10_000); - expect(evaluate(violationsEventually.body)).toBeTrue(); - - const stableAlways = properties.stableCatalog; - expect(stableAlways?.kind).toBe("always"); - const stableEventually = requireFormula(stableAlways?.body as FakeFormula); - expect(stableEventually.kind).toBe("eventually"); - expect(stableEventually.milliseconds).toBe(10_000); - expect(evaluate(stableEventually.body)).toBeTrue(); - cell.current = readDirectBombadilObservation(contractFixture({ - catalogHash: "", + sample(contractFixture({ + catalogHash: "fnv1a-64:ffffffffffffffff", })); - expect(evaluate(stableEventually.body)).toBeFalse(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeFalse(); + + sample(contractFixture({ violations: { console: 1 } })); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeFalse(); const outerAlways = properties.eventualQuiescence; expect(outerAlways?.kind).toBe("always"); @@ -492,32 +602,28 @@ describe("Direct Bombadil formulas", () => { const properties = createDirectBombadilProperties() as unknown as Record; const cell = cells.at(-1); if (cell === undefined) throw new Error("Expected one Direct extractor cell"); - cell.current = readDirectBombadilObservation({}); - - const exactEventually = requireFormula(properties.exactContract?.body as FakeFormula); - const violationsEventually = requireFormula( - properties.noDeclaredViolations?.body as FakeFormula, - ); - const stableEventually = requireFormula(properties.stableCatalog?.body as FakeFormula); - expect(evaluate(exactEventually.body)).toBeFalse(); - expect(evaluate(violationsEventually.body)).toBeFalse(); - expect(evaluate(stableEventually.body)).toBeFalse(); + const sample = (window: unknown): void => { + cell.current = cell.read({ window }); + }; + sample({}); + expect(evaluate(properties.startupContract?.body)).toBeFalse(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); const eventual = properties.eventualQuiescence?.body as FakeFormula; expect(evaluate(eventual.body)).toBeFalse(); - cell.current = readDirectBombadilObservation(contractFixture()); - expect(evaluate(exactEventually.body)).toBeTrue(); - expect(evaluate(violationsEventually.body)).toBeTrue(); - expect(evaluate(stableEventually.body)).toBeTrue(); + sample(contractFixture()); + expect(evaluate(properties.startupContract?.body)).toBeTrue(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); expect(evaluate(eventual.body)).toBeTrue(); - cell.current = readDirectBombadilObservation(contractFixture({ - catalogHash: "", - violations: { console: 1 }, - })); - expect(evaluate(exactEventually.body)).toBeFalse(); - expect(evaluate(stableEventually.body)).toBeFalse(); - expect(evaluate(violationsEventually.body)).toBeFalse(); + sample({}); + expect(evaluate(properties.exactContract?.body)).toBeFalse(); + expect(evaluate(properties.stableCatalog?.body)).toBeFalse(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeFalse(); }); }); diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index a9eb154..8cf4512 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -28,6 +28,8 @@ const DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; const DIRECT_SESSION_MANIFEST_SCHEMA = "direct.session-manifest/v1"; const DIRECT_PROBE_SCHEMA = "direct.probe/v1"; const MAX_RAW_CONTRACT_CHARACTERS = 2_000_000; +const MAX_NAMED_SNAPSHOT_CANONICAL_BYTES = 2 * 1024 * 1024; +const MAX_NAMED_SNAPSHOT_JSON_DEPTH = 64; const BRIDGE_KEYS = new Set(["manifest", "reset", "schema", "snapshot"]); const UNSAFE_CLICK_INPUT_TYPES = new Set(["image", "reset", "submit"]); const UNSAFE_CLICK_LABEL_PHRASES = [ @@ -43,6 +45,12 @@ const UNSAFE_CLICK_LABEL_PHRASES = [ "unlink", ] as const; const SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +const RESERVED_SNAPSHOT_NAMES = new Set([ + "direct", + "__proto__", + "constructor", + "prototype", +]); const RESOURCE_LEAK_OPTION_KEYS = new Set(["growthLimit", "metric", "windowMillis"]); const RESOURCE_METRICS = [ "dom_nodes", @@ -71,12 +79,24 @@ export interface DirectBombadilObservation { } export interface DirectBombadilProperties { + readonly startupContract: Formula; readonly exactContract: Formula; readonly stableCatalog: Formula; readonly noDeclaredViolations: Formula; readonly eventualQuiescence: Formula; } +function observationHasExactContract( + observation: DirectBombadilObservation, +): boolean { + return observation.contractValid + && observation.activeSource === "scenario" + && observation.activeScenario.length > 0 + && observation.activeRoute.length > 0 + && observation.activationHash.length > 0 + && observation.catalogHash.length > 0; +} + export type DirectBombadilResourceMetric = (typeof RESOURCE_METRICS)[number]; export interface DirectBombadilResourceLeakOptions { @@ -108,6 +128,7 @@ function safeClickAction(action: ActionTemplate): boolean { }); return fingerprint.href === null && tag !== "a" + && tag !== "label" && fingerprint.role?.toLowerCase() !== "link" && !hasUnsafeLabel && !UNSAFE_CLICK_INPUT_TYPES.has(inputType) @@ -255,10 +276,53 @@ function boundedJsonClone(value: unknown): BombadilJson | null { return JSON.parse(source) as BombadilJson; } -function optionalBoundedJsonClone(value: unknown): BombadilJson | undefined { - const source = JSON.stringify(value); - if (source === undefined || source.length > MAX_RAW_CONTRACT_CHARACTERS) return undefined; - return JSON.parse(source) as BombadilJson; +function cloneNamedSnapshotJson( + value: unknown, + depth = 0, + ancestors: WeakSet = new WeakSet(), +): BombadilJson | undefined { + if (depth > MAX_NAMED_SNAPSHOT_JSON_DEPTH) return undefined; + if (value === null || typeof value === "boolean" || typeof value === "string") { + return value; + } + if (typeof value === "number") return Number.isFinite(value) ? value : undefined; + if (typeof value !== "object") return undefined; + if (ancestors.has(value)) return undefined; + ancestors.add(value); + try { + if (Array.isArray(value)) { + const cloned: BombadilJson[] = []; + for (const entry of value) { + const child = cloneNamedSnapshotJson(entry, depth + 1, ancestors); + if (child === undefined) return undefined; + cloned.push(child); + } + return cloned; + } + const clonedEntries: [string, BombadilJson][] = []; + for (const key of Object.keys(value)) { + const child = cloneNamedSnapshotJson( + Reflect.get(value, key), + depth + 1, + ancestors, + ); + if (child === undefined) return undefined; + clonedEntries.push([key, child]); + } + return Object.fromEntries(clonedEntries) as BombadilJson; + } finally { + ancestors.delete(value); + } +} + +function boundedNamedSnapshotJson(value: unknown): BombadilJson | undefined { + const cloned = cloneNamedSnapshotJson(value); + if (cloned === undefined) return undefined; + const source = JSON.stringify(cloned); + return new TextEncoder().encode(source).byteLength + <= MAX_NAMED_SNAPSHOT_CANONICAL_BYTES + ? cloned + : undefined; } /** @@ -268,24 +332,43 @@ function optionalBoundedJsonClone(value: unknown): BombadilJson | undefined { export function createDirectBombadilNamedSnapshot(options: { readonly fallback: T; readonly name: string; + readonly parse: (value: BombadilJson) => T | null; readonly read: (state: BombadilBrowserState) => unknown; }): Cell { if ( options.name.length === 0 || options.name.length > 128 || !SNAPSHOT_NAME_PATTERN.test(options.name) + || RESERVED_SNAPSHOT_NAMES.has(options.name) ) { - throw new Error("Bombadil snapshot name must be a safe 1-128 character identifier"); + throw new Error( + "Bombadil snapshot name must be a safe, unreserved 1-128 character identifier", + ); + } + const parse = (value: unknown): T | null => { + const cloned = boundedNamedSnapshotJson(value); + if (cloned === undefined) return null; + const parsed = options.parse(cloned); + return parsed !== null && boundedNamedSnapshotJson(parsed) !== undefined + ? parsed + : null; + }; + let fallback: T | null = null; + try { + fallback = parse(options.fallback); + } catch { + fallback = null; } - const fallback = optionalBoundedJsonClone(options.fallback); - if (fallback === undefined) { - throw new Error("Bombadil snapshot fallback must be bounded JSON"); + if (fallback === null) { + throw new Error( + "Bombadil snapshot fallback must be bounded JSON accepted by parse", + ); } return extract((state) => { try { - return (optionalBoundedJsonClone(options.read(state)) ?? fallback) as T; + return parse(options.read(state)) ?? fallback; } catch { - return fallback as T; + return fallback; } }).named(options.name); } @@ -387,39 +470,50 @@ export function readDirectBombadilObservation( } } -/** Builds the four Direct invariants used by Bombadil browser campaigns. */ +/** Builds bounded startup plus strict recurring Direct browser invariants. */ export function createDirectBombadilProperties(): DirectBombadilProperties { - const direct = extract((state) => - readDirectBombadilObservation(state.window) - ).named("direct"); + let initial: Readonly<{ + activationHash: string; + activeRoute: string; + activeScenario: string; + catalogHash: string; + }> | null = null; + const direct = extract((state) => { + const observation = readDirectBombadilObservation(state.window); + if (initial === null && observationHasExactContract(observation)) { + initial = Object.freeze({ + activationHash: observation.activationHash, + activeRoute: observation.activeRoute, + activeScenario: observation.activeScenario, + catalogHash: observation.catalogHash, + }); + } + return observation; + }).named("direct"); - const exactContract = always( - eventually(() => - direct.current.contractValid - && direct.current.activeSource === "scenario" - && direct.current.activeScenario.length > 0 - && direct.current.activeRoute.length > 0 - && direct.current.activationHash.length > 0 - ).within(10, "seconds"), - ); - const stableCatalog = always( - eventually(() => - direct.current.contractValid - && direct.current.catalogHash.length > 0 - ).within(10, "seconds"), - ); - const noDeclaredViolations = always( - eventually(() => - direct.current.contractValid - && direct.current.violationsValid - && direct.current.violations.every((value: number) => value === 0) - ).within(10, "seconds"), - ); + const startupContract = eventually(() => initial !== null).within(10, "seconds"); + const exactContract = always(() => initial === null || ( + observationHasExactContract(direct.current) + && direct.current.activationHash === initial.activationHash + && direct.current.activeRoute === initial.activeRoute + && direct.current.activeScenario === initial.activeScenario + )); + const stableCatalog = always(() => initial === null || ( + observationHasExactContract(direct.current) + && direct.current.catalogHash === initial.catalogHash + )); + const noDeclaredViolations = always(() => initial === null || ( + observationHasExactContract(direct.current) + && direct.current.violationsValid + && direct.current.violations.every((value: number) => value === 0) + )); const eventualQuiescence = always( - eventually(() => direct.current.isQuiescent).within(10, "seconds"), + eventually(() => initial !== null && direct.current.isQuiescent) + .within(10, "seconds"), ); return Object.freeze({ + startupContract, exactContract, stableCatalog, noDeclaredViolations, diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index 82e9595..d7711f9 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -448,6 +448,46 @@ describe("Direct Bombadil configuration and invocation", () => { ); }); + test("orders query and policy artifacts by explicit code units", async () => { + const { config } = await fixture(); + const validated = validateDirectBombadilFuzzConfig({ + ...config, + targetQuery: { z: "4", "a.": "3", A: "1", "a-": "2" }, + explorationPolicy: { + minDistinctNamedSnapshotValues: { + z: 4, + "a.": 3, + A: 1, + "a-": 2, + }, + minNamedSnapshotChangesAfterNonWait: { + z: 4, + "a.": 3, + A: 1, + "a-": 2, + }, + requiredActionKinds: ["Wait", "TypeText", "Click"], + requiredNamedSnapshots: ["z", "a.", "A", "a-"], + }, + }); + const expectedNames = ["A", "a-", "a.", "z"]; + expect(Object.keys(validated.targetQuery)).toEqual(expectedNames); + expect(validated.explorationPolicy?.requiredActionKinds).toEqual([ + "Click", + "TypeText", + "Wait", + ]); + expect(validated.explorationPolicy?.requiredNamedSnapshots).toEqual( + expectedNames, + ); + expect(Object.keys( + validated.explorationPolicy?.minDistinctNamedSnapshotValues ?? {}, + )).toEqual(expectedNames); + expect(Object.keys( + validated.explorationPolicy?.minNamedSnapshotChangesAfterNonWait ?? {}, + )).toEqual(expectedNames); + }); + test("rejects unsafe artifact, scenario, route, path, readiness, and server command inputs", async () => { const { config, repositoryRoot } = await fixture(); expect(() => validateDirectBombadilFuzzConfig({ @@ -925,6 +965,103 @@ describe("Direct Bombadil exploration summary", () => { }); }); + test("uses the first exact Direct observation only as the policy baseline", async () => { + const tracePath = await summaryTrace([ + traceLine(absentObservation(), 1, { + namedSnapshots: [{ name: "phase", value: "loading" }], + }), + traceLine(absentObservation(), 2, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: "pre-handshake-click" }], + }), + traceLine(directObservation(), 3, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: "ready" }], + }), + ]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + minNonWaitActions: 1, + requiredActionKinds: ["Click"], + requiredNamedSnapshots: ["phase"], + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.actions).toMatchObject({ + byKind: {}, + nonWaitCount: 0, + total: 0, + }); + expect(summary.urls.observationCount).toBe(1); + expect(summary.transitions).toEqual({ + distinctNonNullHashCount: 1, + nonNullHashCount: 1, + }); + expect(summary.namedSnapshots.find((entry) => entry.name === "phase")) + .toMatchObject({ + changeAfterNonWaitCount: 0, + distinctValueCount: 1, + observationCount: 1, + }); + expect(summary.policy).toMatchObject({ + failures: [ + expect.stringContaining("minimum non-Wait"), + expect.stringContaining("required action kind Click"), + expect.stringContaining("distinct-value minimum"), + expect.stringContaining("post-non-Wait change minimum"), + ], + satisfied: false, + }); + }); + + test("orders mixed-case and punctuation summary names by code unit", async () => { + const observation = directObservation(); + const tracePath = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: [ + { name: "z", value: 4 }, + { name: "a.", value: 3 }, + { name: "A", value: 1 }, + { name: "a-", value: 2 }, + ], + violations: [ + { name: "z", violation: { False: {} } }, + { name: "a.", violation: { False: {} } }, + { name: "A", violation: { False: {} } }, + { name: "a-", violation: { False: {} } }, + ], + })]); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.map(({ name }) => name)).toEqual([ + "A", + "a-", + "a.", + "direct", + "z", + ]); + expect(Object.keys(summary.propertyViolations.byName)).toEqual([ + "A", + "a-", + "a.", + "z", + ]); + }); + test("rejects hostile envelopes, action targets, and excessively deep snapshot JSON", async () => { const observation = directObservation(); const badTarget = await summaryTrace([traceLine(observation, 1, { diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index a76deaf..b53c3a7 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -457,6 +457,12 @@ function hasExactKeys( return keys.length === expected.size && keys.every((key) => expected.has(key)); } +function compareCodeUnits(left: string, right: string): number { + if (left < right) return -1; + if (left > right) return 1; + return 0; +} + function parseTraceDirectObservation(value: unknown): TraceDirectObservation { if (!isRecord(value) || !hasExactKeys(value, DIRECT_OBSERVATION_KEYS)) { throw new Error("Bombadil trace has an invalid named direct observation"); @@ -650,7 +656,7 @@ function canonicalJson(value: unknown, depth = 0): string { return `[${value.map((entry) => canonicalJson(entry, depth + 1)).join(",")}]`; } if (!isRecord(value)) throw new Error("Bombadil named snapshot is not JSON"); - const entries = Object.keys(value).sort().map((key) => + const entries = Object.keys(value).sort(compareCodeUnits).map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key], depth + 1)}` ); return `{${entries.join(",")}}`; @@ -975,7 +981,7 @@ function sortedCountRecord( values: ReadonlyMap, ): Readonly>> { return Object.freeze(Object.fromEntries( - [...values.entries()].sort(([left], [right]) => left.localeCompare(right)), + [...values.entries()].sort(([left], [right]) => compareCodeUnits(left, right)), ) as Partial>); } @@ -1030,6 +1036,9 @@ export async function summarizeDirectBombadilTrace(options: { let waitStreak = 0; let maxWaitStreak = 0; let nonNullHashCount = 0; + let policyObservationCount = 0; + let reachedExactObservation = false; + let previousObservationWasExact = false; let stableTarget = true; const stream = createReadStream(options.tracePath, { encoding: "utf8" }); const lines = createInterface({ input: stream, crlfDelay: Infinity }); @@ -1043,7 +1052,17 @@ export async function summarizeDirectBombadilTrace(options: { throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); } const parsed = parseTraceLine(line, lineCount); - if (parsed.action !== null) { + const currentObservationIsExact = + exactTraceDirectObservation(parsed.directObservation) !== null; + if (!reachedExactObservation && !currentObservationIsExact) { + previousObservationWasExact = false; + continue; + } + if (!reachedExactObservation) reachedExactObservation = true; + policyObservationCount += 1; + const actionFollowsExactObservation = previousObservationWasExact; + + if (actionFollowsExactObservation && parsed.action !== null) { totalActions += 1; actionCounts.set(parsed.action.kind, (actionCounts.get(parsed.action.kind) ?? 0) + 1); if (parsed.action.kind === "Wait") { @@ -1062,7 +1081,7 @@ export async function summarizeDirectBombadilTrace(options: { (targetTags.get(parsed.action.targetTag) ?? 0) + 1, ); } - } else { + } else if (actionFollowsExactObservation) { waitStreak = 0; } @@ -1096,7 +1115,8 @@ export async function summarizeDirectBombadilTrace(options: { snapshots.set(snapshot.name, entry); } if ( - parsed.action !== null + actionFollowsExactObservation + && parsed.action !== null && parsed.action.kind !== "Wait" && entry.lastValueSha256 !== null && entry.lastValueSha256 !== snapshot.valueSha256 @@ -1126,6 +1146,7 @@ export async function summarizeDirectBombadilTrace(options: { parsed.state.resources[sourceName as keyof typeof RESOURCE_FIELD_MAP], ); } + previousObservationWasExact = currentObservationIsExact; } } finally { lines.close(); @@ -1183,8 +1204,8 @@ export async function summarizeDirectBombadilTrace(options: { }), urls: Object.freeze({ distinctFingerprintCount: urlFingerprints.size, - fingerprintSha256: Object.freeze([...urlFingerprints].sort()), - observationCount: lineCount, + fingerprintSha256: Object.freeze([...urlFingerprints].sort(compareCodeUnits)), + observationCount: policyObservationCount, stableTarget, }), transitions: Object.freeze({ @@ -1192,11 +1213,11 @@ export async function summarizeDirectBombadilTrace(options: { nonNullHashCount, }), namedSnapshots: Object.freeze([...snapshots.entries()] - .sort(([left], [right]) => left.localeCompare(right)) + .sort(([left], [right]) => compareCodeUnits(left, right)) .map(([name, entry]) => Object.freeze({ changeAfterNonWaitCount: entry.changeAfterNonWaitCount, distinctValueCount: entry.values.size, - distinctValueSha256: Object.freeze([...entry.values].sort()), + distinctValueSha256: Object.freeze([...entry.values].sort(compareCodeUnits)), name, observationCount: entry.observationCount, }))), @@ -1325,7 +1346,7 @@ function validateTargetQuery(value: unknown): Readonly> { } const validated: Record = {}; for (const [name, queryValue] of [...entries].sort(([left], [right]) => - left.localeCompare(right) + compareCodeUnits(left, right) )) { if ( name.length === 0 @@ -1410,7 +1431,7 @@ function validateSnapshotMinimumMap(options: { } const validated: Record = {}; for (const [rawName, minimum] of Object.entries(options.value).sort(([left], [right]) => - left.localeCompare(right) + compareCodeUnits(left, right) )) { const name = validateSnapshotName(rawName, `${options.label} key`); if ( @@ -1462,7 +1483,7 @@ function validateExplorationPolicy( ) { throw new Error("explorationPolicy.requiredActionKinds contains an unknown or duplicate kind"); } - requiredActionKinds.sort(); + requiredActionKinds.sort(compareCodeUnits); const requiredNamedSnapshotsInput = value.requiredNamedSnapshots ?? []; if (!Array.isArray(requiredNamedSnapshotsInput) || requiredNamedSnapshotsInput.length > 32) { @@ -1474,7 +1495,7 @@ function validateExplorationPolicy( if (new Set(requiredNamedSnapshots).size !== requiredNamedSnapshots.length) { throw new Error("explorationPolicy.requiredNamedSnapshots contains a duplicate name"); } - requiredNamedSnapshots.sort(); + requiredNamedSnapshots.sort(compareCodeUnits); const minDistinctNamedSnapshotValues = validateSnapshotMinimumMap({ label: "explorationPolicy.minDistinctNamedSnapshotValues", From 6ccc6bb4cd3b75cb9a57415eddf8af048f183cce Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 12:53:19 -0400 Subject: [PATCH 11/20] fix: remove stale Bombadil test helper --- src/tooling/bombadil-campaign.test.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 86c94cd..2f47923 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -167,11 +167,6 @@ function evaluate(body: unknown): boolean { return (body as () => boolean)(); } -function requireFormula(value: boolean | FakeFormula): FakeFormula { - if (typeof value === "boolean") throw new Error("Expected a nested formula"); - return value; -} - describe("Direct Bombadil observation", () => { test("accepts the exact current bridge, manifest, and probe contract", () => { const observation = readDirectBombadilObservation(contractFixture()); From 89dfd30e1036ad0fc52c355a944315a4d131b423 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:00:51 -0400 Subject: [PATCH 12/20] fix: accept null Bombadil snapshot values --- docs/verification.md | 2 +- src/tooling/bombadil-campaign.test.ts | 10 +++++----- src/tooling/bombadil-campaign.ts | 16 ++++++++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/verification.md b/docs/verification.md index cb765f9..2aa45be 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -272,7 +272,7 @@ const direct = createDirectBombadilProperties(); const phase = createDirectBombadilNamedSnapshot({ fallback: "unavailable", name: "todos.phase", - parse: (value) => typeof value === "string" ? value : null, + parse: (value) => typeof value === "string" ? value : undefined, read: ({ window }) => Reflect.get(window, "__todosPhase"), }); diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 2f47923..027bc8f 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -242,10 +242,10 @@ describe("Direct Bombadil named snapshots", () => { name: "product.phase", parse: (value) => { if (typeof value !== "object" || value === null || Array.isArray(value)) { - return null; + return undefined; } const status = Reflect.get(value, "status"); - return typeof status === "string" ? { status } : null; + return typeof status === "string" ? { status } : undefined; }, read: (state) => Reflect.get(state.window, "phase"), }) as unknown as FakeCell; @@ -275,10 +275,10 @@ describe("Direct Bombadil named snapshots", () => { name: "product.compat", parse: (value) => { if (typeof value !== "object" || value === null || Array.isArray(value)) { - return null; + return undefined; } const status = Reflect.get(value, "status"); - return typeof status === "string" ? { status } : null; + return typeof status === "string" ? { status } : undefined; }, read: (state) => Reflect.get(state.window, "phase"), }) as unknown as FakeCell; @@ -362,7 +362,7 @@ describe("Direct Bombadil named snapshots", () => { expect(() => createDirectBombadilNamedSnapshot({ fallback: null, name: "safe", - parse: () => null, + parse: () => undefined, read: () => null, })).toThrow("accepted by parse"); expect(() => createDirectBombadilNamedSnapshot({ diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index 8cf4512..6d44f2b 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -332,7 +332,7 @@ function boundedNamedSnapshotJson(value: unknown): BombadilJson | undefined { export function createDirectBombadilNamedSnapshot(options: { readonly fallback: T; readonly name: string; - readonly parse: (value: BombadilJson) => T | null; + readonly parse: (value: BombadilJson) => T | undefined; readonly read: (state: BombadilBrowserState) => unknown; }): Cell { if ( @@ -345,21 +345,21 @@ export function createDirectBombadilNamedSnapshot(option "Bombadil snapshot name must be a safe, unreserved 1-128 character identifier", ); } - const parse = (value: unknown): T | null => { + const parse = (value: unknown): T | undefined => { const cloned = boundedNamedSnapshotJson(value); - if (cloned === undefined) return null; + if (cloned === undefined) return undefined; const parsed = options.parse(cloned); - return parsed !== null && boundedNamedSnapshotJson(parsed) !== undefined + return parsed !== undefined && boundedNamedSnapshotJson(parsed) !== undefined ? parsed - : null; + : undefined; }; - let fallback: T | null = null; + let fallback: T | undefined; try { fallback = parse(options.fallback); } catch { - fallback = null; + fallback = undefined; } - if (fallback === null) { + if (fallback === undefined) { throw new Error( "Bombadil snapshot fallback must be bounded JSON accepted by parse", ); From c06b641aa5aaac59ae47831e51756955123d1aa6 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:02:16 -0400 Subject: [PATCH 13/20] fix: own parsed Bombadil snapshots --- src/tooling/bombadil-campaign.test.ts | 19 +++++++++++++++++-- src/tooling/bombadil-campaign.ts | 6 +++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 027bc8f..c9d4f2d 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -270,6 +270,14 @@ describe("Direct Bombadil named snapshots", () => { }); test("produces a named value accepted by the host summary contract", async () => { + class MutableParsedValue { + status = "ready"; + + toJSON() { + return { status: "é".repeat(1_100_000) }; + } + } + const parserOutput = new MutableParsedValue(); const snapshot = createDirectBombadilNamedSnapshot({ fallback: { status: "unavailable" }, name: "product.compat", @@ -278,11 +286,18 @@ describe("Direct Bombadil named snapshots", () => { return undefined; } const status = Reflect.get(value, "status"); - return typeof status === "string" ? { status } : undefined; + return typeof status === "string" ? parserOutput : undefined; }, read: (state) => Reflect.get(state.window, "phase"), }) as unknown as FakeCell; - const value = snapshot.read({ window: { phase: { status: "ready" } } }); + const value = snapshot.read({ + window: { phase: { status: "ready" } }, + }) as { status: string }; + expect(value).toEqual({ status: "ready" }); + expect(value).not.toBe(parserOutput); + expect(Object.getPrototypeOf(value)).toBe(Object.prototype); + parserOutput.status = "mutated"; + expect(value).toEqual({ status: "ready" }); const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-helper-summary-")); const tracePath = join(directory, "trace.jsonl"); try { diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index 6d44f2b..c564f76 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -349,9 +349,9 @@ export function createDirectBombadilNamedSnapshot(option const cloned = boundedNamedSnapshotJson(value); if (cloned === undefined) return undefined; const parsed = options.parse(cloned); - return parsed !== undefined && boundedNamedSnapshotJson(parsed) !== undefined - ? parsed - : undefined; + if (parsed === undefined) return undefined; + const owned = boundedNamedSnapshotJson(parsed); + return owned === undefined ? undefined : owned as T; }; let fallback: T | undefined; try { From 9c63cd4672b544580b70da6d37feec2310d64ad7 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:05:09 -0400 Subject: [PATCH 14/20] fix: validate owned Bombadil JSON --- docs/verification.md | 2 +- src/tooling/bombadil-campaign.test.ts | 50 +++++++++++++-------------- src/tooling/bombadil-campaign.ts | 18 ++++------ 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/docs/verification.md b/docs/verification.md index 2aa45be..0ac8180 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -272,8 +272,8 @@ const direct = createDirectBombadilProperties(); const phase = createDirectBombadilNamedSnapshot({ fallback: "unavailable", name: "todos.phase", - parse: (value) => typeof value === "string" ? value : undefined, read: ({ window }) => Reflect.get(window, "__todosPhase"), + validate: (value): value is string => typeof value === "string", }); export const direct_safe_actions = createDirectBombadilActions(); diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index c9d4f2d..a469a25 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -240,14 +240,13 @@ describe("Direct Bombadil named snapshots", () => { const snapshot = createDirectBombadilNamedSnapshot({ fallback: { status: "unavailable" }, name: "product.phase", - parse: (value) => { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - return undefined; - } - const status = Reflect.get(value, "status"); - return typeof status === "string" ? { status } : undefined; - }, read: (state) => Reflect.get(state.window, "phase"), + validate: (value): value is { readonly status: string } => ( + typeof value === "object" + && value !== null + && !Array.isArray(value) + && typeof Reflect.get(value, "status") === "string" + ), }) as unknown as FakeCell; expect(snapshot.name).toBe("product.phase"); expect(snapshot.read({ window: { phase: { status: "ready" } } })).toEqual({ @@ -270,33 +269,32 @@ describe("Direct Bombadil named snapshots", () => { }); test("produces a named value accepted by the host summary contract", async () => { - class MutableParsedValue { + class MutablePageValue { status = "ready"; toJSON() { return { status: "é".repeat(1_100_000) }; } } - const parserOutput = new MutableParsedValue(); + const pageValue = new MutablePageValue(); const snapshot = createDirectBombadilNamedSnapshot({ fallback: { status: "unavailable" }, name: "product.compat", - parse: (value) => { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - return undefined; - } - const status = Reflect.get(value, "status"); - return typeof status === "string" ? parserOutput : undefined; - }, - read: (state) => Reflect.get(state.window, "phase"), + read: () => pageValue, + validate: (value): value is { readonly status: string } => ( + typeof value === "object" + && value !== null + && !Array.isArray(value) + && typeof Reflect.get(value, "status") === "string" + ), }) as unknown as FakeCell; const value = snapshot.read({ window: { phase: { status: "ready" } }, }) as { status: string }; expect(value).toEqual({ status: "ready" }); - expect(value).not.toBe(parserOutput); + expect(value).not.toBe(pageValue); expect(Object.getPrototypeOf(value)).toBe(Object.prototype); - parserOutput.status = "mutated"; + pageValue.status = "mutated"; expect(value).toEqual({ status: "ready" }); const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-helper-summary-")); const tracePath = join(directory, "trace.jsonl"); @@ -355,16 +353,16 @@ describe("Direct Bombadil named snapshots", () => { expect(() => createDirectBombadilNamedSnapshot({ fallback: null, name, - parse: (value) => value, read: () => null, + validate: (_value): _value is BombadilJson => true, })).toThrow("safe, unreserved"); } const snapshot = createDirectBombadilNamedSnapshot({ fallback: null, name: "safe", - parse: (value) => value, read: (state) => Reflect.get(state.window, "phase"), + validate: (_value): _value is BombadilJson => true, }) as unknown as FakeCell; let atLimit: BombadilJson = null; for (let index = 0; index < 64; index += 1) atLimit = [atLimit]; @@ -373,19 +371,19 @@ describe("Direct Bombadil named snapshots", () => { expect(snapshot.read({ window: { phase: beyondLimit } })).toBeNull(); }); - test("rejects non-JSON or parser-invalid fallbacks before registering an extractor", () => { + test("rejects non-JSON or predicate-invalid fallbacks before registering an extractor", () => { expect(() => createDirectBombadilNamedSnapshot({ fallback: null, name: "safe", - parse: () => undefined, read: () => null, - })).toThrow("accepted by parse"); + validate: (_value): _value is never => false, + })).toThrow("accepted by validate"); expect(() => createDirectBombadilNamedSnapshot({ fallback: undefined as never, name: "safe", - parse: (value) => value, read: () => null, - })).toThrow("fallback must be bounded JSON accepted by parse"); + validate: (_value): _value is BombadilJson => true, + })).toThrow("fallback must be bounded JSON accepted by validate"); }); }); diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index c564f76..85d3010 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -332,8 +332,8 @@ function boundedNamedSnapshotJson(value: unknown): BombadilJson | undefined { export function createDirectBombadilNamedSnapshot(options: { readonly fallback: T; readonly name: string; - readonly parse: (value: BombadilJson) => T | undefined; readonly read: (state: BombadilBrowserState) => unknown; + readonly validate: (value: BombadilJson) => value is T; }): Cell { if ( options.name.length === 0 @@ -345,28 +345,24 @@ export function createDirectBombadilNamedSnapshot(option "Bombadil snapshot name must be a safe, unreserved 1-128 character identifier", ); } - const parse = (value: unknown): T | undefined => { - const cloned = boundedNamedSnapshotJson(value); - if (cloned === undefined) return undefined; - const parsed = options.parse(cloned); - if (parsed === undefined) return undefined; - const owned = boundedNamedSnapshotJson(parsed); - return owned === undefined ? undefined : owned as T; + const validate = (value: unknown): T | undefined => { + const owned = boundedNamedSnapshotJson(value); + return owned !== undefined && options.validate(owned) ? owned : undefined; }; let fallback: T | undefined; try { - fallback = parse(options.fallback); + fallback = validate(options.fallback); } catch { fallback = undefined; } if (fallback === undefined) { throw new Error( - "Bombadil snapshot fallback must be bounded JSON accepted by parse", + "Bombadil snapshot fallback must be bounded JSON accepted by validate", ); } return extract((state) => { try { - return parse(options.read(state)) ?? fallback; + return validate(options.read(state)) ?? fallback; } catch { return fallback; } From 08a6cd8a0596a01800e17c8553ecac509eb98b60 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:11:27 -0400 Subject: [PATCH 15/20] fix: name Bombadil test predicates --- src/tooling/bombadil-campaign.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index a469a25..812be74 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -167,6 +167,14 @@ function evaluate(body: unknown): boolean { return (body as () => boolean)(); } +function acceptBombadilJson(value: BombadilJson): value is BombadilJson { + return value !== undefined; +} + +function rejectBombadilJson(value: BombadilJson): value is never { + return value === undefined; +} + describe("Direct Bombadil observation", () => { test("accepts the exact current bridge, manifest, and probe contract", () => { const observation = readDirectBombadilObservation(contractFixture()); @@ -354,7 +362,7 @@ describe("Direct Bombadil named snapshots", () => { fallback: null, name, read: () => null, - validate: (_value): _value is BombadilJson => true, + validate: acceptBombadilJson, })).toThrow("safe, unreserved"); } @@ -362,7 +370,7 @@ describe("Direct Bombadil named snapshots", () => { fallback: null, name: "safe", read: (state) => Reflect.get(state.window, "phase"), - validate: (_value): _value is BombadilJson => true, + validate: acceptBombadilJson, }) as unknown as FakeCell; let atLimit: BombadilJson = null; for (let index = 0; index < 64; index += 1) atLimit = [atLimit]; @@ -376,13 +384,13 @@ describe("Direct Bombadil named snapshots", () => { fallback: null, name: "safe", read: () => null, - validate: (_value): _value is never => false, + validate: rejectBombadilJson, })).toThrow("accepted by validate"); expect(() => createDirectBombadilNamedSnapshot({ fallback: undefined as never, name: "safe", read: () => null, - validate: (_value): _value is BombadilJson => true, + validate: acceptBombadilJson, })).toThrow("fallback must be bounded JSON accepted by validate"); }); }); From ffee66bf5f8a6cc69b72f5856c39b9482c230d01 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:29:17 -0400 Subject: [PATCH 16/20] feat: attribute Bombadil exploration actions --- README.md | 6 +- docs/verification.md | 29 ++- src/tooling/bombadil-runner.test.ts | 297 ++++++++++++++++++++++- src/tooling/bombadil-runner.ts | 363 ++++++++++++++++++++++++---- 4 files changed, 628 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index b461193..0f1f7be 100644 --- a/README.md +++ b/README.md @@ -290,8 +290,10 @@ snapshots that expose semantic state without retaining page content. Run short scheduled diagnostic lane. Inspect and replay a retained failing trace, then promote the smallest readable failure to a deterministic product regression. When a campaign must exercise an interaction, require a named product value to -change after a non-Wait action so bootstrap and idle transitions do not satisfy -the exploration policy. +change after the intended action kind, as well as after a non-Wait action, so +bootstrap, idle, prerequisite, and unrelated transitions do not satisfy the +exploration policy. Attribution requires adjacent exact Direct observations; +it is temporal response evidence rather than proof of causality. If the full product snapshot includes viewport dimensions, put that requirement on a separate interaction snapshot without viewport fields and require an opposite-size `SetViewport` independently. Latch the first ready product state diff --git a/docs/verification.md b/docs/verification.md index 0ac8180..f2a4252 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -310,9 +310,10 @@ scenario boundary and exercises behavior the product can interpret. `createDirectBombadilNamedSnapshot` gives product properties and post-run diagnostics a small semantic signal. It requires a safe bounded name distinct -from `direct` and JavaScript prototype names, an explicit product parser, at +from `direct` and JavaScript prototype names, an explicit product type +predicate over Direct's owned plain-JSON clone, at most 64 JSON levels, and at most 2 MiB of UTF-8 JSON. It fails closed to the -parsed fallback when a page getter throws or returns unsuitable data. Extract +validated fallback when a page getter throws or returns unsuitable data. Extract state, not page content or credentials. Named values are represented only by canonical SHA-256 hashes in the exploration summary; the authoritative raw trace still contains the original values. @@ -355,6 +356,9 @@ await runDirectBombadilFuzz({ minNonWaitActions: 1, requiredNamedSnapshots: ["direct", "todos.phase"], minDistinctNamedSnapshotValues: { "todos.phase": 2 }, + minNamedSnapshotChangesAfterActionKind: { + "todos.phase": { Click: 1 }, + }, minNamedSnapshotChangesAfterNonWait: { "todos.phase": 1 }, requireStableTargetUrl: true, }, @@ -399,11 +403,15 @@ responsive behavior that needs separate exploration. `explorationPolicy` is optional. When present, it can require a minimum count of non-Wait actions, particular action kinds and named snapshots, minimum distinct hashed values for named snapshots, named-value changes observed after -a non-Wait action, and an exact stable target URL. The change requirement keeps +a non-Wait action or one particular action kind, and an exact stable target +URL. The change requirement keeps bootstrap or Wait-only transitions from satisfying a product-interaction -threshold. The trace records the last action with the resulting state, so this -count identifies a named value transition associated with an active product -action. It does not prove causality. The policy detects a campaign that passed +threshold. Per-kind attribution requires exact Direct observations and the +same named snapshot in both immediately adjacent samples. The trace records +the last action with the resulting state, so this count identifies a named +value transition associated with an active product action. It does not prove +causality. The runner strictly validates every 0.7.2 action payload before +crediting its kind. The policy detects a campaign that passed its properties without doing the intended exploration. It is a diagnostic sufficiency check, not a coverage claim. Start with observed stable behavior and raise thresholds only when the action generator makes them reliably @@ -500,12 +508,15 @@ Each attempt writes `run.json`, `exploration-summary.json`, `bombadil.log`, and `artifacts/direct-bombadil///`, including failures. The rolling `manifest.json` points to the latest record. `rawTracePath` reports a regular nonempty trace even if attestation fails; `tracePath` is present only -after exact attestation. The summary strictly parses the 0.7.2 envelopes and +after exact attestation. The v2 summary strictly parses the 0.7.2 envelopes and records the raw trace SHA-256, action-kind and safe target-tag counts, non-Wait count and longest Wait streak, origin-relative URL fingerprints, non-null transition-hash cardinality, canonical named-snapshot value hashes, property -violation names and counts, named-value changes after non-Wait actions, and -browser resource high-water marks. It excludes +violation names and counts, named-value changes after non-Wait actions and +specific action kinds, and browser resource high-water marks. Action, URL, +transition, and named-snapshot policy evidence starts at the first exact Direct +observation. Separate raw URL and transition hashes, property violations, and +resource maxima retain strictly parsed startup-prefix diagnostics. It excludes typed text, accessible names, snapshot values, URLs, screenshots, and absolute paths. These diagnostics describe what Bombadil happened to explore. They do not measure code, state, interaction, or Direct catalog coverage, and the raw diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index d7711f9..ace1c47 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -460,6 +460,12 @@ describe("Direct Bombadil configuration and invocation", () => { A: 1, "a-": 2, }, + minNamedSnapshotChangesAfterActionKind: { + z: { Wait: 4, Click: 3 }, + "a.": { SetViewport: 3 }, + A: { Wait: 1, Click: 2 }, + "a-": { TypeText: 2 }, + }, minNamedSnapshotChangesAfterNonWait: { z: 4, "a.": 3, @@ -483,6 +489,12 @@ describe("Direct Bombadil configuration and invocation", () => { expect(Object.keys( validated.explorationPolicy?.minDistinctNamedSnapshotValues ?? {}, )).toEqual(expectedNames); + expect(Object.keys( + validated.explorationPolicy?.minNamedSnapshotChangesAfterActionKind ?? {}, + )).toEqual(expectedNames); + expect(Object.keys( + validated.explorationPolicy?.minNamedSnapshotChangesAfterActionKind.A ?? {}, + )).toEqual(["Click", "Wait"]); expect(Object.keys( validated.explorationPolicy?.minNamedSnapshotChangesAfterNonWait ?? {}, )).toEqual(expectedNames); @@ -546,6 +558,30 @@ describe("Direct Bombadil configuration and invocation", () => { ...config, explorationPolicy: { minNamedSnapshotChangesAfterNonWait: { phase: 10_001 } }, })).toThrow("between 1 and 10000"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { phase: {} }, + }, + })).toThrow("bounded action map"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { Unknown: 1 } as never, + }, + }, + })).toThrow("unknown action kind"); + for (const minimum of [0, 10_001]) { + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { Click: minimum }, + }, + }, + })).toThrow("between 1 and 10000"); + } }); test("rejects specification, server cwd, and replay symlinks that escape the repository", async () => { @@ -871,6 +907,7 @@ describe("Direct Bombadil exploration summary", () => { const options = { explorationPolicy: { minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterActionKind: { phase: { Click: 1 } }, minNamedSnapshotChangesAfterNonWait: { phase: 1 }, minNonWaitActions: 2, requireStableTargetUrl: true, @@ -884,7 +921,7 @@ describe("Direct Bombadil exploration summary", () => { const second = await summarizeDirectBombadilTrace(options); expect(second).toEqual(first); expect(first).toMatchObject({ - schema: "direct.bombadil-exploration-summary/v1", + schema: "direct.bombadil-exploration-summary/v2", actions: { byKind: { Click: 1, TypeText: 1, Wait: 1 }, maxWaitStreak: 1, @@ -895,9 +932,16 @@ describe("Direct Bombadil exploration summary", () => { urls: { distinctFingerprintCount: 1, observationCount: 4, + rawDistinctFingerprintCount: 1, + rawObservationCount: 4, stableTarget: true, }, - transitions: { distinctNonNullHashCount: 4, nonNullHashCount: 4 }, + transitions: { + distinctNonNullHashCount: 4, + nonNullHashCount: 4, + rawDistinctNonNullHashCount: 4, + rawNonNullHashCount: 4, + }, propertyViolations: { byName: { noConsoleErrors: 1 }, total: 1 }, resourceHighWaterMarks: { domNodes: 4, @@ -906,6 +950,7 @@ describe("Direct Bombadil exploration summary", () => { policy: { configured: true, failures: [], satisfied: true }, }); expect(first.namedSnapshots.find((entry) => entry.name === "phase")).toMatchObject({ + changeAfterActionKind: { Click: 1 }, changeAfterNonWaitCount: 1, distinctValueCount: 2, observationCount: 4, @@ -919,6 +964,82 @@ describe("Direct Bombadil exploration summary", () => { expect(serialized).not.toContain("/tmp/"); }); + test("accepts every exact Bombadil 0.7.2 browser action payload", async () => { + const actions = [ + "Back", + { + Click: { + fingerprint: { accessible_name: "private click label", tag: "button" }, + point: { x: 1, y: 2 }, + }, + }, + { + DoubleClick: { + delay_millis: 10, + fingerprint: { + structural_path: "private/path", + tag: "x-private.widget_\u00e9", + }, + point: { x: 2, y: 3 }, + }, + }, + "Forward", + { + MouseDrag: { + delay_millis: 5, + from: { x: 3, y: 4 }, + steps: 10, + to: { x: 30, y: 40 }, + }, + }, + { PressKey: { code: 13 } }, + "Reload", + { ScrollDown: { distance: 100, origin: { x: 5, y: 6 } } }, + { ScrollUp: { distance: 100, origin: { x: 5, y: 6 } } }, + { + SetFileInputFiles: { + files: ["/private/file.txt"], + selector: "#private-file-input", + }, + }, + { SetViewport: { height: 720, width: 1_280 } }, + { TypeText: { delay_millis: 5, text: "private typed text" } }, + "Wait", + ] as const; + const observation = directObservation(); + const tracePath = await summaryTrace([ + traceLine(observation, 1), + ...actions.map((action, index) => traceLine(observation, index + 2, { + action, + })), + ]); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(Object.keys(summary.actions.byKind)).toEqual([ + "Back", + "Click", + "DoubleClick", + "Forward", + "MouseDrag", + "PressKey", + "Reload", + "ScrollDown", + "ScrollUp", + "SetFileInputFiles", + "SetViewport", + "TypeText", + "Wait", + ]); + expect(summary.actions.total).toBe(actions.length); + expect(Object.keys(summary.actions.targetTags)).toContain("button"); + expect(Object.keys(summary.actions.targetTags).some((tag) => + /^sha256:[a-f0-9]{64}$/u.test(tag) + )).toBeTrue(); + expect(JSON.stringify(summary)).not.toContain("private"); + }); + test("reports strict policy misses while keeping the diagnostic summary", async () => { const tracePath = await summaryTrace([traceLine(directObservation(), 1)]); const summary = await summarizeDirectBombadilTrace({ @@ -965,10 +1086,133 @@ describe("Direct Bombadil exploration summary", () => { }); }); + test("attributes named changes to the exact action kind", async () => { + const observation = directObservation(); + const noOpClickTrace = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(observation, 2, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(observation, 3, { + action: { SetViewport: { height: 720, width: 1_280 } }, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + ]); + const failed = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { Click: 1, SetViewport: 1 }, + }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: noOpClickTrace, + }); + expect(failed.namedSnapshots.find((entry) => entry.name === "phase")) + .toMatchObject({ + changeAfterActionKind: { SetViewport: 1 }, + changeAfterNonWaitCount: 1, + }); + expect(failed.policy).toMatchObject({ + failures: [expect.stringContaining("post-Click change minimum")], + satisfied: false, + }); + + const attributedTrace = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(observation, 2, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + traceLine(observation, 3, { + action: { SetViewport: { height: 720, width: 1_280 } }, + namedSnapshots: [{ name: "phase", value: 2 }], + }), + ]); + const attributed = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { SetViewport: 1, Click: 1 }, + }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: attributedTrace, + }); + const phase = attributed.namedSnapshots.find((entry) => entry.name === "phase"); + expect(phase?.changeAfterActionKind).toEqual({ Click: 1, SetViewport: 1 }); + expect(phase?.changeAfterNonWaitCount).toBe(2); + expect(phase?.changeAfterNonWaitCount).toBe( + Object.entries(phase?.changeAfterActionKind ?? {}) + .filter(([kind]) => kind !== "Wait") + .reduce((total, [, count]) => total + count, 0), + ); + expect(attributed.policy.satisfied).toBeTrue(); + }); + + test("requires adjacent exact observations for action attribution", async () => { + const exact = directObservation(); + const click = { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + } as const; + const missingSnapshotTrace = await summaryTrace([ + traceLine(exact, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(exact, 2), + traceLine(exact, 3, { + action: click, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + ]); + const invalidCurrentTrace = await summaryTrace([ + traceLine(exact, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(absentObservation(), 2, { + action: click, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + ]); + for (const tracePath of [missingSnapshotTrace, invalidCurrentTrace]) { + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { phase: { Click: 1 } }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.find((entry) => entry.name === "phase") + ?.changeAfterActionKind).toEqual({}); + expect(summary.policy.satisfied).toBeFalse(); + } + }); + test("uses the first exact Direct observation only as the policy baseline", async () => { const tracePath = await summaryTrace([ - traceLine(absentObservation(), 1, { + traceLine(absentObservation(), 20, { namedSnapshots: [{ name: "phase", value: "loading" }], + violations: [{ + name: "startup_contract", + violation: { False: {} }, + }], }), traceLine(absentObservation(), 2, { action: { @@ -992,6 +1236,7 @@ describe("Direct Bombadil exploration summary", () => { const summary = await summarizeDirectBombadilTrace({ explorationPolicy: { minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterActionKind: { phase: { Click: 1 } }, minNamedSnapshotChangesAfterNonWait: { phase: 1 }, minNonWaitActions: 1, requiredActionKinds: ["Click"], @@ -1006,10 +1251,18 @@ describe("Direct Bombadil exploration summary", () => { total: 0, }); expect(summary.urls.observationCount).toBe(1); + expect(summary.urls.rawObservationCount).toBe(3); expect(summary.transitions).toEqual({ distinctNonNullHashCount: 1, nonNullHashCount: 1, + rawDistinctNonNullHashCount: 3, + rawNonNullHashCount: 3, + }); + expect(summary.propertyViolations).toEqual({ + byName: { startup_contract: 1 }, + total: 1, }); + expect(summary.resourceHighWaterMarks.domNodes).toBe(20); expect(summary.namedSnapshots.find((entry) => entry.name === "phase")) .toMatchObject({ changeAfterNonWaitCount: 0, @@ -1022,6 +1275,7 @@ describe("Direct Bombadil exploration summary", () => { expect.stringContaining("required action kind Click"), expect.stringContaining("distinct-value minimum"), expect.stringContaining("post-non-Wait change minimum"), + expect.stringContaining("post-Click change minimum"), ], satisfied: false, }); @@ -1065,13 +1319,42 @@ describe("Direct Bombadil exploration summary", () => { test("rejects hostile envelopes, action targets, and excessively deep snapshot JSON", async () => { const observation = directObservation(); const badTarget = await summaryTrace([traceLine(observation, 1, { - action: { Click: { fingerprint: { tag: "unsafe tag" } } }, + action: { + Click: { + fingerprint: { tag: "unsafe tag" }, + point: { x: 1, y: 1 }, + }, + }, })]); expect((await rejection(summarizeDirectBombadilTrace({ targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", tracePath: badTarget, }))).message).toContain("invalid action target tag"); + for (const action of [ + { PressKey: {} }, + { PressKey: { code: 256 } }, + { SetViewport: {} }, + { SetViewport: { height: 720, width: 0 } }, + { + MouseDrag: { + delay_millis: 0, + from: { x: 0, y: 0 }, + steps: 0, + to: { x: 1, y: 1 }, + }, + }, + { TypeText: { delay_millis: 0, text: "safe", unexpected: true } }, + ]) { + const malformedAction = await summaryTrace([traceLine(observation, 1, { + action, + })]); + expect((await rejection(summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: malformedAction, + }))).message).toContain("invalid action"); + } + let deep: unknown = null; for (let index = 0; index < 70; index += 1) deep = [deep]; const deepTrace = await summaryTrace([traceLine(observation, 1, { @@ -1218,7 +1501,7 @@ describe("Direct Bombadil run lifecycle", () => { initialDirect: { source: "scenario", scenario: "surface.ready", route: "/surface" }, server: { logPresent: true }, explorationSummary: { - schema: "direct.bombadil-exploration-summary/v1", + schema: "direct.bombadil-exploration-summary/v2", policy: { configured: false, satisfied: true }, }, viewport: { deviceScaleFactor: 2, height: 768, width: 1_024 }, @@ -1234,7 +1517,7 @@ describe("Direct Bombadil run lifecycle", () => { expect(await readFile(String(server.logPath), "utf8")).toContain("server output"); const summaryPath = String(manifest.explorationSummaryPath); expect(JSON.parse(await readFile(summaryPath, "utf8"))).toMatchObject({ - schema: "direct.bombadil-exploration-summary/v1", + schema: "direct.bombadil-exploration-summary/v2", trace: { lineCount: 2 }, }); }); @@ -1260,7 +1543,7 @@ describe("Direct Bombadil run lifecycle", () => { }, }); expect(await readFile(String(manifest.explorationSummaryPath), "utf8")) - .toContain("direct.bombadil-exploration-summary/v1"); + .toContain("direct.bombadil-exploration-summary/v2"); }); test("retains an attested failure artifact and server log for a nonzero exit", async () => { diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index b53c3a7..ddad076 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -79,9 +79,41 @@ const TRACE_RESOURCE_KEYS = new Set([ "timestamp", ]); const TRACE_VIOLATION_KEYS = new Set(["name", "violation"]); +const TRACE_POINT_KEYS = new Set(["x", "y"]); +const TRACE_FINGERPRINT_KEYS = new Set([ + "accessible_name", + "href", + "id", + "input_type", + "name_attr", + "placeholder", + "role", + "structural_path", + "tag", + "test_id", + "text_content", +]); +const TRACE_CLICK_ACTION_KEYS = new Set(["fingerprint", "point"]); +const TRACE_DOUBLE_CLICK_ACTION_KEYS = new Set([ + "delay_millis", + "fingerprint", + "point", +]); +const TRACE_TYPE_TEXT_ACTION_KEYS = new Set(["delay_millis", "text"]); +const TRACE_PRESS_KEY_ACTION_KEYS = new Set(["code"]); +const TRACE_SCROLL_ACTION_KEYS = new Set(["distance", "origin"]); +const TRACE_FILE_INPUT_ACTION_KEYS = new Set(["files", "selector"]); +const TRACE_MOUSE_DRAG_ACTION_KEYS = new Set([ + "delay_millis", + "from", + "steps", + "to", +]); +const TRACE_VIEWPORT_ACTION_KEYS = new Set(["height", "width"]); const VIEWPORT_KEYS = new Set(["deviceScaleFactor", "height", "width"]); const EXPLORATION_POLICY_KEYS = new Set([ "minDistinctNamedSnapshotValues", + "minNamedSnapshotChangesAfterActionKind", "minNamedSnapshotChangesAfterNonWait", "minNonWaitActions", "requireStableTargetUrl", @@ -115,10 +147,6 @@ const UNIT_ACTION_KINDS = new Set([ "Reload", "Wait", ]); -const TARGETED_ACTION_KINDS = new Set([ - "Click", - "DoubleClick", -]); const DIRECT_OBSERVATION_KEYS = new Set([ "activationHash", "activeRoute", @@ -153,6 +181,10 @@ export interface DirectBombadilViewportConfig { export interface DirectBombadilExplorationPolicy { readonly minDistinctNamedSnapshotValues?: Readonly>; + readonly minNamedSnapshotChangesAfterActionKind?: Readonly>> + >>; readonly minNamedSnapshotChangesAfterNonWait?: Readonly>; readonly minNonWaitActions?: number; readonly requireStableTargetUrl?: boolean; @@ -161,7 +193,7 @@ export interface DirectBombadilExplorationPolicy { } export interface DirectBombadilExplorationSummary { - readonly schema: "direct.bombadil-exploration-summary/v1"; + readonly schema: "direct.bombadil-exploration-summary/v2"; readonly trace: { readonly bytes: number; readonly lineCount: number; @@ -178,13 +210,21 @@ export interface DirectBombadilExplorationSummary { readonly distinctFingerprintCount: number; readonly fingerprintSha256: readonly string[]; readonly observationCount: number; + readonly rawDistinctFingerprintCount: number; + readonly rawFingerprintSha256: readonly string[]; + readonly rawObservationCount: number; readonly stableTarget: boolean; }; readonly transitions: { readonly distinctNonNullHashCount: number; readonly nonNullHashCount: number; + readonly rawDistinctNonNullHashCount: number; + readonly rawNonNullHashCount: number; }; readonly namedSnapshots: readonly { + readonly changeAfterActionKind: Readonly< + Partial> + >; readonly changeAfterNonWaitCount: number; readonly distinctValueCount: number; readonly distinctValueSha256: readonly string[]; @@ -364,6 +404,10 @@ interface ValidatedViewport { interface ValidatedExplorationPolicy { readonly minDistinctNamedSnapshotValues: Readonly>; + readonly minNamedSnapshotChangesAfterActionKind: Readonly>> + >>; readonly minNamedSnapshotChangesAfterNonWait: Readonly>; readonly minNonWaitActions: number; readonly requireStableTargetUrl: boolean; @@ -676,43 +720,163 @@ function namedSnapshotValueSha256(value: unknown): string { return sha256(canonical); } +function validTracePoint(value: unknown): boolean { + return isRecord(value) + && hasExactKeys(value, TRACE_POINT_KEYS) + && typeof value.x === "number" + && Number.isFinite(value.x) + && typeof value.y === "number" + && Number.isFinite(value.y); +} + +function parseTraceFingerprintTag(value: unknown, lineNumber: number): string { + if ( + !isRecord(value) + || !Object.keys(value).every((key) => TRACE_FINGERPRINT_KEYS.has(key)) + ) { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target`, + ); + } + for (const [key, candidate] of Object.entries(value)) { + if (key !== "tag" && typeof candidate !== "string") { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target`, + ); + } + } + const tag = value.tag; + if ( + typeof tag !== "string" + || tag.length === 0 + ) { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target tag`, + ); + } + if ( + typeof value.structural_path === "string" + && Object.keys(value).some((key) => ( + key !== "tag" && key !== "structural_path" + )) + ) { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target`, + ); + } + return tag.length <= 64 && TARGET_TAG_PATTERN.test(tag) + ? tag + : `sha256:${sha256(tag)}`; +} + +function isSafeIntegerBetween( + value: unknown, + minimum: number, + maximum: number, +): value is number { + return typeof value === "number" + && Number.isSafeInteger(value) + && value >= minimum + && value <= maximum; +} + +function invalidTraceAction(lineNumber: number): never { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action`, + ); +} + function parseTraceAction(value: unknown, lineNumber: number): ParsedTraceAction | null { if (value === null) return null; if (typeof value === "string") { if (!ACTION_KIND_SET.has(value) || !UNIT_ACTION_KINDS.has(value as DirectBombadilActionKind)) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + return invalidTraceAction(lineNumber); } return { kind: value as DirectBombadilActionKind, targetTag: null }; } if (!isRecord(value) || Object.keys(value).length !== 1) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + return invalidTraceAction(lineNumber); } const kind = Object.keys(value)[0]; + const payload = kind === undefined ? undefined : value[kind]; if ( kind === undefined || !ACTION_KIND_SET.has(kind) || UNIT_ACTION_KINDS.has(kind as DirectBombadilActionKind) - || !isRecord(value[kind]) + || !isRecord(payload) ) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); + return invalidTraceAction(lineNumber); } const actionKind = kind as DirectBombadilActionKind; let targetTag: string | null = null; - if (TARGETED_ACTION_KINDS.has(actionKind)) { - const fingerprint = value[kind].fingerprint; - if (!isRecord(fingerprint)) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); - } - const tag = fingerprint.tag; - if ( - typeof tag !== "string" - || tag.length === 0 - || tag.length > 64 - || !TARGET_TAG_PATTERN.test(tag) - ) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target tag`); - } - targetTag = tag; + switch (actionKind) { + case "Click": + if ( + !hasExactKeys(payload, TRACE_CLICK_ACTION_KEYS) + || !validTracePoint(payload.point) + ) { + return invalidTraceAction(lineNumber); + } + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "DoubleClick": + if ( + !hasExactKeys(payload, TRACE_DOUBLE_CLICK_ACTION_KEYS) + || !isSafeIntegerBetween(payload.delay_millis, 0, 1_000) + || !validTracePoint(payload.point) + ) return invalidTraceAction(lineNumber); + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "TypeText": + if ( + !hasExactKeys(payload, TRACE_TYPE_TEXT_ACTION_KEYS) + || !isSafeIntegerBetween(payload.delay_millis, 0, Number.MAX_SAFE_INTEGER) + || typeof payload.text !== "string" + ) return invalidTraceAction(lineNumber); + break; + case "PressKey": + if ( + !hasExactKeys(payload, TRACE_PRESS_KEY_ACTION_KEYS) + || !isSafeIntegerBetween(payload.code, 0, 255) + ) { + return invalidTraceAction(lineNumber); + } + break; + case "ScrollDown": + case "ScrollUp": + if ( + !hasExactKeys(payload, TRACE_SCROLL_ACTION_KEYS) + || typeof payload.distance !== "number" + || !Number.isFinite(payload.distance) + || !validTracePoint(payload.origin) + ) return invalidTraceAction(lineNumber); + break; + case "SetFileInputFiles": + if ( + !hasExactKeys(payload, TRACE_FILE_INPUT_ACTION_KEYS) + || typeof payload.selector !== "string" + || !Array.isArray(payload.files) + || !payload.files.every((file) => typeof file === "string") + ) return invalidTraceAction(lineNumber); + break; + case "MouseDrag": + if ( + !hasExactKeys(payload, TRACE_MOUSE_DRAG_ACTION_KEYS) + || !isSafeIntegerBetween(payload.delay_millis, 0, 1_000) + || !isSafeIntegerBetween(payload.steps, 1, 255) + || !validTracePoint(payload.from) + || !validTracePoint(payload.to) + ) return invalidTraceAction(lineNumber); + break; + case "SetViewport": + if ( + !hasExactKeys(payload, TRACE_VIEWPORT_ACTION_KEYS) + || !isSafeIntegerBetween(payload.height, 1, 10_000) + || !isSafeIntegerBetween(payload.width, 1, 10_000) + ) return invalidTraceAction(lineNumber); + break; + default: + return invalidTraceAction(lineNumber); } return { kind: actionKind, targetTag }; } @@ -1011,9 +1175,13 @@ export async function summarizeDirectBombadilTrace(options: { const actionCounts = new Map(); const targetTags = new Map(); const urlFingerprints = new Set(); + const rawUrlFingerprints = new Set(); const transitionHashes = new Set(); + const rawTransitionHashes = new Set(); const snapshots = new Map; changeAfterNonWaitCount: number; + lastObservationIndex: number | null; lastValueSha256: string | null; observationCount: number; readonly values: Set; @@ -1036,8 +1204,8 @@ export async function summarizeDirectBombadilTrace(options: { let waitStreak = 0; let maxWaitStreak = 0; let nonNullHashCount = 0; + let rawNonNullHashCount = 0; let policyObservationCount = 0; - let reachedExactObservation = false; let previousObservationWasExact = false; let stableTarget = true; const stream = createReadStream(options.tracePath, { encoding: "utf8" }); @@ -1052,15 +1220,43 @@ export async function summarizeDirectBombadilTrace(options: { throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); } const parsed = parseTraceLine(line, lineCount); + const rawRelativeUrl = + `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + rawUrlFingerprints.add(sha256(rawRelativeUrl)); + if (rawUrlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct raw URL fingerprints`, + ); + } + if (parsed.state.currentHash !== null) { + rawNonNullHashCount += 1; + rawTransitionHashes.add(String(parsed.state.currentHash)); + } + for (const name of parsed.propertyViolationNames) { + if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`, + ); + } + propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); + } + for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { + resources[outputName] = Math.max( + resources[outputName], + parsed.state.resources[sourceName as keyof typeof RESOURCE_FIELD_MAP], + ); + } const currentObservationIsExact = exactTraceDirectObservation(parsed.directObservation) !== null; - if (!reachedExactObservation && !currentObservationIsExact) { + if (!currentObservationIsExact) { previousObservationWasExact = false; continue; } - if (!reachedExactObservation) reachedExactObservation = true; policyObservationCount += 1; const actionFollowsExactObservation = previousObservationWasExact; + const recordedActionKind = actionFollowsExactObservation + ? parsed.action?.kind ?? null + : null; if (actionFollowsExactObservation && parsed.action !== null) { totalActions += 1; @@ -1107,22 +1303,30 @@ export async function summarizeDirectBombadilTrace(options: { ); } entry = { + changeAfterActionKind: new Map(), changeAfterNonWaitCount: 0, + lastObservationIndex: null, lastValueSha256: null, observationCount: 0, values: new Set(), }; snapshots.set(snapshot.name, entry); } - if ( - actionFollowsExactObservation - && parsed.action !== null - && parsed.action.kind !== "Wait" + const changedAfterRecordedAction = + recordedActionKind !== null + && entry.lastObservationIndex === policyObservationCount - 1 && entry.lastValueSha256 !== null - && entry.lastValueSha256 !== snapshot.valueSha256 - ) { + && entry.lastValueSha256 !== snapshot.valueSha256; + if (changedAfterRecordedAction) { + entry.changeAfterActionKind.set( + recordedActionKind, + (entry.changeAfterActionKind.get(recordedActionKind) ?? 0) + 1, + ); + } + if (changedAfterRecordedAction && recordedActionKind !== "Wait") { entry.changeAfterNonWaitCount += 1; } + entry.lastObservationIndex = policyObservationCount; entry.lastValueSha256 = snapshot.valueSha256; entry.observationCount += 1; entry.values.add(snapshot.valueSha256); @@ -1132,21 +1336,7 @@ export async function summarizeDirectBombadilTrace(options: { ); } } - for (const name of parsed.propertyViolationNames) { - if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { - throw new Error( - `Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`, - ); - } - propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); - } - for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { - resources[outputName] = Math.max( - resources[outputName], - parsed.state.resources[sourceName as keyof typeof RESOURCE_FIELD_MAP], - ); - } - previousObservationWasExact = currentObservationIsExact; + previousObservationWasExact = true; } } finally { lines.close(); @@ -1183,13 +1373,27 @@ export async function summarizeDirectBombadilTrace(options: { ); } } + for (const [name, minimumByKind] of Object.entries( + policy.minNamedSnapshotChangesAfterActionKind, + )) { + for (const [kind, minimum] of Object.entries(minimumByKind) as [ + DirectBombadilActionKind, + number, + ][]) { + if ((snapshots.get(name)?.changeAfterActionKind.get(kind) ?? 0) < minimum) { + policyFailures.push( + `named snapshot ${name} did not reach its post-${kind} change minimum`, + ); + } + } + } if (policy.requireStableTargetUrl && !stableTarget) { policyFailures.push("the browser did not remain on the exact target URL"); } } const traceBytes = await readFile(options.tracePath); return Object.freeze({ - schema: "direct.bombadil-exploration-summary/v1", + schema: "direct.bombadil-exploration-summary/v2", trace: Object.freeze({ bytes: metadata.size, lineCount, @@ -1206,15 +1410,23 @@ export async function summarizeDirectBombadilTrace(options: { distinctFingerprintCount: urlFingerprints.size, fingerprintSha256: Object.freeze([...urlFingerprints].sort(compareCodeUnits)), observationCount: policyObservationCount, + rawDistinctFingerprintCount: rawUrlFingerprints.size, + rawFingerprintSha256: Object.freeze( + [...rawUrlFingerprints].sort(compareCodeUnits), + ), + rawObservationCount: lineCount, stableTarget, }), transitions: Object.freeze({ distinctNonNullHashCount: transitionHashes.size, nonNullHashCount, + rawDistinctNonNullHashCount: rawTransitionHashes.size, + rawNonNullHashCount, }), namedSnapshots: Object.freeze([...snapshots.entries()] .sort(([left], [right]) => compareCodeUnits(left, right)) .map(([name, entry]) => Object.freeze({ + changeAfterActionKind: sortedCountRecord(entry.changeAfterActionKind), changeAfterNonWaitCount: entry.changeAfterNonWaitCount, distinctValueCount: entry.values.size, distinctValueSha256: Object.freeze([...entry.values].sort(compareCodeUnits)), @@ -1449,6 +1661,53 @@ function validateSnapshotMinimumMap(options: { return Object.freeze(validated); } +function validateSnapshotActionMinimumMap(options: { + readonly label: string; + readonly value: unknown; +}): Readonly>> +>> { + if (!isRecord(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated: Record< + string, + Readonly>> + > = {}; + for (const [rawName, rawMinimumByKind] of Object.entries(options.value) + .sort(([left], [right]) => compareCodeUnits(left, right))) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if ( + !isRecord(rawMinimumByKind) + || Object.keys(rawMinimumByKind).length === 0 + || Object.keys(rawMinimumByKind).length > ACTION_KINDS.length + ) { + throw new Error(`${options.label} ${name} must be a bounded action map`); + } + const minimumByKind: Partial> = {}; + for (const [rawKind, minimum] of Object.entries(rawMinimumByKind) + .sort(([left], [right]) => compareCodeUnits(left, right))) { + if (!ACTION_KIND_SET.has(rawKind)) { + throw new Error(`${options.label} ${name} contains an unknown action kind`); + } + if ( + typeof minimum !== "number" + || !Number.isSafeInteger(minimum) + || minimum < 1 + || minimum > TRACE_MAX_LINES + ) { + throw new Error( + `${options.label} ${name}.${rawKind} must be an integer between 1 and ${String(TRACE_MAX_LINES)}`, + ); + } + minimumByKind[rawKind as DirectBombadilActionKind] = minimum; + } + validated[name] = Object.freeze(minimumByKind); + } + return Object.freeze(validated); +} + function validateExplorationPolicy( value: unknown, ): ValidatedExplorationPolicy | null { @@ -1502,6 +1761,11 @@ function validateExplorationPolicy( maximum: TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME, value: value.minDistinctNamedSnapshotValues ?? {}, }); + const minNamedSnapshotChangesAfterActionKind = + validateSnapshotActionMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterActionKind", + value: value.minNamedSnapshotChangesAfterActionKind ?? {}, + }); const minNamedSnapshotChangesAfterNonWait = validateSnapshotMinimumMap({ label: "explorationPolicy.minNamedSnapshotChangesAfterNonWait", maximum: TRACE_MAX_LINES, @@ -1513,6 +1777,7 @@ function validateExplorationPolicy( } return Object.freeze({ minDistinctNamedSnapshotValues, + minNamedSnapshotChangesAfterActionKind, minNamedSnapshotChangesAfterNonWait, minNonWaitActions, requireStableTargetUrl, From 82353396640765c077cd4b98db2d3bc8fe9fd92f Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 13:31:26 -0400 Subject: [PATCH 17/20] test: reject empty Bombadil target tags --- src/tooling/bombadil-runner.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index ace1c47..fb2ccf4 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -1321,7 +1321,7 @@ describe("Direct Bombadil exploration summary", () => { const badTarget = await summaryTrace([traceLine(observation, 1, { action: { Click: { - fingerprint: { tag: "unsafe tag" }, + fingerprint: { tag: "" }, point: { x: 1, y: 1 }, }, }, From be71a7fc4bc68e948aba5786a4b51097e811d3bd Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 14:29:56 -0400 Subject: [PATCH 18/20] build: refresh Bombadil tooling bundle --- dist/tooling/bombadil.js | 816 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 810 insertions(+), 6 deletions(-) diff --git a/dist/tooling/bombadil.js b/dist/tooling/bombadil.js index 3e4afaf..9f094fd 100644 --- a/dist/tooling/bombadil.js +++ b/dist/tooling/bombadil.js @@ -5,6 +5,7 @@ import { readFile, realpath, stat, writeFile as writeFile2 } from "fs/promises"; import { isAbsolute, join as join2, relative, resolve } from "path"; import process2 from "process"; import { createInterface } from "readline"; +import { createHash } from "crypto"; // src/core/result.ts function ok(value) { @@ -1134,6 +1135,12 @@ var TRACE_MAX_BYTES = 64 * 1024 * 1024; var TRACE_MAX_LINE_BYTES = 16 * 1024 * 1024; var TRACE_MAX_LINES = 1e4; var TRACE_MAX_SNAPSHOTS_PER_LINE = 4096; +var TRACE_MAX_NAMED_SNAPSHOT_NAMES = 128; +var TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME = 1024; +var TRACE_MAX_DISTINCT_URLS = 1024; +var TRACE_MAX_PROPERTY_NAMES = 128; +var TRACE_MAX_CANONICAL_SNAPSHOT_BYTES = 2 * 1024 * 1024; +var TRACE_MAX_JSON_DEPTH = 64; var RANDOM_RUN_OVERHEAD_MS = 30000; var REPLAY_WALL_CLOCK_TIMEOUT_MS = MAX_TIME_LIMIT_SECONDS * 1000 + RANDOM_RUN_OVERHEAD_MS; var PROCESS_TERMINATION_GRACE_MS = 5000; @@ -1142,6 +1149,94 @@ var SERVER_OUTPUT_TIMEOUT_MS = 3000; var DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; var TRACE_LINE_KEYS = new Set(["action", "snapshots", "state", "timestamp", "violations"]); var TRACE_SNAPSHOT_KEYS = new Set(["index", "name", "time", "value"]); +var TRACE_STATE_KEYS = new Set([ + "hash_current", + "hash_previous", + "resources", + "screenshot", + "url" +]); +var TRACE_RESOURCE_KEYS = new Set([ + "documents", + "dom_nodes", + "js_event_listeners", + "js_heap_total", + "js_heap_used", + "layout_objects", + "script_duration", + "task_duration", + "thread_time", + "timestamp" +]); +var TRACE_VIOLATION_KEYS = new Set(["name", "violation"]); +var TRACE_POINT_KEYS = new Set(["x", "y"]); +var TRACE_FINGERPRINT_KEYS = new Set([ + "accessible_name", + "href", + "id", + "input_type", + "name_attr", + "placeholder", + "role", + "structural_path", + "tag", + "test_id", + "text_content" +]); +var TRACE_CLICK_ACTION_KEYS = new Set(["fingerprint", "point"]); +var TRACE_DOUBLE_CLICK_ACTION_KEYS = new Set([ + "delay_millis", + "fingerprint", + "point" +]); +var TRACE_TYPE_TEXT_ACTION_KEYS = new Set(["delay_millis", "text"]); +var TRACE_PRESS_KEY_ACTION_KEYS = new Set(["code"]); +var TRACE_SCROLL_ACTION_KEYS = new Set(["distance", "origin"]); +var TRACE_FILE_INPUT_ACTION_KEYS = new Set(["files", "selector"]); +var TRACE_MOUSE_DRAG_ACTION_KEYS = new Set([ + "delay_millis", + "from", + "steps", + "to" +]); +var TRACE_VIEWPORT_ACTION_KEYS = new Set(["height", "width"]); +var VIEWPORT_KEYS = new Set(["deviceScaleFactor", "height", "width"]); +var EXPLORATION_POLICY_KEYS = new Set([ + "minDistinctNamedSnapshotValues", + "minNamedSnapshotChangesAfterActionKind", + "minNamedSnapshotChangesAfterNonWait", + "minNonWaitActions", + "requireStableTargetUrl", + "requiredActionKinds", + "requiredNamedSnapshots" +]); +var DEFAULT_VIEWPORT_WIDTH = 1024; +var DEFAULT_VIEWPORT_HEIGHT = 768; +var DEFAULT_DEVICE_SCALE_FACTOR = 2; +var SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +var TARGET_TAG_PATTERN = /^[a-z][a-z0-9-]*$/u; +var ACTION_KINDS = [ + "Back", + "Click", + "DoubleClick", + "Forward", + "MouseDrag", + "PressKey", + "Reload", + "ScrollDown", + "ScrollUp", + "SetFileInputFiles", + "SetViewport", + "TypeText", + "Wait" +]; +var ACTION_KIND_SET = new Set(ACTION_KINDS); +var UNIT_ACTION_KINDS = new Set([ + "Back", + "Forward", + "Reload", + "Wait" +]); var DIRECT_OBSERVATION_KEYS = new Set([ "activationHash", "activeRoute", @@ -1217,6 +1312,13 @@ function hasExactKeys(value, expected) { const keys = Object.keys(value); return keys.length === expected.size && keys.every((key) => expected.has(key)); } +function compareCodeUnits(left, right) { + if (left < right) + return -1; + if (left > right) + return 1; + return 0; +} function parseTraceDirectObservation(value) { if (!isRecord2(value) || !hasExactKeys(value, DIRECT_OBSERVATION_KEYS)) { throw new Error("Bombadil trace has an invalid named direct observation"); @@ -1311,6 +1413,181 @@ function exactTraceDirectObservation(observation) { isQuiescent: probe2.value.isQuiescent }; } +var RESOURCE_FIELD_MAP = { + documents: "documents", + dom_nodes: "domNodes", + js_event_listeners: "jsEventListeners", + js_heap_total: "jsHeapTotalBytes", + js_heap_used: "jsHeapUsedBytes", + layout_objects: "layoutObjects", + script_duration: "scriptDurationSeconds", + task_duration: "taskDurationSeconds", + thread_time: "threadTimeSeconds" +}; +function canonicalJson2(value, depth = 0) { + if (depth > TRACE_MAX_JSON_DEPTH) { + throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(TRACE_MAX_JSON_DEPTH)}`); + } + if (value === null || typeof value === "boolean" || typeof value === "string") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new Error("Bombadil named snapshot has a non-finite number"); + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((entry) => canonicalJson2(entry, depth + 1)).join(",")}]`; + } + if (!isRecord2(value)) + throw new Error("Bombadil named snapshot is not JSON"); + const entries = Object.keys(value).sort(compareCodeUnits).map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key], depth + 1)}`); + return `{${entries.join(",")}}`; +} +function sha256(value) { + return createHash("sha256").update(value).digest("hex"); +} +function namedSnapshotValueSha256(value) { + const canonical = canonicalJson2(value); + if (Buffer.byteLength(canonical, "utf8") > TRACE_MAX_CANONICAL_SNAPSHOT_BYTES) { + throw new Error(`Bombadil named snapshot exceeds ${String(TRACE_MAX_CANONICAL_SNAPSHOT_BYTES)} canonical bytes`); + } + return sha256(canonical); +} +function validTracePoint(value) { + return isRecord2(value) && hasExactKeys(value, TRACE_POINT_KEYS) && typeof value.x === "number" && Number.isFinite(value.x) && typeof value.y === "number" && Number.isFinite(value.y); +} +function parseTraceFingerprintTag(value, lineNumber) { + if (!isRecord2(value) || !Object.keys(value).every((key) => TRACE_FINGERPRINT_KEYS.has(key))) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + for (const [key, candidate] of Object.entries(value)) { + if (key !== "tag" && typeof candidate !== "string") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + } + const tag = value.tag; + if (typeof tag !== "string" || tag.length === 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target tag`); + } + if (typeof value.structural_path === "string" && Object.keys(value).some((key) => key !== "tag" && key !== "structural_path")) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + return tag.length <= 64 && TARGET_TAG_PATTERN.test(tag) ? tag : `sha256:${sha256(tag)}`; +} +function isSafeIntegerBetween(value, minimum, maximum) { + return typeof value === "number" && Number.isSafeInteger(value) && value >= minimum && value <= maximum; +} +function invalidTraceAction(lineNumber) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); +} +function parseTraceAction(value, lineNumber) { + if (value === null) + return null; + if (typeof value === "string") { + if (!ACTION_KIND_SET.has(value) || !UNIT_ACTION_KINDS.has(value)) { + return invalidTraceAction(lineNumber); + } + return { kind: value, targetTag: null }; + } + if (!isRecord2(value) || Object.keys(value).length !== 1) { + return invalidTraceAction(lineNumber); + } + const kind = Object.keys(value)[0]; + const payload = kind === undefined ? undefined : value[kind]; + if (kind === undefined || !ACTION_KIND_SET.has(kind) || UNIT_ACTION_KINDS.has(kind) || !isRecord2(payload)) { + return invalidTraceAction(lineNumber); + } + const actionKind = kind; + let targetTag = null; + switch (actionKind) { + case "Click": + if (!hasExactKeys(payload, TRACE_CLICK_ACTION_KEYS) || !validTracePoint(payload.point)) { + return invalidTraceAction(lineNumber); + } + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "DoubleClick": + if (!hasExactKeys(payload, TRACE_DOUBLE_CLICK_ACTION_KEYS) || !isSafeIntegerBetween(payload.delay_millis, 0, 1000) || !validTracePoint(payload.point)) + return invalidTraceAction(lineNumber); + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "TypeText": + if (!hasExactKeys(payload, TRACE_TYPE_TEXT_ACTION_KEYS) || !isSafeIntegerBetween(payload.delay_millis, 0, Number.MAX_SAFE_INTEGER) || typeof payload.text !== "string") + return invalidTraceAction(lineNumber); + break; + case "PressKey": + if (!hasExactKeys(payload, TRACE_PRESS_KEY_ACTION_KEYS) || !isSafeIntegerBetween(payload.code, 0, 255)) { + return invalidTraceAction(lineNumber); + } + break; + case "ScrollDown": + case "ScrollUp": + if (!hasExactKeys(payload, TRACE_SCROLL_ACTION_KEYS) || typeof payload.distance !== "number" || !Number.isFinite(payload.distance) || !validTracePoint(payload.origin)) + return invalidTraceAction(lineNumber); + break; + case "SetFileInputFiles": + if (!hasExactKeys(payload, TRACE_FILE_INPUT_ACTION_KEYS) || typeof payload.selector !== "string" || !Array.isArray(payload.files) || !payload.files.every((file) => typeof file === "string")) + return invalidTraceAction(lineNumber); + break; + case "MouseDrag": + if (!hasExactKeys(payload, TRACE_MOUSE_DRAG_ACTION_KEYS) || !isSafeIntegerBetween(payload.delay_millis, 0, 1000) || !isSafeIntegerBetween(payload.steps, 1, 255) || !validTracePoint(payload.from) || !validTracePoint(payload.to)) + return invalidTraceAction(lineNumber); + break; + case "SetViewport": + if (!hasExactKeys(payload, TRACE_VIEWPORT_ACTION_KEYS) || !isSafeIntegerBetween(payload.height, 1, 1e4) || !isSafeIntegerBetween(payload.width, 1, 1e4)) + return invalidTraceAction(lineNumber); + break; + default: + return invalidTraceAction(lineNumber); + } + return { kind: actionKind, targetTag }; +} +function parseNonNegativeFiniteNumber(value, lineNumber, field) { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + return value; +} +function parseTraceState(value, lineNumber) { + if (!isRecord2(value) || !hasExactKeys(value, TRACE_STATE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser state`); + } + if (typeof value.url !== "string" || value.url.length === 0 || value.url.length > 8192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + let url; + try { + url = new URL(value.url); + } catch { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL protocol`); + } + if (typeof value.screenshot !== "string" || value.screenshot.length > 8192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid screenshot path`); + } + for (const field of ["hash_previous", "hash_current"]) { + const hash = value[field]; + if (hash !== null && (typeof hash !== "number" || !Number.isFinite(hash) || !Number.isInteger(hash) || hash < 0)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + } + if (!isRecord2(value.resources) || !hasExactKeys(value.resources, TRACE_RESOURCE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid browser resources`); + } + const resources = {}; + for (const field of Object.keys(RESOURCE_FIELD_MAP)) { + resources[field] = parseNonNegativeFiniteNumber(value.resources[field], lineNumber, `resources.${field}`); + } + parseNonNegativeFiniteNumber(value.resources.timestamp, lineNumber, "resources.timestamp"); + return { + currentHash: value.hash_current, + resources, + url + }; +} function parseTraceLine(line, lineNumber) { let input; try { @@ -1324,7 +1601,27 @@ function parseTraceLine(line, lineNumber) { if (!Number.isSafeInteger(input.timestamp) || typeof input.timestamp !== "number" || input.timestamp < 0 || !Array.isArray(input.snapshots) || input.snapshots.length > TRACE_MAX_SNAPSHOTS_PER_LINE || !Array.isArray(input.violations)) { throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid state fields`); } + const state = parseTraceState(input.state, lineNumber); + const action = parseTraceAction(input.action, lineNumber); const snapshots = input.snapshots; + const namedSnapshots = []; + const namedSnapshotNames = new Set; + for (const snapshotValue of snapshots) { + if (!isRecord2(snapshotValue) || !hasExactKeys(snapshotValue, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshotValue.index) || !Number.isSafeInteger(snapshotValue.time) || snapshotValue.index < 0 || snapshotValue.time < 0 || snapshotValue.name !== null && typeof snapshotValue.name !== "string") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid snapshot`); + } + if (snapshotValue.name !== null) { + const name = validateSnapshotName(snapshotValue.name, `Bombadil trace line ${String(lineNumber)} snapshot name`); + if (namedSnapshotNames.has(name)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} repeats named snapshot ${name}`); + } + namedSnapshotNames.add(name); + namedSnapshots.push({ + name, + valueSha256: namedSnapshotValueSha256(snapshotValue.value) + }); + } + } const directSnapshots = snapshots.filter((snapshot2) => isRecord2(snapshot2) && snapshot2.name === "direct"); if (directSnapshots.length !== 1) { throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); @@ -1333,7 +1630,23 @@ function parseTraceLine(line, lineNumber) { if (snapshot === undefined || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshot.index) || !Number.isSafeInteger(snapshot.time) || snapshot.index < 0 || snapshot.time < 0) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); } - return parseTraceDirectObservation(snapshot.value); + const propertyViolationNames = []; + for (const violation of input.violations) { + if (!isRecord2(violation) || !hasExactKeys(violation, TRACE_VIOLATION_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + propertyViolationNames.push(validateSnapshotName(violation.name, `Bombadil trace line ${String(lineNumber)} property violation name`)); + if (!isRecord2(violation.violation) || Object.keys(violation.violation).length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + } + return { + action, + directObservation: parseTraceDirectObservation(snapshot.value), + namedSnapshots, + propertyViolationNames, + state + }; } async function attestDirectBombadilTrace(options) { const metadata = await stat(options.tracePath).catch(() => null); @@ -1360,7 +1673,7 @@ async function attestDirectBombadilTrace(options) { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); } - const observation = parseTraceLine(line, observationCount); + const observation = parseTraceLine(line, observationCount).directObservation; const exact = exactTraceDirectObservation(observation); if (exact === null) { if (initial !== null) { @@ -1428,6 +1741,245 @@ async function attestDirectBombadilTrace(options) { validObservationCount }; } +function sortedCountRecord(values) { + return Object.freeze(Object.fromEntries([...values.entries()].sort(([left], [right]) => compareCodeUnits(left, right)))); +} +async function summarizeDirectBombadilTrace(options) { + const metadata = await stat(options.tracePath).catch(() => null); + if (metadata === null || !metadata.isFile() || metadata.size === 0) { + throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + } + if (metadata.size > TRACE_MAX_BYTES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_BYTES)} bytes`); + } + let targetUrl; + try { + targetUrl = new URL(options.targetUrl); + } catch { + throw new Error("targetUrl must be an absolute URL"); + } + const policy = validateExplorationPolicy(options.explorationPolicy); + const actionCounts = new Map; + const targetTags = new Map; + const urlFingerprints = new Set; + const rawUrlFingerprints = new Set; + const transitionHashes = new Set; + const rawTransitionHashes = new Set; + const snapshots = new Map; + const propertyViolations = new Map; + const resources = { + documents: 0, + domNodes: 0, + jsEventListeners: 0, + jsHeapTotalBytes: 0, + jsHeapUsedBytes: 0, + layoutObjects: 0, + scriptDurationSeconds: 0, + taskDurationSeconds: 0, + threadTimeSeconds: 0 + }; + let lineCount = 0; + let totalActions = 0; + let nonWaitCount = 0; + let waitStreak = 0; + let maxWaitStreak = 0; + let nonNullHashCount = 0; + let rawNonNullHashCount = 0; + let policyObservationCount = 0; + let previousObservationWasExact = false; + let stableTarget = true; + const stream = createReadStream(options.tracePath, { encoding: "utf8" }); + const lines = createInterface({ input: stream, crlfDelay: Infinity }); + try { + for await (const line of lines) { + lineCount += 1; + if (lineCount > TRACE_MAX_LINES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); + } + if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { + throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); + } + const parsed = parseTraceLine(line, lineCount); + const rawRelativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + rawUrlFingerprints.add(sha256(rawRelativeUrl)); + if (rawUrlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct raw URL fingerprints`); + } + if (parsed.state.currentHash !== null) { + rawNonNullHashCount += 1; + rawTransitionHashes.add(String(parsed.state.currentHash)); + } + for (const name of parsed.propertyViolationNames) { + if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`); + } + propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); + } + for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { + resources[outputName] = Math.max(resources[outputName], parsed.state.resources[sourceName]); + } + const currentObservationIsExact = exactTraceDirectObservation(parsed.directObservation) !== null; + if (!currentObservationIsExact) { + previousObservationWasExact = false; + continue; + } + policyObservationCount += 1; + const actionFollowsExactObservation = previousObservationWasExact; + const recordedActionKind = actionFollowsExactObservation ? parsed.action?.kind ?? null : null; + if (actionFollowsExactObservation && parsed.action !== null) { + totalActions += 1; + actionCounts.set(parsed.action.kind, (actionCounts.get(parsed.action.kind) ?? 0) + 1); + if (parsed.action.kind === "Wait") { + waitStreak += 1; + maxWaitStreak = Math.max(maxWaitStreak, waitStreak); + } else { + nonWaitCount += 1; + waitStreak = 0; + } + if (parsed.action.targetTag !== null) { + if (!targetTags.has(parsed.action.targetTag) && targetTags.size >= 128) { + throw new Error("Bombadil trace exceeds 128 distinct action target tags"); + } + targetTags.set(parsed.action.targetTag, (targetTags.get(parsed.action.targetTag) ?? 0) + 1); + } + } else if (actionFollowsExactObservation) { + waitStreak = 0; + } + const relativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + urlFingerprints.add(sha256(relativeUrl)); + if (urlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct URL fingerprints`); + } + stableTarget &&= parsed.state.url.href === targetUrl.href; + if (parsed.state.currentHash !== null) { + nonNullHashCount += 1; + transitionHashes.add(String(parsed.state.currentHash)); + } + for (const snapshot of parsed.namedSnapshots) { + let entry = snapshots.get(snapshot.name); + if (entry === undefined) { + if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`); + } + entry = { + changeAfterActionKind: new Map, + changeAfterNonWaitCount: 0, + lastObservationIndex: null, + lastValueSha256: null, + observationCount: 0, + values: new Set + }; + snapshots.set(snapshot.name, entry); + } + const changedAfterRecordedAction = recordedActionKind !== null && entry.lastObservationIndex === policyObservationCount - 1 && entry.lastValueSha256 !== null && entry.lastValueSha256 !== snapshot.valueSha256; + if (changedAfterRecordedAction) { + entry.changeAfterActionKind.set(recordedActionKind, (entry.changeAfterActionKind.get(recordedActionKind) ?? 0) + 1); + } + if (changedAfterRecordedAction && recordedActionKind !== "Wait") { + entry.changeAfterNonWaitCount += 1; + } + entry.lastObservationIndex = policyObservationCount; + entry.lastValueSha256 = snapshot.valueSha256; + entry.observationCount += 1; + entry.values.add(snapshot.valueSha256); + if (entry.values.size > TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { + throw new Error(`Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`); + } + } + previousObservationWasExact = true; + } + } finally { + lines.close(); + stream.destroy(); + } + if (lineCount === 0) + throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + const policyFailures = []; + if (policy !== null) { + if (nonWaitCount < policy.minNonWaitActions) { + policyFailures.push("minimum non-Wait action count was not reached"); + } + for (const kind of policy.requiredActionKinds) { + if ((actionCounts.get(kind) ?? 0) === 0) { + policyFailures.push(`required action kind ${kind} was not observed`); + } + } + for (const name of policy.requiredNamedSnapshots) { + if (!snapshots.has(name)) { + policyFailures.push(`required named snapshot ${name} was not observed`); + } + } + for (const [name, minimum] of Object.entries(policy.minDistinctNamedSnapshotValues)) { + if ((snapshots.get(name)?.values.size ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its distinct-value minimum`); + } + } + for (const [name, minimum] of Object.entries(policy.minNamedSnapshotChangesAfterNonWait)) { + if ((snapshots.get(name)?.changeAfterNonWaitCount ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its post-non-Wait change minimum`); + } + } + for (const [name, minimumByKind] of Object.entries(policy.minNamedSnapshotChangesAfterActionKind)) { + for (const [kind, minimum] of Object.entries(minimumByKind)) { + if ((snapshots.get(name)?.changeAfterActionKind.get(kind) ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its post-${kind} change minimum`); + } + } + } + if (policy.requireStableTargetUrl && !stableTarget) { + policyFailures.push("the browser did not remain on the exact target URL"); + } + } + const traceBytes = await readFile(options.tracePath); + return Object.freeze({ + schema: "direct.bombadil-exploration-summary/v2", + trace: Object.freeze({ + bytes: metadata.size, + lineCount, + sha256: sha256(traceBytes) + }), + actions: Object.freeze({ + byKind: sortedCountRecord(actionCounts), + maxWaitStreak, + nonWaitCount, + targetTags: sortedCountRecord(targetTags), + total: totalActions + }), + urls: Object.freeze({ + distinctFingerprintCount: urlFingerprints.size, + fingerprintSha256: Object.freeze([...urlFingerprints].sort(compareCodeUnits)), + observationCount: policyObservationCount, + rawDistinctFingerprintCount: rawUrlFingerprints.size, + rawFingerprintSha256: Object.freeze([...rawUrlFingerprints].sort(compareCodeUnits)), + rawObservationCount: lineCount, + stableTarget + }), + transitions: Object.freeze({ + distinctNonNullHashCount: transitionHashes.size, + nonNullHashCount, + rawDistinctNonNullHashCount: rawTransitionHashes.size, + rawNonNullHashCount + }), + namedSnapshots: Object.freeze([...snapshots.entries()].sort(([left], [right]) => compareCodeUnits(left, right)).map(([name, entry]) => Object.freeze({ + changeAfterActionKind: sortedCountRecord(entry.changeAfterActionKind), + changeAfterNonWaitCount: entry.changeAfterNonWaitCount, + distinctValueCount: entry.values.size, + distinctValueSha256: Object.freeze([...entry.values].sort(compareCodeUnits)), + name, + observationCount: entry.observationCount + }))), + propertyViolations: Object.freeze({ + byName: sortedCountRecord(propertyViolations), + total: [...propertyViolations.values()].reduce((total, value) => total + value, 0) + }), + resourceHighWaterMarks: Object.freeze(resources), + policy: Object.freeze({ + configured: policy !== null, + failures: Object.freeze(policyFailures), + satisfied: policyFailures.length === 0 + }) + }); +} function parseDirectBombadilFuzzArguments(arguments_, defaultBaseUrl) { let baseUrl = defaultBaseUrl; let timeLimitSeconds = DEFAULT_TIME_LIMIT_SECONDS; @@ -1527,7 +2079,7 @@ function validateTargetQuery(value) { throw new Error("targetQuery may contain at most 16 parameters"); } const validated = {}; - for (const [name, queryValue] of [...entries].sort(([left], [right]) => left.localeCompare(right))) { + for (const [name, queryValue] of [...entries].sort(([left], [right]) => compareCodeUnits(left, right))) { if (name.length === 0 || name.length > 128 || !QUERY_PARAMETER_NAME_PATTERN.test(name) || PROTOTYPE_PROPERTY_NAMES.has(name) || hasControlCharacters3(name) || name === SCENARIO_QUERY_KEY2 || name === FIXTURE_QUERY_KEY2) { throw new Error("targetQuery contains an invalid or reserved parameter name"); } @@ -1538,6 +2090,131 @@ function validateTargetQuery(value) { } return Object.freeze(validated); } +function validateSnapshotName(value, label) { + if (typeof value !== "string" || value.length === 0 || value.length > 128 || !SNAPSHOT_NAME_PATTERN.test(value) || PROTOTYPE_PROPERTY_NAMES.has(value) || hasControlCharacters3(value)) { + throw new Error(`${label} must be a safe bounded snapshot name`); + } + return value; +} +function validateViewport(value) { + if (value === undefined) { + return Object.freeze({ + deviceScaleFactor: DEFAULT_DEVICE_SCALE_FACTOR, + height: DEFAULT_VIEWPORT_HEIGHT, + width: DEFAULT_VIEWPORT_WIDTH + }); + } + if (!isRecord2(value) || !Object.keys(value).every((key) => VIEWPORT_KEYS.has(key))) { + throw new Error("viewport must contain only width, height, and deviceScaleFactor"); + } + const validateDimension = (name, input) => { + if (typeof input !== "number" || !Number.isSafeInteger(input) || input < 1 || input > 65535) { + throw new Error(`viewport.${name} must be an integer between 1 and 65535`); + } + return input; + }; + const width = validateDimension("width", value.width ?? DEFAULT_VIEWPORT_WIDTH); + const height = validateDimension("height", value.height ?? DEFAULT_VIEWPORT_HEIGHT); + const deviceScaleFactor = value.deviceScaleFactor ?? DEFAULT_DEVICE_SCALE_FACTOR; + if (typeof deviceScaleFactor !== "number" || !Number.isFinite(deviceScaleFactor) || deviceScaleFactor < 0.1 || deviceScaleFactor > 10) { + throw new Error("viewport.deviceScaleFactor must be a finite number between 0.1 and 10"); + } + return Object.freeze({ deviceScaleFactor, height, width }); +} +function validateSnapshotMinimumMap(options) { + if (!isRecord2(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated = {}; + for (const [rawName, minimum] of Object.entries(options.value).sort(([left], [right]) => compareCodeUnits(left, right))) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if (typeof minimum !== "number" || !Number.isSafeInteger(minimum) || minimum < 1 || minimum > options.maximum) { + throw new Error(`${options.label} ${name} must be an integer between 1 and ${String(options.maximum)}`); + } + validated[name] = minimum; + } + return Object.freeze(validated); +} +function validateSnapshotActionMinimumMap(options) { + if (!isRecord2(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated = {}; + for (const [rawName, rawMinimumByKind] of Object.entries(options.value).sort(([left], [right]) => compareCodeUnits(left, right))) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if (!isRecord2(rawMinimumByKind) || Object.keys(rawMinimumByKind).length === 0 || Object.keys(rawMinimumByKind).length > ACTION_KINDS.length) { + throw new Error(`${options.label} ${name} must be a bounded action map`); + } + const minimumByKind = {}; + for (const [rawKind, minimum] of Object.entries(rawMinimumByKind).sort(([left], [right]) => compareCodeUnits(left, right))) { + if (!ACTION_KIND_SET.has(rawKind)) { + throw new Error(`${options.label} ${name} contains an unknown action kind`); + } + if (typeof minimum !== "number" || !Number.isSafeInteger(minimum) || minimum < 1 || minimum > TRACE_MAX_LINES) { + throw new Error(`${options.label} ${name}.${rawKind} must be an integer between 1 and ${String(TRACE_MAX_LINES)}`); + } + minimumByKind[rawKind] = minimum; + } + validated[name] = Object.freeze(minimumByKind); + } + return Object.freeze(validated); +} +function validateExplorationPolicy(value) { + if (value === undefined) + return null; + if (!isRecord2(value) || !Object.keys(value).every((key) => EXPLORATION_POLICY_KEYS.has(key))) { + throw new Error("explorationPolicy contains an unknown field"); + } + const minNonWaitActions = value.minNonWaitActions ?? 0; + if (typeof minNonWaitActions !== "number" || !Number.isSafeInteger(minNonWaitActions) || minNonWaitActions < 0 || minNonWaitActions > TRACE_MAX_LINES) { + throw new Error(`explorationPolicy.minNonWaitActions must be an integer between 0 and ${String(TRACE_MAX_LINES)}`); + } + const requiredActionKindsInput = value.requiredActionKinds ?? []; + if (!Array.isArray(requiredActionKindsInput) || requiredActionKindsInput.length > ACTION_KINDS.length) { + throw new Error("explorationPolicy.requiredActionKinds must be a bounded array"); + } + const requiredActionKinds = [...requiredActionKindsInput]; + if (!requiredActionKinds.every((kind) => typeof kind === "string" && ACTION_KIND_SET.has(kind)) || new Set(requiredActionKinds).size !== requiredActionKinds.length) { + throw new Error("explorationPolicy.requiredActionKinds contains an unknown or duplicate kind"); + } + requiredActionKinds.sort(compareCodeUnits); + const requiredNamedSnapshotsInput = value.requiredNamedSnapshots ?? []; + if (!Array.isArray(requiredNamedSnapshotsInput) || requiredNamedSnapshotsInput.length > 32) { + throw new Error("explorationPolicy.requiredNamedSnapshots must be a bounded array"); + } + const requiredNamedSnapshots = requiredNamedSnapshotsInput.map((name) => validateSnapshotName(name, "explorationPolicy.requiredNamedSnapshots entry")); + if (new Set(requiredNamedSnapshots).size !== requiredNamedSnapshots.length) { + throw new Error("explorationPolicy.requiredNamedSnapshots contains a duplicate name"); + } + requiredNamedSnapshots.sort(compareCodeUnits); + const minDistinctNamedSnapshotValues = validateSnapshotMinimumMap({ + label: "explorationPolicy.minDistinctNamedSnapshotValues", + maximum: TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME, + value: value.minDistinctNamedSnapshotValues ?? {} + }); + const minNamedSnapshotChangesAfterActionKind = validateSnapshotActionMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterActionKind", + value: value.minNamedSnapshotChangesAfterActionKind ?? {} + }); + const minNamedSnapshotChangesAfterNonWait = validateSnapshotMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterNonWait", + maximum: TRACE_MAX_LINES, + value: value.minNamedSnapshotChangesAfterNonWait ?? {} + }); + const requireStableTargetUrl = value.requireStableTargetUrl ?? false; + if (typeof requireStableTargetUrl !== "boolean") { + throw new Error("explorationPolicy.requireStableTargetUrl must be a boolean"); + } + return Object.freeze({ + minDistinctNamedSnapshotValues, + minNamedSnapshotChangesAfterActionKind, + minNamedSnapshotChangesAfterNonWait, + minNonWaitActions, + requireStableTargetUrl, + requiredActionKinds: Object.freeze(requiredActionKinds), + requiredNamedSnapshots: Object.freeze(requiredNamedSnapshots) + }); +} function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { const repositoryRoot = resolve(config.repositoryRoot); if (!isAbsolute(config.repositoryRoot) || repositoryRoot !== config.repositoryRoot) { @@ -1590,6 +2267,8 @@ function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { const entryPath = config.entryPath ?? "/"; validateEntryPath(entryPath); const targetQuery = validateTargetQuery(config.targetQuery ?? {}); + const viewport = validateViewport(config.viewport); + const explorationPolicy = validateExplorationPolicy(config.explorationPolicy); const startupTimeoutMs = config.server.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; if (!Number.isSafeInteger(startupTimeoutMs) || startupTimeoutMs < 1000 || startupTimeoutMs > MAX_STARTUP_TIMEOUT_MS) { throw new Error(`server.startupTimeoutMs must be an integer between 1000 and ${String(MAX_STARTUP_TIMEOUT_MS)}`); @@ -1604,8 +2283,10 @@ function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { artifactRoot: join2(repositoryRoot, "artifacts", "direct-bombadil", config.artifactName), bombadilExecutable: bombadilNativeBinary(repositoryRoot), entryPath, + explorationPolicy, port, targetQuery, + viewport, server: { ...config.server, cwd: serverCwd, @@ -1624,6 +2305,7 @@ function resolveReplayPath(repositoryRoot, replayPath) { return resolved; } function createDirectBombadilInvocation(options) { + const viewport = validateViewport(options.viewport); const target = new URL(options.entryPath ?? "/", `${options.baseUrl}/`); target.searchParams.set(SCENARIO_QUERY_KEY2, options.scenario); for (const [name, value] of Object.entries(options.targetQuery ?? {})) { @@ -1638,7 +2320,13 @@ function createDirectBombadilInvocation(options) { "--output-path", options.outputPath, "--headless", - "--instrument-javascript=" + "--instrument-javascript=", + "--width", + String(viewport.width), + "--height", + String(viewport.height), + "--device-scale-factor", + String(viewport.deviceScaleFactor) ]; if (options.replayPath === null) { command.push("--exit-on-violation", "--time-limit", `${String(options.timeLimitSeconds)}s`); @@ -1892,6 +2580,78 @@ function helpText(defaultBaseUrl) { ].join(` `); } +function parseMatrixCampaignArgument(arguments_) { + const forwarded = []; + let campaignId = null; + let help = false; + for (let index = 0;index < arguments_.length; index += 1) { + const argument = arguments_[index]; + if (argument === undefined) + continue; + if (argument === "--help" || argument === "-h") + help = true; + if (argument === "--campaign" || argument.startsWith("--campaign=")) { + if (campaignId !== null) + throw new Error("--campaign may be provided only once"); + if (argument === "--campaign") { + const next = readOptionValue(arguments_, index, "--campaign"); + campaignId = next.value; + index = next.index; + } else { + campaignId = argument.slice("--campaign=".length); + } + if (campaignId.length === 0) + throw new Error("--campaign requires a value"); + continue; + } + forwarded.push(argument); + } + return { arguments: Object.freeze(forwarded), campaignId, help }; +} +function validateCampaignMatrix(campaigns) { + if (campaigns.length === 0 || campaigns.length > 32) { + throw new Error("Bombadil campaign matrix must contain 1-32 campaigns"); + } + const ids = new Set; + for (const campaign of campaigns) { + if (!ARTIFACT_NAME_PATTERN.test(campaign.id) || ids.has(campaign.id)) { + throw new Error("Bombadil campaign IDs must be unique lowercase kebab identifiers"); + } + ids.add(campaign.id); + } + return campaigns; +} +async function runDirectBombadilFuzzMatrix(campaignsInput, arguments_ = process2.argv.slice(2), dependencyOverrides = {}) { + const campaigns = validateCampaignMatrix(campaignsInput); + const parsed = parseMatrixCampaignArgument(arguments_); + if (parsed.help) { + process2.stdout.write(`${[ + helpText(campaigns[0]?.config.baseUrl ?? ""), + " --campaign Run one campaign; required with --replay", + "", + `Campaigns: ${campaigns.map((campaign) => campaign.id).join(", ")}` + ].join(` +`)} +`); + return { kind: "help" }; + } + const selected = parsed.campaignId === null ? campaigns : campaigns.filter((campaign) => campaign.id === parsed.campaignId); + if (selected.length === 0) { + throw new Error(`Unknown Bombadil campaign ${parsed.campaignId ?? ""}`); + } + if (parsed.campaignId === null && parsed.arguments.some((argument) => argument === "--replay" || argument.startsWith("--replay="))) { + throw new Error("--replay requires exactly one --campaign in matrix mode"); + } + const results = []; + for (const campaign of selected) { + const result = await runDirectBombadilFuzz(campaign.config, parsed.arguments, dependencyOverrides); + if (result.kind !== "run") { + throw new Error("Bombadil campaign unexpectedly returned help during matrix execution"); + } + results.push({ campaignId: campaign.id, result }); + } + return { kind: "matrix", results: Object.freeze(results) }; +} async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2), dependencyOverrides = {}) { const parsed = parseDirectBombadilFuzzArguments(arguments_, config.baseUrl); if (parsed.kind === "help") { @@ -1923,7 +2683,8 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) scenario: validated.scenario, specificationPath: validated.specificationPath, targetQuery: validated.targetQuery, - timeLimitSeconds: parsed.timeLimitSeconds + timeLimitSeconds: parsed.timeLimitSeconds, + viewport: validated.viewport }); const abortableInvocation = { ...invocation, abortSignal: abortController.signal }; const serverCommand = validated.server.command.map((argument) => argument === "{port}" ? validated.port : argument); @@ -1933,6 +2694,8 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) let processResult = null; let attestation = null; let attestationFailure = null; + let explorationSummary = null; + let explorationSummaryFailure = null; let rawTracePath = null; let serverOutput = ""; let serverOutputFailure = null; @@ -1988,6 +2751,15 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) } catch (error) { attestationFailure = error; } + try { + explorationSummary = await summarizeDirectBombadilTrace({ + ...validated.explorationPolicy === null ? {} : { explorationPolicy: validated.explorationPolicy }, + targetUrl: invocation.targetUrl, + tracePath + }); + } catch (error) { + explorationSummaryFailure = error; + } if (processFailure !== null) { throw processFailure instanceof Error ? processFailure : new Error(renderUnknown(processFailure)); } @@ -2005,6 +2777,12 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) if (attestationFailure !== null) { throw attestationFailure instanceof Error ? attestationFailure : new Error(renderUnknown(attestationFailure)); } + if (explorationSummaryFailure !== null) { + throw explorationSummaryFailure instanceof Error ? explorationSummaryFailure : new Error(renderUnknown(explorationSummaryFailure)); + } + if (explorationSummary?.policy.satisfied !== true) { + throw new Error(`Bombadil exploration policy was not satisfied: ${explorationSummary?.policy.failures.join("; ") ?? "summary unavailable"}`); + } } catch (error) { failure = error; } @@ -2038,6 +2816,7 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) const status = failure === null ? "passed" : "failed"; const logPath = join2(artifactRun.runDirectory, "bombadil.log"); const serverLogPath = join2(artifactRun.runDirectory, "server.log"); + const explorationSummaryPath = join2(artifactRun.runDirectory, "exploration-summary.json"); const record = { schema: ARTIFACT_SCHEMA, evidenceClass: "diagnostic-fuzz", @@ -2053,6 +2832,8 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) entryPath: validated.entryPath, targetQuery: validated.targetQuery, targetUrl: invocation.targetUrl, + viewport: validated.viewport, + explorationPolicy: validated.explorationPolicy, specificationPath: validated.specificationPath, replayPath, timeLimitSeconds: replayPath === null ? parsed.timeLimitSeconds : null, @@ -2074,6 +2855,9 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) }, attestation, attestationFailure: attestationFailure === null ? null : renderUnknown(attestationFailure), + explorationSummary, + explorationSummaryPath: explorationSummary === null ? null : explorationSummaryPath, + explorationSummaryFailure: explorationSummaryFailure === null ? null : renderUnknown(explorationSummaryFailure), initialDirect: attestation?.initial ?? null, interruptedSignal: capturedSignal, failure: failure === null ? null : renderUnknown(failure) @@ -2085,9 +2869,23 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) ` : ""}`, "utf8"); await writeFile2(serverLogPath, `${serverOutput}${serverOutput.length > 0 ? ` ` : ""}`, "utf8"); + if (explorationSummary !== null) { + await writeJsonAtomically(explorationSummaryPath, explorationSummary); + } await writeJsonAtomically(join2(artifactRun.runDirectory, "run.json"), record); await writeJsonAtomically(artifactRun.manifestPath, record); - const summary = `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}; artifacts: ${artifactRun.runDirectory}; log: ${logPath}`; + const exploration = explorationSummary === null ? "exploration=unavailable" : [ + `nonWait=${String(explorationSummary.actions.nonWaitCount)}`, + `maxWaitStreak=${String(explorationSummary.actions.maxWaitStreak)}`, + `namedChanges=${explorationSummary.namedSnapshots.map((snapshot) => `${snapshot.name}:${String(snapshot.changeAfterNonWaitCount)}`).join(",") || "none"}`, + `policy=${explorationSummary.policy.satisfied ? "satisfied" : "failed"}` + ].join("; "); + const summary = [ + `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}`, + exploration, + `artifacts: ${artifactRun.runDirectory}`, + `log: ${logPath}` + ].join("; "); (status === "passed" ? process2.stdout : process2.stderr).write(`${summary} `); if (failure !== null) { @@ -2108,10 +2906,16 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) // src/tooling/bombadil.ts var attestDirectBombadilTrace2 = attestDirectBombadilTrace; +var summarizeDirectBombadilTrace2 = summarizeDirectBombadilTrace; function runDirectBombadilFuzz2(config, arguments_) { return arguments_ === undefined ? runDirectBombadilFuzz(config) : runDirectBombadilFuzz(config, arguments_); } +function runDirectBombadilFuzzMatrix2(campaigns, arguments_) { + return arguments_ === undefined ? runDirectBombadilFuzzMatrix(campaigns) : runDirectBombadilFuzzMatrix(campaigns, arguments_); +} export { + summarizeDirectBombadilTrace2 as summarizeDirectBombadilTrace, + runDirectBombadilFuzzMatrix2 as runDirectBombadilFuzzMatrix, runDirectBombadilFuzz2 as runDirectBombadilFuzz, attestDirectBombadilTrace2 as attestDirectBombadilTrace }; From a4fa0492457c1a6d25b373bc1f8dbe145243b82e Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 15:39:33 -0400 Subject: [PATCH 19/20] fix: preserve Bombadil trace and cancellation contracts --- dist/tooling/bombadil.js | 252 ++++++++++---- dist/tooling/browser-verification-entry.js | 42 ++- src/tooling/bombadil-runner.test.ts | 364 ++++++++++++++++++++- src/tooling/bombadil-runner.ts | 278 ++++++++++++---- src/tooling/browser-verification.test.ts | 62 ++++ src/tooling/browser-verification.ts | 60 +++- 6 files changed, 920 insertions(+), 138 deletions(-) diff --git a/dist/tooling/bombadil.js b/dist/tooling/bombadil.js index 9f094fd..341bc26 100644 --- a/dist/tooling/bombadil.js +++ b/dist/tooling/bombadil.js @@ -1046,23 +1046,54 @@ async function stopVerificationServerWithOutput(server, stopTimeoutMs = DEFAULT_ async function stopVerificationServer(server, stopTimeoutMs = DEFAULT_STOP_TIMEOUT_MS) { await stopVerificationServerWithOutput(server, stopTimeoutMs); } +function verificationServerAcquisitionAbortError() { + return new Error("Verification server acquisition was aborted"); +} +function throwIfVerificationServerAcquisitionAborted(signal) { + if (signal?.aborted === true) + throw verificationServerAcquisitionAbortError(); +} +async function waitForVerificationServerAcquisitionStep(promise, signal) { + if (signal === undefined) + return await promise; + throwIfVerificationServerAcquisitionAborted(signal); + let abortListener; + const aborted = new Promise((_resolve, reject) => { + abortListener = () => reject(verificationServerAcquisitionAbortError()); + signal.addEventListener("abort", abortListener, { once: true }); + if (signal.aborted) + abortListener(); + }); + let value; + try { + value = await Promise.race([promise, aborted]); + } finally { + if (abortListener !== undefined) + signal.removeEventListener("abort", abortListener); + } + throwIfVerificationServerAcquisitionAborted(signal); + return value; +} async function acquireVerificationServer(options) { const probeTimeoutMs = options.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; const readinessPath = options.readinessPath ?? "/"; const isReachable = options.isReachable ?? serverIsReachable; const canStartLocally = canAutomaticallyStartLocalServer(options.baseUrl, options.localHosts); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { if (canStartLocally && options.reuseExistingLocalServer === false) { throw new Error(`A local server is already reachable at ${options.baseUrl}; ` + "verification will not reuse a server whose worktree ownership is unknown"); } - await Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS), options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "reused" }; } } if (!canStartLocally) { throw new Error(`No server is reachable at ${options.baseUrl}; automatic startup is limited to local HTTP URLs`); } + throwIfVerificationServerAcquisitionAborted(options.abortSignal); const server = options.startServer(); let exitedWithCode = null; try { @@ -1073,10 +1104,11 @@ async function acquireVerificationServer(options) { exitedWithCode = exitCode; break; } - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "started", server }; } - await Bun.sleep(options.pollIntervalMs ?? 200); + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.pollIntervalMs ?? 200), options.abortSignal); } } catch (error) { await stopVerificationServer(server); @@ -1424,9 +1456,9 @@ var RESOURCE_FIELD_MAP = { task_duration: "taskDurationSeconds", thread_time: "threadTimeSeconds" }; -function canonicalJson2(value, depth = 0) { - if (depth > TRACE_MAX_JSON_DEPTH) { - throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(TRACE_MAX_JSON_DEPTH)}`); +function canonicalJson2(value, depth = 0, maximumDepth = TRACE_MAX_JSON_DEPTH) { + if (depth > maximumDepth) { + throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(maximumDepth)}`); } if (value === null || typeof value === "boolean" || typeof value === "string") { return JSON.stringify(value); @@ -1437,20 +1469,21 @@ function canonicalJson2(value, depth = 0) { return JSON.stringify(value); } if (Array.isArray(value)) { - return `[${value.map((entry) => canonicalJson2(entry, depth + 1)).join(",")}]`; + return `[${value.map((entry) => canonicalJson2(entry, depth + 1, maximumDepth)).join(",")}]`; } if (!isRecord2(value)) throw new Error("Bombadil named snapshot is not JSON"); - const entries = Object.keys(value).sort(compareCodeUnits).map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key], depth + 1)}`); + const entries = Object.keys(value).sort(compareCodeUnits).map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key], depth + 1, maximumDepth)}`); return `{${entries.join(",")}}`; } function sha256(value) { return createHash("sha256").update(value).digest("hex"); } -function namedSnapshotValueSha256(value) { - const canonical = canonicalJson2(value); - if (Buffer.byteLength(canonical, "utf8") > TRACE_MAX_CANONICAL_SNAPSHOT_BYTES) { - throw new Error(`Bombadil named snapshot exceeds ${String(TRACE_MAX_CANONICAL_SNAPSHOT_BYTES)} canonical bytes`); +function namedSnapshotValueSha256(value, options = {}) { + const maximumBytes = options.maximumBytes ?? TRACE_MAX_CANONICAL_SNAPSHOT_BYTES; + const canonical = canonicalJson2(value, 0, options.maximumDepth ?? TRACE_MAX_JSON_DEPTH); + if (Buffer.byteLength(canonical, "utf8") > maximumBytes) { + throw new Error(`Bombadil named snapshot exceeds ${String(maximumBytes)} canonical bytes`); } return sha256(canonical); } @@ -1588,7 +1621,7 @@ function parseTraceState(value, lineNumber) { url }; } -function parseTraceLine(line, lineNumber) { +function parseTraceEnvelope(line, lineNumber) { let input; try { input = JSON.parse(line); @@ -1601,37 +1634,82 @@ function parseTraceLine(line, lineNumber) { if (!Number.isSafeInteger(input.timestamp) || typeof input.timestamp !== "number" || input.timestamp < 0 || !Array.isArray(input.snapshots) || input.snapshots.length > TRACE_MAX_SNAPSHOTS_PER_LINE || !Array.isArray(input.violations)) { throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid state fields`); } - const state = parseTraceState(input.state, lineNumber); - const action = parseTraceAction(input.action, lineNumber); - const snapshots = input.snapshots; - const namedSnapshots = []; - const namedSnapshotNames = new Set; + return { + action: input.action, + snapshots: input.snapshots, + state: input.state, + timestamp: input.timestamp, + violations: input.violations + }; +} +function parseDirectTraceObservation(envelope, lineNumber) { + const directSnapshots = envelope.snapshots.filter((snapshot2) => isRecord2(snapshot2) && snapshot2.name === "direct"); + if (directSnapshots.length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); + } + const snapshot = directSnapshots[0]; + if (snapshot === undefined || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshot.index) || !Number.isSafeInteger(snapshot.time) || snapshot.index < 0 || snapshot.time < 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); + } + return { + observation: parseTraceDirectObservation(snapshot.value), + value: snapshot.value + }; +} +function parseDirectTraceLine(line, lineNumber) { + return parseDirectTraceObservation(parseTraceEnvelope(line, lineNumber), lineNumber).observation; +} +function parseTraceLine(line, lineNumber, strictDiagnosticSnapshotNames) { + const envelope = parseTraceEnvelope(line, lineNumber); + const state = parseTraceState(envelope.state, lineNumber); + const action = parseTraceAction(envelope.action, lineNumber); + const snapshots = envelope.snapshots; + const direct = parseDirectTraceObservation(envelope, lineNumber); + const namedSnapshots = [{ + name: "direct", + valueSha256: namedSnapshotValueSha256(direct.value, { + maximumBytes: TRACE_MAX_LINE_BYTES, + maximumDepth: TRACE_MAX_JSON_DEPTH + 4 + }) + }]; + const diagnosticSnapshotValues = new Map; for (const snapshotValue of snapshots) { if (!isRecord2(snapshotValue) || !hasExactKeys(snapshotValue, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshotValue.index) || !Number.isSafeInteger(snapshotValue.time) || snapshotValue.index < 0 || snapshotValue.time < 0 || snapshotValue.name !== null && typeof snapshotValue.name !== "string") { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid snapshot`); } - if (snapshotValue.name !== null) { - const name = validateSnapshotName(snapshotValue.name, `Bombadil trace line ${String(lineNumber)} snapshot name`); - if (namedSnapshotNames.has(name)) { + if (snapshotValue.name === null || snapshotValue.name === "direct") + continue; + let name; + try { + name = validateSnapshotName(snapshotValue.name, `Bombadil trace line ${String(lineNumber)} snapshot name`); + } catch (error) { + if (strictDiagnosticSnapshotNames.has(snapshotValue.name)) + throw error; + continue; + } + const values = diagnosticSnapshotValues.get(name) ?? []; + values.push(snapshotValue.value); + diagnosticSnapshotValues.set(name, values); + } + for (const [name, values] of diagnosticSnapshotValues) { + if (values.length !== 1) { + if (strictDiagnosticSnapshotNames.has(name)) { throw new Error(`Bombadil trace line ${String(lineNumber)} repeats named snapshot ${name}`); } - namedSnapshotNames.add(name); + continue; + } + try { namedSnapshots.push({ name, - valueSha256: namedSnapshotValueSha256(snapshotValue.value) + valueSha256: namedSnapshotValueSha256(values[0]) }); + } catch (error) { + if (strictDiagnosticSnapshotNames.has(name)) + throw error; } } - const directSnapshots = snapshots.filter((snapshot2) => isRecord2(snapshot2) && snapshot2.name === "direct"); - if (directSnapshots.length !== 1) { - throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); - } - const snapshot = directSnapshots[0]; - if (snapshot === undefined || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshot.index) || !Number.isSafeInteger(snapshot.time) || snapshot.index < 0 || snapshot.time < 0) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); - } const propertyViolationNames = []; - for (const violation of input.violations) { + for (const violation of envelope.violations) { if (!isRecord2(violation) || !hasExactKeys(violation, TRACE_VIOLATION_KEYS)) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); } @@ -1642,7 +1720,7 @@ function parseTraceLine(line, lineNumber) { } return { action, - directObservation: parseTraceDirectObservation(snapshot.value), + directObservation: direct.observation, namedSnapshots, propertyViolationNames, state @@ -1673,7 +1751,7 @@ async function attestDirectBombadilTrace(options) { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); } - const observation = parseTraceLine(line, observationCount).directObservation; + const observation = parseDirectTraceLine(line, observationCount); const exact = exactTraceDirectObservation(observation); if (exact === null) { if (initial !== null) { @@ -1759,6 +1837,7 @@ async function summarizeDirectBombadilTrace(options) { throw new Error("targetUrl must be an absolute URL"); } const policy = validateExplorationPolicy(options.explorationPolicy); + const strictDiagnosticSnapshotNames = explorationPolicySnapshotNames(policy); const actionCounts = new Map; const targetTags = new Map; const urlFingerprints = new Set; @@ -1788,6 +1867,8 @@ async function summarizeDirectBombadilTrace(options) { let policyObservationCount = 0; let previousObservationWasExact = false; let stableTarget = true; + let trackedUnrelatedSnapshotNameCount = 0; + const unrelatedSnapshotNameLimit = Math.max(0, TRACE_MAX_NAMED_SNAPSHOT_NAMES - strictDiagnosticSnapshotNames.size); const stream = createReadStream(options.tracePath, { encoding: "utf8" }); const lines = createInterface({ input: stream, crlfDelay: Infinity }); try { @@ -1799,7 +1880,7 @@ async function summarizeDirectBombadilTrace(options) { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); } - const parsed = parseTraceLine(line, lineCount); + const parsed = parseTraceLine(line, lineCount, strictDiagnosticSnapshotNames); const rawRelativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; rawUrlFingerprints.add(sha256(rawRelativeUrl)); if (rawUrlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { @@ -1858,6 +1939,10 @@ async function summarizeDirectBombadilTrace(options) { for (const snapshot of parsed.namedSnapshots) { let entry = snapshots.get(snapshot.name); if (entry === undefined) { + const isStrictSnapshot = snapshot.name === "direct" || strictDiagnosticSnapshotNames.has(snapshot.name); + if (!isStrictSnapshot && trackedUnrelatedSnapshotNameCount >= unrelatedSnapshotNameLimit) { + continue; + } if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`); } @@ -1870,6 +1955,14 @@ async function summarizeDirectBombadilTrace(options) { values: new Set }; snapshots.set(snapshot.name, entry); + if (!isStrictSnapshot) + trackedUnrelatedSnapshotNameCount += 1; + } + if (!entry.values.has(snapshot.valueSha256) && entry.values.size >= TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { + if (snapshot.name === "direct" || strictDiagnosticSnapshotNames.has(snapshot.name)) { + throw new Error(`Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`); + } + continue; } const changedAfterRecordedAction = recordedActionKind !== null && entry.lastObservationIndex === policyObservationCount - 1 && entry.lastValueSha256 !== null && entry.lastValueSha256 !== snapshot.valueSha256; if (changedAfterRecordedAction) { @@ -1882,9 +1975,6 @@ async function summarizeDirectBombadilTrace(options) { entry.lastValueSha256 = snapshot.valueSha256; entry.observationCount += 1; entry.values.add(snapshot.valueSha256); - if (entry.values.size > TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { - throw new Error(`Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`); - } } previousObservationWasExact = true; } @@ -2159,6 +2249,20 @@ function validateSnapshotActionMinimumMap(options) { } return Object.freeze(validated); } +function explorationPolicySnapshotNames(policy) { + const names = new Set(["direct"]); + if (policy === null) + return names; + for (const name of policy.requiredNamedSnapshots) + names.add(name); + for (const name of Object.keys(policy.minDistinctNamedSnapshotValues)) + names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterNonWait)) + names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterActionKind)) + names.add(name); + return names; +} function validateExplorationPolicy(value) { if (value === undefined) return null; @@ -2205,7 +2309,7 @@ function validateExplorationPolicy(value) { if (typeof requireStableTargetUrl !== "boolean") { throw new Error("explorationPolicy.requireStableTargetUrl must be a boolean"); } - return Object.freeze({ + const validated = Object.freeze({ minDistinctNamedSnapshotValues, minNamedSnapshotChangesAfterActionKind, minNamedSnapshotChangesAfterNonWait, @@ -2214,6 +2318,10 @@ function validateExplorationPolicy(value) { requiredActionKinds: Object.freeze(requiredActionKinds), requiredNamedSnapshots: Object.freeze(requiredNamedSnapshots) }); + if (explorationPolicySnapshotNames(validated).size > TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error(`explorationPolicy may reference at most ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES - 1)} distinct non-Direct snapshots`); + } + return validated; } function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { const repositoryRoot = resolve(config.repositoryRoot); @@ -2450,6 +2558,7 @@ async function runBombadilNativeProcess(invocation) { } var defaultDependencies = { acquireServer: acquireVerificationServer, + createAbortController: () => new AbortController, now: () => new Date, runBombadil: runBombadilNativeProcess, serverOutputTimeoutMs: SERVER_OUTPUT_TIMEOUT_MS, @@ -2652,6 +2761,17 @@ async function runDirectBombadilFuzzMatrix(campaignsInput, arguments_ = process2 } return { kind: "matrix", results: Object.freeze(results) }; } +function throwIfBombadilRunAborted(signal) { + if (signal.aborted) + throw new Error("Bombadil fuzzing was interrupted"); +} +function terminateAbortedOwnedServer(signal, server) { + if (!signal.aborted) + return; + if (server.exitCode() === null) + server.terminate(); + throwIfBombadilRunAborted(signal); +} async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2), dependencyOverrides = {}) { const parsed = parseDirectBombadilFuzzArguments(arguments_, config.baseUrl); if (parsed.kind === "help") { @@ -2672,7 +2792,7 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) }); const outputPath = join2(artifactRun.runDirectory, "bombadil"); const tracePath = join2(outputPath, "trace.jsonl"); - const abortController = new AbortController; + const abortController = dependencies.createAbortController?.() ?? new AbortController; const invocation = createDirectBombadilInvocation({ baseUrl: validated.baseUrl, bombadilExecutable: validated.bombadilExecutable, @@ -2715,23 +2835,37 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) try { await requireRegularFile(validated.bombadilExecutable, "The root Bombadil executable"); bombadilVersion = await readExactBombadilVersion(validated.repositoryRoot); - if (abortController.signal.aborted) - throw new Error("Bombadil fuzzing was interrupted"); - lease = await dependencies.acquireServer({ - baseUrl: validated.baseUrl, - label: validated.label, - readinessPath: validated.server.readinessPath, - reuseExistingLocalServer: false, - startupTimeoutMs: validated.server.startupTimeoutMs, - startServer: () => { - ownedServer = dependencies.spawnServer({ - command: serverCommand, - cwd: validated.server.cwd, - ...validated.server.env === undefined ? {} : { env: validated.server.env } - }); - return ownedServer; - } - }); + throwIfBombadilRunAborted(abortController.signal); + try { + lease = await dependencies.acquireServer({ + abortSignal: abortController.signal, + baseUrl: validated.baseUrl, + label: validated.label, + readinessPath: validated.server.readinessPath, + reuseExistingLocalServer: false, + startupTimeoutMs: validated.server.startupTimeoutMs, + startServer: () => { + throwIfBombadilRunAborted(abortController.signal); + ownedServer = dependencies.spawnServer({ + command: serverCommand, + cwd: validated.server.cwd, + ...validated.server.env === undefined ? {} : { env: validated.server.env } + }); + terminateAbortedOwnedServer(abortController.signal, ownedServer); + return ownedServer; + } + }); + } catch (error) { + if (abortController.signal.aborted) + throwIfBombadilRunAborted(abortController.signal); + throw error; + } + if (abortController.signal.aborted) { + const acquiredOwnedServer = ownedServer; + if (acquiredOwnedServer?.exitCode() === null) + acquiredOwnedServer.terminate(); + throwIfBombadilRunAborted(abortController.signal); + } let processFailure = null; try { processResult = await dependencies.runBombadil(abortableInvocation); diff --git a/dist/tooling/browser-verification-entry.js b/dist/tooling/browser-verification-entry.js index d8a4582..77fa758 100644 --- a/dist/tooling/browser-verification-entry.js +++ b/dist/tooling/browser-verification-entry.js @@ -1393,23 +1393,54 @@ async function stopVerificationServerWithOutput(server, stopTimeoutMs = DEFAULT_ async function stopVerificationServer(server, stopTimeoutMs = DEFAULT_STOP_TIMEOUT_MS) { await stopVerificationServerWithOutput(server, stopTimeoutMs); } +function verificationServerAcquisitionAbortError() { + return new Error("Verification server acquisition was aborted"); +} +function throwIfVerificationServerAcquisitionAborted(signal) { + if (signal?.aborted === true) + throw verificationServerAcquisitionAbortError(); +} +async function waitForVerificationServerAcquisitionStep(promise, signal) { + if (signal === undefined) + return await promise; + throwIfVerificationServerAcquisitionAborted(signal); + let abortListener; + const aborted = new Promise((_resolve, reject) => { + abortListener = () => reject(verificationServerAcquisitionAbortError()); + signal.addEventListener("abort", abortListener, { once: true }); + if (signal.aborted) + abortListener(); + }); + let value; + try { + value = await Promise.race([promise, aborted]); + } finally { + if (abortListener !== undefined) + signal.removeEventListener("abort", abortListener); + } + throwIfVerificationServerAcquisitionAborted(signal); + return value; +} async function acquireVerificationServer(options) { const probeTimeoutMs = options.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; const readinessPath = options.readinessPath ?? "/"; const isReachable = options.isReachable ?? serverIsReachable; const canStartLocally = canAutomaticallyStartLocalServer(options.baseUrl, options.localHosts); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { if (canStartLocally && options.reuseExistingLocalServer === false) { throw new Error(`A local server is already reachable at ${options.baseUrl}; ` + "verification will not reuse a server whose worktree ownership is unknown"); } - await Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS), options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "reused" }; } } if (!canStartLocally) { throw new Error(`No server is reachable at ${options.baseUrl}; automatic startup is limited to local HTTP URLs`); } + throwIfVerificationServerAcquisitionAborted(options.abortSignal); const server = options.startServer(); let exitedWithCode = null; try { @@ -1420,10 +1451,11 @@ async function acquireVerificationServer(options) { exitedWithCode = exitCode; break; } - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "started", server }; } - await Bun.sleep(options.pollIntervalMs ?? 200); + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.pollIntervalMs ?? 200), options.abortSignal); } } catch (error) { await stopVerificationServer(server); diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index fb2ccf4..45e668c 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -171,6 +171,67 @@ function directObservation(options: { }; } +function largeDirectObservation(): Record { + const scenarioIds = Array.from({ length: 256 }, (_, index) => + `s${String(index).padStart(3, "0")}.${"x".repeat(115)}` + ); + const firstScenario = scenarioIds[0]; + if (firstScenario === undefined) throw new Error("large Direct fixture needs a scenario"); + const citedScenarios: [string, ...string[]] = [firstScenario, ...scenarioIds.slice(1)]; + const definition = defineDirect({ + parseWorld: (input) => { + if ( + typeof input !== "object" + || input === null + || Array.isArray(input) + || typeof Reflect.get(input, "count") !== "number" + ) { + throw new Error("count is required"); + } + return { count: Reflect.get(input, "count") as number }; + }, + defaultScenario: firstScenario, + scenarios: scenarioIds.map((id, index) => ({ + description: "d".repeat(2_000), + id, + route: `/surface/${String(index)}`, + title: "t".repeat(160), + world: { count: index }, + })), + coverage: Array.from({ length: 256 }, (_, index) => ({ + claim: "c".repeat(1_000), + key: `coverage.${String(index)}`, + mode: "fixture" as const, + scenarios: citedScenarios, + })), + }); + const session = createDirectSession({ + definition, + activation: { kind: "scenario", scenario: firstScenario }, + create: () => ({}), + }); + if (!session.ok) throw new Error(session.error.message); + const snapshot = session.value.probe.snapshot(); + if (!snapshot.ok) throw new Error(snapshot.error.message); + const manifest = jsonClone(session.value.manifest); + const probe = jsonClone(snapshot.value); + return { + activationHash: manifest.active.activationHash, + activeRoute: manifest.active.route, + activeScenario: manifest.active.scenario, + activeSource: manifest.active.source, + bridgePresent: true, + bridgeSchema: "direct.browser-bridge/v2", + catalogHash: manifest.catalogHash, + contractValid: true, + isQuiescent: probe.isQuiescent, + manifest, + probe, + violations: Object.values(probe.violations), + violationsValid: true, + }; +} + function absentObservation(): Record { return { activationHash: "", @@ -238,15 +299,24 @@ function traceLine( async function writeTrace( tracePath: string, observations: readonly unknown[], + lineOptions: readonly TraceLineOptions[] = [], ): Promise { await mkdir(join(tracePath, ".."), { recursive: true }); await writeFile( tracePath, - `${observations.map((observation, index) => traceLine(observation, index + 1)).join("\n")}\n`, + `${observations.map((observation, index) => + traceLine(observation, index + 1, lineOptions[index]) + ).join("\n")}\n`, "utf8", ); } +function nestedJson(depth: number): unknown { + let value: unknown = null; + for (let index = 0; index < depth; index += 1) value = [value]; + return value; +} + function fakeServer( calls: string[], output: Promise = Promise.resolve("server output"), @@ -288,6 +358,7 @@ function dependencies(options: { readonly noTrace?: boolean; readonly neverServerOutput?: boolean; readonly observations?: readonly unknown[]; + readonly traceLineOptions?: readonly TraceLineOptions[]; readonly serverOutputTimeoutMs?: number; readonly stopFailure?: boolean; readonly termination?: "aborted" | "timeout"; @@ -336,6 +407,7 @@ function dependencies(options: { await writeTrace( join(invocation.outputPath, "trace.jsonl"), options.observations ?? [absentObservation(), directObservation()], + options.traceLineOptions, ); } return { @@ -354,7 +426,7 @@ function dependencies(options: { if (options.stopFailure === true) { throw new Error("server cleanup failed"); } - ownedServer.terminate(); + if (ownedServer.exitCode() === null) ownedServer.terminate(); await ownedServer.exited; }, }, @@ -582,6 +654,31 @@ describe("Direct Bombadil configuration and invocation", () => { }, })).toThrow("between 1 and 10000"); } + const snapshotNames = (prefix: string): string[] => + Array.from({ length: 32 }, (_, index) => `${prefix}${String(index)}`); + const maximumPolicy = { + minDistinctNamedSnapshotValues: Object.fromEntries( + snapshotNames("d").map((name) => [name, 1]), + ), + minNamedSnapshotChangesAfterActionKind: Object.fromEntries( + snapshotNames("a").map((name) => [name, { Click: 1 }]), + ), + minNamedSnapshotChangesAfterNonWait: Object.fromEntries( + snapshotNames("n").map((name) => [name, 1]), + ), + requiredNamedSnapshots: snapshotNames("r").slice(0, 31), + } as const; + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: maximumPolicy, + })).not.toThrow(); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + ...maximumPolicy, + requiredNamedSnapshots: snapshotNames("r"), + }, + })).toThrow("at most 127 distinct non-Direct snapshots"); }); test("rejects specification, server cwd, and replay symlinks that escape the repository", async () => { @@ -740,11 +837,14 @@ describe("Direct Bombadil campaign matrix", () => { }); describe("Direct Bombadil trace attestation", () => { - async function attest(observations: readonly unknown[]) { + async function attest( + observations: readonly unknown[], + lineOptions: readonly TraceLineOptions[] = [], + ) { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-trace-")); temporaryDirectories.push(directory); const tracePath = join(directory, "trace.jsonl"); - await writeTrace(tracePath, observations); + await writeTrace(tracePath, observations, lineOptions); return attestDirectBombadilTrace({ expectedRoute: "/surface", expectedScenario: "surface.ready", @@ -774,6 +874,72 @@ describe("Direct Bombadil trace attestation", () => { }); }); + test("ignores unrelated Bombadil snapshots outside the Direct attestation contract", async () => { + const oversizedValue = "x".repeat(2 * 1024 * 1024 + 1); + const cases: readonly { + readonly label: string; + readonly snapshots: NonNullable; + }[] = [{ + label: "arbitrary name", + snapshots: [{ name: "phase state", value: "ready" }], + }, { + label: "duplicate unrelated name", + snapshots: [ + { name: "phase", value: "loading" }, + { name: "phase", value: "ready" }, + ], + }, { + label: "depth 65 value", + snapshots: [{ name: "deep", value: nestedJson(65) }], + }, { + label: "value above two MiB", + snapshots: [{ name: "large", value: oversizedValue }], + }]; + + for (const testCase of cases) { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-compatible-trace-")); + temporaryDirectories.push(directory); + const tracePath = join(directory, "trace.jsonl"); + await writeTrace(tracePath, [directObservation()], [{ + namedSnapshots: testCase.snapshots, + }]); + const result = await attestDirectBombadilTrace({ + expectedRoute: "/surface", + expectedScenario: "surface.ready", + tracePath, + }); + expect(result.validObservationCount, testCase.label).toBe(1); + } + }); + + test("keeps exact Direct uniqueness and schema bounds independent of diagnostic limits", async () => { + const duplicateError = await rejection(attest([directObservation()], [{ + namedSnapshots: [{ name: "direct", value: directObservation() }], + }])); + expect(duplicateError.message).toContain("must contain one named direct snapshot"); + + const observation = largeDirectObservation(); + expect(Buffer.byteLength(JSON.stringify(observation), "utf8")) + .toBeGreaterThan(2 * 1024 * 1024); + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-large-direct-trace-")); + temporaryDirectories.push(directory); + const tracePath = join(directory, "trace.jsonl"); + await writeTrace(tracePath, [observation]); + const result = await attestDirectBombadilTrace({ + expectedRoute: String(observation.activeRoute), + expectedScenario: String(observation.activeScenario), + tracePath, + }); + expect(result.validObservationCount).toBe(1); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots).toEqual([ + expect.objectContaining({ name: "direct", observationCount: 1 }), + ]); + }); + test("rejects a vacuous, wrongly activated, or post-activation missing trace", async () => { expect((await rejection(attest([absentObservation()]))).message) .toContain("never reached a valid Direct contract"); @@ -964,6 +1130,60 @@ describe("Direct Bombadil exploration summary", () => { expect(serialized).not.toContain("/tmp/"); }); + test("omits out-of-contract unrelated snapshots while keeping policy snapshots strict", async () => { + const observation = directObservation(); + const oversizedValue = "x".repeat(2 * 1024 * 1024 + 1); + const tracePath = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: [ + { name: "owned", value: "ready" }, + { name: "phase state", value: "ready" }, + { name: "duplicate", value: 0 }, + { name: "duplicate", value: 1 }, + { name: "deep", value: nestedJson(65) }, + { name: "large", value: oversizedValue }, + ], + })]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { requiredNamedSnapshots: ["owned"] }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.map(({ name }) => name)).toEqual(["direct", "owned"]); + expect(summary.policy).toMatchObject({ configured: true, failures: [], satisfied: true }); + + const strictCases: readonly { + readonly label: string; + readonly snapshots: NonNullable; + readonly expected: string; + }[] = [{ + label: "duplicate", + snapshots: [ + { name: "owned", value: 0 }, + { name: "owned", value: 1 }, + ], + expected: "repeats named snapshot owned", + }, { + label: "depth", + snapshots: [{ name: "owned", value: nestedJson(65) }], + expected: "exceeds JSON depth", + }, { + label: "size", + snapshots: [{ name: "owned", value: oversizedValue }], + expected: "exceeds 2097152 canonical bytes", + }]; + for (const testCase of strictCases) { + const strictTracePath = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: testCase.snapshots, + })]); + const error = await rejection(summarizeDirectBombadilTrace({ + explorationPolicy: { requiredNamedSnapshots: ["owned"] }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: strictTracePath, + })); + expect(error.message, testCase.label).toContain(testCase.expected); + } + }); + test("accepts every exact Bombadil 0.7.2 browser action payload", async () => { const actions = [ "Back", @@ -1361,6 +1581,7 @@ describe("Direct Bombadil exploration summary", () => { namedSnapshots: [{ name: "deep", value: deep }], })]); expect((await rejection(summarizeDirectBombadilTrace({ + explorationPolicy: { requiredNamedSnapshots: ["deep"] }, targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", tracePath: deepTrace, }))).message).toContain("exceeds JSON depth"); @@ -1522,6 +1743,141 @@ describe("Direct Bombadil run lifecycle", () => { }); }); + test("runs with policy-owned evidence despite arbitrary unrelated named snapshots", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies({ + observations: [directObservation()], + traceLineOptions: [{ + namedSnapshots: [ + { name: "owned", value: "ready" }, + { name: "phase state", value: "ready" }, + { name: "duplicate", value: 0 }, + { name: "duplicate", value: 1 }, + { name: "deep", value: nestedJson(65) }, + { name: "large", value: "x".repeat(2 * 1024 * 1024 + 1) }, + ], + }], + }); + const result = await runDirectBombadilFuzz({ + ...config, + explorationPolicy: { requiredNamedSnapshots: ["owned"] }, + }, [], runtime.overrides); + expect(result.kind).toBe("run"); + const manifest = JSON.parse(await readFile( + join(repositoryRoot, "artifacts", "direct-bombadil", "fixture-product", "manifest.json"), + "utf8", + )) as Record; + expect(manifest).toMatchObject({ + status: "passed", + attestation: { validObservationCount: 1 }, + explorationSummary: { + namedSnapshots: [{ name: "direct" }, { name: "owned" }], + policy: { configured: true, failures: [], satisfied: true }, + }, + }); + }); + + test("does not spawn when cancellation wins before server startup", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + const error = await rejection(runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + acquireServer: (options): Promise => { + runtime.calls.push("acquire-server"); + controller.abort(); + return Promise.resolve({ source: "started", server: options.startServer() }); + }, + })); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual(["acquire-server"]); + }); + + test("terminates a server when cancellation wins during spawn", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + const server = fakeServer(runtime.calls); + const error = await rejection(runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + spawnServer: () => { + runtime.calls.push("spawn-server"); + controller.abort(); + return server; + }, + })); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual([ + "acquire-server", + "spawn-server", + "terminate", + "stop-server", + ]); + expect(server.exitCode()).toBe(0); + }); + + test("cleans a just-acquired server before Bombadil can run", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + const error = await rejection(runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + acquireServer: (options): Promise => { + runtime.calls.push("acquire-server"); + const server = options.startServer(); + controller.abort(); + return Promise.resolve({ source: "started", server }); + }, + })); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual([ + "acquire-server", + "spawn-server", + "terminate", + "stop-server", + ]); + expect(runtime.calls).not.toContain("run-bombadil"); + }); + + test("unwinds a pending server acquisition when cancellation arrives", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + let markAcquiring!: () => void; + const acquiring = new Promise((resolve) => { + markAcquiring = resolve; + }); + const run = runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + acquireServer: async (options): Promise => { + runtime.calls.push("acquire-server"); + const server = options.startServer(); + markAcquiring(); + await new Promise((_resolve, reject) => { + const abort = (): void => reject(new Error("acquisition aborted")); + options.abortSignal?.addEventListener("abort", abort, { once: true }); + if (options.abortSignal?.aborted === true) abort(); + }); + return { source: "started", server }; + }, + }); + await acquiring; + controller.abort(); + const error = await rejection(run); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual([ + "acquire-server", + "spawn-server", + "stop-server", + "terminate", + ]); + expect(runtime.calls).not.toContain("run-bombadil"); + }); + test("fails a configured exploration policy while retaining the derived sidecar", async () => { const { config, repositoryRoot } = await fixture(); const runtime = dependencies(); diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index ddad076..57107e8 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -354,6 +354,7 @@ export interface DirectBombadilTraceAttestation { export interface DirectBombadilRunnerDependencies { readonly acquireServer: typeof acquireVerificationServer; + readonly createAbortController?: () => AbortController; readonly now: () => Date; readonly runBombadil: ( invocation: DirectBombadilInvocation, @@ -673,6 +674,19 @@ interface ParsedTraceLine { readonly state: ParsedTraceState; } +interface ParsedTraceEnvelope { + readonly action: unknown; + readonly snapshots: readonly unknown[]; + readonly state: unknown; + readonly timestamp: number; + readonly violations: readonly unknown[]; +} + +interface ParsedDirectTraceObservation { + readonly observation: TraceDirectObservation; + readonly value: unknown; +} + const RESOURCE_FIELD_MAP = { documents: "documents", dom_nodes: "domNodes", @@ -685,9 +699,13 @@ const RESOURCE_FIELD_MAP = { thread_time: "threadTimeSeconds", } as const; -function canonicalJson(value: unknown, depth = 0): string { - if (depth > TRACE_MAX_JSON_DEPTH) { - throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(TRACE_MAX_JSON_DEPTH)}`); +function canonicalJson( + value: unknown, + depth = 0, + maximumDepth = TRACE_MAX_JSON_DEPTH, +): string { + if (depth > maximumDepth) { + throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(maximumDepth)}`); } if (value === null || typeof value === "boolean" || typeof value === "string") { return JSON.stringify(value); @@ -697,11 +715,11 @@ function canonicalJson(value: unknown, depth = 0): string { return JSON.stringify(value); } if (Array.isArray(value)) { - return `[${value.map((entry) => canonicalJson(entry, depth + 1)).join(",")}]`; + return `[${value.map((entry) => canonicalJson(entry, depth + 1, maximumDepth)).join(",")}]`; } if (!isRecord(value)) throw new Error("Bombadil named snapshot is not JSON"); const entries = Object.keys(value).sort(compareCodeUnits).map((key) => - `${JSON.stringify(key)}:${canonicalJson(value[key], depth + 1)}` + `${JSON.stringify(key)}:${canonicalJson(value[key], depth + 1, maximumDepth)}` ); return `{${entries.join(",")}}`; } @@ -710,11 +728,22 @@ function sha256(value: string | Uint8Array): string { return createHash("sha256").update(value).digest("hex"); } -function namedSnapshotValueSha256(value: unknown): string { - const canonical = canonicalJson(value); - if (Buffer.byteLength(canonical, "utf8") > TRACE_MAX_CANONICAL_SNAPSHOT_BYTES) { +function namedSnapshotValueSha256( + value: unknown, + options: { + readonly maximumBytes?: number; + readonly maximumDepth?: number; + } = {}, +): string { + const maximumBytes = options.maximumBytes ?? TRACE_MAX_CANONICAL_SNAPSHOT_BYTES; + const canonical = canonicalJson( + value, + 0, + options.maximumDepth ?? TRACE_MAX_JSON_DEPTH, + ); + if (Buffer.byteLength(canonical, "utf8") > maximumBytes) { throw new Error( - `Bombadil named snapshot exceeds ${String(TRACE_MAX_CANONICAL_SNAPSHOT_BYTES)} canonical bytes`, + `Bombadil named snapshot exceeds ${String(maximumBytes)} canonical bytes`, ); } return sha256(canonical); @@ -941,7 +970,7 @@ function parseTraceState(value: unknown, lineNumber: number): ParsedTraceState { }; } -function parseTraceLine(line: string, lineNumber: number): ParsedTraceLine { +function parseTraceEnvelope(line: string, lineNumber: number): ParsedTraceEnvelope { let input: unknown; try { input = JSON.parse(line) as unknown; @@ -961,11 +990,65 @@ function parseTraceLine(line: string, lineNumber: number): ParsedTraceLine { ) { throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid state fields`); } - const state = parseTraceState(input.state, lineNumber); - const action = parseTraceAction(input.action, lineNumber); - const snapshots = input.snapshots as unknown[]; - const namedSnapshots: Array<{ readonly name: string; readonly valueSha256: string }> = []; - const namedSnapshotNames = new Set(); + return { + action: input.action, + snapshots: input.snapshots as unknown[], + state: input.state, + timestamp: input.timestamp, + violations: input.violations, + }; +} + +function parseDirectTraceObservation( + envelope: ParsedTraceEnvelope, + lineNumber: number, +): ParsedDirectTraceObservation { + const directSnapshots = envelope.snapshots.filter( + (snapshot): snapshot is Readonly> => + isRecord(snapshot) && snapshot.name === "direct", + ); + if (directSnapshots.length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); + } + const snapshot = directSnapshots[0]; + if ( + snapshot === undefined + || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) + || !Number.isSafeInteger(snapshot.index) + || !Number.isSafeInteger(snapshot.time) + || (snapshot.index as number) < 0 + || (snapshot.time as number) < 0 + ) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); + } + return { + observation: parseTraceDirectObservation(snapshot.value), + value: snapshot.value, + }; +} + +function parseDirectTraceLine(line: string, lineNumber: number): TraceDirectObservation { + return parseDirectTraceObservation(parseTraceEnvelope(line, lineNumber), lineNumber).observation; +} + +function parseTraceLine( + line: string, + lineNumber: number, + strictDiagnosticSnapshotNames: ReadonlySet, +): ParsedTraceLine { + const envelope = parseTraceEnvelope(line, lineNumber); + const state = parseTraceState(envelope.state, lineNumber); + const action = parseTraceAction(envelope.action, lineNumber); + const snapshots = envelope.snapshots; + const direct = parseDirectTraceObservation(envelope, lineNumber); + const namedSnapshots: Array<{ readonly name: string; readonly valueSha256: string }> = [{ + name: "direct", + valueSha256: namedSnapshotValueSha256(direct.value, { + maximumBytes: TRACE_MAX_LINE_BYTES, + maximumDepth: TRACE_MAX_JSON_DEPTH + 4, + }), + }]; + const diagnosticSnapshotValues = new Map(); for (const snapshotValue of snapshots) { if ( !isRecord(snapshotValue) @@ -978,40 +1061,39 @@ function parseTraceLine(line: string, lineNumber: number): ParsedTraceLine { ) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid snapshot`); } - if (snapshotValue.name !== null) { - const name = validateSnapshotName( + if (snapshotValue.name === null || snapshotValue.name === "direct") continue; + let name: string; + try { + name = validateSnapshotName( snapshotValue.name, `Bombadil trace line ${String(lineNumber)} snapshot name`, ); - if (namedSnapshotNames.has(name)) { + } catch (error) { + if (strictDiagnosticSnapshotNames.has(snapshotValue.name)) throw error; + continue; + } + const values = diagnosticSnapshotValues.get(name) ?? []; + values.push(snapshotValue.value); + diagnosticSnapshotValues.set(name, values); + } + for (const [name, values] of diagnosticSnapshotValues) { + if (values.length !== 1) { + if (strictDiagnosticSnapshotNames.has(name)) { throw new Error(`Bombadil trace line ${String(lineNumber)} repeats named snapshot ${name}`); } - namedSnapshotNames.add(name); + continue; + } + try { namedSnapshots.push({ name, - valueSha256: namedSnapshotValueSha256(snapshotValue.value), + valueSha256: namedSnapshotValueSha256(values[0]), }); + } catch (error) { + if (strictDiagnosticSnapshotNames.has(name)) throw error; } } - const directSnapshots = snapshots.filter((snapshot): snapshot is Readonly> => - isRecord(snapshot) && snapshot.name === "direct" - ); - if (directSnapshots.length !== 1) { - throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); - } - const snapshot = directSnapshots[0]; - if ( - snapshot === undefined - || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) - || !Number.isSafeInteger(snapshot.index) - || !Number.isSafeInteger(snapshot.time) - || (snapshot.index as number) < 0 - || (snapshot.time as number) < 0 - ) { - throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); - } const propertyViolationNames: string[] = []; - for (const violation of input.violations) { + for (const violation of envelope.violations) { if (!isRecord(violation) || !hasExactKeys(violation, TRACE_VIOLATION_KEYS)) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); } @@ -1025,7 +1107,7 @@ function parseTraceLine(line: string, lineNumber: number): ParsedTraceLine { } return { action, - directObservation: parseTraceDirectObservation(snapshot.value), + directObservation: direct.observation, namedSnapshots, propertyViolationNames, state, @@ -1063,7 +1145,7 @@ export async function attestDirectBombadilTrace(options: { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); } - const observation = parseTraceLine(line, observationCount).directObservation; + const observation = parseDirectTraceLine(line, observationCount); const exact = exactTraceDirectObservation(observation); if (exact === null) { if (initial !== null) { @@ -1172,6 +1254,7 @@ export async function summarizeDirectBombadilTrace(options: { throw new Error("targetUrl must be an absolute URL"); } const policy = validateExplorationPolicy(options.explorationPolicy); + const strictDiagnosticSnapshotNames = explorationPolicySnapshotNames(policy); const actionCounts = new Map(); const targetTags = new Map(); const urlFingerprints = new Set(); @@ -1208,6 +1291,11 @@ export async function summarizeDirectBombadilTrace(options: { let policyObservationCount = 0; let previousObservationWasExact = false; let stableTarget = true; + let trackedUnrelatedSnapshotNameCount = 0; + const unrelatedSnapshotNameLimit = Math.max( + 0, + TRACE_MAX_NAMED_SNAPSHOT_NAMES - strictDiagnosticSnapshotNames.size, + ); const stream = createReadStream(options.tracePath, { encoding: "utf8" }); const lines = createInterface({ input: stream, crlfDelay: Infinity }); try { @@ -1219,7 +1307,7 @@ export async function summarizeDirectBombadilTrace(options: { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); } - const parsed = parseTraceLine(line, lineCount); + const parsed = parseTraceLine(line, lineCount, strictDiagnosticSnapshotNames); const rawRelativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; rawUrlFingerprints.add(sha256(rawRelativeUrl)); @@ -1297,6 +1385,11 @@ export async function summarizeDirectBombadilTrace(options: { for (const snapshot of parsed.namedSnapshots) { let entry = snapshots.get(snapshot.name); if (entry === undefined) { + const isStrictSnapshot = snapshot.name === "direct" + || strictDiagnosticSnapshotNames.has(snapshot.name); + if (!isStrictSnapshot && trackedUnrelatedSnapshotNameCount >= unrelatedSnapshotNameLimit) { + continue; + } if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { throw new Error( `Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`, @@ -1311,6 +1404,21 @@ export async function summarizeDirectBombadilTrace(options: { values: new Set(), }; snapshots.set(snapshot.name, entry); + if (!isStrictSnapshot) trackedUnrelatedSnapshotNameCount += 1; + } + if ( + !entry.values.has(snapshot.valueSha256) + && entry.values.size >= TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME + ) { + if ( + snapshot.name === "direct" + || strictDiagnosticSnapshotNames.has(snapshot.name) + ) { + throw new Error( + `Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`, + ); + } + continue; } const changedAfterRecordedAction = recordedActionKind !== null @@ -1330,11 +1438,6 @@ export async function summarizeDirectBombadilTrace(options: { entry.lastValueSha256 = snapshot.valueSha256; entry.observationCount += 1; entry.values.add(snapshot.valueSha256); - if (entry.values.size > TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { - throw new Error( - `Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`, - ); - } } previousObservationWasExact = true; } @@ -1708,6 +1811,18 @@ function validateSnapshotActionMinimumMap(options: { return Object.freeze(validated); } +function explorationPolicySnapshotNames( + policy: ValidatedExplorationPolicy | null, +): ReadonlySet { + const names = new Set(["direct"]); + if (policy === null) return names; + for (const name of policy.requiredNamedSnapshots) names.add(name); + for (const name of Object.keys(policy.minDistinctNamedSnapshotValues)) names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterNonWait)) names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterActionKind)) names.add(name); + return names; +} + function validateExplorationPolicy( value: unknown, ): ValidatedExplorationPolicy | null { @@ -1775,7 +1890,7 @@ function validateExplorationPolicy( if (typeof requireStableTargetUrl !== "boolean") { throw new Error("explorationPolicy.requireStableTargetUrl must be a boolean"); } - return Object.freeze({ + const validated: ValidatedExplorationPolicy = Object.freeze({ minDistinctNamedSnapshotValues, minNamedSnapshotChangesAfterActionKind, minNamedSnapshotChangesAfterNonWait, @@ -1784,6 +1899,12 @@ function validateExplorationPolicy( requiredActionKinds: Object.freeze(requiredActionKinds), requiredNamedSnapshots: Object.freeze(requiredNamedSnapshots), }); + if (explorationPolicySnapshotNames(validated).size > TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error( + `explorationPolicy may reference at most ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES - 1)} distinct non-Direct snapshots`, + ); + } + return validated; } export function validateDirectBombadilFuzzConfig( @@ -2091,6 +2212,7 @@ export async function runBombadilNativeProcess( const defaultDependencies: DirectBombadilRunnerDependencies = { acquireServer: acquireVerificationServer, + createAbortController: () => new AbortController(), now: () => new Date(), runBombadil: runBombadilNativeProcess, serverOutputTimeoutMs: SERVER_OUTPUT_TIMEOUT_MS, @@ -2359,6 +2481,19 @@ export async function runDirectBombadilFuzzMatrix( return { kind: "matrix", results: Object.freeze(results) }; } +function throwIfBombadilRunAborted(signal: AbortSignal): void { + if (signal.aborted) throw new Error("Bombadil fuzzing was interrupted"); +} + +function terminateAbortedOwnedServer( + signal: AbortSignal, + server: ManagedVerificationServer, +): void { + if (!signal.aborted) return; + if (server.exitCode() === null) server.terminate(); + throwIfBombadilRunAborted(signal); +} + /** Runs one bounded diagnostic Bombadil campaign and always releases its server lease. */ export async function runDirectBombadilFuzz( config: DirectBombadilFuzzConfig, @@ -2390,7 +2525,7 @@ export async function runDirectBombadilFuzz( }); const outputPath = join(artifactRun.runDirectory, "bombadil"); const tracePath = join(outputPath, "trace.jsonl"); - const abortController = new AbortController(); + const abortController = dependencies.createAbortController?.() ?? new AbortController(); const invocation = createDirectBombadilInvocation({ baseUrl: validated.baseUrl, bombadilExecutable: validated.bombadilExecutable, @@ -2436,23 +2571,36 @@ export async function runDirectBombadilFuzz( try { await requireRegularFile(validated.bombadilExecutable, "The root Bombadil executable"); bombadilVersion = await readExactBombadilVersion(validated.repositoryRoot); - if (abortController.signal.aborted) throw new Error("Bombadil fuzzing was interrupted"); - - lease = await dependencies.acquireServer({ - baseUrl: validated.baseUrl, - label: validated.label, - readinessPath: validated.server.readinessPath, - reuseExistingLocalServer: false, - startupTimeoutMs: validated.server.startupTimeoutMs, - startServer: () => { - ownedServer = dependencies.spawnServer({ - command: serverCommand, - cwd: validated.server.cwd, - ...(validated.server.env === undefined ? {} : { env: validated.server.env }), - }); - return ownedServer; - }, - }); + throwIfBombadilRunAborted(abortController.signal); + + try { + lease = await dependencies.acquireServer({ + abortSignal: abortController.signal, + baseUrl: validated.baseUrl, + label: validated.label, + readinessPath: validated.server.readinessPath, + reuseExistingLocalServer: false, + startupTimeoutMs: validated.server.startupTimeoutMs, + startServer: () => { + throwIfBombadilRunAborted(abortController.signal); + ownedServer = dependencies.spawnServer({ + command: serverCommand, + cwd: validated.server.cwd, + ...(validated.server.env === undefined ? {} : { env: validated.server.env }), + }); + terminateAbortedOwnedServer(abortController.signal, ownedServer); + return ownedServer; + }, + }); + } catch (error) { + if (abortController.signal.aborted) throwIfBombadilRunAborted(abortController.signal); + throw error; + } + if (abortController.signal.aborted) { + const acquiredOwnedServer = ownedServer as ManagedVerificationServer | null; + if (acquiredOwnedServer?.exitCode() === null) acquiredOwnedServer.terminate(); + throwIfBombadilRunAborted(abortController.signal); + } let processFailure: unknown = null; try { processResult = await dependencies.runBombadil(abortableInvocation); diff --git a/src/tooling/browser-verification.test.ts b/src/tooling/browser-verification.test.ts index 2e84b3d..13e36e6 100644 --- a/src/tooling/browser-verification.test.ts +++ b/src/tooling/browser-verification.test.ts @@ -730,6 +730,68 @@ describe("server leases", () => { expect(fixture.calls).toEqual(["terminate"]); }); + test("aborts a pending readiness probe and terminates the owned server", async () => { + const fixture = fakeServer(); + const controller = new AbortController(); + let markPendingProbe!: () => void; + const pendingProbe = new Promise((resolve) => { + markPendingProbe = resolve; + }); + const neverReachable = new Promise(() => undefined); + let probes = 0; + const acquisition = acquireVerificationServer({ + abortSignal: controller.signal, + baseUrl: "http://localhost:8080", + label: "Fixture server", + startupTimeoutMs: 120_000, + startServer: () => fixture.server, + isReachable: () => { + probes += 1; + if (probes === 1) return false; + markPendingProbe(); + return neverReachable; + }, + }); + await pendingProbe; + controller.abort(); + const failure = await rejection(acquisition); + expect(failure.message).toBe("Verification server acquisition was aborted"); + expect(fixture.calls).toEqual(["terminate"]); + }, 1_000); + + test("does not return a lease when cancellation follows readiness", async () => { + const fixture = fakeServer(); + const controller = new AbortController(); + let markPendingProbe!: () => void; + const pendingProbe = new Promise((resolve) => { + markPendingProbe = resolve; + }); + let resolveReachability!: (reachable: boolean) => void; + const reachability = new Promise((resolve) => { + resolveReachability = resolve; + }); + let probes = 0; + const acquisition = acquireVerificationServer({ + abortSignal: controller.signal, + baseUrl: "http://localhost:8080", + label: "Fixture server", + startupTimeoutMs: 120_000, + startServer: () => fixture.server, + isReachable: () => { + probes += 1; + if (probes === 1) return false; + markPendingProbe(); + return reachability; + }, + }); + await pendingProbe; + resolveReachability(true); + queueMicrotask(() => controller.abort()); + const failure = await rejection(acquisition); + expect(failure.message).toBe("Verification server acquisition was aborted"); + expect(fixture.calls).toEqual(["terminate"]); + }); + test("bounds cleanup when a server never exits after SIGKILL", async () => { const calls: string[] = []; const never = new Promise(() => undefined); diff --git a/src/tooling/browser-verification.ts b/src/tooling/browser-verification.ts index 95da4e9..46ca7f0 100644 --- a/src/tooling/browser-verification.ts +++ b/src/tooling/browser-verification.ts @@ -813,7 +813,38 @@ export async function stopVerificationServer( await stopVerificationServerWithOutput(server, stopTimeoutMs); } +function verificationServerAcquisitionAbortError(): Error { + return new Error("Verification server acquisition was aborted"); +} + +function throwIfVerificationServerAcquisitionAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted === true) throw verificationServerAcquisitionAbortError(); +} + +async function waitForVerificationServerAcquisitionStep( + promise: Promise, + signal: AbortSignal | undefined, +): Promise { + if (signal === undefined) return await promise; + throwIfVerificationServerAcquisitionAborted(signal); + let abortListener: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + abortListener = () => reject(verificationServerAcquisitionAbortError()); + signal.addEventListener("abort", abortListener, { once: true }); + if (signal.aborted) abortListener(); + }); + let value: Value; + try { + value = await Promise.race([promise, aborted]); + } finally { + if (abortListener !== undefined) signal.removeEventListener("abort", abortListener); + } + throwIfVerificationServerAcquisitionAborted(signal); + return value; +} + export async function acquireVerificationServer(options: { + readonly abortSignal?: AbortSignal; readonly baseUrl: string; readonly label: string; readonly localHosts?: ReadonlySet; @@ -837,7 +868,11 @@ export async function acquireVerificationServer(options: { options.baseUrl, options.localHosts, ); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); + if (await waitForVerificationServerAcquisitionStep( + Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), + options.abortSignal, + )) { if (canStartLocally && options.reuseExistingLocalServer === false) { throw new Error( `A local server is already reachable at ${options.baseUrl}; ` @@ -847,8 +882,15 @@ export async function acquireVerificationServer(options: { // A verifier-owned command can exit before its child listener has finished // shutting down. Require the listener to survive a bounded interval before // another verifier trusts it as independently managed infrastructure. - await Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + await waitForVerificationServerAcquisitionStep( + Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS), + options.abortSignal, + ); + if (await waitForVerificationServerAcquisitionStep( + Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), + options.abortSignal, + )) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "reused" }; } } @@ -858,6 +900,7 @@ export async function acquireVerificationServer(options: { ); } + throwIfVerificationServerAcquisitionAborted(options.abortSignal); const server = options.startServer(); let exitedWithCode: number | null = null; try { @@ -868,10 +911,17 @@ export async function acquireVerificationServer(options: { exitedWithCode = exitCode; break; } - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + if (await waitForVerificationServerAcquisitionStep( + Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), + options.abortSignal, + )) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "started", server }; } - await Bun.sleep(options.pollIntervalMs ?? 200); + await waitForVerificationServerAcquisitionStep( + Bun.sleep(options.pollIntervalMs ?? 200), + options.abortSignal, + ); } } catch (error) { await stopVerificationServer(server); From 5d2f6343e73d5d3e0b4edee6a620e2c983f04895 Mon Sep 17 00:00:00 2001 From: 0thernet Date: Fri, 28 Aug 2026 17:39:03 -0400 Subject: [PATCH 20/20] Fix npm publish package budget --- .github/workflows/npm-publish.yml | 2 +- scripts/npm-publish-workflow.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 8f172af..6bd70e6 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -275,7 +275,7 @@ jobs: const minimumPackedBytes = 140_000; const maximumPackedBytes = 180_000; const minimumUnpackedBytes = 650_000; - const maximumUnpackedBytes = 750_000; + const maximumUnpackedBytes = 810_000; const maximumMetadataBytes = 250_000; const expectedName = "@hraness/direct"; const expectedVersion = process.env.EXPECTED_VERSION; diff --git a/scripts/npm-publish-workflow.test.ts b/scripts/npm-publish-workflow.test.ts index efe7abf..91deb0d 100644 --- a/scripts/npm-publish-workflow.test.ts +++ b/scripts/npm-publish-workflow.test.ts @@ -267,7 +267,7 @@ describe("npm release workflows", () => { "const minimumPackedBytes = 140_000", "const maximumPackedBytes = 180_000", "const minimumUnpackedBytes = 650_000", - "const maximumUnpackedBytes = 750_000", + "const maximumUnpackedBytes = 810_000", "record.files.length !== record.entryCount", "unpackedSize !== record.unpackedSize", 'createHash("sha1")',