diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 8f172af..6bd70e6 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -275,7 +275,7 @@ jobs: const minimumPackedBytes = 140_000; const maximumPackedBytes = 180_000; const minimumUnpackedBytes = 650_000; - const maximumUnpackedBytes = 750_000; + const maximumUnpackedBytes = 810_000; const maximumMetadataBytes = 250_000; const expectedName = "@hraness/direct"; const expectedVersion = process.env.EXPECTED_VERSION; diff --git a/README.md b/README.md index 066e2fc..84c4353 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ state with predictable local fixtures. it does not click through the browser or test the systems it replaces. ```sh -bun add --dev @hraness/direct@0.7.6 +bun add --dev @hraness/direct@0.7.7 # or -npm install --save-dev @hraness/direct@0.7.6 +npm install --save-dev @hraness/direct@0.7.7 ``` [Install @hraness/direct from npm](https://www.npmjs.com/package/@hraness/direct) · @@ -53,7 +53,7 @@ Copy this prompt into Codex, Claude Code, or another coding agent: ```text Use $direct to install hraness/direct from -the npm registry at the exact 0.7.6 version. Follow the repository README, add +the npm registry at the exact 0.7.7 version. Follow the repository README, add `@hraness/direct` to devDependencies only, and verify that the production dependency graph excludes Direct. Do not add a fixture composition until I ask. @@ -69,7 +69,7 @@ Pin the public npm package to an exact immutable version: ```json { "devDependencies": { - "@hraness/direct": "0.7.6" + "@hraness/direct": "0.7.7" } } ``` @@ -218,7 +218,7 @@ A quiet probe means the declared deterministic work settled. It does not prove t | `@hraness/direct/web` | Atomic browser installation, with low-level bridge and firewall escape hatches | Browser only | | `@hraness/direct/tooling/browser-verification` | Protocol-bound bridge reads, bounded agent-browser commands, local server leases, and artifact writes | Bun 1.3.14 with Node APIs | | `@hraness/direct/tooling/bombadil-campaign` | Direct property and conservative action factories for a Bombadil specification | Bombadil 0.7.2 specification compiler | -| `@hraness/direct/tooling/bombadil` | Local server ownership, native Bombadil lifecycle, trace attestation, replay, and diagnostic artifacts | Bun 1.3.14 with Node APIs | +| `@hraness/direct/tooling/bombadil` | Local server ownership, native Bombadil lifecycle, serial campaign matrices, trace attestation and summaries, replay, and diagnostic artifacts | Bun 1.3.14 with Node APIs | | `@hraness/direct/tooling/bundle-boundary` | Deterministic emitted-file scans and exact versioned-wire evidence | Bun 1.3.14 with Node APIs | The tooling subpaths are development-only. They are built separately from the @@ -264,6 +264,7 @@ export * from "@antithesishq/bombadil/browser/defaults/properties"; const direct = createDirectBombadilProperties(); export const direct_safe_actions = createDirectBombadilActions(); +export const direct_startup_contract = direct.startupContract; export const direct_exact_contract = direct.exactContract; export const direct_stable_catalog = direct.stableCatalog; export const direct_no_declared_violations = direct.noDeclaredViolations; @@ -275,7 +276,34 @@ path, and any additional safe actions. Call `runDirectBombadilFuzz` from `@hraness/direct/tooling/bombadil` in a small Bun wrapper. The runner accepts only an explicit local HTTP origin, starts an argv-only server command, invokes the exact native 0.7.2 binary, attests the bounded trace with Direct's canonical -parsers, writes pass or failure artifacts, and releases its owned processes. +parsers, writes pass or failure artifacts plus a compact exploration summary, +and releases its owned processes. Use `runDirectBombadilFuzzMatrix` when a +product owns several scenarios; it runs them serially and requires one exact +campaign selector for replay. + +Startup is the only repairable contract phase. It must reach one exact Direct +observation within ten seconds. From that sample onward, activation identity, +route, scenario, catalog, and zero declared violations are immediate safety +invariants; only quiescence remains bounded liveness. + +Keep liveness formulas time-bounded. Prefer guarded product actions with +explicit weights over unrestricted browser actions, and name small JSON +snapshots that expose semantic state without retaining page content. Run short +12–30 second campaigns while editing and longer 60–300 second matrices in a +scheduled diagnostic lane. Inspect and replay a retained failing trace, then +promote the smallest readable failure to a deterministic product regression. +When a campaign must exercise an interaction, require a named product value to +change after the intended action kind, as well as after a non-Wait action, so +bootstrap, idle, prerequisite, and unrelated transitions do not satisfy the +exploration policy. Attribution requires adjacent exact Direct observations; +it is temporal response evidence rather than proof of causality. +If the full product snapshot includes viewport dimensions, put that requirement +on a separate interaction snapshot without viewport fields and require an +opposite-size `SetViewport` independently. Latch the first ready product state +for initial-world properties so later actions cannot repair a bad initial state. +The raw trace remains authoritative and may contain screenshots, URLs, typed +text, accessible labels, and local paths; treat it as potentially sensitive. +Summary counts and hashes help triage exploration but are not Direct coverage. See [Verification](./docs/verification.md#run-a-bounded-bombadil-campaign) for the complete configuration and proof limits. diff --git a/dist/tooling/bombadil.js b/dist/tooling/bombadil.js index 3e4afaf..341bc26 100644 --- a/dist/tooling/bombadil.js +++ b/dist/tooling/bombadil.js @@ -5,6 +5,7 @@ import { readFile, realpath, stat, writeFile as writeFile2 } from "fs/promises"; import { isAbsolute, join as join2, relative, resolve } from "path"; import process2 from "process"; import { createInterface } from "readline"; +import { createHash } from "crypto"; // src/core/result.ts function ok(value) { @@ -1045,23 +1046,54 @@ async function stopVerificationServerWithOutput(server, stopTimeoutMs = DEFAULT_ async function stopVerificationServer(server, stopTimeoutMs = DEFAULT_STOP_TIMEOUT_MS) { await stopVerificationServerWithOutput(server, stopTimeoutMs); } +function verificationServerAcquisitionAbortError() { + return new Error("Verification server acquisition was aborted"); +} +function throwIfVerificationServerAcquisitionAborted(signal) { + if (signal?.aborted === true) + throw verificationServerAcquisitionAbortError(); +} +async function waitForVerificationServerAcquisitionStep(promise, signal) { + if (signal === undefined) + return await promise; + throwIfVerificationServerAcquisitionAborted(signal); + let abortListener; + const aborted = new Promise((_resolve, reject) => { + abortListener = () => reject(verificationServerAcquisitionAbortError()); + signal.addEventListener("abort", abortListener, { once: true }); + if (signal.aborted) + abortListener(); + }); + let value; + try { + value = await Promise.race([promise, aborted]); + } finally { + if (abortListener !== undefined) + signal.removeEventListener("abort", abortListener); + } + throwIfVerificationServerAcquisitionAborted(signal); + return value; +} async function acquireVerificationServer(options) { const probeTimeoutMs = options.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; const readinessPath = options.readinessPath ?? "/"; const isReachable = options.isReachable ?? serverIsReachable; const canStartLocally = canAutomaticallyStartLocalServer(options.baseUrl, options.localHosts); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { if (canStartLocally && options.reuseExistingLocalServer === false) { throw new Error(`A local server is already reachable at ${options.baseUrl}; ` + "verification will not reuse a server whose worktree ownership is unknown"); } - await Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS), options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "reused" }; } } if (!canStartLocally) { throw new Error(`No server is reachable at ${options.baseUrl}; automatic startup is limited to local HTTP URLs`); } + throwIfVerificationServerAcquisitionAborted(options.abortSignal); const server = options.startServer(); let exitedWithCode = null; try { @@ -1072,10 +1104,11 @@ async function acquireVerificationServer(options) { exitedWithCode = exitCode; break; } - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "started", server }; } - await Bun.sleep(options.pollIntervalMs ?? 200); + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.pollIntervalMs ?? 200), options.abortSignal); } } catch (error) { await stopVerificationServer(server); @@ -1134,6 +1167,12 @@ var TRACE_MAX_BYTES = 64 * 1024 * 1024; var TRACE_MAX_LINE_BYTES = 16 * 1024 * 1024; var TRACE_MAX_LINES = 1e4; var TRACE_MAX_SNAPSHOTS_PER_LINE = 4096; +var TRACE_MAX_NAMED_SNAPSHOT_NAMES = 128; +var TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME = 1024; +var TRACE_MAX_DISTINCT_URLS = 1024; +var TRACE_MAX_PROPERTY_NAMES = 128; +var TRACE_MAX_CANONICAL_SNAPSHOT_BYTES = 2 * 1024 * 1024; +var TRACE_MAX_JSON_DEPTH = 64; var RANDOM_RUN_OVERHEAD_MS = 30000; var REPLAY_WALL_CLOCK_TIMEOUT_MS = MAX_TIME_LIMIT_SECONDS * 1000 + RANDOM_RUN_OVERHEAD_MS; var PROCESS_TERMINATION_GRACE_MS = 5000; @@ -1142,6 +1181,94 @@ var SERVER_OUTPUT_TIMEOUT_MS = 3000; var DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; var TRACE_LINE_KEYS = new Set(["action", "snapshots", "state", "timestamp", "violations"]); var TRACE_SNAPSHOT_KEYS = new Set(["index", "name", "time", "value"]); +var TRACE_STATE_KEYS = new Set([ + "hash_current", + "hash_previous", + "resources", + "screenshot", + "url" +]); +var TRACE_RESOURCE_KEYS = new Set([ + "documents", + "dom_nodes", + "js_event_listeners", + "js_heap_total", + "js_heap_used", + "layout_objects", + "script_duration", + "task_duration", + "thread_time", + "timestamp" +]); +var TRACE_VIOLATION_KEYS = new Set(["name", "violation"]); +var TRACE_POINT_KEYS = new Set(["x", "y"]); +var TRACE_FINGERPRINT_KEYS = new Set([ + "accessible_name", + "href", + "id", + "input_type", + "name_attr", + "placeholder", + "role", + "structural_path", + "tag", + "test_id", + "text_content" +]); +var TRACE_CLICK_ACTION_KEYS = new Set(["fingerprint", "point"]); +var TRACE_DOUBLE_CLICK_ACTION_KEYS = new Set([ + "delay_millis", + "fingerprint", + "point" +]); +var TRACE_TYPE_TEXT_ACTION_KEYS = new Set(["delay_millis", "text"]); +var TRACE_PRESS_KEY_ACTION_KEYS = new Set(["code"]); +var TRACE_SCROLL_ACTION_KEYS = new Set(["distance", "origin"]); +var TRACE_FILE_INPUT_ACTION_KEYS = new Set(["files", "selector"]); +var TRACE_MOUSE_DRAG_ACTION_KEYS = new Set([ + "delay_millis", + "from", + "steps", + "to" +]); +var TRACE_VIEWPORT_ACTION_KEYS = new Set(["height", "width"]); +var VIEWPORT_KEYS = new Set(["deviceScaleFactor", "height", "width"]); +var EXPLORATION_POLICY_KEYS = new Set([ + "minDistinctNamedSnapshotValues", + "minNamedSnapshotChangesAfterActionKind", + "minNamedSnapshotChangesAfterNonWait", + "minNonWaitActions", + "requireStableTargetUrl", + "requiredActionKinds", + "requiredNamedSnapshots" +]); +var DEFAULT_VIEWPORT_WIDTH = 1024; +var DEFAULT_VIEWPORT_HEIGHT = 768; +var DEFAULT_DEVICE_SCALE_FACTOR = 2; +var SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +var TARGET_TAG_PATTERN = /^[a-z][a-z0-9-]*$/u; +var ACTION_KINDS = [ + "Back", + "Click", + "DoubleClick", + "Forward", + "MouseDrag", + "PressKey", + "Reload", + "ScrollDown", + "ScrollUp", + "SetFileInputFiles", + "SetViewport", + "TypeText", + "Wait" +]; +var ACTION_KIND_SET = new Set(ACTION_KINDS); +var UNIT_ACTION_KINDS = new Set([ + "Back", + "Forward", + "Reload", + "Wait" +]); var DIRECT_OBSERVATION_KEYS = new Set([ "activationHash", "activeRoute", @@ -1217,6 +1344,13 @@ function hasExactKeys(value, expected) { const keys = Object.keys(value); return keys.length === expected.size && keys.every((key) => expected.has(key)); } +function compareCodeUnits(left, right) { + if (left < right) + return -1; + if (left > right) + return 1; + return 0; +} function parseTraceDirectObservation(value) { if (!isRecord2(value) || !hasExactKeys(value, DIRECT_OBSERVATION_KEYS)) { throw new Error("Bombadil trace has an invalid named direct observation"); @@ -1311,7 +1445,183 @@ function exactTraceDirectObservation(observation) { isQuiescent: probe2.value.isQuiescent }; } -function parseTraceLine(line, lineNumber) { +var RESOURCE_FIELD_MAP = { + documents: "documents", + dom_nodes: "domNodes", + js_event_listeners: "jsEventListeners", + js_heap_total: "jsHeapTotalBytes", + js_heap_used: "jsHeapUsedBytes", + layout_objects: "layoutObjects", + script_duration: "scriptDurationSeconds", + task_duration: "taskDurationSeconds", + thread_time: "threadTimeSeconds" +}; +function canonicalJson2(value, depth = 0, maximumDepth = TRACE_MAX_JSON_DEPTH) { + if (depth > maximumDepth) { + throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(maximumDepth)}`); + } + if (value === null || typeof value === "boolean" || typeof value === "string") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) + throw new Error("Bombadil named snapshot has a non-finite number"); + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((entry) => canonicalJson2(entry, depth + 1, maximumDepth)).join(",")}]`; + } + if (!isRecord2(value)) + throw new Error("Bombadil named snapshot is not JSON"); + const entries = Object.keys(value).sort(compareCodeUnits).map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key], depth + 1, maximumDepth)}`); + return `{${entries.join(",")}}`; +} +function sha256(value) { + return createHash("sha256").update(value).digest("hex"); +} +function namedSnapshotValueSha256(value, options = {}) { + const maximumBytes = options.maximumBytes ?? TRACE_MAX_CANONICAL_SNAPSHOT_BYTES; + const canonical = canonicalJson2(value, 0, options.maximumDepth ?? TRACE_MAX_JSON_DEPTH); + if (Buffer.byteLength(canonical, "utf8") > maximumBytes) { + throw new Error(`Bombadil named snapshot exceeds ${String(maximumBytes)} canonical bytes`); + } + return sha256(canonical); +} +function validTracePoint(value) { + return isRecord2(value) && hasExactKeys(value, TRACE_POINT_KEYS) && typeof value.x === "number" && Number.isFinite(value.x) && typeof value.y === "number" && Number.isFinite(value.y); +} +function parseTraceFingerprintTag(value, lineNumber) { + if (!isRecord2(value) || !Object.keys(value).every((key) => TRACE_FINGERPRINT_KEYS.has(key))) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + for (const [key, candidate] of Object.entries(value)) { + if (key !== "tag" && typeof candidate !== "string") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + } + const tag = value.tag; + if (typeof tag !== "string" || tag.length === 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target tag`); + } + if (typeof value.structural_path === "string" && Object.keys(value).some((key) => key !== "tag" && key !== "structural_path")) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action target`); + } + return tag.length <= 64 && TARGET_TAG_PATTERN.test(tag) ? tag : `sha256:${sha256(tag)}`; +} +function isSafeIntegerBetween(value, minimum, maximum) { + return typeof value === "number" && Number.isSafeInteger(value) && value >= minimum && value <= maximum; +} +function invalidTraceAction(lineNumber) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid action`); +} +function parseTraceAction(value, lineNumber) { + if (value === null) + return null; + if (typeof value === "string") { + if (!ACTION_KIND_SET.has(value) || !UNIT_ACTION_KINDS.has(value)) { + return invalidTraceAction(lineNumber); + } + return { kind: value, targetTag: null }; + } + if (!isRecord2(value) || Object.keys(value).length !== 1) { + return invalidTraceAction(lineNumber); + } + const kind = Object.keys(value)[0]; + const payload = kind === undefined ? undefined : value[kind]; + if (kind === undefined || !ACTION_KIND_SET.has(kind) || UNIT_ACTION_KINDS.has(kind) || !isRecord2(payload)) { + return invalidTraceAction(lineNumber); + } + const actionKind = kind; + let targetTag = null; + switch (actionKind) { + case "Click": + if (!hasExactKeys(payload, TRACE_CLICK_ACTION_KEYS) || !validTracePoint(payload.point)) { + return invalidTraceAction(lineNumber); + } + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "DoubleClick": + if (!hasExactKeys(payload, TRACE_DOUBLE_CLICK_ACTION_KEYS) || !isSafeIntegerBetween(payload.delay_millis, 0, 1000) || !validTracePoint(payload.point)) + return invalidTraceAction(lineNumber); + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "TypeText": + if (!hasExactKeys(payload, TRACE_TYPE_TEXT_ACTION_KEYS) || !isSafeIntegerBetween(payload.delay_millis, 0, Number.MAX_SAFE_INTEGER) || typeof payload.text !== "string") + return invalidTraceAction(lineNumber); + break; + case "PressKey": + if (!hasExactKeys(payload, TRACE_PRESS_KEY_ACTION_KEYS) || !isSafeIntegerBetween(payload.code, 0, 255)) { + return invalidTraceAction(lineNumber); + } + break; + case "ScrollDown": + case "ScrollUp": + if (!hasExactKeys(payload, TRACE_SCROLL_ACTION_KEYS) || typeof payload.distance !== "number" || !Number.isFinite(payload.distance) || !validTracePoint(payload.origin)) + return invalidTraceAction(lineNumber); + break; + case "SetFileInputFiles": + if (!hasExactKeys(payload, TRACE_FILE_INPUT_ACTION_KEYS) || typeof payload.selector !== "string" || !Array.isArray(payload.files) || !payload.files.every((file) => typeof file === "string")) + return invalidTraceAction(lineNumber); + break; + case "MouseDrag": + if (!hasExactKeys(payload, TRACE_MOUSE_DRAG_ACTION_KEYS) || !isSafeIntegerBetween(payload.delay_millis, 0, 1000) || !isSafeIntegerBetween(payload.steps, 1, 255) || !validTracePoint(payload.from) || !validTracePoint(payload.to)) + return invalidTraceAction(lineNumber); + break; + case "SetViewport": + if (!hasExactKeys(payload, TRACE_VIEWPORT_ACTION_KEYS) || !isSafeIntegerBetween(payload.height, 1, 1e4) || !isSafeIntegerBetween(payload.width, 1, 1e4)) + return invalidTraceAction(lineNumber); + break; + default: + return invalidTraceAction(lineNumber); + } + return { kind: actionKind, targetTag }; +} +function parseNonNegativeFiniteNumber(value, lineNumber, field) { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + return value; +} +function parseTraceState(value, lineNumber) { + if (!isRecord2(value) || !hasExactKeys(value, TRACE_STATE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser state`); + } + if (typeof value.url !== "string" || value.url.length === 0 || value.url.length > 8192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + let url; + try { + url = new URL(value.url); + } catch { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL protocol`); + } + if (typeof value.screenshot !== "string" || value.screenshot.length > 8192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid screenshot path`); + } + for (const field of ["hash_previous", "hash_current"]) { + const hash = value[field]; + if (hash !== null && (typeof hash !== "number" || !Number.isFinite(hash) || !Number.isInteger(hash) || hash < 0)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + } + if (!isRecord2(value.resources) || !hasExactKeys(value.resources, TRACE_RESOURCE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid browser resources`); + } + const resources = {}; + for (const field of Object.keys(RESOURCE_FIELD_MAP)) { + resources[field] = parseNonNegativeFiniteNumber(value.resources[field], lineNumber, `resources.${field}`); + } + parseNonNegativeFiniteNumber(value.resources.timestamp, lineNumber, "resources.timestamp"); + return { + currentHash: value.hash_current, + resources, + url + }; +} +function parseTraceEnvelope(line, lineNumber) { let input; try { input = JSON.parse(line); @@ -1324,8 +1634,16 @@ function parseTraceLine(line, lineNumber) { if (!Number.isSafeInteger(input.timestamp) || typeof input.timestamp !== "number" || input.timestamp < 0 || !Array.isArray(input.snapshots) || input.snapshots.length > TRACE_MAX_SNAPSHOTS_PER_LINE || !Array.isArray(input.violations)) { throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid state fields`); } - const snapshots = input.snapshots; - const directSnapshots = snapshots.filter((snapshot2) => isRecord2(snapshot2) && snapshot2.name === "direct"); + return { + action: input.action, + snapshots: input.snapshots, + state: input.state, + timestamp: input.timestamp, + violations: input.violations + }; +} +function parseDirectTraceObservation(envelope, lineNumber) { + const directSnapshots = envelope.snapshots.filter((snapshot2) => isRecord2(snapshot2) && snapshot2.name === "direct"); if (directSnapshots.length !== 1) { throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); } @@ -1333,7 +1651,80 @@ function parseTraceLine(line, lineNumber) { if (snapshot === undefined || !hasExactKeys(snapshot, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshot.index) || !Number.isSafeInteger(snapshot.time) || snapshot.index < 0 || snapshot.time < 0) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); } - return parseTraceDirectObservation(snapshot.value); + return { + observation: parseTraceDirectObservation(snapshot.value), + value: snapshot.value + }; +} +function parseDirectTraceLine(line, lineNumber) { + return parseDirectTraceObservation(parseTraceEnvelope(line, lineNumber), lineNumber).observation; +} +function parseTraceLine(line, lineNumber, strictDiagnosticSnapshotNames) { + const envelope = parseTraceEnvelope(line, lineNumber); + const state = parseTraceState(envelope.state, lineNumber); + const action = parseTraceAction(envelope.action, lineNumber); + const snapshots = envelope.snapshots; + const direct = parseDirectTraceObservation(envelope, lineNumber); + const namedSnapshots = [{ + name: "direct", + valueSha256: namedSnapshotValueSha256(direct.value, { + maximumBytes: TRACE_MAX_LINE_BYTES, + maximumDepth: TRACE_MAX_JSON_DEPTH + 4 + }) + }]; + const diagnosticSnapshotValues = new Map; + for (const snapshotValue of snapshots) { + if (!isRecord2(snapshotValue) || !hasExactKeys(snapshotValue, TRACE_SNAPSHOT_KEYS) || !Number.isSafeInteger(snapshotValue.index) || !Number.isSafeInteger(snapshotValue.time) || snapshotValue.index < 0 || snapshotValue.time < 0 || snapshotValue.name !== null && typeof snapshotValue.name !== "string") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid snapshot`); + } + if (snapshotValue.name === null || snapshotValue.name === "direct") + continue; + let name; + try { + name = validateSnapshotName(snapshotValue.name, `Bombadil trace line ${String(lineNumber)} snapshot name`); + } catch (error) { + if (strictDiagnosticSnapshotNames.has(snapshotValue.name)) + throw error; + continue; + } + const values = diagnosticSnapshotValues.get(name) ?? []; + values.push(snapshotValue.value); + diagnosticSnapshotValues.set(name, values); + } + for (const [name, values] of diagnosticSnapshotValues) { + if (values.length !== 1) { + if (strictDiagnosticSnapshotNames.has(name)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} repeats named snapshot ${name}`); + } + continue; + } + try { + namedSnapshots.push({ + name, + valueSha256: namedSnapshotValueSha256(values[0]) + }); + } catch (error) { + if (strictDiagnosticSnapshotNames.has(name)) + throw error; + } + } + const propertyViolationNames = []; + for (const violation of envelope.violations) { + if (!isRecord2(violation) || !hasExactKeys(violation, TRACE_VIOLATION_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + propertyViolationNames.push(validateSnapshotName(violation.name, `Bombadil trace line ${String(lineNumber)} property violation name`)); + if (!isRecord2(violation.violation) || Object.keys(violation.violation).length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + } + return { + action, + directObservation: direct.observation, + namedSnapshots, + propertyViolationNames, + state + }; } async function attestDirectBombadilTrace(options) { const metadata = await stat(options.tracePath).catch(() => null); @@ -1360,7 +1751,7 @@ async function attestDirectBombadilTrace(options) { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); } - const observation = parseTraceLine(line, observationCount); + const observation = parseDirectTraceLine(line, observationCount); const exact = exactTraceDirectObservation(observation); if (exact === null) { if (initial !== null) { @@ -1428,6 +1819,257 @@ async function attestDirectBombadilTrace(options) { validObservationCount }; } +function sortedCountRecord(values) { + return Object.freeze(Object.fromEntries([...values.entries()].sort(([left], [right]) => compareCodeUnits(left, right)))); +} +async function summarizeDirectBombadilTrace(options) { + const metadata = await stat(options.tracePath).catch(() => null); + if (metadata === null || !metadata.isFile() || metadata.size === 0) { + throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + } + if (metadata.size > TRACE_MAX_BYTES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_BYTES)} bytes`); + } + let targetUrl; + try { + targetUrl = new URL(options.targetUrl); + } catch { + throw new Error("targetUrl must be an absolute URL"); + } + const policy = validateExplorationPolicy(options.explorationPolicy); + const strictDiagnosticSnapshotNames = explorationPolicySnapshotNames(policy); + const actionCounts = new Map; + const targetTags = new Map; + const urlFingerprints = new Set; + const rawUrlFingerprints = new Set; + const transitionHashes = new Set; + const rawTransitionHashes = new Set; + const snapshots = new Map; + const propertyViolations = new Map; + const resources = { + documents: 0, + domNodes: 0, + jsEventListeners: 0, + jsHeapTotalBytes: 0, + jsHeapUsedBytes: 0, + layoutObjects: 0, + scriptDurationSeconds: 0, + taskDurationSeconds: 0, + threadTimeSeconds: 0 + }; + let lineCount = 0; + let totalActions = 0; + let nonWaitCount = 0; + let waitStreak = 0; + let maxWaitStreak = 0; + let nonNullHashCount = 0; + let rawNonNullHashCount = 0; + let policyObservationCount = 0; + let previousObservationWasExact = false; + let stableTarget = true; + let trackedUnrelatedSnapshotNameCount = 0; + const unrelatedSnapshotNameLimit = Math.max(0, TRACE_MAX_NAMED_SNAPSHOT_NAMES - strictDiagnosticSnapshotNames.size); + const stream = createReadStream(options.tracePath, { encoding: "utf8" }); + const lines = createInterface({ input: stream, crlfDelay: Infinity }); + try { + for await (const line of lines) { + lineCount += 1; + if (lineCount > TRACE_MAX_LINES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); + } + if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { + throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); + } + const parsed = parseTraceLine(line, lineCount, strictDiagnosticSnapshotNames); + const rawRelativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + rawUrlFingerprints.add(sha256(rawRelativeUrl)); + if (rawUrlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct raw URL fingerprints`); + } + if (parsed.state.currentHash !== null) { + rawNonNullHashCount += 1; + rawTransitionHashes.add(String(parsed.state.currentHash)); + } + for (const name of parsed.propertyViolationNames) { + if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`); + } + propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); + } + for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { + resources[outputName] = Math.max(resources[outputName], parsed.state.resources[sourceName]); + } + const currentObservationIsExact = exactTraceDirectObservation(parsed.directObservation) !== null; + if (!currentObservationIsExact) { + previousObservationWasExact = false; + continue; + } + policyObservationCount += 1; + const actionFollowsExactObservation = previousObservationWasExact; + const recordedActionKind = actionFollowsExactObservation ? parsed.action?.kind ?? null : null; + if (actionFollowsExactObservation && parsed.action !== null) { + totalActions += 1; + actionCounts.set(parsed.action.kind, (actionCounts.get(parsed.action.kind) ?? 0) + 1); + if (parsed.action.kind === "Wait") { + waitStreak += 1; + maxWaitStreak = Math.max(maxWaitStreak, waitStreak); + } else { + nonWaitCount += 1; + waitStreak = 0; + } + if (parsed.action.targetTag !== null) { + if (!targetTags.has(parsed.action.targetTag) && targetTags.size >= 128) { + throw new Error("Bombadil trace exceeds 128 distinct action target tags"); + } + targetTags.set(parsed.action.targetTag, (targetTags.get(parsed.action.targetTag) ?? 0) + 1); + } + } else if (actionFollowsExactObservation) { + waitStreak = 0; + } + const relativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + urlFingerprints.add(sha256(relativeUrl)); + if (urlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct URL fingerprints`); + } + stableTarget &&= parsed.state.url.href === targetUrl.href; + if (parsed.state.currentHash !== null) { + nonNullHashCount += 1; + transitionHashes.add(String(parsed.state.currentHash)); + } + for (const snapshot of parsed.namedSnapshots) { + let entry = snapshots.get(snapshot.name); + if (entry === undefined) { + const isStrictSnapshot = snapshot.name === "direct" || strictDiagnosticSnapshotNames.has(snapshot.name); + if (!isStrictSnapshot && trackedUnrelatedSnapshotNameCount >= unrelatedSnapshotNameLimit) { + continue; + } + if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`); + } + entry = { + changeAfterActionKind: new Map, + changeAfterNonWaitCount: 0, + lastObservationIndex: null, + lastValueSha256: null, + observationCount: 0, + values: new Set + }; + snapshots.set(snapshot.name, entry); + if (!isStrictSnapshot) + trackedUnrelatedSnapshotNameCount += 1; + } + if (!entry.values.has(snapshot.valueSha256) && entry.values.size >= TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME) { + if (snapshot.name === "direct" || strictDiagnosticSnapshotNames.has(snapshot.name)) { + throw new Error(`Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`); + } + continue; + } + const changedAfterRecordedAction = recordedActionKind !== null && entry.lastObservationIndex === policyObservationCount - 1 && entry.lastValueSha256 !== null && entry.lastValueSha256 !== snapshot.valueSha256; + if (changedAfterRecordedAction) { + entry.changeAfterActionKind.set(recordedActionKind, (entry.changeAfterActionKind.get(recordedActionKind) ?? 0) + 1); + } + if (changedAfterRecordedAction && recordedActionKind !== "Wait") { + entry.changeAfterNonWaitCount += 1; + } + entry.lastObservationIndex = policyObservationCount; + entry.lastValueSha256 = snapshot.valueSha256; + entry.observationCount += 1; + entry.values.add(snapshot.valueSha256); + } + previousObservationWasExact = true; + } + } finally { + lines.close(); + stream.destroy(); + } + if (lineCount === 0) + throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + const policyFailures = []; + if (policy !== null) { + if (nonWaitCount < policy.minNonWaitActions) { + policyFailures.push("minimum non-Wait action count was not reached"); + } + for (const kind of policy.requiredActionKinds) { + if ((actionCounts.get(kind) ?? 0) === 0) { + policyFailures.push(`required action kind ${kind} was not observed`); + } + } + for (const name of policy.requiredNamedSnapshots) { + if (!snapshots.has(name)) { + policyFailures.push(`required named snapshot ${name} was not observed`); + } + } + for (const [name, minimum] of Object.entries(policy.minDistinctNamedSnapshotValues)) { + if ((snapshots.get(name)?.values.size ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its distinct-value minimum`); + } + } + for (const [name, minimum] of Object.entries(policy.minNamedSnapshotChangesAfterNonWait)) { + if ((snapshots.get(name)?.changeAfterNonWaitCount ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its post-non-Wait change minimum`); + } + } + for (const [name, minimumByKind] of Object.entries(policy.minNamedSnapshotChangesAfterActionKind)) { + for (const [kind, minimum] of Object.entries(minimumByKind)) { + if ((snapshots.get(name)?.changeAfterActionKind.get(kind) ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its post-${kind} change minimum`); + } + } + } + if (policy.requireStableTargetUrl && !stableTarget) { + policyFailures.push("the browser did not remain on the exact target URL"); + } + } + const traceBytes = await readFile(options.tracePath); + return Object.freeze({ + schema: "direct.bombadil-exploration-summary/v2", + trace: Object.freeze({ + bytes: metadata.size, + lineCount, + sha256: sha256(traceBytes) + }), + actions: Object.freeze({ + byKind: sortedCountRecord(actionCounts), + maxWaitStreak, + nonWaitCount, + targetTags: sortedCountRecord(targetTags), + total: totalActions + }), + urls: Object.freeze({ + distinctFingerprintCount: urlFingerprints.size, + fingerprintSha256: Object.freeze([...urlFingerprints].sort(compareCodeUnits)), + observationCount: policyObservationCount, + rawDistinctFingerprintCount: rawUrlFingerprints.size, + rawFingerprintSha256: Object.freeze([...rawUrlFingerprints].sort(compareCodeUnits)), + rawObservationCount: lineCount, + stableTarget + }), + transitions: Object.freeze({ + distinctNonNullHashCount: transitionHashes.size, + nonNullHashCount, + rawDistinctNonNullHashCount: rawTransitionHashes.size, + rawNonNullHashCount + }), + namedSnapshots: Object.freeze([...snapshots.entries()].sort(([left], [right]) => compareCodeUnits(left, right)).map(([name, entry]) => Object.freeze({ + changeAfterActionKind: sortedCountRecord(entry.changeAfterActionKind), + changeAfterNonWaitCount: entry.changeAfterNonWaitCount, + distinctValueCount: entry.values.size, + distinctValueSha256: Object.freeze([...entry.values].sort(compareCodeUnits)), + name, + observationCount: entry.observationCount + }))), + propertyViolations: Object.freeze({ + byName: sortedCountRecord(propertyViolations), + total: [...propertyViolations.values()].reduce((total, value) => total + value, 0) + }), + resourceHighWaterMarks: Object.freeze(resources), + policy: Object.freeze({ + configured: policy !== null, + failures: Object.freeze(policyFailures), + satisfied: policyFailures.length === 0 + }) + }); +} function parseDirectBombadilFuzzArguments(arguments_, defaultBaseUrl) { let baseUrl = defaultBaseUrl; let timeLimitSeconds = DEFAULT_TIME_LIMIT_SECONDS; @@ -1527,7 +2169,7 @@ function validateTargetQuery(value) { throw new Error("targetQuery may contain at most 16 parameters"); } const validated = {}; - for (const [name, queryValue] of [...entries].sort(([left], [right]) => left.localeCompare(right))) { + for (const [name, queryValue] of [...entries].sort(([left], [right]) => compareCodeUnits(left, right))) { if (name.length === 0 || name.length > 128 || !QUERY_PARAMETER_NAME_PATTERN.test(name) || PROTOTYPE_PROPERTY_NAMES.has(name) || hasControlCharacters3(name) || name === SCENARIO_QUERY_KEY2 || name === FIXTURE_QUERY_KEY2) { throw new Error("targetQuery contains an invalid or reserved parameter name"); } @@ -1538,6 +2180,149 @@ function validateTargetQuery(value) { } return Object.freeze(validated); } +function validateSnapshotName(value, label) { + if (typeof value !== "string" || value.length === 0 || value.length > 128 || !SNAPSHOT_NAME_PATTERN.test(value) || PROTOTYPE_PROPERTY_NAMES.has(value) || hasControlCharacters3(value)) { + throw new Error(`${label} must be a safe bounded snapshot name`); + } + return value; +} +function validateViewport(value) { + if (value === undefined) { + return Object.freeze({ + deviceScaleFactor: DEFAULT_DEVICE_SCALE_FACTOR, + height: DEFAULT_VIEWPORT_HEIGHT, + width: DEFAULT_VIEWPORT_WIDTH + }); + } + if (!isRecord2(value) || !Object.keys(value).every((key) => VIEWPORT_KEYS.has(key))) { + throw new Error("viewport must contain only width, height, and deviceScaleFactor"); + } + const validateDimension = (name, input) => { + if (typeof input !== "number" || !Number.isSafeInteger(input) || input < 1 || input > 65535) { + throw new Error(`viewport.${name} must be an integer between 1 and 65535`); + } + return input; + }; + const width = validateDimension("width", value.width ?? DEFAULT_VIEWPORT_WIDTH); + const height = validateDimension("height", value.height ?? DEFAULT_VIEWPORT_HEIGHT); + const deviceScaleFactor = value.deviceScaleFactor ?? DEFAULT_DEVICE_SCALE_FACTOR; + if (typeof deviceScaleFactor !== "number" || !Number.isFinite(deviceScaleFactor) || deviceScaleFactor < 0.1 || deviceScaleFactor > 10) { + throw new Error("viewport.deviceScaleFactor must be a finite number between 0.1 and 10"); + } + return Object.freeze({ deviceScaleFactor, height, width }); +} +function validateSnapshotMinimumMap(options) { + if (!isRecord2(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated = {}; + for (const [rawName, minimum] of Object.entries(options.value).sort(([left], [right]) => compareCodeUnits(left, right))) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if (typeof minimum !== "number" || !Number.isSafeInteger(minimum) || minimum < 1 || minimum > options.maximum) { + throw new Error(`${options.label} ${name} must be an integer between 1 and ${String(options.maximum)}`); + } + validated[name] = minimum; + } + return Object.freeze(validated); +} +function validateSnapshotActionMinimumMap(options) { + if (!isRecord2(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated = {}; + for (const [rawName, rawMinimumByKind] of Object.entries(options.value).sort(([left], [right]) => compareCodeUnits(left, right))) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if (!isRecord2(rawMinimumByKind) || Object.keys(rawMinimumByKind).length === 0 || Object.keys(rawMinimumByKind).length > ACTION_KINDS.length) { + throw new Error(`${options.label} ${name} must be a bounded action map`); + } + const minimumByKind = {}; + for (const [rawKind, minimum] of Object.entries(rawMinimumByKind).sort(([left], [right]) => compareCodeUnits(left, right))) { + if (!ACTION_KIND_SET.has(rawKind)) { + throw new Error(`${options.label} ${name} contains an unknown action kind`); + } + if (typeof minimum !== "number" || !Number.isSafeInteger(minimum) || minimum < 1 || minimum > TRACE_MAX_LINES) { + throw new Error(`${options.label} ${name}.${rawKind} must be an integer between 1 and ${String(TRACE_MAX_LINES)}`); + } + minimumByKind[rawKind] = minimum; + } + validated[name] = Object.freeze(minimumByKind); + } + return Object.freeze(validated); +} +function explorationPolicySnapshotNames(policy) { + const names = new Set(["direct"]); + if (policy === null) + return names; + for (const name of policy.requiredNamedSnapshots) + names.add(name); + for (const name of Object.keys(policy.minDistinctNamedSnapshotValues)) + names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterNonWait)) + names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterActionKind)) + names.add(name); + return names; +} +function validateExplorationPolicy(value) { + if (value === undefined) + return null; + if (!isRecord2(value) || !Object.keys(value).every((key) => EXPLORATION_POLICY_KEYS.has(key))) { + throw new Error("explorationPolicy contains an unknown field"); + } + const minNonWaitActions = value.minNonWaitActions ?? 0; + if (typeof minNonWaitActions !== "number" || !Number.isSafeInteger(minNonWaitActions) || minNonWaitActions < 0 || minNonWaitActions > TRACE_MAX_LINES) { + throw new Error(`explorationPolicy.minNonWaitActions must be an integer between 0 and ${String(TRACE_MAX_LINES)}`); + } + const requiredActionKindsInput = value.requiredActionKinds ?? []; + if (!Array.isArray(requiredActionKindsInput) || requiredActionKindsInput.length > ACTION_KINDS.length) { + throw new Error("explorationPolicy.requiredActionKinds must be a bounded array"); + } + const requiredActionKinds = [...requiredActionKindsInput]; + if (!requiredActionKinds.every((kind) => typeof kind === "string" && ACTION_KIND_SET.has(kind)) || new Set(requiredActionKinds).size !== requiredActionKinds.length) { + throw new Error("explorationPolicy.requiredActionKinds contains an unknown or duplicate kind"); + } + requiredActionKinds.sort(compareCodeUnits); + const requiredNamedSnapshotsInput = value.requiredNamedSnapshots ?? []; + if (!Array.isArray(requiredNamedSnapshotsInput) || requiredNamedSnapshotsInput.length > 32) { + throw new Error("explorationPolicy.requiredNamedSnapshots must be a bounded array"); + } + const requiredNamedSnapshots = requiredNamedSnapshotsInput.map((name) => validateSnapshotName(name, "explorationPolicy.requiredNamedSnapshots entry")); + if (new Set(requiredNamedSnapshots).size !== requiredNamedSnapshots.length) { + throw new Error("explorationPolicy.requiredNamedSnapshots contains a duplicate name"); + } + requiredNamedSnapshots.sort(compareCodeUnits); + const minDistinctNamedSnapshotValues = validateSnapshotMinimumMap({ + label: "explorationPolicy.minDistinctNamedSnapshotValues", + maximum: TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME, + value: value.minDistinctNamedSnapshotValues ?? {} + }); + const minNamedSnapshotChangesAfterActionKind = validateSnapshotActionMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterActionKind", + value: value.minNamedSnapshotChangesAfterActionKind ?? {} + }); + const minNamedSnapshotChangesAfterNonWait = validateSnapshotMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterNonWait", + maximum: TRACE_MAX_LINES, + value: value.minNamedSnapshotChangesAfterNonWait ?? {} + }); + const requireStableTargetUrl = value.requireStableTargetUrl ?? false; + if (typeof requireStableTargetUrl !== "boolean") { + throw new Error("explorationPolicy.requireStableTargetUrl must be a boolean"); + } + const validated = Object.freeze({ + minDistinctNamedSnapshotValues, + minNamedSnapshotChangesAfterActionKind, + minNamedSnapshotChangesAfterNonWait, + minNonWaitActions, + requireStableTargetUrl, + requiredActionKinds: Object.freeze(requiredActionKinds), + requiredNamedSnapshots: Object.freeze(requiredNamedSnapshots) + }); + if (explorationPolicySnapshotNames(validated).size > TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error(`explorationPolicy may reference at most ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES - 1)} distinct non-Direct snapshots`); + } + return validated; +} function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { const repositoryRoot = resolve(config.repositoryRoot); if (!isAbsolute(config.repositoryRoot) || repositoryRoot !== config.repositoryRoot) { @@ -1590,6 +2375,8 @@ function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { const entryPath = config.entryPath ?? "/"; validateEntryPath(entryPath); const targetQuery = validateTargetQuery(config.targetQuery ?? {}); + const viewport = validateViewport(config.viewport); + const explorationPolicy = validateExplorationPolicy(config.explorationPolicy); const startupTimeoutMs = config.server.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; if (!Number.isSafeInteger(startupTimeoutMs) || startupTimeoutMs < 1000 || startupTimeoutMs > MAX_STARTUP_TIMEOUT_MS) { throw new Error(`server.startupTimeoutMs must be an integer between 1000 and ${String(MAX_STARTUP_TIMEOUT_MS)}`); @@ -1604,8 +2391,10 @@ function validateDirectBombadilFuzzConfig(config, baseUrlOverride) { artifactRoot: join2(repositoryRoot, "artifacts", "direct-bombadil", config.artifactName), bombadilExecutable: bombadilNativeBinary(repositoryRoot), entryPath, + explorationPolicy, port, targetQuery, + viewport, server: { ...config.server, cwd: serverCwd, @@ -1624,6 +2413,7 @@ function resolveReplayPath(repositoryRoot, replayPath) { return resolved; } function createDirectBombadilInvocation(options) { + const viewport = validateViewport(options.viewport); const target = new URL(options.entryPath ?? "/", `${options.baseUrl}/`); target.searchParams.set(SCENARIO_QUERY_KEY2, options.scenario); for (const [name, value] of Object.entries(options.targetQuery ?? {})) { @@ -1638,7 +2428,13 @@ function createDirectBombadilInvocation(options) { "--output-path", options.outputPath, "--headless", - "--instrument-javascript=" + "--instrument-javascript=", + "--width", + String(viewport.width), + "--height", + String(viewport.height), + "--device-scale-factor", + String(viewport.deviceScaleFactor) ]; if (options.replayPath === null) { command.push("--exit-on-violation", "--time-limit", `${String(options.timeLimitSeconds)}s`); @@ -1762,6 +2558,7 @@ async function runBombadilNativeProcess(invocation) { } var defaultDependencies = { acquireServer: acquireVerificationServer, + createAbortController: () => new AbortController, now: () => new Date, runBombadil: runBombadilNativeProcess, serverOutputTimeoutMs: SERVER_OUTPUT_TIMEOUT_MS, @@ -1892,6 +2689,89 @@ function helpText(defaultBaseUrl) { ].join(` `); } +function parseMatrixCampaignArgument(arguments_) { + const forwarded = []; + let campaignId = null; + let help = false; + for (let index = 0;index < arguments_.length; index += 1) { + const argument = arguments_[index]; + if (argument === undefined) + continue; + if (argument === "--help" || argument === "-h") + help = true; + if (argument === "--campaign" || argument.startsWith("--campaign=")) { + if (campaignId !== null) + throw new Error("--campaign may be provided only once"); + if (argument === "--campaign") { + const next = readOptionValue(arguments_, index, "--campaign"); + campaignId = next.value; + index = next.index; + } else { + campaignId = argument.slice("--campaign=".length); + } + if (campaignId.length === 0) + throw new Error("--campaign requires a value"); + continue; + } + forwarded.push(argument); + } + return { arguments: Object.freeze(forwarded), campaignId, help }; +} +function validateCampaignMatrix(campaigns) { + if (campaigns.length === 0 || campaigns.length > 32) { + throw new Error("Bombadil campaign matrix must contain 1-32 campaigns"); + } + const ids = new Set; + for (const campaign of campaigns) { + if (!ARTIFACT_NAME_PATTERN.test(campaign.id) || ids.has(campaign.id)) { + throw new Error("Bombadil campaign IDs must be unique lowercase kebab identifiers"); + } + ids.add(campaign.id); + } + return campaigns; +} +async function runDirectBombadilFuzzMatrix(campaignsInput, arguments_ = process2.argv.slice(2), dependencyOverrides = {}) { + const campaigns = validateCampaignMatrix(campaignsInput); + const parsed = parseMatrixCampaignArgument(arguments_); + if (parsed.help) { + process2.stdout.write(`${[ + helpText(campaigns[0]?.config.baseUrl ?? ""), + " --campaign Run one campaign; required with --replay", + "", + `Campaigns: ${campaigns.map((campaign) => campaign.id).join(", ")}` + ].join(` +`)} +`); + return { kind: "help" }; + } + const selected = parsed.campaignId === null ? campaigns : campaigns.filter((campaign) => campaign.id === parsed.campaignId); + if (selected.length === 0) { + throw new Error(`Unknown Bombadil campaign ${parsed.campaignId ?? ""}`); + } + if (parsed.campaignId === null && parsed.arguments.some((argument) => argument === "--replay" || argument.startsWith("--replay="))) { + throw new Error("--replay requires exactly one --campaign in matrix mode"); + } + const results = []; + for (const campaign of selected) { + const result = await runDirectBombadilFuzz(campaign.config, parsed.arguments, dependencyOverrides); + if (result.kind !== "run") { + throw new Error("Bombadil campaign unexpectedly returned help during matrix execution"); + } + results.push({ campaignId: campaign.id, result }); + } + return { kind: "matrix", results: Object.freeze(results) }; +} +function throwIfBombadilRunAborted(signal) { + if (signal.aborted) + throw new Error("Bombadil fuzzing was interrupted"); +} +function terminateAbortedOwnedServer(signal, server) { + if (!signal.aborted) + return; + if (server.exitCode() === null) + server.terminate(); + throwIfBombadilRunAborted(signal); +} async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2), dependencyOverrides = {}) { const parsed = parseDirectBombadilFuzzArguments(arguments_, config.baseUrl); if (parsed.kind === "help") { @@ -1912,7 +2792,7 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) }); const outputPath = join2(artifactRun.runDirectory, "bombadil"); const tracePath = join2(outputPath, "trace.jsonl"); - const abortController = new AbortController; + const abortController = dependencies.createAbortController?.() ?? new AbortController; const invocation = createDirectBombadilInvocation({ baseUrl: validated.baseUrl, bombadilExecutable: validated.bombadilExecutable, @@ -1923,7 +2803,8 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) scenario: validated.scenario, specificationPath: validated.specificationPath, targetQuery: validated.targetQuery, - timeLimitSeconds: parsed.timeLimitSeconds + timeLimitSeconds: parsed.timeLimitSeconds, + viewport: validated.viewport }); const abortableInvocation = { ...invocation, abortSignal: abortController.signal }; const serverCommand = validated.server.command.map((argument) => argument === "{port}" ? validated.port : argument); @@ -1933,6 +2814,8 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) let processResult = null; let attestation = null; let attestationFailure = null; + let explorationSummary = null; + let explorationSummaryFailure = null; let rawTracePath = null; let serverOutput = ""; let serverOutputFailure = null; @@ -1952,23 +2835,37 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) try { await requireRegularFile(validated.bombadilExecutable, "The root Bombadil executable"); bombadilVersion = await readExactBombadilVersion(validated.repositoryRoot); - if (abortController.signal.aborted) - throw new Error("Bombadil fuzzing was interrupted"); - lease = await dependencies.acquireServer({ - baseUrl: validated.baseUrl, - label: validated.label, - readinessPath: validated.server.readinessPath, - reuseExistingLocalServer: false, - startupTimeoutMs: validated.server.startupTimeoutMs, - startServer: () => { - ownedServer = dependencies.spawnServer({ - command: serverCommand, - cwd: validated.server.cwd, - ...validated.server.env === undefined ? {} : { env: validated.server.env } - }); - return ownedServer; - } - }); + throwIfBombadilRunAborted(abortController.signal); + try { + lease = await dependencies.acquireServer({ + abortSignal: abortController.signal, + baseUrl: validated.baseUrl, + label: validated.label, + readinessPath: validated.server.readinessPath, + reuseExistingLocalServer: false, + startupTimeoutMs: validated.server.startupTimeoutMs, + startServer: () => { + throwIfBombadilRunAborted(abortController.signal); + ownedServer = dependencies.spawnServer({ + command: serverCommand, + cwd: validated.server.cwd, + ...validated.server.env === undefined ? {} : { env: validated.server.env } + }); + terminateAbortedOwnedServer(abortController.signal, ownedServer); + return ownedServer; + } + }); + } catch (error) { + if (abortController.signal.aborted) + throwIfBombadilRunAborted(abortController.signal); + throw error; + } + if (abortController.signal.aborted) { + const acquiredOwnedServer = ownedServer; + if (acquiredOwnedServer?.exitCode() === null) + acquiredOwnedServer.terminate(); + throwIfBombadilRunAborted(abortController.signal); + } let processFailure = null; try { processResult = await dependencies.runBombadil(abortableInvocation); @@ -1988,6 +2885,15 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) } catch (error) { attestationFailure = error; } + try { + explorationSummary = await summarizeDirectBombadilTrace({ + ...validated.explorationPolicy === null ? {} : { explorationPolicy: validated.explorationPolicy }, + targetUrl: invocation.targetUrl, + tracePath + }); + } catch (error) { + explorationSummaryFailure = error; + } if (processFailure !== null) { throw processFailure instanceof Error ? processFailure : new Error(renderUnknown(processFailure)); } @@ -2005,6 +2911,12 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) if (attestationFailure !== null) { throw attestationFailure instanceof Error ? attestationFailure : new Error(renderUnknown(attestationFailure)); } + if (explorationSummaryFailure !== null) { + throw explorationSummaryFailure instanceof Error ? explorationSummaryFailure : new Error(renderUnknown(explorationSummaryFailure)); + } + if (explorationSummary?.policy.satisfied !== true) { + throw new Error(`Bombadil exploration policy was not satisfied: ${explorationSummary?.policy.failures.join("; ") ?? "summary unavailable"}`); + } } catch (error) { failure = error; } @@ -2038,6 +2950,7 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) const status = failure === null ? "passed" : "failed"; const logPath = join2(artifactRun.runDirectory, "bombadil.log"); const serverLogPath = join2(artifactRun.runDirectory, "server.log"); + const explorationSummaryPath = join2(artifactRun.runDirectory, "exploration-summary.json"); const record = { schema: ARTIFACT_SCHEMA, evidenceClass: "diagnostic-fuzz", @@ -2053,6 +2966,8 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) entryPath: validated.entryPath, targetQuery: validated.targetQuery, targetUrl: invocation.targetUrl, + viewport: validated.viewport, + explorationPolicy: validated.explorationPolicy, specificationPath: validated.specificationPath, replayPath, timeLimitSeconds: replayPath === null ? parsed.timeLimitSeconds : null, @@ -2074,6 +2989,9 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) }, attestation, attestationFailure: attestationFailure === null ? null : renderUnknown(attestationFailure), + explorationSummary, + explorationSummaryPath: explorationSummary === null ? null : explorationSummaryPath, + explorationSummaryFailure: explorationSummaryFailure === null ? null : renderUnknown(explorationSummaryFailure), initialDirect: attestation?.initial ?? null, interruptedSignal: capturedSignal, failure: failure === null ? null : renderUnknown(failure) @@ -2085,9 +3003,23 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) ` : ""}`, "utf8"); await writeFile2(serverLogPath, `${serverOutput}${serverOutput.length > 0 ? ` ` : ""}`, "utf8"); + if (explorationSummary !== null) { + await writeJsonAtomically(explorationSummaryPath, explorationSummary); + } await writeJsonAtomically(join2(artifactRun.runDirectory, "run.json"), record); await writeJsonAtomically(artifactRun.manifestPath, record); - const summary = `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}; artifacts: ${artifactRun.runDirectory}; log: ${logPath}`; + const exploration = explorationSummary === null ? "exploration=unavailable" : [ + `nonWait=${String(explorationSummary.actions.nonWaitCount)}`, + `maxWaitStreak=${String(explorationSummary.actions.maxWaitStreak)}`, + `namedChanges=${explorationSummary.namedSnapshots.map((snapshot) => `${snapshot.name}:${String(snapshot.changeAfterNonWaitCount)}`).join(",") || "none"}`, + `policy=${explorationSummary.policy.satisfied ? "satisfied" : "failed"}` + ].join("; "); + const summary = [ + `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}`, + exploration, + `artifacts: ${artifactRun.runDirectory}`, + `log: ${logPath}` + ].join("; "); (status === "passed" ? process2.stdout : process2.stderr).write(`${summary} `); if (failure !== null) { @@ -2108,10 +3040,16 @@ async function runDirectBombadilFuzz(config, arguments_ = process2.argv.slice(2) // src/tooling/bombadil.ts var attestDirectBombadilTrace2 = attestDirectBombadilTrace; +var summarizeDirectBombadilTrace2 = summarizeDirectBombadilTrace; function runDirectBombadilFuzz2(config, arguments_) { return arguments_ === undefined ? runDirectBombadilFuzz(config) : runDirectBombadilFuzz(config, arguments_); } +function runDirectBombadilFuzzMatrix2(campaigns, arguments_) { + return arguments_ === undefined ? runDirectBombadilFuzzMatrix(campaigns) : runDirectBombadilFuzzMatrix(campaigns, arguments_); +} export { + summarizeDirectBombadilTrace2 as summarizeDirectBombadilTrace, + runDirectBombadilFuzzMatrix2 as runDirectBombadilFuzzMatrix, runDirectBombadilFuzz2 as runDirectBombadilFuzz, attestDirectBombadilTrace2 as attestDirectBombadilTrace }; diff --git a/dist/tooling/browser-verification-entry.js b/dist/tooling/browser-verification-entry.js index d8a4582..77fa758 100644 --- a/dist/tooling/browser-verification-entry.js +++ b/dist/tooling/browser-verification-entry.js @@ -1393,23 +1393,54 @@ async function stopVerificationServerWithOutput(server, stopTimeoutMs = DEFAULT_ async function stopVerificationServer(server, stopTimeoutMs = DEFAULT_STOP_TIMEOUT_MS) { await stopVerificationServerWithOutput(server, stopTimeoutMs); } +function verificationServerAcquisitionAbortError() { + return new Error("Verification server acquisition was aborted"); +} +function throwIfVerificationServerAcquisitionAborted(signal) { + if (signal?.aborted === true) + throw verificationServerAcquisitionAbortError(); +} +async function waitForVerificationServerAcquisitionStep(promise, signal) { + if (signal === undefined) + return await promise; + throwIfVerificationServerAcquisitionAborted(signal); + let abortListener; + const aborted = new Promise((_resolve, reject) => { + abortListener = () => reject(verificationServerAcquisitionAbortError()); + signal.addEventListener("abort", abortListener, { once: true }); + if (signal.aborted) + abortListener(); + }); + let value; + try { + value = await Promise.race([promise, aborted]); + } finally { + if (abortListener !== undefined) + signal.removeEventListener("abort", abortListener); + } + throwIfVerificationServerAcquisitionAborted(signal); + return value; +} async function acquireVerificationServer(options) { const probeTimeoutMs = options.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS; const readinessPath = options.readinessPath ?? "/"; const isReachable = options.isReachable ?? serverIsReachable; const canStartLocally = canAutomaticallyStartLocalServer(options.baseUrl, options.localHosts); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { if (canStartLocally && options.reuseExistingLocalServer === false) { throw new Error(`A local server is already reachable at ${options.baseUrl}; ` + "verification will not reuse a server whose worktree ownership is unknown"); } - await Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS), options.abortSignal); + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "reused" }; } } if (!canStartLocally) { throw new Error(`No server is reachable at ${options.baseUrl}; automatic startup is limited to local HTTP URLs`); } + throwIfVerificationServerAcquisitionAborted(options.abortSignal); const server = options.startServer(); let exitedWithCode = null; try { @@ -1420,10 +1451,11 @@ async function acquireVerificationServer(options) { exitedWithCode = exitCode; break; } - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + if (await waitForVerificationServerAcquisitionStep(Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), options.abortSignal)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "started", server }; } - await Bun.sleep(options.pollIntervalMs ?? 200); + await waitForVerificationServerAcquisitionStep(Bun.sleep(options.pollIntervalMs ?? 200), options.abortSignal); } } catch (error) { await stopVerificationServer(server); diff --git a/docs/publishing.md b/docs/publishing.md index 5facc0a..64c18bd 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -62,8 +62,8 @@ cookie, one-time password, recovery code, or write token to GitHub. `v`. ```sh - git tag v0.7.6 - git push origin refs/tags/v0.7.6 + git tag v0.7.7 + git push origin refs/tags/v0.7.7 ``` 3. Wait for **Release**. The workflow runs these boundaries in order: @@ -123,7 +123,7 @@ Merge the fix to `main`, then dispatch **Release** from current `main` with the exact existing stable tag: ```sh -gh workflow run release.yml --ref main -f tag=v0.7.6 +gh workflow run release.yml --ref main -f tag=v0.7.7 ``` The recovery path skips npm publication. It accepts only the newest stable diff --git a/docs/verification.md b/docs/verification.md index 52c3e58..f2a4252 100644 --- a/docs/verification.md +++ b/docs/verification.md @@ -237,6 +237,11 @@ deterministic checks. This path does not replace product-owned semantic or accessibility assertions. It also does not prove a substituted service, adapter, browser host, operating system, or device. +Bombadil is an experimental 0.x tool. Review the official +[manual](https://antithesishq.github.io/bombadil/) when changing its pinned +version, specification, actions, properties, or CLI invocation; minor releases +may change interfaces or trace details. + Install the one supported release directly in the consumer. Direct declares it as an exact optional peer so products that do not use fuzzing do not install a browser tool: @@ -253,28 +258,77 @@ TypeScript source. Import it only from a Bombadil specification; use the built Create a specification such as `direct/bombadil-campaign.ts`: ```ts +import { always, eventually } from "@antithesishq/bombadil"; import { createDirectBombadilActions, + createDirectBombadilNamedSnapshot, createDirectBombadilProperties, + createDirectBombadilResourceLeakProperty, } from "@hraness/direct/tooling/bombadil-campaign"; export * from "@antithesishq/bombadil/browser/defaults/properties"; const direct = createDirectBombadilProperties(); +const phase = createDirectBombadilNamedSnapshot({ + fallback: "unavailable", + name: "todos.phase", + read: ({ window }) => Reflect.get(window, "__todosPhase"), + validate: (value): value is string => typeof value === "string", +}); export const direct_safe_actions = createDirectBombadilActions(); +export const direct_startup_contract = direct.startupContract; export const direct_exact_contract = direct.exactContract; export const direct_stable_catalog = direct.stableCatalog; export const direct_no_declared_violations = direct.noDeclaredViolations; export const direct_eventual_quiescence = direct.eventualQuiescence; +export const todos_phase_is_known = always( + eventually(() => phase.current !== "unavailable").within(5, "seconds"), +); +export const no_dom_node_leak = createDirectBombadilResourceLeakProperty({ + metric: "dom_nodes", + growthLimit: 500, + windowMillis: 10_000, +}); ``` The Direct action generator deliberately excludes reload, history traversal, -visible links, anchors, href targets, form submission, reset controls, and the -Enter key. It retains ordinary buttons, text input, scrolling, and an +visible links, anchors, href targets, form submission, reset controls, +destructive labels such as delete, remove, clear, discard, unlink, and close, +all labels because their fingerprints do not expose the associated control's +submit, reset, or button type, and the Enter key. It retains ordinary buttons, +text input, scrolling, and an always-eligible low-weight wait. This preserves the post-handshake contract -within one document. Add product actions only when their navigation and form -effects are understood, and keep product-specific assertions in the campaign. +within one document. Add product actions only when their navigation, form, and +destructive effects are understood, and keep product-specific assertions in +the campaign. +Guard domain actions on the state that makes them valid, then weight valuable +state-changing actions above Wait. Do not increase throughput by admitting +reload, navigation, submission, destructive controls, or arbitrary generated +input. A generated action sequence is useful only while it preserves the +scenario boundary and exercises behavior the product can interpret. + +`createDirectBombadilNamedSnapshot` gives product properties and post-run +diagnostics a small semantic signal. It requires a safe bounded name distinct +from `direct` and JavaScript prototype names, an explicit product type +predicate over Direct's owned plain-JSON clone, at +most 64 JSON levels, and at most 2 MiB of UTF-8 JSON. It fails closed to the +validated fallback when a page getter throws or returns unsuitable data. Extract +state, not page content or credentials. Named values are represented only by +canonical SHA-256 hashes in the exploration summary; the authoritative raw +trace still contains the original values. + +Bombadil's 0.7.2 manual documents a sliding-window resource property at +`@antithesishq/bombadil/browser/extras/resources`, but the published npm +package omits that subpath from its `exports` map. Use Direct's +`createDirectBombadilResourceLeakProperty` implementation until a reviewed +Bombadil release exports the official helper. Add a tuned property when +repeated product actions allocate DOM nodes, listeners, layout objects, or +heap. Prefer DOM-node or listener thresholds when they express the defect +because heap samples move with garbage collection. Measure a normal run in +Inspect before setting a growth limit, and keep the window longer than ordinary +rendering bursts. The summary's resource high-water marks help choose and +review those thresholds; a maximum alone does not prove or disprove a leak. Create a Bun wrapper such as `direct/fuzz-browser.ts`: @@ -297,6 +351,17 @@ await runDirectBombadilFuzz({ repositoryRoot, scenario: "todos.populated", specificationPath: resolve(directRoot, "bombadil-campaign.ts"), + viewport: { width: 1_024, height: 768, deviceScaleFactor: 2 }, + explorationPolicy: { + minNonWaitActions: 1, + requiredNamedSnapshots: ["direct", "todos.phase"], + minDistinctNamedSnapshotValues: { "todos.phase": 2 }, + minNamedSnapshotChangesAfterActionKind: { + "todos.phase": { Click: 1 }, + }, + minNamedSnapshotChangesAfterNonWait: { "todos.phase": 1 }, + requireStableTargetUrl: true, + }, server: { command: [ process.execPath, @@ -327,6 +392,39 @@ the runner rejects Direct's reserved scenario and fixture keys there because it binds the requested scenario itself. The repository root, campaign path, server working directory, and replay trace are resolved canonically before use; a symlink that escapes the configured repository is rejected. +`viewport` is optional and defaults to Bombadil 0.7.2's 1024×768 viewport at a +device scale factor of 2. The runner always passes the validated exact values +to random and replay invocations and records them in `run.json`. +Assign one reviewed viewport to each existing scenario campaign. Alternate a +stable wide and narrow viewport across the matrix instead of duplicating every +scenario at every size; add a second size only when the scenario owns a +responsive behavior that needs separate exploration. + +`explorationPolicy` is optional. When present, it can require a minimum count +of non-Wait actions, particular action kinds and named snapshots, minimum +distinct hashed values for named snapshots, named-value changes observed after +a non-Wait action or one particular action kind, and an exact stable target +URL. The change requirement keeps +bootstrap or Wait-only transitions from satisfying a product-interaction +threshold. Per-kind attribution requires exact Direct observations and the +same named snapshot in both immediately adjacent samples. The trace records +the last action with the resulting state, so this count identifies a named +value transition associated with an active product action. It does not prove +causality. The runner strictly validates every 0.7.2 action payload before +crediting its kind. The policy detects a campaign that passed +its properties without doing the intended exploration. It is a diagnostic +sufficiency check, not a coverage claim. Start with observed stable behavior +and raise thresholds only when the action generator makes them reliably +reachable. + +Do not let responsive evidence stand in for product interaction. When a full +product snapshot includes viewport dimensions, name a second compact +interaction snapshot that excludes them and put the distinct-value and +post-action-change requirements on that snapshot. Require `SetViewport` +separately, and generate only a width or height different from the current +viewport. Likewise, latch the first product-ready observation for initial-world +properties so a later generated action cannot repair an incorrect initial +state before the bounded formula completes. Run random exploration for 12 to 300 seconds. The default is 20 seconds: @@ -334,19 +432,56 @@ Run random exploration for 12 to 300 seconds. The default is 20 seconds: bun direct/fuzz-browser.ts --time-limit 20s ``` +Use 12–30 seconds in the edit loop. A scheduled diagnostic lane can run each +campaign for 60–300 seconds, serially, with an outer job timeout and retained +failure artifacts. Random exploration should supplement the deterministic +required gate rather than make an otherwise healthy pull request depend on one +particular random path. + +For multiple product scenarios, pass a bounded matrix to the shared runner: + +```ts +import { runDirectBombadilFuzzMatrix } from "@hraness/direct/tooling/bombadil"; + +await runDirectBombadilFuzzMatrix([ + { id: "populated", config: populatedCampaign }, + { id: "empty", config: emptyCampaign }, +], process.argv.slice(2)); +``` + +Without `--campaign`, the matrix runs every unique campaign serially. Select +one for focused work with `--campaign empty`. Replay is intentionally rejected +without that selector so a trace cannot be applied to the wrong scenario. + Use `--base-url` to select another local root origin. Use `--replay` with a repository-local `.jsonl` trace instead of `--time-limit` to reproduce a prior run. The native runner supports Bombadil 0.7.2 on Apple silicon macOS and x64 or arm64 Linux. Unsupported platform and architecture pairs fail before a server starts. -The browser formulas require an exact scenario contract, a nonempty catalog, -zero declared violations, and quiescence to recur within ten seconds from every -observed state. Bombadil's temporal engine cannot decide an unbounded `always` -inside a bounded `eventually`, so continuous identity and catalog binding stay -in the host attestation below. A formula result and Bombadil exit status are not -sufficient evidence because a short or incomplete trace could otherwise pass -vacuously. +Bombadil's browser Inspect UI is the fastest way to examine the actions, +screenshots, resource timeline, snapshots, and violations in a retained run: + +```sh +bunx bombadil browser inspect artifacts/direct-bombadil/todos//bombadil +bun direct/fuzz-browser.ts --replay artifacts/direct-bombadil/todos//bombadil/trace.jsonl +``` + +Use the same campaign, viewport, specification, scenario, application tree, +and server configuration for replay. Bombadil rejects a replay that diverges, +so reproduction is strong debugging evidence but not guaranteed after the +product changes. + +The startup formula requires an exact scenario contract within ten seconds. +After that first exact sample, the browser formulas require continuous +activation identity, catalog identity, and zero declared violations. Only +quiescence may recover within a bounded ten-second liveness window. Keep every +liveness obligation bounded to a product latency budget. A finite run cannot +decide an unbounded future obligation, and nesting unbounded `always` inside +bounded `eventually` does not make it decidable. The host attestation below +rechecks the full trace independently. A formula result and Bombadil exit status +are not sufficient evidence because a short or incomplete trace could otherwise +pass vacuously. After every random run or replay, the host runner streams the bounded 0.7.2 JSONL trace from foreign input and requires one named `direct` observation per @@ -369,12 +504,33 @@ configured local server is always stopped through the shared browser-verification lease helpers, and its output drain remains bounded even when cleanup itself fails. -Each attempt writes `run.json`, `bombadil.log`, and `server.log` below +Each attempt writes `run.json`, `exploration-summary.json`, `bombadil.log`, and `server.log` below `artifacts/direct-bombadil///`, including failures. The rolling `manifest.json` points to the latest record. `rawTracePath` reports a regular nonempty trace even if attestation fails; `tracePath` is present only -after exact attestation. Keep these diagnostic artifacts out of source control -unless the consumer explicitly reviews them as fixtures. +after exact attestation. The v2 summary strictly parses the 0.7.2 envelopes and +records the raw trace SHA-256, action-kind and safe target-tag counts, non-Wait +count and longest Wait streak, origin-relative URL fingerprints, non-null +transition-hash cardinality, canonical named-snapshot value hashes, property +violation names and counts, named-value changes after non-Wait actions and +specific action kinds, and browser resource high-water marks. Action, URL, +transition, and named-snapshot policy evidence starts at the first exact Direct +observation. Separate raw URL and transition hashes, property violations, and +resource maxima retain strictly parsed startup-prefix diagnostics. It excludes +typed text, accessible names, snapshot values, URLs, screenshots, and absolute +paths. These diagnostics describe what Bombadil happened to explore. They do +not measure code, state, interaction, or Direct catalog coverage, and the raw +trace remains authoritative. + +Keep all generated artifacts out of source control by default. Upload a failed +scheduled run to access-controlled CI storage with a bounded retention period; +the raw trace can contain screenshots, query values, typed text, accessibility +labels, extracted values, and local paths. Preserve it long enough to inspect +and replay. Once the defect is understood, add the smallest deterministic +regression at the owning parser, reducer, port, component, semantic browser, or +Direct scenario boundary. Verify that regression fails before the fix and +passes after it. Retain a reviewed trace fixture only when replay itself adds +durable value; otherwise remove the sensitive trace after promotion. ## Report coverage without promotion diff --git a/package.json b/package.json index b07430a..b53f7d6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hraness/direct", - "version": "0.7.6", + "version": "0.7.7", "description": "A TypeScript harness for deterministic frontend development with repeatable scenarios, local fixtures, and browser verification for coding agents.", "license": "MIT", "type": "module", diff --git a/scripts/npm-publish-workflow.test.ts b/scripts/npm-publish-workflow.test.ts index b853b85..91deb0d 100644 --- a/scripts/npm-publish-workflow.test.ts +++ b/scripts/npm-publish-workflow.test.ts @@ -159,7 +159,7 @@ describe("npm release workflows", () => { readonly version?: unknown; }; expect(manifest).toEqual(expect.objectContaining({ - version: "0.7.6", + version: "0.7.7", description: "A TypeScript harness for deterministic frontend development with repeatable scenarios, local fixtures, and browser verification for coding agents.", keywords: [ "frontend-development", @@ -267,7 +267,7 @@ describe("npm release workflows", () => { "const minimumPackedBytes = 140_000", "const maximumPackedBytes = 180_000", "const minimumUnpackedBytes = 650_000", - "const maximumUnpackedBytes = 750_000", + "const maximumUnpackedBytes = 810_000", "record.files.length !== record.entryCount", "unpackedSize !== record.unpackedSize", 'createHash("sha1")', @@ -357,7 +357,7 @@ describe("npm release workflows", () => { const binaryDirectory = join(directory, "bin"); const commandLog = join(directory, "commands.log"); const publishMarker = join(directory, "published.txt"); - const tarball = join(directory, "hraness-direct-0.7.6.tgz"); + const tarball = join(directory, "hraness-direct-0.7.7.tgz"); const metadata = join(directory, "npm-pack.json"); const digest = join(directory, "npm-package.sha256"); const sourceSha = "b".repeat(40); @@ -375,7 +375,7 @@ describe("npm release workflows", () => { writeFile(metadata, "reviewed metadata fixture\n", "utf8"), writeFile(digest, "reviewed digest fixture\n", "utf8"), ]); - await writeFile(gitStub, `#!/bin/bash\nset -euo pipefail\nprintf 'git %s\\n' "$*" >> "$COMMAND_LOG"\ncase "$*" in\n *"rev-parse refs/heads/main"*) printf '%s\\n' "$DEFAULT_SHA" ;;\n *"rev-parse refs/tags/v0.7.6^{commit}"*) printf '%s\\n' "$TAG_SHA" ;;\n *"merge-base --is-ancestor"*) [[ "$ANCESTRY_STATE" == ancestor ]] ;;\n *"tag --list v*"*) printf '%s\\n' "$REMOTE_TAGS" ;;\nesac\n`, "utf8"); + await writeFile(gitStub, `#!/bin/bash\nset -euo pipefail\nprintf 'git %s\\n' "$*" >> "$COMMAND_LOG"\ncase "$*" in\n *"rev-parse refs/heads/main"*) printf '%s\\n' "$DEFAULT_SHA" ;;\n *"rev-parse refs/tags/v0.7.7^{commit}"*) printf '%s\\n' "$TAG_SHA" ;;\n *"merge-base --is-ancestor"*) [[ "$ANCESTRY_STATE" == ancestor ]] ;;\n *"tag --list v*"*) printf '%s\\n' "$REMOTE_TAGS" ;;\nesac\n`, "utf8"); await writeFile(sha256Stub, `#!/bin/bash\nset -euo pipefail\nprintf 'sha256sum %s\\n' "$*" >> "$COMMAND_LOG"\ncase "$1" in\n "$TARBALL") value="$EXPECTED_ARCHIVE_SHA256" ;;\n "$METADATA") value="$EXPECTED_METADATA_SHA256" ;;\n "$DIGEST") value="$EXPECTED_DIGEST_SHA256" ;;\n *) echo "unexpected hash target: $1" >&2; exit 1 ;;\nesac\nprintf '%s %s\\n' "$value" "$1"\n`, "utf8"); await writeFile(npmStub, `#!/bin/bash\nset -euo pipefail\nprintf 'npm %s\\n' "$*" >> "$COMMAND_LOG"\nif [[ "\${1-}" == view ]]; then\n printf '%s\\n' "$PUBLISHED_VERSIONS_JSON"\n exit 0\nfi\nprintf 'published\\n' > "$PUBLISH_MARKER"\n`, "utf8"); await Promise.all([chmod(gitStub, 0o755), chmod(npmStub, 0o755), chmod(sha256Stub, 0o755)]); @@ -391,15 +391,15 @@ describe("npm release workflows", () => { EXPECTED_DIGEST_SHA256: digestSha256, EXPECTED_METADATA_SHA256: metadataSha256, EXPECTED_SOURCE_SHA: sourceSha, - EXPECTED_VERSION: "0.7.6", - GITHUB_REF: "refs/tags/v0.7.6", + EXPECTED_VERSION: "0.7.7", + GITHUB_REF: "refs/tags/v0.7.7", GITHUB_REPOSITORY: "hraness/direct", GITHUB_SHA: sourceSha, METADATA: metadata, PATH: `${binaryDirectory}:${process.env.PATH ?? ""}`, - PUBLISHED_VERSIONS_JSON: '["0.7.4","0.7.5"]', + PUBLISHED_VERSIONS_JSON: '["0.7.4","0.7.5","0.7.6"]', PUBLISH_MARKER: publishMarker, - REMOTE_TAGS: "v0.7.4\nv0.7.5\nv0.7.6", + REMOTE_TAGS: "v0.7.4\nv0.7.5\nv0.7.6\nv0.7.7", RUNNER_TEMP: directory, TAG_SHA: sourceSha, TARBALL: tarball, @@ -430,7 +430,7 @@ describe("npm release workflows", () => { }); expect(moved.exitCode).not.toBe(0); expect(`${moved.stdout}${moved.stderr}`).toContain( - "Tag v0.7.6 changed after artifact verification", + "Tag v0.7.7 changed after artifact verification", ); expect(await Bun.file(publishMarker).exists()).toBe(false); @@ -441,18 +441,18 @@ describe("npm release workflows", () => { }); expect(detached.exitCode).not.toBe(0); expect(`${detached.stdout}${detached.stderr}`).toContain( - "Tag v0.7.6 is no longer reachable from main", + "Tag v0.7.7 is no longer reachable from main", ); expect(await Bun.file(publishMarker).exists()).toBe(false); await rm(commandLog, { force: true }); const superseded = await runWorkflowScript(script, { ...baseEnvironment, - REMOTE_TAGS: "v0.7.4\nv0.7.5\nv0.7.6\nv0.7.7", + REMOTE_TAGS: "v0.7.4\nv0.7.5\nv0.7.6\nv0.7.7\nv0.7.8", }); expect(superseded.exitCode).not.toBe(0); expect(`${superseded.stdout}${superseded.stderr}`).toContain( - "Tag v0.7.6 is not the newest stable tag v0.7.7", + "Tag v0.7.7 is not the newest stable tag v0.7.8", ); expect(await readFile(commandLog, "utf8")).not.toContain("npm publish"); expect(await Bun.file(publishMarker).exists()).toBe(false); @@ -460,11 +460,11 @@ describe("npm release workflows", () => { await rm(commandLog, { force: true }); const staleVersion = await runWorkflowScript(script, { ...baseEnvironment, - PUBLISHED_VERSIONS_JSON: '["0.7.4","0.7.7"]', + PUBLISHED_VERSIONS_JSON: '["0.7.4","0.7.8"]', }); expect(staleVersion.exitCode).not.toBe(0); expect(`${staleVersion.stdout}${staleVersion.stderr}`).toContain( - "@hraness/direct@0.7.6 is not newer than published stable 0.7.7", + "@hraness/direct@0.7.7 is not newer than published stable 0.7.8", ); expect(await Bun.file(publishMarker).exists()).toBe(false); diff --git a/scripts/package-artifact.ts b/scripts/package-artifact.ts index c8b667f..759445b 100644 --- a/scripts/package-artifact.ts +++ b/scripts/package-artifact.ts @@ -10,7 +10,7 @@ const packageBudget = Object.freeze({ entryCount: { min: 50, max: 120 }, fileCount: { min: 50, max: 60 }, packedBytes: { min: 140_000, max: 180_000 }, - unpackedBytes: { min: 650_000, max: 750_000 }, + unpackedBytes: { min: 650_000, max: 810_000 }, }); const requiredPaths = Object.freeze([ diff --git a/skills/direct/references/install.md b/skills/direct/references/install.md index 6d240e2..854b885 100644 --- a/skills/direct/references/install.md +++ b/skills/direct/references/install.md @@ -21,9 +21,9 @@ global `direct` CLI. For a new installation, pin the reviewed public release: ```sh -bun add --dev @hraness/direct@0.7.6 +bun add --dev @hraness/direct@0.7.7 # or, in an npm project -npm install --save-dev @hraness/direct@0.7.6 +npm install --save-dev @hraness/direct@0.7.7 ``` The equivalent manifest entry is: @@ -31,7 +31,7 @@ The equivalent manifest entry is: ```json { "devDependencies": { - "@hraness/direct": "0.7.6" + "@hraness/direct": "0.7.7" } } ``` diff --git a/src/exports.test.ts b/src/exports.test.ts index 5d42c43..6672cd7 100644 --- a/src/exports.test.ts +++ b/src/exports.test.ts @@ -61,12 +61,16 @@ describe("public package exports", () => { expect(Object.keys(bombadil).toSorted()).toEqual([ "attestDirectBombadilTrace", "runDirectBombadilFuzz", + "runDirectBombadilFuzzMatrix", + "summarizeDirectBombadilTrace", ]); expect(typeof browserVerification.createAgentBrowser).toBe("function"); expect(typeof browserVerification.createDirectBrowserContractReader).toBe("function"); expect(typeof browserVerification.readDirectBrowserContract).toBe("function"); expect(typeof bombadil.runDirectBombadilFuzz).toBe("function"); expect(typeof bombadil.attestDirectBombadilTrace).toBe("function"); + expect(typeof bombadil.runDirectBombadilFuzzMatrix).toBe("function"); + expect(typeof bombadil.summarizeDirectBombadilTrace).toBe("function"); expect(typeof bundleBoundary.checkBundleBoundary).toBe("function"); expect(typeof bundleBoundary.findForbiddenMarkers).toBe("function"); expect("createAgentBrowser" in root).toBeFalse(); diff --git a/src/tooling/bombadil-campaign.test.ts b/src/tooling/bombadil-campaign.test.ts index 22db343..812be74 100644 --- a/src/tooling/bombadil-campaign.test.ts +++ b/src/tooling/bombadil-campaign.test.ts @@ -1,7 +1,13 @@ import { describe, expect, mock, test } from "bun:test"; +import type { JSON as BombadilJson } from "@antithesishq/bombadil"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { defineDirect } from "@hraness/direct"; import { createDirectSession } from "@hraness/direct/testing"; +import { summarizeDirectBombadilTrace } from "./bombadil-runner.js"; + interface FakeFormula { readonly body: unknown; readonly kind: "always" | "eventually"; @@ -13,7 +19,10 @@ interface FakeCell { current: unknown; name: string | null; readonly named: (name: string) => FakeCell; - readonly read: (state: { readonly window: unknown }) => unknown; + readonly read: (state: { + readonly resources?: Readonly>; + readonly window: unknown; + }) => unknown; } interface FakeActionGenerator { @@ -68,7 +77,9 @@ void mock.module("@antithesishq/bombadil/browser/defaults/actions", () => ({ const { createDirectBombadilActions, + createDirectBombadilNamedSnapshot, createDirectBombadilProperties, + createDirectBombadilResourceLeakProperty, readDirectBombadilObservation, } = await import("./bombadil-campaign.js"); @@ -156,9 +167,12 @@ function evaluate(body: unknown): boolean { return (body as () => boolean)(); } -function requireFormula(value: boolean | FakeFormula): FakeFormula { - if (typeof value === "boolean") throw new Error("Expected a nested formula"); - return value; +function acceptBombadilJson(value: BombadilJson): value is BombadilJson { + return value !== undefined; +} + +function rejectBombadilJson(value: BombadilJson): value is never { + return value === undefined; } describe("Direct Bombadil observation", () => { @@ -229,6 +243,214 @@ describe("Direct Bombadil observation", () => { }); }); +describe("Direct Bombadil named snapshots", () => { + test("names bounded JSON and fails closed around hostile or oversized page values", () => { + const snapshot = createDirectBombadilNamedSnapshot({ + fallback: { status: "unavailable" }, + name: "product.phase", + read: (state) => Reflect.get(state.window, "phase"), + validate: (value): value is { readonly status: string } => ( + typeof value === "object" + && value !== null + && !Array.isArray(value) + && typeof Reflect.get(value, "status") === "string" + ), + }) as unknown as FakeCell; + expect(snapshot.name).toBe("product.phase"); + expect(snapshot.read({ window: { phase: { status: "ready" } } })).toEqual({ + status: "ready", + }); + + const hostile = {}; + Object.defineProperty(hostile, "phase", { + get: () => { + throw new Error("hostile getter"); + }, + }); + expect(snapshot.read({ window: hostile })).toEqual({ status: "unavailable" }); + expect(snapshot.read({ + window: { phase: { status: "é".repeat(1_100_000) } }, + })).toEqual({ status: "unavailable" }); + expect(snapshot.read({ window: { phase: null } })).toEqual({ + status: "unavailable", + }); + }); + + test("produces a named value accepted by the host summary contract", async () => { + class MutablePageValue { + status = "ready"; + + toJSON() { + return { status: "é".repeat(1_100_000) }; + } + } + const pageValue = new MutablePageValue(); + const snapshot = createDirectBombadilNamedSnapshot({ + fallback: { status: "unavailable" }, + name: "product.compat", + read: () => pageValue, + validate: (value): value is { readonly status: string } => ( + typeof value === "object" + && value !== null + && !Array.isArray(value) + && typeof Reflect.get(value, "status") === "string" + ), + }) as unknown as FakeCell; + const value = snapshot.read({ + window: { phase: { status: "ready" } }, + }) as { status: string }; + expect(value).toEqual({ status: "ready" }); + expect(value).not.toBe(pageValue); + expect(Object.getPrototypeOf(value)).toBe(Object.prototype); + pageValue.status = "mutated"; + expect(value).toEqual({ status: "ready" }); + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-helper-summary-")); + const tracePath = join(directory, "trace.jsonl"); + try { + await writeFile(tracePath, `${JSON.stringify({ + action: null, + snapshots: [ + { + index: 0, + name: "direct", + time: 1, + value: readDirectBombadilObservation(contractFixture()), + }, + { index: 1, name: snapshot.name, time: 1, value }, + ], + state: { + hash_current: 1, + hash_previous: null, + resources: { + documents: 1, + dom_nodes: 1, + js_event_listeners: 1, + js_heap_total: 1, + js_heap_used: 1, + layout_objects: 1, + script_duration: 0, + task_duration: 0, + thread_time: 0, + timestamp: 1, + }, + screenshot: "1.png", + url: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + }, + timestamp: 1, + violations: [], + })}\n`, "utf8"); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.find(({ name }) => name === snapshot.name)) + .toMatchObject({ distinctValueCount: 1, observationCount: 1 }); + } finally { + await rm(directory, { force: true, recursive: true }); + } + }); + + test("matches the host name, UTF-8 size, and JSON depth boundary", () => { + for (const name of [ + "direct", + "__proto__", + "constructor", + "prototype", + "unsafe name", + ]) { + expect(() => createDirectBombadilNamedSnapshot({ + fallback: null, + name, + read: () => null, + validate: acceptBombadilJson, + })).toThrow("safe, unreserved"); + } + + const snapshot = createDirectBombadilNamedSnapshot({ + fallback: null, + name: "safe", + read: (state) => Reflect.get(state.window, "phase"), + validate: acceptBombadilJson, + }) as unknown as FakeCell; + let atLimit: BombadilJson = null; + for (let index = 0; index < 64; index += 1) atLimit = [atLimit]; + expect(snapshot.read({ window: { phase: atLimit } })).toEqual(atLimit); + const beyondLimit: BombadilJson = [atLimit]; + expect(snapshot.read({ window: { phase: beyondLimit } })).toBeNull(); + }); + + test("rejects non-JSON or predicate-invalid fallbacks before registering an extractor", () => { + expect(() => createDirectBombadilNamedSnapshot({ + fallback: null, + name: "safe", + read: () => null, + validate: rejectBombadilJson, + })).toThrow("accepted by validate"); + expect(() => createDirectBombadilNamedSnapshot({ + fallback: undefined as never, + name: "safe", + read: () => null, + validate: acceptBombadilJson, + })).toThrow("fallback must be bounded JSON accepted by validate"); + }); +}); + +describe("Direct Bombadil resource properties", () => { + const resources = (timestamp: number, domNodes: number) => ({ + documents: 1, + dom_nodes: domNodes, + js_event_listeners: 2, + js_heap_total: 2_000, + js_heap_used: 1_000, + layout_objects: 10, + script_duration: 0, + task_duration: 0, + thread_time: 0, + timestamp, + }); + + test("detects excessive growth across a bounded sliding resource window", () => { + const property = createDirectBombadilResourceLeakProperty({ + growthLimit: 10, + metric: "dom_nodes", + windowMillis: 1_000, + }) as unknown as FakeFormula; + const cell = cells.at(-1); + if (cell === undefined) throw new Error("resource extractor was not registered"); + + cell.current = cell.read({ resources: resources(1, 100), window: {} }); + expect(evaluate(property.body)).toBeTrue(); + cell.current = cell.read({ resources: resources(1.5, 105), window: {} }); + expect(evaluate(property.body)).toBeTrue(); + cell.current = cell.read({ resources: resources(1.75, 120), window: {} }); + expect(evaluate(property.body)).toBeFalse(); + }); + + test("rejects unknown fields, metrics, and unsafe numeric bounds", () => { + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: 1, + metric: "dom_nodes", + windowMillis: 1_000, + extra: true, + } as never)).toThrow("must contain metric"); + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: 1, + metric: "documents" as never, + windowMillis: 1_000, + })).toThrow("metric is unsupported"); + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: Number.POSITIVE_INFINITY, + metric: "dom_nodes", + windowMillis: 1_000, + })).toThrow("positive finite safe number"); + expect(() => createDirectBombadilResourceLeakProperty({ + growthLimit: 1, + metric: "dom_nodes", + windowMillis: 300_001, + })).toThrow("between 1 and 300000"); + }); +}); + function clickAction(overrides: Readonly> = {}): unknown { return { Click: { @@ -252,7 +474,7 @@ function clickAction(overrides: Readonly> = {}): u } describe("Direct Bombadil actions", () => { - test("prunes nested visible navigation and submission clicks", () => { + test("prunes nested navigation, submission, and destructive clicks", () => { fakeClicks.generate = () => ({ branches: [ [7, { value: clickAction() }], @@ -266,10 +488,16 @@ describe("Direct Bombadil actions", () => { [2, { value: clickAction({ textContent: "ReSeT" }) }], [2, { value: clickAction({ accessibleName: "RESET", textContent: "Again" }) }], [2, { value: clickAction({ accessibleName: "", textContent: "Reset" }) }], + [2, { value: clickAction({ textContent: "Delete track" }) }], + [2, { value: clickAction({ accessibleName: "Close editor", textContent: "Done" }) }], + [2, { value: clickAction({ textContent: "Sign-out" }) }], + [2, { value: clickAction({ textContent: "Remove from playlist" }) }], + [2, { value: clickAction({ textContent: "Unclear status" }) }], [1, { branches: [ [9, { value: clickAction({ tag: "input", inputType: "text" }) }], [8, { value: clickAction({ tag: "A" }) }], + [7, { value: clickAction({ tag: "label", textContent: "Continue" }) }], ], }], ], @@ -284,6 +512,7 @@ describe("Direct Bombadil actions", () => { expect(filtered?.generate()).toEqual({ branches: [ [7, { value: clickAction() }], + [2, { value: clickAction({ textContent: "Unclear status" }) }], [1, { branches: [ [9, { value: clickAction({ tag: "input", inputType: "text" }) }], @@ -325,25 +554,38 @@ describe("Direct Bombadil actions", () => { }); describe("Direct Bombadil formulas", () => { - test("returns four recurring bounded health formulas", () => { + test("splits bounded startup from strict recurring health", () => { const properties = createDirectBombadilProperties() as unknown as Record; const cell = cells.at(-1); if (cell === undefined) throw new Error("Expected one Direct extractor cell"); expect(cell.name).toBe("direct"); - cell.current = readDirectBombadilObservation(contractFixture()); + const sample = (window: unknown): void => { + cell.current = cell.read({ window }); + }; + sample({}); expect(Object.keys(properties).sort()).toEqual([ "eventualQuiescence", "exactContract", "noDeclaredViolations", "stableCatalog", + "startupContract", ]); - const exactContract = properties.exactContract; - expect(exactContract?.kind).toBe("always"); - const exactEventually = requireFormula(exactContract?.body as FakeFormula); - expect(exactEventually.kind).toBe("eventually"); - expect(exactEventually.milliseconds).toBe(10_000); - expect(evaluate(exactEventually.body)).toBeTrue(); + expect(properties.startupContract?.kind).toBe("eventually"); + expect(properties.startupContract?.milliseconds).toBe(10_000); + expect(evaluate(properties.startupContract?.body)).toBeFalse(); + expect(properties.exactContract?.kind).toBe("always"); + expect(properties.stableCatalog?.kind).toBe("always"); + expect(properties.noDeclaredViolations?.kind).toBe("always"); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); + + sample(contractFixture()); + expect(evaluate(properties.startupContract?.body)).toBeTrue(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); const initial = cell.current as Readonly>; for (const [key, value] of [ ["activeScenario", ""], @@ -352,29 +594,18 @@ describe("Direct Bombadil formulas", () => { ["activeSource", "fixture"], ] as const) { cell.current = { ...initial, [key]: value }; - expect(evaluate(exactEventually.body), key).toBeFalse(); + expect(evaluate(properties.exactContract?.body), key).toBeFalse(); } cell.current = initial; - const noDeclaredViolations = properties.noDeclaredViolations; - expect(noDeclaredViolations?.kind).toBe("always"); - const violationsEventually = requireFormula( - noDeclaredViolations?.body as FakeFormula, - ); - expect(violationsEventually.kind).toBe("eventually"); - expect(violationsEventually.milliseconds).toBe(10_000); - expect(evaluate(violationsEventually.body)).toBeTrue(); - - const stableAlways = properties.stableCatalog; - expect(stableAlways?.kind).toBe("always"); - const stableEventually = requireFormula(stableAlways?.body as FakeFormula); - expect(stableEventually.kind).toBe("eventually"); - expect(stableEventually.milliseconds).toBe(10_000); - expect(evaluate(stableEventually.body)).toBeTrue(); - cell.current = readDirectBombadilObservation(contractFixture({ - catalogHash: "", + sample(contractFixture({ + catalogHash: "fnv1a-64:ffffffffffffffff", })); - expect(evaluate(stableEventually.body)).toBeFalse(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeFalse(); + + sample(contractFixture({ violations: { console: 1 } })); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeFalse(); const outerAlways = properties.eventualQuiescence; expect(outerAlways?.kind).toBe("always"); @@ -387,32 +618,28 @@ describe("Direct Bombadil formulas", () => { const properties = createDirectBombadilProperties() as unknown as Record; const cell = cells.at(-1); if (cell === undefined) throw new Error("Expected one Direct extractor cell"); - cell.current = readDirectBombadilObservation({}); - - const exactEventually = requireFormula(properties.exactContract?.body as FakeFormula); - const violationsEventually = requireFormula( - properties.noDeclaredViolations?.body as FakeFormula, - ); - const stableEventually = requireFormula(properties.stableCatalog?.body as FakeFormula); - expect(evaluate(exactEventually.body)).toBeFalse(); - expect(evaluate(violationsEventually.body)).toBeFalse(); - expect(evaluate(stableEventually.body)).toBeFalse(); + const sample = (window: unknown): void => { + cell.current = cell.read({ window }); + }; + sample({}); + expect(evaluate(properties.startupContract?.body)).toBeFalse(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); const eventual = properties.eventualQuiescence?.body as FakeFormula; expect(evaluate(eventual.body)).toBeFalse(); - cell.current = readDirectBombadilObservation(contractFixture()); - expect(evaluate(exactEventually.body)).toBeTrue(); - expect(evaluate(violationsEventually.body)).toBeTrue(); - expect(evaluate(stableEventually.body)).toBeTrue(); + sample(contractFixture()); + expect(evaluate(properties.startupContract?.body)).toBeTrue(); + expect(evaluate(properties.exactContract?.body)).toBeTrue(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeTrue(); + expect(evaluate(properties.stableCatalog?.body)).toBeTrue(); expect(evaluate(eventual.body)).toBeTrue(); - cell.current = readDirectBombadilObservation(contractFixture({ - catalogHash: "", - violations: { console: 1 }, - })); - expect(evaluate(exactEventually.body)).toBeFalse(); - expect(evaluate(stableEventually.body)).toBeFalse(); - expect(evaluate(violationsEventually.body)).toBeFalse(); + sample({}); + expect(evaluate(properties.exactContract?.body)).toBeFalse(); + expect(evaluate(properties.stableCatalog?.body)).toBeFalse(); + expect(evaluate(properties.noDeclaredViolations?.body)).toBeFalse(); }); }); diff --git a/src/tooling/bombadil-campaign.ts b/src/tooling/bombadil-campaign.ts index 5a932d4..85d3010 100644 --- a/src/tooling/bombadil-campaign.ts +++ b/src/tooling/bombadil-campaign.ts @@ -9,6 +9,7 @@ import { extract, weighted, type ActionGenerator, + type Cell, type Formula, type JSON as BombadilJson, type Tree, @@ -27,8 +28,38 @@ const DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; const DIRECT_SESSION_MANIFEST_SCHEMA = "direct.session-manifest/v1"; const DIRECT_PROBE_SCHEMA = "direct.probe/v1"; const MAX_RAW_CONTRACT_CHARACTERS = 2_000_000; +const MAX_NAMED_SNAPSHOT_CANONICAL_BYTES = 2 * 1024 * 1024; +const MAX_NAMED_SNAPSHOT_JSON_DEPTH = 64; const BRIDGE_KEYS = new Set(["manifest", "reset", "schema", "snapshot"]); const UNSAFE_CLICK_INPUT_TYPES = new Set(["image", "reset", "submit"]); +const UNSAFE_CLICK_LABEL_PHRASES = [ + "clear", + "close", + "delete", + "discard", + "erase", + "log out", + "remove", + "reset", + "sign out", + "unlink", +] as const; +const SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +const RESERVED_SNAPSHOT_NAMES = new Set([ + "direct", + "__proto__", + "constructor", + "prototype", +]); +const RESOURCE_LEAK_OPTION_KEYS = new Set(["growthLimit", "metric", "windowMillis"]); +const RESOURCE_METRICS = [ + "dom_nodes", + "js_event_listeners", + "js_heap_total", + "js_heap_used", + "layout_objects", +] as const; +const RESOURCE_METRIC_SET = new Set(RESOURCE_METRICS); export interface DirectBombadilObservation { readonly [key: string | number | symbol]: BombadilJson; @@ -48,12 +79,32 @@ export interface DirectBombadilObservation { } export interface DirectBombadilProperties { + readonly startupContract: Formula; readonly exactContract: Formula; readonly stableCatalog: Formula; readonly noDeclaredViolations: Formula; readonly eventualQuiescence: Formula; } +function observationHasExactContract( + observation: DirectBombadilObservation, +): boolean { + return observation.contractValid + && observation.activeSource === "scenario" + && observation.activeScenario.length > 0 + && observation.activeRoute.length > 0 + && observation.activationHash.length > 0 + && observation.catalogHash.length > 0; +} + +export type DirectBombadilResourceMetric = (typeof RESOURCE_METRICS)[number]; + +export interface DirectBombadilResourceLeakOptions { + readonly growthLimit: number; + readonly metric: DirectBombadilResourceMetric; + readonly windowMillis: number; +} + function safeClickAction(action: ActionTemplate): boolean { if (typeof action !== "object" || action === null) return false; const candidate = "Click" in action @@ -67,11 +118,19 @@ function safeClickAction(action: ActionTemplate): boolean { const inputType = fingerprint.inputType?.toLowerCase() ?? ""; const labels = [fingerprint.accessibleName, fingerprint.textContent] .filter((label): label is string => label !== null) - .map((label) => label.trim().toLowerCase()); + .map((label) => label + .trim() + .toLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, " ")); + const hasUnsafeLabel = labels.some((label) => { + const padded = ` ${label} `; + return UNSAFE_CLICK_LABEL_PHRASES.some((phrase) => padded.includes(` ${phrase} `)); + }); return fingerprint.href === null && tag !== "a" + && tag !== "label" && fingerprint.role?.toLowerCase() !== "link" - && !labels.includes("reset") + && !hasUnsafeLabel && !UNSAFE_CLICK_INPUT_TYPES.has(inputType) && (tag !== "button" || inputType === "button"); } @@ -100,7 +159,8 @@ function pruneActionTree( /** * Builds a browser action generator without reload/history actions or visible - * navigation and submission click targets, keeping Direct continuously bound. + * navigation, submission, reset, or destructive click targets, keeping Direct + * continuously bound. */ export function createDirectBombadilActions(): ActionGenerator { const safeClicks = actions(() => pruneActionTree(clicks.generate(), safeClickAction) ?? []); @@ -126,6 +186,72 @@ function hasExactKeys( return keys.length === expected.size && keys.every((key) => expected.has(key)); } +/** + * Builds Bombadil's documented sliding-window resource growth invariant from + * the public browser state because 0.7.2 omits its extras module from exports. + */ +export function createDirectBombadilResourceLeakProperty( + options: DirectBombadilResourceLeakOptions, +): Formula { + if (!isRecord(options) || !hasExactKeys(options, RESOURCE_LEAK_OPTION_KEYS)) { + throw new Error("Bombadil resource leak options must contain metric, growthLimit, and windowMillis"); + } + if (typeof options.metric !== "string" || !RESOURCE_METRIC_SET.has(options.metric)) { + throw new Error("Bombadil resource leak metric is unsupported"); + } + if ( + typeof options.growthLimit !== "number" + || !Number.isFinite(options.growthLimit) + || options.growthLimit <= 0 + || options.growthLimit > Number.MAX_SAFE_INTEGER + ) { + throw new Error("Bombadil resource leak growthLimit must be a positive finite safe number"); + } + if ( + typeof options.windowMillis !== "number" + || !Number.isSafeInteger(options.windowMillis) + || options.windowMillis < 1 + || options.windowMillis > 300_000 + ) { + throw new Error("Bombadil resource leak windowMillis must be an integer between 1 and 300000"); + } + const metric = options.metric; + const samples: Array<{ readonly timestamp: number; readonly value: number }> = []; + let previousTimestamp = -1; + const window = extract((state) => { + const timestamp = state.resources.timestamp * 1_000; + const value = state.resources[metric]; + if ( + !Number.isFinite(timestamp) + || timestamp < 0 + || timestamp < previousTimestamp + || !Number.isFinite(value) + || value < 0 + ) { + return { baseline: 0, valid: false, value: 0 }; + } + previousTimestamp = timestamp; + samples.push({ timestamp, value }); + const cutoff = timestamp - options.windowMillis; + while (samples.length > 2 && (samples[1]?.timestamp ?? Number.POSITIVE_INFINITY) <= cutoff) { + samples.shift(); + } + return { + baseline: samples[0]?.value ?? value, + valid: true, + value, + }; + }); + return always(() => + window.current.valid + && window.current.value - window.current.baseline <= options.growthLimit + ); +} + function invalidObservation(bridgePresent = false): DirectBombadilObservation { return { activationHash: "", @@ -150,6 +276,99 @@ function boundedJsonClone(value: unknown): BombadilJson | null { return JSON.parse(source) as BombadilJson; } +function cloneNamedSnapshotJson( + value: unknown, + depth = 0, + ancestors: WeakSet = new WeakSet(), +): BombadilJson | undefined { + if (depth > MAX_NAMED_SNAPSHOT_JSON_DEPTH) return undefined; + if (value === null || typeof value === "boolean" || typeof value === "string") { + return value; + } + if (typeof value === "number") return Number.isFinite(value) ? value : undefined; + if (typeof value !== "object") return undefined; + if (ancestors.has(value)) return undefined; + ancestors.add(value); + try { + if (Array.isArray(value)) { + const cloned: BombadilJson[] = []; + for (const entry of value) { + const child = cloneNamedSnapshotJson(entry, depth + 1, ancestors); + if (child === undefined) return undefined; + cloned.push(child); + } + return cloned; + } + const clonedEntries: [string, BombadilJson][] = []; + for (const key of Object.keys(value)) { + const child = cloneNamedSnapshotJson( + Reflect.get(value, key), + depth + 1, + ancestors, + ); + if (child === undefined) return undefined; + clonedEntries.push([key, child]); + } + return Object.fromEntries(clonedEntries) as BombadilJson; + } finally { + ancestors.delete(value); + } +} + +function boundedNamedSnapshotJson(value: unknown): BombadilJson | undefined { + const cloned = cloneNamedSnapshotJson(value); + if (cloned === undefined) return undefined; + const source = JSON.stringify(cloned); + return new TextEncoder().encode(source).byteLength + <= MAX_NAMED_SNAPSHOT_CANONICAL_BYTES + ? cloned + : undefined; +} + +/** + * Creates a named, bounded JSON extractor that fails closed to an explicit + * fallback when a page getter throws or returns non-JSON or oversized data. + */ +export function createDirectBombadilNamedSnapshot(options: { + readonly fallback: T; + readonly name: string; + readonly read: (state: BombadilBrowserState) => unknown; + readonly validate: (value: BombadilJson) => value is T; +}): Cell { + if ( + options.name.length === 0 + || options.name.length > 128 + || !SNAPSHOT_NAME_PATTERN.test(options.name) + || RESERVED_SNAPSHOT_NAMES.has(options.name) + ) { + throw new Error( + "Bombadil snapshot name must be a safe, unreserved 1-128 character identifier", + ); + } + const validate = (value: unknown): T | undefined => { + const owned = boundedNamedSnapshotJson(value); + return owned !== undefined && options.validate(owned) ? owned : undefined; + }; + let fallback: T | undefined; + try { + fallback = validate(options.fallback); + } catch { + fallback = undefined; + } + if (fallback === undefined) { + throw new Error( + "Bombadil snapshot fallback must be bounded JSON accepted by validate", + ); + } + return extract((state) => { + try { + return validate(options.read(state)) ?? fallback; + } catch { + return fallback; + } + }).named(options.name); +} + function readNonNegativeCounters(value: unknown): { readonly valid: boolean; readonly values: number[]; @@ -247,39 +466,50 @@ export function readDirectBombadilObservation( } } -/** Builds the four Direct invariants used by Bombadil browser campaigns. */ +/** Builds bounded startup plus strict recurring Direct browser invariants. */ export function createDirectBombadilProperties(): DirectBombadilProperties { - const direct = extract((state) => - readDirectBombadilObservation(state.window) - ).named("direct"); + let initial: Readonly<{ + activationHash: string; + activeRoute: string; + activeScenario: string; + catalogHash: string; + }> | null = null; + const direct = extract((state) => { + const observation = readDirectBombadilObservation(state.window); + if (initial === null && observationHasExactContract(observation)) { + initial = Object.freeze({ + activationHash: observation.activationHash, + activeRoute: observation.activeRoute, + activeScenario: observation.activeScenario, + catalogHash: observation.catalogHash, + }); + } + return observation; + }).named("direct"); - const exactContract = always( - eventually(() => - direct.current.contractValid - && direct.current.activeSource === "scenario" - && direct.current.activeScenario.length > 0 - && direct.current.activeRoute.length > 0 - && direct.current.activationHash.length > 0 - ).within(10, "seconds"), - ); - const stableCatalog = always( - eventually(() => - direct.current.contractValid - && direct.current.catalogHash.length > 0 - ).within(10, "seconds"), - ); - const noDeclaredViolations = always( - eventually(() => - direct.current.contractValid - && direct.current.violationsValid - && direct.current.violations.every((value: number) => value === 0) - ).within(10, "seconds"), - ); + const startupContract = eventually(() => initial !== null).within(10, "seconds"); + const exactContract = always(() => initial === null || ( + observationHasExactContract(direct.current) + && direct.current.activationHash === initial.activationHash + && direct.current.activeRoute === initial.activeRoute + && direct.current.activeScenario === initial.activeScenario + )); + const stableCatalog = always(() => initial === null || ( + observationHasExactContract(direct.current) + && direct.current.catalogHash === initial.catalogHash + )); + const noDeclaredViolations = always(() => initial === null || ( + observationHasExactContract(direct.current) + && direct.current.violationsValid + && direct.current.violations.every((value: number) => value === 0) + )); const eventualQuiescence = always( - eventually(() => direct.current.isQuiescent).within(10, "seconds"), + eventually(() => initial !== null && direct.current.isQuiescent) + .within(10, "seconds"), ); return Object.freeze({ + startupContract, exactContract, stableCatalog, noDeclaredViolations, diff --git a/src/tooling/bombadil-runner.test.ts b/src/tooling/bombadil-runner.test.ts index a3e0709..45e668c 100644 --- a/src/tooling/bombadil-runner.test.ts +++ b/src/tooling/bombadil-runner.test.ts @@ -11,6 +11,8 @@ import { parseDirectBombadilFuzzArguments, runBombadilNativeProcess, runDirectBombadilFuzz, + runDirectBombadilFuzzMatrix, + summarizeDirectBombadilTrace, validateDirectBombadilFuzzConfig, type DirectBombadilFuzzConfig, type DirectBombadilInvocation, @@ -169,6 +171,67 @@ function directObservation(options: { }; } +function largeDirectObservation(): Record { + const scenarioIds = Array.from({ length: 256 }, (_, index) => + `s${String(index).padStart(3, "0")}.${"x".repeat(115)}` + ); + const firstScenario = scenarioIds[0]; + if (firstScenario === undefined) throw new Error("large Direct fixture needs a scenario"); + const citedScenarios: [string, ...string[]] = [firstScenario, ...scenarioIds.slice(1)]; + const definition = defineDirect({ + parseWorld: (input) => { + if ( + typeof input !== "object" + || input === null + || Array.isArray(input) + || typeof Reflect.get(input, "count") !== "number" + ) { + throw new Error("count is required"); + } + return { count: Reflect.get(input, "count") as number }; + }, + defaultScenario: firstScenario, + scenarios: scenarioIds.map((id, index) => ({ + description: "d".repeat(2_000), + id, + route: `/surface/${String(index)}`, + title: "t".repeat(160), + world: { count: index }, + })), + coverage: Array.from({ length: 256 }, (_, index) => ({ + claim: "c".repeat(1_000), + key: `coverage.${String(index)}`, + mode: "fixture" as const, + scenarios: citedScenarios, + })), + }); + const session = createDirectSession({ + definition, + activation: { kind: "scenario", scenario: firstScenario }, + create: () => ({}), + }); + if (!session.ok) throw new Error(session.error.message); + const snapshot = session.value.probe.snapshot(); + if (!snapshot.ok) throw new Error(snapshot.error.message); + const manifest = jsonClone(session.value.manifest); + const probe = jsonClone(snapshot.value); + return { + activationHash: manifest.active.activationHash, + activeRoute: manifest.active.route, + activeScenario: manifest.active.scenario, + activeSource: manifest.active.source, + bridgePresent: true, + bridgeSchema: "direct.browser-bridge/v2", + catalogHash: manifest.catalogHash, + contractValid: true, + isQuiescent: probe.isQuiescent, + manifest, + probe, + violations: Object.values(probe.violations), + violationsValid: true, + }; +} + function absentObservation(): Record { return { activationHash: "", @@ -187,28 +250,73 @@ function absentObservation(): Record { }; } -function traceLine(observation: unknown, timestamp: number): string { +interface TraceLineOptions { + readonly action?: unknown; + readonly namedSnapshots?: readonly { readonly name: string; readonly value: unknown }[]; + readonly url?: string; + readonly violations?: readonly unknown[]; +} + +function traceLine( + observation: unknown, + timestamp: number, + options: TraceLineOptions = {}, +): string { return JSON.stringify({ timestamp, - action: null, - state: {}, - snapshots: [{ index: 0, name: "direct", value: observation, time: timestamp }], - violations: [], + action: options.action ?? null, + state: { + url: options.url ?? "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + hash_previous: timestamp === 1 ? null : timestamp - 1, + hash_current: timestamp, + screenshot: `/tmp/${String(timestamp)}.png`, + resources: { + js_heap_used: timestamp * 10, + js_heap_total: timestamp * 20, + dom_nodes: timestamp, + documents: 1, + js_event_listeners: timestamp * 2, + layout_objects: timestamp * 3, + timestamp, + thread_time: timestamp / 10, + task_duration: timestamp / 20, + script_duration: timestamp / 30, + }, + }, + snapshots: [ + { index: 0, name: "direct", value: observation, time: timestamp }, + ...(options.namedSnapshots ?? []).map((snapshot, index) => ({ + index: index + 1, + name: snapshot.name, + value: snapshot.value, + time: timestamp, + })), + ], + violations: options.violations ?? [], }); } async function writeTrace( tracePath: string, observations: readonly unknown[], + lineOptions: readonly TraceLineOptions[] = [], ): Promise { await mkdir(join(tracePath, ".."), { recursive: true }); await writeFile( tracePath, - `${observations.map((observation, index) => traceLine(observation, index + 1)).join("\n")}\n`, + `${observations.map((observation, index) => + traceLine(observation, index + 1, lineOptions[index]) + ).join("\n")}\n`, "utf8", ); } +function nestedJson(depth: number): unknown { + let value: unknown = null; + for (let index = 0; index < depth; index += 1) value = [value]; + return value; +} + function fakeServer( calls: string[], output: Promise = Promise.resolve("server output"), @@ -250,6 +358,7 @@ function dependencies(options: { readonly noTrace?: boolean; readonly neverServerOutput?: boolean; readonly observations?: readonly unknown[]; + readonly traceLineOptions?: readonly TraceLineOptions[]; readonly serverOutputTimeoutMs?: number; readonly stopFailure?: boolean; readonly termination?: "aborted" | "timeout"; @@ -298,6 +407,7 @@ function dependencies(options: { await writeTrace( join(invocation.outputPath, "trace.jsonl"), options.observations ?? [absentObservation(), directObservation()], + options.traceLineOptions, ); } return { @@ -316,7 +426,7 @@ function dependencies(options: { if (options.stopFailure === true) { throw new Error("server cleanup failed"); } - ownedServer.terminate(); + if (ownedServer.exitCode() === null) ownedServer.terminate(); await ownedServer.exited; }, }, @@ -392,11 +502,76 @@ describe("Direct Bombadil configuration and invocation", () => { expect(validated.entryPath).toBe("/"); expect(validated.targetQuery).toEqual({}); expect(validated.server.readinessPath).toBe("/ready"); + expect(validated.viewport).toEqual({ + deviceScaleFactor: 2, + height: 768, + width: 1_024, + }); + expect(validateDirectBombadilFuzzConfig({ + ...config, + viewport: { deviceScaleFactor: 1.5, height: 720, width: 1_280 }, + }).viewport).toEqual({ + deviceScaleFactor: 1.5, + height: 720, + width: 1_280, + }); expect(validated.bombadilExecutable).toEndWith( `node_modules/@antithesishq/bombadil/binaries/${nativeBinaryName()}`, ); }); + test("orders query and policy artifacts by explicit code units", async () => { + const { config } = await fixture(); + const validated = validateDirectBombadilFuzzConfig({ + ...config, + targetQuery: { z: "4", "a.": "3", A: "1", "a-": "2" }, + explorationPolicy: { + minDistinctNamedSnapshotValues: { + z: 4, + "a.": 3, + A: 1, + "a-": 2, + }, + minNamedSnapshotChangesAfterActionKind: { + z: { Wait: 4, Click: 3 }, + "a.": { SetViewport: 3 }, + A: { Wait: 1, Click: 2 }, + "a-": { TypeText: 2 }, + }, + minNamedSnapshotChangesAfterNonWait: { + z: 4, + "a.": 3, + A: 1, + "a-": 2, + }, + requiredActionKinds: ["Wait", "TypeText", "Click"], + requiredNamedSnapshots: ["z", "a.", "A", "a-"], + }, + }); + const expectedNames = ["A", "a-", "a.", "z"]; + expect(Object.keys(validated.targetQuery)).toEqual(expectedNames); + expect(validated.explorationPolicy?.requiredActionKinds).toEqual([ + "Click", + "TypeText", + "Wait", + ]); + expect(validated.explorationPolicy?.requiredNamedSnapshots).toEqual( + expectedNames, + ); + expect(Object.keys( + validated.explorationPolicy?.minDistinctNamedSnapshotValues ?? {}, + )).toEqual(expectedNames); + expect(Object.keys( + validated.explorationPolicy?.minNamedSnapshotChangesAfterActionKind ?? {}, + )).toEqual(expectedNames); + expect(Object.keys( + validated.explorationPolicy?.minNamedSnapshotChangesAfterActionKind.A ?? {}, + )).toEqual(["Click", "Wait"]); + expect(Object.keys( + validated.explorationPolicy?.minNamedSnapshotChangesAfterNonWait ?? {}, + )).toEqual(expectedNames); + }); + test("rejects unsafe artifact, scenario, route, path, readiness, and server command inputs", async () => { const { config, repositoryRoot } = await fixture(); expect(() => validateDirectBombadilFuzzConfig({ @@ -435,6 +610,75 @@ describe("Direct Bombadil configuration and invocation", () => { ...config, server: { ...config.server, readinessPath: "//external.test" }, })).toThrow("origin-relative"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + viewport: { width: 0 }, + })).toThrow("viewport.width"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + viewport: { deviceScaleFactor: Number.POSITIVE_INFINITY }, + })).toThrow("deviceScaleFactor"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { requiredActionKinds: ["Wait", "Wait"] }, + })).toThrow("duplicate kind"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { minDistinctNamedSnapshotValues: { "unsafe name": 2 } }, + })).toThrow("safe bounded snapshot name"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { minNamedSnapshotChangesAfterNonWait: { phase: 10_001 } }, + })).toThrow("between 1 and 10000"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { phase: {} }, + }, + })).toThrow("bounded action map"); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { Unknown: 1 } as never, + }, + }, + })).toThrow("unknown action kind"); + for (const minimum of [0, 10_001]) { + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { Click: minimum }, + }, + }, + })).toThrow("between 1 and 10000"); + } + const snapshotNames = (prefix: string): string[] => + Array.from({ length: 32 }, (_, index) => `${prefix}${String(index)}`); + const maximumPolicy = { + minDistinctNamedSnapshotValues: Object.fromEntries( + snapshotNames("d").map((name) => [name, 1]), + ), + minNamedSnapshotChangesAfterActionKind: Object.fromEntries( + snapshotNames("a").map((name) => [name, { Click: 1 }]), + ), + minNamedSnapshotChangesAfterNonWait: Object.fromEntries( + snapshotNames("n").map((name) => [name, 1]), + ), + requiredNamedSnapshots: snapshotNames("r").slice(0, 31), + } as const; + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: maximumPolicy, + })).not.toThrow(); + expect(() => validateDirectBombadilFuzzConfig({ + ...config, + explorationPolicy: { + ...maximumPolicy, + requiredNamedSnapshots: snapshotNames("r"), + }, + })).toThrow("at most 127 distinct non-Direct snapshots"); }); test("rejects specification, server cwd, and replay symlinks that escape the repository", async () => { @@ -493,6 +737,7 @@ describe("Direct Bombadil configuration and invocation", () => { specificationPath: "/repo/spec.ts", targetQuery: { workbench: "frame" }, timeLimitSeconds: 20, + viewport: { deviceScaleFactor: 1.25, height: 720, width: 1_280 }, }); expect(invocation.targetUrl).toBe( "http://127.0.0.1:5184/direct/?__direct_scenario=surface.ready&workbench=frame", @@ -508,6 +753,12 @@ describe("Direct Bombadil configuration and invocation", () => { "/repo/artifacts/run/bombadil", "--headless", "--instrument-javascript=", + "--width", + "1280", + "--height", + "720", + "--device-scale-factor", + "1.25", "--exit-on-violation", "--time-limit", "20s", @@ -526,18 +777,74 @@ describe("Direct Bombadil configuration and invocation", () => { timeLimitSeconds: 20, }); expect(invocation.command).toContain("--reproduce"); + expect(invocation.command).toEqual(expect.arrayContaining([ + "--width", "1024", "--height", "768", "--device-scale-factor", "2", + ])); expect(invocation.command).not.toContain("--time-limit"); expect(invocation.command).not.toContain("--exit-on-violation"); expect(invocation.wallClockTimeoutMs).toBe(330_000); }); }); +describe("Direct Bombadil campaign matrix", () => { + test("runs unique bounded campaigns serially and selects exactly one", async () => { + const { config } = await fixture(); + const campaigns = [{ id: "primary", config }, { + id: "secondary", + config: { ...config, artifactName: "fixture-secondary" }, + }] as const; + const allRuntime = dependencies(); + const all = await runDirectBombadilFuzzMatrix( + campaigns, + ["--time-limit=12s"], + allRuntime.overrides, + ); + expect(all).toMatchObject({ + kind: "matrix", + results: [{ campaignId: "primary" }, { campaignId: "secondary" }], + }); + expect(allRuntime.calls.filter((call) => call === "run-bombadil")).toHaveLength(2); + + const selectedRuntime = dependencies(); + const selected = await runDirectBombadilFuzzMatrix( + campaigns, + ["--campaign=secondary", "--time-limit=12s"], + selectedRuntime.overrides, + ); + expect(selected).toMatchObject({ + kind: "matrix", + results: [{ campaignId: "secondary" }], + }); + expect(selectedRuntime.calls.filter((call) => call === "run-bombadil")).toHaveLength(1); + }); + + test("rejects ambiguous replay, duplicate IDs, and unknown selection", async () => { + const { config } = await fixture(); + const campaigns = [{ id: "primary", config }] as const; + expect((await rejection(runDirectBombadilFuzzMatrix( + campaigns, + ["--replay=artifacts/trace.jsonl"], + ))).message).toContain("requires exactly one --campaign"); + expect((await rejection(runDirectBombadilFuzzMatrix( + campaigns, + ["--campaign=missing"], + ))).message).toContain("Unknown Bombadil campaign"); + expect((await rejection(runDirectBombadilFuzzMatrix([ + { id: "same", config }, + { id: "same", config: { ...config, artifactName: "other" } }, + ], []))).message).toContain("unique lowercase kebab"); + }); +}); + describe("Direct Bombadil trace attestation", () => { - async function attest(observations: readonly unknown[]) { + async function attest( + observations: readonly unknown[], + lineOptions: readonly TraceLineOptions[] = [], + ) { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-trace-")); temporaryDirectories.push(directory); const tracePath = join(directory, "trace.jsonl"); - await writeTrace(tracePath, observations); + await writeTrace(tracePath, observations, lineOptions); return attestDirectBombadilTrace({ expectedRoute: "/surface", expectedScenario: "surface.ready", @@ -567,6 +874,72 @@ describe("Direct Bombadil trace attestation", () => { }); }); + test("ignores unrelated Bombadil snapshots outside the Direct attestation contract", async () => { + const oversizedValue = "x".repeat(2 * 1024 * 1024 + 1); + const cases: readonly { + readonly label: string; + readonly snapshots: NonNullable; + }[] = [{ + label: "arbitrary name", + snapshots: [{ name: "phase state", value: "ready" }], + }, { + label: "duplicate unrelated name", + snapshots: [ + { name: "phase", value: "loading" }, + { name: "phase", value: "ready" }, + ], + }, { + label: "depth 65 value", + snapshots: [{ name: "deep", value: nestedJson(65) }], + }, { + label: "value above two MiB", + snapshots: [{ name: "large", value: oversizedValue }], + }]; + + for (const testCase of cases) { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-compatible-trace-")); + temporaryDirectories.push(directory); + const tracePath = join(directory, "trace.jsonl"); + await writeTrace(tracePath, [directObservation()], [{ + namedSnapshots: testCase.snapshots, + }]); + const result = await attestDirectBombadilTrace({ + expectedRoute: "/surface", + expectedScenario: "surface.ready", + tracePath, + }); + expect(result.validObservationCount, testCase.label).toBe(1); + } + }); + + test("keeps exact Direct uniqueness and schema bounds independent of diagnostic limits", async () => { + const duplicateError = await rejection(attest([directObservation()], [{ + namedSnapshots: [{ name: "direct", value: directObservation() }], + }])); + expect(duplicateError.message).toContain("must contain one named direct snapshot"); + + const observation = largeDirectObservation(); + expect(Buffer.byteLength(JSON.stringify(observation), "utf8")) + .toBeGreaterThan(2 * 1024 * 1024); + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-large-direct-trace-")); + temporaryDirectories.push(directory); + const tracePath = join(directory, "trace.jsonl"); + await writeTrace(tracePath, [observation]); + const result = await attestDirectBombadilTrace({ + expectedRoute: String(observation.activeRoute), + expectedScenario: String(observation.activeScenario), + tracePath, + }); + expect(result.validObservationCount).toBe(1); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots).toEqual([ + expect.objectContaining({ name: "direct", observationCount: 1 }), + ]); + }); + test("rejects a vacuous, wrongly activated, or post-activation missing trace", async () => { expect((await rejection(attest([absentObservation()]))).message) .toContain("never reached a valid Direct contract"); @@ -657,6 +1030,564 @@ describe("Direct Bombadil trace attestation", () => { }); }); +describe("Direct Bombadil exploration summary", () => { + async function summaryTrace(lines: readonly string[]): Promise { + const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-summary-")); + temporaryDirectories.push(directory); + const tracePath = join(directory, "trace.jsonl"); + await writeFile(tracePath, `${lines.join("\n")}\n`, "utf8"); + return tracePath; + } + + test("derives deterministic bounded diagnostics without retaining typed text or labels", async () => { + const observation = directObservation(); + const tracePath = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: { b: 2, a: 1 } }], + }), + traceLine(observation, 2, { + action: { + Click: { + fingerprint: { + accessible_name: "sensitive button label", + tag: "button", + }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: { a: 2 } }], + }), + traceLine(observation, 3, { + action: "Wait", + namedSnapshots: [{ name: "phase", value: { a: 2 } }], + }), + traceLine(observation, 4, { + action: { TypeText: { delay_millis: 0, text: "sensitive typed text" } }, + namedSnapshots: [{ name: "phase", value: { a: 2 } }], + violations: [{ + name: "noConsoleErrors", + violation: { False: { condition: "sensitive formula source" } }, + }], + }), + ]); + const options = { + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterActionKind: { phase: { Click: 1 } }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + minNonWaitActions: 2, + requireStableTargetUrl: true, + requiredActionKinds: ["Click", "TypeText"] as const, + requiredNamedSnapshots: ["phase"], + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }; + const first = await summarizeDirectBombadilTrace(options); + const second = await summarizeDirectBombadilTrace(options); + expect(second).toEqual(first); + expect(first).toMatchObject({ + schema: "direct.bombadil-exploration-summary/v2", + actions: { + byKind: { Click: 1, TypeText: 1, Wait: 1 }, + maxWaitStreak: 1, + nonWaitCount: 2, + targetTags: { button: 1 }, + total: 3, + }, + urls: { + distinctFingerprintCount: 1, + observationCount: 4, + rawDistinctFingerprintCount: 1, + rawObservationCount: 4, + stableTarget: true, + }, + transitions: { + distinctNonNullHashCount: 4, + nonNullHashCount: 4, + rawDistinctNonNullHashCount: 4, + rawNonNullHashCount: 4, + }, + propertyViolations: { byName: { noConsoleErrors: 1 }, total: 1 }, + resourceHighWaterMarks: { + domNodes: 4, + jsHeapUsedBytes: 40, + }, + policy: { configured: true, failures: [], satisfied: true }, + }); + expect(first.namedSnapshots.find((entry) => entry.name === "phase")).toMatchObject({ + changeAfterActionKind: { Click: 1 }, + changeAfterNonWaitCount: 1, + distinctValueCount: 2, + observationCount: 4, + }); + expect(first.trace.sha256).toHaveLength(64); + const serialized = JSON.stringify(first); + expect(serialized).not.toContain("sensitive button label"); + expect(serialized).not.toContain("sensitive typed text"); + expect(serialized).not.toContain("sensitive formula source"); + expect(serialized).not.toContain("127.0.0.1"); + expect(serialized).not.toContain("/tmp/"); + }); + + test("omits out-of-contract unrelated snapshots while keeping policy snapshots strict", async () => { + const observation = directObservation(); + const oversizedValue = "x".repeat(2 * 1024 * 1024 + 1); + const tracePath = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: [ + { name: "owned", value: "ready" }, + { name: "phase state", value: "ready" }, + { name: "duplicate", value: 0 }, + { name: "duplicate", value: 1 }, + { name: "deep", value: nestedJson(65) }, + { name: "large", value: oversizedValue }, + ], + })]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { requiredNamedSnapshots: ["owned"] }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.map(({ name }) => name)).toEqual(["direct", "owned"]); + expect(summary.policy).toMatchObject({ configured: true, failures: [], satisfied: true }); + + const strictCases: readonly { + readonly label: string; + readonly snapshots: NonNullable; + readonly expected: string; + }[] = [{ + label: "duplicate", + snapshots: [ + { name: "owned", value: 0 }, + { name: "owned", value: 1 }, + ], + expected: "repeats named snapshot owned", + }, { + label: "depth", + snapshots: [{ name: "owned", value: nestedJson(65) }], + expected: "exceeds JSON depth", + }, { + label: "size", + snapshots: [{ name: "owned", value: oversizedValue }], + expected: "exceeds 2097152 canonical bytes", + }]; + for (const testCase of strictCases) { + const strictTracePath = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: testCase.snapshots, + })]); + const error = await rejection(summarizeDirectBombadilTrace({ + explorationPolicy: { requiredNamedSnapshots: ["owned"] }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: strictTracePath, + })); + expect(error.message, testCase.label).toContain(testCase.expected); + } + }); + + test("accepts every exact Bombadil 0.7.2 browser action payload", async () => { + const actions = [ + "Back", + { + Click: { + fingerprint: { accessible_name: "private click label", tag: "button" }, + point: { x: 1, y: 2 }, + }, + }, + { + DoubleClick: { + delay_millis: 10, + fingerprint: { + structural_path: "private/path", + tag: "x-private.widget_\u00e9", + }, + point: { x: 2, y: 3 }, + }, + }, + "Forward", + { + MouseDrag: { + delay_millis: 5, + from: { x: 3, y: 4 }, + steps: 10, + to: { x: 30, y: 40 }, + }, + }, + { PressKey: { code: 13 } }, + "Reload", + { ScrollDown: { distance: 100, origin: { x: 5, y: 6 } } }, + { ScrollUp: { distance: 100, origin: { x: 5, y: 6 } } }, + { + SetFileInputFiles: { + files: ["/private/file.txt"], + selector: "#private-file-input", + }, + }, + { SetViewport: { height: 720, width: 1_280 } }, + { TypeText: { delay_millis: 5, text: "private typed text" } }, + "Wait", + ] as const; + const observation = directObservation(); + const tracePath = await summaryTrace([ + traceLine(observation, 1), + ...actions.map((action, index) => traceLine(observation, index + 2, { + action, + })), + ]); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(Object.keys(summary.actions.byKind)).toEqual([ + "Back", + "Click", + "DoubleClick", + "Forward", + "MouseDrag", + "PressKey", + "Reload", + "ScrollDown", + "ScrollUp", + "SetFileInputFiles", + "SetViewport", + "TypeText", + "Wait", + ]); + expect(summary.actions.total).toBe(actions.length); + expect(Object.keys(summary.actions.targetTags)).toContain("button"); + expect(Object.keys(summary.actions.targetTags).some((tag) => + /^sha256:[a-f0-9]{64}$/u.test(tag) + )).toBeTrue(); + expect(JSON.stringify(summary)).not.toContain("private"); + }); + + test("reports strict policy misses while keeping the diagnostic summary", async () => { + const tracePath = await summaryTrace([traceLine(directObservation(), 1)]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNonWaitActions: 1, + requireStableTargetUrl: true, + requiredActionKinds: ["Click"], + requiredNamedSnapshots: ["phase"], + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.policy.satisfied).toBeFalse(); + expect(summary.policy.failures).toHaveLength(4); + }); + + test("does not credit bootstrap or Wait-only state changes to product actions", async () => { + const observation = directObservation(); + const tracePath = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: "loading" }], + }), + traceLine(observation, 2, { + action: "Wait", + namedSnapshots: [{ name: "phase", value: "ready" }], + }), + ]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.find((entry) => entry.name === "phase")).toMatchObject({ + changeAfterNonWaitCount: 0, + distinctValueCount: 2, + }); + expect(summary.policy).toMatchObject({ + failures: [expect.stringContaining("post-non-Wait change minimum")], + satisfied: false, + }); + }); + + test("attributes named changes to the exact action kind", async () => { + const observation = directObservation(); + const noOpClickTrace = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(observation, 2, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(observation, 3, { + action: { SetViewport: { height: 720, width: 1_280 } }, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + ]); + const failed = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { Click: 1, SetViewport: 1 }, + }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: noOpClickTrace, + }); + expect(failed.namedSnapshots.find((entry) => entry.name === "phase")) + .toMatchObject({ + changeAfterActionKind: { SetViewport: 1 }, + changeAfterNonWaitCount: 1, + }); + expect(failed.policy).toMatchObject({ + failures: [expect.stringContaining("post-Click change minimum")], + satisfied: false, + }); + + const attributedTrace = await summaryTrace([ + traceLine(observation, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(observation, 2, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + traceLine(observation, 3, { + action: { SetViewport: { height: 720, width: 1_280 } }, + namedSnapshots: [{ name: "phase", value: 2 }], + }), + ]); + const attributed = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { + phase: { SetViewport: 1, Click: 1 }, + }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: attributedTrace, + }); + const phase = attributed.namedSnapshots.find((entry) => entry.name === "phase"); + expect(phase?.changeAfterActionKind).toEqual({ Click: 1, SetViewport: 1 }); + expect(phase?.changeAfterNonWaitCount).toBe(2); + expect(phase?.changeAfterNonWaitCount).toBe( + Object.entries(phase?.changeAfterActionKind ?? {}) + .filter(([kind]) => kind !== "Wait") + .reduce((total, [, count]) => total + count, 0), + ); + expect(attributed.policy.satisfied).toBeTrue(); + }); + + test("requires adjacent exact observations for action attribution", async () => { + const exact = directObservation(); + const click = { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + } as const; + const missingSnapshotTrace = await summaryTrace([ + traceLine(exact, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(exact, 2), + traceLine(exact, 3, { + action: click, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + ]); + const invalidCurrentTrace = await summaryTrace([ + traceLine(exact, 1, { + namedSnapshots: [{ name: "phase", value: 0 }], + }), + traceLine(absentObservation(), 2, { + action: click, + namedSnapshots: [{ name: "phase", value: 1 }], + }), + ]); + for (const tracePath of [missingSnapshotTrace, invalidCurrentTrace]) { + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minNamedSnapshotChangesAfterActionKind: { phase: { Click: 1 } }, + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.find((entry) => entry.name === "phase") + ?.changeAfterActionKind).toEqual({}); + expect(summary.policy.satisfied).toBeFalse(); + } + }); + + test("uses the first exact Direct observation only as the policy baseline", async () => { + const tracePath = await summaryTrace([ + traceLine(absentObservation(), 20, { + namedSnapshots: [{ name: "phase", value: "loading" }], + violations: [{ + name: "startup_contract", + violation: { False: {} }, + }], + }), + traceLine(absentObservation(), 2, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: "pre-handshake-click" }], + }), + traceLine(directObservation(), 3, { + action: { + Click: { + fingerprint: { tag: "button" }, + point: { x: 1, y: 1 }, + }, + }, + namedSnapshots: [{ name: "phase", value: "ready" }], + }), + ]); + const summary = await summarizeDirectBombadilTrace({ + explorationPolicy: { + minDistinctNamedSnapshotValues: { phase: 2 }, + minNamedSnapshotChangesAfterActionKind: { phase: { Click: 1 } }, + minNamedSnapshotChangesAfterNonWait: { phase: 1 }, + minNonWaitActions: 1, + requiredActionKinds: ["Click"], + requiredNamedSnapshots: ["phase"], + }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.actions).toMatchObject({ + byKind: {}, + nonWaitCount: 0, + total: 0, + }); + expect(summary.urls.observationCount).toBe(1); + expect(summary.urls.rawObservationCount).toBe(3); + expect(summary.transitions).toEqual({ + distinctNonNullHashCount: 1, + nonNullHashCount: 1, + rawDistinctNonNullHashCount: 3, + rawNonNullHashCount: 3, + }); + expect(summary.propertyViolations).toEqual({ + byName: { startup_contract: 1 }, + total: 1, + }); + expect(summary.resourceHighWaterMarks.domNodes).toBe(20); + expect(summary.namedSnapshots.find((entry) => entry.name === "phase")) + .toMatchObject({ + changeAfterNonWaitCount: 0, + distinctValueCount: 1, + observationCount: 1, + }); + expect(summary.policy).toMatchObject({ + failures: [ + expect.stringContaining("minimum non-Wait"), + expect.stringContaining("required action kind Click"), + expect.stringContaining("distinct-value minimum"), + expect.stringContaining("post-non-Wait change minimum"), + expect.stringContaining("post-Click change minimum"), + ], + satisfied: false, + }); + }); + + test("orders mixed-case and punctuation summary names by code unit", async () => { + const observation = directObservation(); + const tracePath = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: [ + { name: "z", value: 4 }, + { name: "a.", value: 3 }, + { name: "A", value: 1 }, + { name: "a-", value: 2 }, + ], + violations: [ + { name: "z", violation: { False: {} } }, + { name: "a.", violation: { False: {} } }, + { name: "A", violation: { False: {} } }, + { name: "a-", violation: { False: {} } }, + ], + })]); + const summary = await summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath, + }); + expect(summary.namedSnapshots.map(({ name }) => name)).toEqual([ + "A", + "a-", + "a.", + "direct", + "z", + ]); + expect(Object.keys(summary.propertyViolations.byName)).toEqual([ + "A", + "a-", + "a.", + "z", + ]); + }); + + test("rejects hostile envelopes, action targets, and excessively deep snapshot JSON", async () => { + const observation = directObservation(); + const badTarget = await summaryTrace([traceLine(observation, 1, { + action: { + Click: { + fingerprint: { tag: "" }, + point: { x: 1, y: 1 }, + }, + }, + })]); + expect((await rejection(summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: badTarget, + }))).message).toContain("invalid action target tag"); + + for (const action of [ + { PressKey: {} }, + { PressKey: { code: 256 } }, + { SetViewport: {} }, + { SetViewport: { height: 720, width: 0 } }, + { + MouseDrag: { + delay_millis: 0, + from: { x: 0, y: 0 }, + steps: 0, + to: { x: 1, y: 1 }, + }, + }, + { TypeText: { delay_millis: 0, text: "safe", unexpected: true } }, + ]) { + const malformedAction = await summaryTrace([traceLine(observation, 1, { + action, + })]); + expect((await rejection(summarizeDirectBombadilTrace({ + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: malformedAction, + }))).message).toContain("invalid action"); + } + + let deep: unknown = null; + for (let index = 0; index < 70; index += 1) deep = [deep]; + const deepTrace = await summaryTrace([traceLine(observation, 1, { + namedSnapshots: [{ name: "deep", value: deep }], + })]); + expect((await rejection(summarizeDirectBombadilTrace({ + explorationPolicy: { requiredNamedSnapshots: ["deep"] }, + targetUrl: "http://127.0.0.1:4919/?__direct_scenario=surface.ready", + tracePath: deepTrace, + }))).message).toContain("exceeds JSON depth"); + }); +}); + describe("Direct Bombadil process lifecycle", () => { test("cleans descendants and inherited pipes after a normal leader exit", async () => { const directory = await mkdtemp(join(tmpdir(), "direct-bombadil-normal-exit-")); @@ -790,6 +1721,11 @@ describe("Direct Bombadil run lifecycle", () => { }, initialDirect: { source: "scenario", scenario: "surface.ready", route: "/surface" }, server: { logPresent: true }, + explorationSummary: { + schema: "direct.bombadil-exploration-summary/v2", + policy: { configured: false, satisfied: true }, + }, + viewport: { deviceScaleFactor: 2, height: 768, width: 1_024 }, }); const bombadil = record(manifest.bombadil, "bombadil"); expect(bombadil.version).toBe("0.7.2"); @@ -800,6 +1736,170 @@ describe("Direct Bombadil run lifecycle", () => { expect(await readFile(String(bombadil.logPath), "utf8")).toContain("bombadil stdout"); const server = record(manifest.server, "server"); expect(await readFile(String(server.logPath), "utf8")).toContain("server output"); + const summaryPath = String(manifest.explorationSummaryPath); + expect(JSON.parse(await readFile(summaryPath, "utf8"))).toMatchObject({ + schema: "direct.bombadil-exploration-summary/v2", + trace: { lineCount: 2 }, + }); + }); + + test("runs with policy-owned evidence despite arbitrary unrelated named snapshots", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies({ + observations: [directObservation()], + traceLineOptions: [{ + namedSnapshots: [ + { name: "owned", value: "ready" }, + { name: "phase state", value: "ready" }, + { name: "duplicate", value: 0 }, + { name: "duplicate", value: 1 }, + { name: "deep", value: nestedJson(65) }, + { name: "large", value: "x".repeat(2 * 1024 * 1024 + 1) }, + ], + }], + }); + const result = await runDirectBombadilFuzz({ + ...config, + explorationPolicy: { requiredNamedSnapshots: ["owned"] }, + }, [], runtime.overrides); + expect(result.kind).toBe("run"); + const manifest = JSON.parse(await readFile( + join(repositoryRoot, "artifacts", "direct-bombadil", "fixture-product", "manifest.json"), + "utf8", + )) as Record; + expect(manifest).toMatchObject({ + status: "passed", + attestation: { validObservationCount: 1 }, + explorationSummary: { + namedSnapshots: [{ name: "direct" }, { name: "owned" }], + policy: { configured: true, failures: [], satisfied: true }, + }, + }); + }); + + test("does not spawn when cancellation wins before server startup", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + const error = await rejection(runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + acquireServer: (options): Promise => { + runtime.calls.push("acquire-server"); + controller.abort(); + return Promise.resolve({ source: "started", server: options.startServer() }); + }, + })); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual(["acquire-server"]); + }); + + test("terminates a server when cancellation wins during spawn", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + const server = fakeServer(runtime.calls); + const error = await rejection(runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + spawnServer: () => { + runtime.calls.push("spawn-server"); + controller.abort(); + return server; + }, + })); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual([ + "acquire-server", + "spawn-server", + "terminate", + "stop-server", + ]); + expect(server.exitCode()).toBe(0); + }); + + test("cleans a just-acquired server before Bombadil can run", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + const error = await rejection(runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + acquireServer: (options): Promise => { + runtime.calls.push("acquire-server"); + const server = options.startServer(); + controller.abort(); + return Promise.resolve({ source: "started", server }); + }, + })); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual([ + "acquire-server", + "spawn-server", + "terminate", + "stop-server", + ]); + expect(runtime.calls).not.toContain("run-bombadil"); + }); + + test("unwinds a pending server acquisition when cancellation arrives", async () => { + const { config } = await fixture(); + const runtime = dependencies(); + const controller = new AbortController(); + let markAcquiring!: () => void; + const acquiring = new Promise((resolve) => { + markAcquiring = resolve; + }); + const run = runDirectBombadilFuzz(config, [], { + ...runtime.overrides, + createAbortController: () => controller, + acquireServer: async (options): Promise => { + runtime.calls.push("acquire-server"); + const server = options.startServer(); + markAcquiring(); + await new Promise((_resolve, reject) => { + const abort = (): void => reject(new Error("acquisition aborted")); + options.abortSignal?.addEventListener("abort", abort, { once: true }); + if (options.abortSignal?.aborted === true) abort(); + }); + return { source: "started", server }; + }, + }); + await acquiring; + controller.abort(); + const error = await rejection(run); + expect(error.message).toContain("Bombadil fuzzing was interrupted"); + expect(runtime.calls).toEqual([ + "acquire-server", + "spawn-server", + "stop-server", + "terminate", + ]); + expect(runtime.calls).not.toContain("run-bombadil"); + }); + + test("fails a configured exploration policy while retaining the derived sidecar", async () => { + const { config, repositoryRoot } = await fixture(); + const runtime = dependencies(); + expect((await rejection(runDirectBombadilFuzz({ + ...config, + explorationPolicy: { + minNonWaitActions: 1, + requiredActionKinds: ["Click"], + }, + }, [], runtime.overrides))).message).toContain("exploration policy was not satisfied"); + const manifest = JSON.parse(await readFile( + join(repositoryRoot, "artifacts", "direct-bombadil", "fixture-product", "manifest.json"), + "utf8", + )) as Record; + expect(manifest).toMatchObject({ + status: "failed", + explorationSummary: { + policy: { configured: true, satisfied: false }, + }, + }); + expect(await readFile(String(manifest.explorationSummaryPath), "utf8")) + .toContain("direct.bombadil-exploration-summary/v2"); }); test("retains an attested failure artifact and server log for a nonzero exit", async () => { diff --git a/src/tooling/bombadil-runner.ts b/src/tooling/bombadil-runner.ts index 61cf990..57107e8 100644 --- a/src/tooling/bombadil-runner.ts +++ b/src/tooling/bombadil-runner.ts @@ -3,6 +3,7 @@ import { readFile, realpath, stat, writeFile } from "node:fs/promises"; import { isAbsolute, join, relative, resolve } from "node:path"; import process from "node:process"; import { createInterface } from "node:readline"; +import { createHash } from "node:crypto"; import { parseDirectProbeSnapshot, @@ -44,6 +45,12 @@ const TRACE_MAX_BYTES = 64 * 1024 * 1024; const TRACE_MAX_LINE_BYTES = 16 * 1024 * 1024; const TRACE_MAX_LINES = 10_000; const TRACE_MAX_SNAPSHOTS_PER_LINE = 4_096; +const TRACE_MAX_NAMED_SNAPSHOT_NAMES = 128; +const TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME = 1_024; +const TRACE_MAX_DISTINCT_URLS = 1_024; +const TRACE_MAX_PROPERTY_NAMES = 128; +const TRACE_MAX_CANONICAL_SNAPSHOT_BYTES = 2 * 1024 * 1024; +const TRACE_MAX_JSON_DEPTH = 64; const RANDOM_RUN_OVERHEAD_MS = 30_000; const REPLAY_WALL_CLOCK_TIMEOUT_MS = MAX_TIME_LIMIT_SECONDS * 1_000 + RANDOM_RUN_OVERHEAD_MS; const PROCESS_TERMINATION_GRACE_MS = 5_000; @@ -52,6 +59,94 @@ const SERVER_OUTPUT_TIMEOUT_MS = 3_000; const DIRECT_BROWSER_BRIDGE_SCHEMA = "direct.browser-bridge/v2"; const TRACE_LINE_KEYS = new Set(["action", "snapshots", "state", "timestamp", "violations"]); const TRACE_SNAPSHOT_KEYS = new Set(["index", "name", "time", "value"]); +const TRACE_STATE_KEYS = new Set([ + "hash_current", + "hash_previous", + "resources", + "screenshot", + "url", +]); +const TRACE_RESOURCE_KEYS = new Set([ + "documents", + "dom_nodes", + "js_event_listeners", + "js_heap_total", + "js_heap_used", + "layout_objects", + "script_duration", + "task_duration", + "thread_time", + "timestamp", +]); +const TRACE_VIOLATION_KEYS = new Set(["name", "violation"]); +const TRACE_POINT_KEYS = new Set(["x", "y"]); +const TRACE_FINGERPRINT_KEYS = new Set([ + "accessible_name", + "href", + "id", + "input_type", + "name_attr", + "placeholder", + "role", + "structural_path", + "tag", + "test_id", + "text_content", +]); +const TRACE_CLICK_ACTION_KEYS = new Set(["fingerprint", "point"]); +const TRACE_DOUBLE_CLICK_ACTION_KEYS = new Set([ + "delay_millis", + "fingerprint", + "point", +]); +const TRACE_TYPE_TEXT_ACTION_KEYS = new Set(["delay_millis", "text"]); +const TRACE_PRESS_KEY_ACTION_KEYS = new Set(["code"]); +const TRACE_SCROLL_ACTION_KEYS = new Set(["distance", "origin"]); +const TRACE_FILE_INPUT_ACTION_KEYS = new Set(["files", "selector"]); +const TRACE_MOUSE_DRAG_ACTION_KEYS = new Set([ + "delay_millis", + "from", + "steps", + "to", +]); +const TRACE_VIEWPORT_ACTION_KEYS = new Set(["height", "width"]); +const VIEWPORT_KEYS = new Set(["deviceScaleFactor", "height", "width"]); +const EXPLORATION_POLICY_KEYS = new Set([ + "minDistinctNamedSnapshotValues", + "minNamedSnapshotChangesAfterActionKind", + "minNamedSnapshotChangesAfterNonWait", + "minNonWaitActions", + "requireStableTargetUrl", + "requiredActionKinds", + "requiredNamedSnapshots", +]); +const DEFAULT_VIEWPORT_WIDTH = 1_024; +const DEFAULT_VIEWPORT_HEIGHT = 768; +const DEFAULT_DEVICE_SCALE_FACTOR = 2; +const SNAPSHOT_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_.:/-]*$/u; +const TARGET_TAG_PATTERN = /^[a-z][a-z0-9-]*$/u; +const ACTION_KINDS = [ + "Back", + "Click", + "DoubleClick", + "Forward", + "MouseDrag", + "PressKey", + "Reload", + "ScrollDown", + "ScrollUp", + "SetFileInputFiles", + "SetViewport", + "TypeText", + "Wait", +] as const; +const ACTION_KIND_SET = new Set(ACTION_KINDS); +const UNIT_ACTION_KINDS = new Set([ + "Back", + "Forward", + "Reload", + "Wait", +]); const DIRECT_OBSERVATION_KEYS = new Set([ "activationHash", "activeRoute", @@ -76,6 +171,88 @@ export interface DirectBombadilServerConfig { readonly startupTimeoutMs?: number; } +export type DirectBombadilActionKind = (typeof ACTION_KINDS)[number]; + +export interface DirectBombadilViewportConfig { + readonly deviceScaleFactor?: number; + readonly height?: number; + readonly width?: number; +} + +export interface DirectBombadilExplorationPolicy { + readonly minDistinctNamedSnapshotValues?: Readonly>; + readonly minNamedSnapshotChangesAfterActionKind?: Readonly>> + >>; + readonly minNamedSnapshotChangesAfterNonWait?: Readonly>; + readonly minNonWaitActions?: number; + readonly requireStableTargetUrl?: boolean; + readonly requiredActionKinds?: readonly DirectBombadilActionKind[]; + readonly requiredNamedSnapshots?: readonly string[]; +} + +export interface DirectBombadilExplorationSummary { + readonly schema: "direct.bombadil-exploration-summary/v2"; + readonly trace: { + readonly bytes: number; + readonly lineCount: number; + readonly sha256: string; + }; + readonly actions: { + readonly byKind: Readonly>>; + readonly maxWaitStreak: number; + readonly nonWaitCount: number; + readonly targetTags: Readonly>; + readonly total: number; + }; + readonly urls: { + readonly distinctFingerprintCount: number; + readonly fingerprintSha256: readonly string[]; + readonly observationCount: number; + readonly rawDistinctFingerprintCount: number; + readonly rawFingerprintSha256: readonly string[]; + readonly rawObservationCount: number; + readonly stableTarget: boolean; + }; + readonly transitions: { + readonly distinctNonNullHashCount: number; + readonly nonNullHashCount: number; + readonly rawDistinctNonNullHashCount: number; + readonly rawNonNullHashCount: number; + }; + readonly namedSnapshots: readonly { + readonly changeAfterActionKind: Readonly< + Partial> + >; + readonly changeAfterNonWaitCount: number; + readonly distinctValueCount: number; + readonly distinctValueSha256: readonly string[]; + readonly name: string; + readonly observationCount: number; + }[]; + readonly propertyViolations: { + readonly byName: Readonly>; + readonly total: number; + }; + readonly resourceHighWaterMarks: { + readonly documents: number; + readonly domNodes: number; + readonly jsEventListeners: number; + readonly jsHeapTotalBytes: number; + readonly jsHeapUsedBytes: number; + readonly layoutObjects: number; + readonly scriptDurationSeconds: number; + readonly taskDurationSeconds: number; + readonly threadTimeSeconds: number; + }; + readonly policy: { + readonly configured: boolean; + readonly failures: readonly string[]; + readonly satisfied: boolean; + }; +} + export interface DirectBombadilFuzzConfig { readonly artifactName: string; readonly baseUrl: string; @@ -86,6 +263,8 @@ export interface DirectBombadilFuzzConfig { readonly scenario: string; readonly specificationPath: string; readonly targetQuery?: Readonly>; + readonly explorationPolicy?: DirectBombadilExplorationPolicy; + readonly viewport?: DirectBombadilViewportConfig; readonly server: DirectBombadilServerConfig; } @@ -107,6 +286,21 @@ export type DirectBombadilFuzzResult = readonly status: "passed"; }; +export interface DirectBombadilFuzzCampaign { + readonly config: DirectBombadilFuzzConfig; + readonly id: string; +} + +export type DirectBombadilFuzzMatrixResult = + | { readonly kind: "help" } + | { + readonly kind: "matrix"; + readonly results: readonly { + readonly campaignId: string; + readonly result: Extract; + }[]; + }; + export interface DirectBombadilInvocation { readonly abortSignal?: AbortSignal; readonly command: readonly string[]; @@ -160,6 +354,7 @@ export interface DirectBombadilTraceAttestation { export interface DirectBombadilRunnerDependencies { readonly acquireServer: typeof acquireVerificationServer; + readonly createAbortController?: () => AbortController; readonly now: () => Date; readonly runBombadil: ( invocation: DirectBombadilInvocation, @@ -184,17 +379,41 @@ interface ProcessSignalEmitter { ) => unknown; } -interface ValidatedConfig extends DirectBombadilFuzzConfig { +type ValidatedConfig = Omit< + DirectBombadilFuzzConfig, + "explorationPolicy" | "server" | "viewport" +> & { readonly artifactRoot: string; readonly baseUrl: string; readonly bombadilExecutable: string; readonly entryPath: `/${string}`; + readonly explorationPolicy: ValidatedExplorationPolicy | null; readonly port: string; readonly targetQuery: Readonly>; + readonly viewport: ValidatedViewport; readonly server: DirectBombadilServerConfig & { readonly readinessPath: `/${string}`; readonly startupTimeoutMs: number; }; +}; + +interface ValidatedViewport { + readonly deviceScaleFactor: number; + readonly height: number; + readonly width: number; +} + +interface ValidatedExplorationPolicy { + readonly minDistinctNamedSnapshotValues: Readonly>; + readonly minNamedSnapshotChangesAfterActionKind: Readonly>> + >>; + readonly minNamedSnapshotChangesAfterNonWait: Readonly>; + readonly minNonWaitActions: number; + readonly requireStableTargetUrl: boolean; + readonly requiredActionKinds: readonly DirectBombadilActionKind[]; + readonly requiredNamedSnapshots: readonly string[]; } function readOptionValue( @@ -283,6 +502,12 @@ function hasExactKeys( return keys.length === expected.size && keys.every((key) => expected.has(key)); } +function compareCodeUnits(left: string, right: string): number { + if (left < right) return -1; + if (left > right) return 1; + return 0; +} + function parseTraceDirectObservation(value: unknown): TraceDirectObservation { if (!isRecord(value) || !hasExactKeys(value, DIRECT_OBSERVATION_KEYS)) { throw new Error("Bombadil trace has an invalid named direct observation"); @@ -427,7 +652,325 @@ function exactTraceDirectObservation( }; } -function parseTraceLine(line: string, lineNumber: number): TraceDirectObservation { +interface ParsedTraceAction { + readonly kind: DirectBombadilActionKind; + readonly targetTag: string | null; +} + +interface ParsedTraceState { + readonly currentHash: number | null; + readonly resources: Readonly>; + readonly url: URL; +} + +interface ParsedTraceLine { + readonly action: ParsedTraceAction | null; + readonly directObservation: TraceDirectObservation; + readonly namedSnapshots: readonly { + readonly name: string; + readonly valueSha256: string; + }[]; + readonly propertyViolationNames: readonly string[]; + readonly state: ParsedTraceState; +} + +interface ParsedTraceEnvelope { + readonly action: unknown; + readonly snapshots: readonly unknown[]; + readonly state: unknown; + readonly timestamp: number; + readonly violations: readonly unknown[]; +} + +interface ParsedDirectTraceObservation { + readonly observation: TraceDirectObservation; + readonly value: unknown; +} + +const RESOURCE_FIELD_MAP = { + documents: "documents", + dom_nodes: "domNodes", + js_event_listeners: "jsEventListeners", + js_heap_total: "jsHeapTotalBytes", + js_heap_used: "jsHeapUsedBytes", + layout_objects: "layoutObjects", + script_duration: "scriptDurationSeconds", + task_duration: "taskDurationSeconds", + thread_time: "threadTimeSeconds", +} as const; + +function canonicalJson( + value: unknown, + depth = 0, + maximumDepth = TRACE_MAX_JSON_DEPTH, +): string { + if (depth > maximumDepth) { + throw new Error(`Bombadil named snapshot exceeds JSON depth ${String(maximumDepth)}`); + } + if (value === null || typeof value === "boolean" || typeof value === "string") { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("Bombadil named snapshot has a non-finite number"); + return JSON.stringify(value); + } + if (Array.isArray(value)) { + return `[${value.map((entry) => canonicalJson(entry, depth + 1, maximumDepth)).join(",")}]`; + } + if (!isRecord(value)) throw new Error("Bombadil named snapshot is not JSON"); + const entries = Object.keys(value).sort(compareCodeUnits).map((key) => + `${JSON.stringify(key)}:${canonicalJson(value[key], depth + 1, maximumDepth)}` + ); + return `{${entries.join(",")}}`; +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +function namedSnapshotValueSha256( + value: unknown, + options: { + readonly maximumBytes?: number; + readonly maximumDepth?: number; + } = {}, +): string { + const maximumBytes = options.maximumBytes ?? TRACE_MAX_CANONICAL_SNAPSHOT_BYTES; + const canonical = canonicalJson( + value, + 0, + options.maximumDepth ?? TRACE_MAX_JSON_DEPTH, + ); + if (Buffer.byteLength(canonical, "utf8") > maximumBytes) { + throw new Error( + `Bombadil named snapshot exceeds ${String(maximumBytes)} canonical bytes`, + ); + } + return sha256(canonical); +} + +function validTracePoint(value: unknown): boolean { + return isRecord(value) + && hasExactKeys(value, TRACE_POINT_KEYS) + && typeof value.x === "number" + && Number.isFinite(value.x) + && typeof value.y === "number" + && Number.isFinite(value.y); +} + +function parseTraceFingerprintTag(value: unknown, lineNumber: number): string { + if ( + !isRecord(value) + || !Object.keys(value).every((key) => TRACE_FINGERPRINT_KEYS.has(key)) + ) { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target`, + ); + } + for (const [key, candidate] of Object.entries(value)) { + if (key !== "tag" && typeof candidate !== "string") { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target`, + ); + } + } + const tag = value.tag; + if ( + typeof tag !== "string" + || tag.length === 0 + ) { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target tag`, + ); + } + if ( + typeof value.structural_path === "string" + && Object.keys(value).some((key) => ( + key !== "tag" && key !== "structural_path" + )) + ) { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action target`, + ); + } + return tag.length <= 64 && TARGET_TAG_PATTERN.test(tag) + ? tag + : `sha256:${sha256(tag)}`; +} + +function isSafeIntegerBetween( + value: unknown, + minimum: number, + maximum: number, +): value is number { + return typeof value === "number" + && Number.isSafeInteger(value) + && value >= minimum + && value <= maximum; +} + +function invalidTraceAction(lineNumber: number): never { + throw new Error( + `Bombadil trace line ${String(lineNumber)} has an invalid action`, + ); +} + +function parseTraceAction(value: unknown, lineNumber: number): ParsedTraceAction | null { + if (value === null) return null; + if (typeof value === "string") { + if (!ACTION_KIND_SET.has(value) || !UNIT_ACTION_KINDS.has(value as DirectBombadilActionKind)) { + return invalidTraceAction(lineNumber); + } + return { kind: value as DirectBombadilActionKind, targetTag: null }; + } + if (!isRecord(value) || Object.keys(value).length !== 1) { + return invalidTraceAction(lineNumber); + } + const kind = Object.keys(value)[0]; + const payload = kind === undefined ? undefined : value[kind]; + if ( + kind === undefined + || !ACTION_KIND_SET.has(kind) + || UNIT_ACTION_KINDS.has(kind as DirectBombadilActionKind) + || !isRecord(payload) + ) { + return invalidTraceAction(lineNumber); + } + const actionKind = kind as DirectBombadilActionKind; + let targetTag: string | null = null; + switch (actionKind) { + case "Click": + if ( + !hasExactKeys(payload, TRACE_CLICK_ACTION_KEYS) + || !validTracePoint(payload.point) + ) { + return invalidTraceAction(lineNumber); + } + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "DoubleClick": + if ( + !hasExactKeys(payload, TRACE_DOUBLE_CLICK_ACTION_KEYS) + || !isSafeIntegerBetween(payload.delay_millis, 0, 1_000) + || !validTracePoint(payload.point) + ) return invalidTraceAction(lineNumber); + targetTag = parseTraceFingerprintTag(payload.fingerprint, lineNumber); + break; + case "TypeText": + if ( + !hasExactKeys(payload, TRACE_TYPE_TEXT_ACTION_KEYS) + || !isSafeIntegerBetween(payload.delay_millis, 0, Number.MAX_SAFE_INTEGER) + || typeof payload.text !== "string" + ) return invalidTraceAction(lineNumber); + break; + case "PressKey": + if ( + !hasExactKeys(payload, TRACE_PRESS_KEY_ACTION_KEYS) + || !isSafeIntegerBetween(payload.code, 0, 255) + ) { + return invalidTraceAction(lineNumber); + } + break; + case "ScrollDown": + case "ScrollUp": + if ( + !hasExactKeys(payload, TRACE_SCROLL_ACTION_KEYS) + || typeof payload.distance !== "number" + || !Number.isFinite(payload.distance) + || !validTracePoint(payload.origin) + ) return invalidTraceAction(lineNumber); + break; + case "SetFileInputFiles": + if ( + !hasExactKeys(payload, TRACE_FILE_INPUT_ACTION_KEYS) + || typeof payload.selector !== "string" + || !Array.isArray(payload.files) + || !payload.files.every((file) => typeof file === "string") + ) return invalidTraceAction(lineNumber); + break; + case "MouseDrag": + if ( + !hasExactKeys(payload, TRACE_MOUSE_DRAG_ACTION_KEYS) + || !isSafeIntegerBetween(payload.delay_millis, 0, 1_000) + || !isSafeIntegerBetween(payload.steps, 1, 255) + || !validTracePoint(payload.from) + || !validTracePoint(payload.to) + ) return invalidTraceAction(lineNumber); + break; + case "SetViewport": + if ( + !hasExactKeys(payload, TRACE_VIEWPORT_ACTION_KEYS) + || !isSafeIntegerBetween(payload.height, 1, 10_000) + || !isSafeIntegerBetween(payload.width, 1, 10_000) + ) return invalidTraceAction(lineNumber); + break; + default: + return invalidTraceAction(lineNumber); + } + return { kind: actionKind, targetTag }; +} + +function parseNonNegativeFiniteNumber( + value: unknown, + lineNumber: number, + field: string, +): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + return value; +} + +function parseTraceState(value: unknown, lineNumber: number): ParsedTraceState { + if (!isRecord(value) || !hasExactKeys(value, TRACE_STATE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser state`); + } + if (typeof value.url !== "string" || value.url.length === 0 || value.url.length > 8_192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + let url: URL; + try { + url = new URL(value.url); + } catch { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL`); + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid browser URL protocol`); + } + if (typeof value.screenshot !== "string" || value.screenshot.length > 8_192) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid screenshot path`); + } + for (const field of ["hash_previous", "hash_current"] as const) { + const hash = value[field]; + if (hash !== null && ( + typeof hash !== "number" + || !Number.isFinite(hash) + || !Number.isInteger(hash) + || hash < 0 + )) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid ${field}`); + } + } + if (!isRecord(value.resources) || !hasExactKeys(value.resources, TRACE_RESOURCE_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid browser resources`); + } + const resources: Record = {}; + for (const field of Object.keys(RESOURCE_FIELD_MAP) as (keyof typeof RESOURCE_FIELD_MAP)[]) { + resources[field] = parseNonNegativeFiniteNumber( + value.resources[field], + lineNumber, + `resources.${field}`, + ); + } + parseNonNegativeFiniteNumber(value.resources.timestamp, lineNumber, "resources.timestamp"); + return { + currentHash: value.hash_current as number | null, + resources: resources as Readonly>, + url, + }; +} + +function parseTraceEnvelope(line: string, lineNumber: number): ParsedTraceEnvelope { let input: unknown; try { input = JSON.parse(line) as unknown; @@ -447,9 +990,22 @@ function parseTraceLine(line: string, lineNumber: number): TraceDirectObservatio ) { throw new Error(`Bombadil trace line ${String(lineNumber)} has invalid state fields`); } - const snapshots = input.snapshots as unknown[]; - const directSnapshots = snapshots.filter((snapshot): snapshot is Readonly> => - isRecord(snapshot) && snapshot.name === "direct" + return { + action: input.action, + snapshots: input.snapshots as unknown[], + state: input.state, + timestamp: input.timestamp, + violations: input.violations, + }; +} + +function parseDirectTraceObservation( + envelope: ParsedTraceEnvelope, + lineNumber: number, +): ParsedDirectTraceObservation { + const directSnapshots = envelope.snapshots.filter( + (snapshot): snapshot is Readonly> => + isRecord(snapshot) && snapshot.name === "direct", ); if (directSnapshots.length !== 1) { throw new Error(`Bombadil trace line ${String(lineNumber)} must contain one named direct snapshot`); @@ -465,7 +1021,97 @@ function parseTraceLine(line: string, lineNumber: number): TraceDirectObservatio ) { throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid direct snapshot`); } - return parseTraceDirectObservation(snapshot.value); + return { + observation: parseTraceDirectObservation(snapshot.value), + value: snapshot.value, + }; +} + +function parseDirectTraceLine(line: string, lineNumber: number): TraceDirectObservation { + return parseDirectTraceObservation(parseTraceEnvelope(line, lineNumber), lineNumber).observation; +} + +function parseTraceLine( + line: string, + lineNumber: number, + strictDiagnosticSnapshotNames: ReadonlySet, +): ParsedTraceLine { + const envelope = parseTraceEnvelope(line, lineNumber); + const state = parseTraceState(envelope.state, lineNumber); + const action = parseTraceAction(envelope.action, lineNumber); + const snapshots = envelope.snapshots; + const direct = parseDirectTraceObservation(envelope, lineNumber); + const namedSnapshots: Array<{ readonly name: string; readonly valueSha256: string }> = [{ + name: "direct", + valueSha256: namedSnapshotValueSha256(direct.value, { + maximumBytes: TRACE_MAX_LINE_BYTES, + maximumDepth: TRACE_MAX_JSON_DEPTH + 4, + }), + }]; + const diagnosticSnapshotValues = new Map(); + for (const snapshotValue of snapshots) { + if ( + !isRecord(snapshotValue) + || !hasExactKeys(snapshotValue, TRACE_SNAPSHOT_KEYS) + || !Number.isSafeInteger(snapshotValue.index) + || !Number.isSafeInteger(snapshotValue.time) + || (snapshotValue.index as number) < 0 + || (snapshotValue.time as number) < 0 + || (snapshotValue.name !== null && typeof snapshotValue.name !== "string") + ) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid snapshot`); + } + if (snapshotValue.name === null || snapshotValue.name === "direct") continue; + let name: string; + try { + name = validateSnapshotName( + snapshotValue.name, + `Bombadil trace line ${String(lineNumber)} snapshot name`, + ); + } catch (error) { + if (strictDiagnosticSnapshotNames.has(snapshotValue.name)) throw error; + continue; + } + const values = diagnosticSnapshotValues.get(name) ?? []; + values.push(snapshotValue.value); + diagnosticSnapshotValues.set(name, values); + } + for (const [name, values] of diagnosticSnapshotValues) { + if (values.length !== 1) { + if (strictDiagnosticSnapshotNames.has(name)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} repeats named snapshot ${name}`); + } + continue; + } + try { + namedSnapshots.push({ + name, + valueSha256: namedSnapshotValueSha256(values[0]), + }); + } catch (error) { + if (strictDiagnosticSnapshotNames.has(name)) throw error; + } + } + const propertyViolationNames: string[] = []; + for (const violation of envelope.violations) { + if (!isRecord(violation) || !hasExactKeys(violation, TRACE_VIOLATION_KEYS)) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + propertyViolationNames.push(validateSnapshotName( + violation.name, + `Bombadil trace line ${String(lineNumber)} property violation name`, + )); + if (!isRecord(violation.violation) || Object.keys(violation.violation).length !== 1) { + throw new Error(`Bombadil trace line ${String(lineNumber)} has an invalid property violation`); + } + } + return { + action, + directObservation: direct.observation, + namedSnapshots, + propertyViolationNames, + state, + }; } /** Exact post-run proof over Bombadil 0.7.2's bounded JSONL trace. */ @@ -499,7 +1145,7 @@ export async function attestDirectBombadilTrace(options: { if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { throw new Error(`Bombadil trace line ${String(observationCount)} is too large`); } - const observation = parseTraceLine(line, observationCount); + const observation = parseDirectTraceLine(line, observationCount); const exact = exactTraceDirectObservation(observation); if (exact === null) { if (initial !== null) { @@ -577,6 +1223,332 @@ export async function attestDirectBombadilTrace(options: { }; } +function sortedCountRecord( + values: ReadonlyMap, +): Readonly>> { + return Object.freeze(Object.fromEntries( + [...values.entries()].sort(([left], [right]) => compareCodeUnits(left, right)), + ) as Partial>); +} + +/** + * Derives bounded diagnostic exploration metadata from the raw Bombadil trace. + * The hashes and counts are navigation aids, not Direct coverage evidence. + */ +export async function summarizeDirectBombadilTrace(options: { + readonly explorationPolicy?: DirectBombadilExplorationPolicy; + readonly targetUrl: string; + readonly tracePath: string; +}): Promise { + const metadata = await stat(options.tracePath).catch(() => null); + if (metadata === null || !metadata.isFile() || metadata.size === 0) { + throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + } + if (metadata.size > TRACE_MAX_BYTES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_BYTES)} bytes`); + } + let targetUrl: URL; + try { + targetUrl = new URL(options.targetUrl); + } catch { + throw new Error("targetUrl must be an absolute URL"); + } + const policy = validateExplorationPolicy(options.explorationPolicy); + const strictDiagnosticSnapshotNames = explorationPolicySnapshotNames(policy); + const actionCounts = new Map(); + const targetTags = new Map(); + const urlFingerprints = new Set(); + const rawUrlFingerprints = new Set(); + const transitionHashes = new Set(); + const rawTransitionHashes = new Set(); + const snapshots = new Map; + changeAfterNonWaitCount: number; + lastObservationIndex: number | null; + lastValueSha256: string | null; + observationCount: number; + readonly values: Set; + }>(); + const propertyViolations = new Map(); + const resources = { + documents: 0, + domNodes: 0, + jsEventListeners: 0, + jsHeapTotalBytes: 0, + jsHeapUsedBytes: 0, + layoutObjects: 0, + scriptDurationSeconds: 0, + taskDurationSeconds: 0, + threadTimeSeconds: 0, + }; + let lineCount = 0; + let totalActions = 0; + let nonWaitCount = 0; + let waitStreak = 0; + let maxWaitStreak = 0; + let nonNullHashCount = 0; + let rawNonNullHashCount = 0; + let policyObservationCount = 0; + let previousObservationWasExact = false; + let stableTarget = true; + let trackedUnrelatedSnapshotNameCount = 0; + const unrelatedSnapshotNameLimit = Math.max( + 0, + TRACE_MAX_NAMED_SNAPSHOT_NAMES - strictDiagnosticSnapshotNames.size, + ); + const stream = createReadStream(options.tracePath, { encoding: "utf8" }); + const lines = createInterface({ input: stream, crlfDelay: Infinity }); + try { + for await (const line of lines) { + lineCount += 1; + if (lineCount > TRACE_MAX_LINES) { + throw new Error(`Bombadil trace exceeds ${String(TRACE_MAX_LINES)} lines`); + } + if (Buffer.byteLength(line, "utf8") > TRACE_MAX_LINE_BYTES) { + throw new Error(`Bombadil trace line ${String(lineCount)} is too large`); + } + const parsed = parseTraceLine(line, lineCount, strictDiagnosticSnapshotNames); + const rawRelativeUrl = + `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + rawUrlFingerprints.add(sha256(rawRelativeUrl)); + if (rawUrlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct raw URL fingerprints`, + ); + } + if (parsed.state.currentHash !== null) { + rawNonNullHashCount += 1; + rawTransitionHashes.add(String(parsed.state.currentHash)); + } + for (const name of parsed.propertyViolationNames) { + if (!propertyViolations.has(name) && propertyViolations.size >= TRACE_MAX_PROPERTY_NAMES) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_PROPERTY_NAMES)} property names`, + ); + } + propertyViolations.set(name, (propertyViolations.get(name) ?? 0) + 1); + } + for (const [sourceName, outputName] of Object.entries(RESOURCE_FIELD_MAP)) { + resources[outputName] = Math.max( + resources[outputName], + parsed.state.resources[sourceName as keyof typeof RESOURCE_FIELD_MAP], + ); + } + const currentObservationIsExact = + exactTraceDirectObservation(parsed.directObservation) !== null; + if (!currentObservationIsExact) { + previousObservationWasExact = false; + continue; + } + policyObservationCount += 1; + const actionFollowsExactObservation = previousObservationWasExact; + const recordedActionKind = actionFollowsExactObservation + ? parsed.action?.kind ?? null + : null; + + if (actionFollowsExactObservation && parsed.action !== null) { + totalActions += 1; + actionCounts.set(parsed.action.kind, (actionCounts.get(parsed.action.kind) ?? 0) + 1); + if (parsed.action.kind === "Wait") { + waitStreak += 1; + maxWaitStreak = Math.max(maxWaitStreak, waitStreak); + } else { + nonWaitCount += 1; + waitStreak = 0; + } + if (parsed.action.targetTag !== null) { + if (!targetTags.has(parsed.action.targetTag) && targetTags.size >= 128) { + throw new Error("Bombadil trace exceeds 128 distinct action target tags"); + } + targetTags.set( + parsed.action.targetTag, + (targetTags.get(parsed.action.targetTag) ?? 0) + 1, + ); + } + } else if (actionFollowsExactObservation) { + waitStreak = 0; + } + + const relativeUrl = `${parsed.state.url.pathname}${parsed.state.url.search}${parsed.state.url.hash}`; + urlFingerprints.add(sha256(relativeUrl)); + if (urlFingerprints.size > TRACE_MAX_DISTINCT_URLS) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_DISTINCT_URLS)} distinct URL fingerprints`, + ); + } + stableTarget &&= parsed.state.url.href === targetUrl.href; + if (parsed.state.currentHash !== null) { + nonNullHashCount += 1; + transitionHashes.add(String(parsed.state.currentHash)); + } + + for (const snapshot of parsed.namedSnapshots) { + let entry = snapshots.get(snapshot.name); + if (entry === undefined) { + const isStrictSnapshot = snapshot.name === "direct" + || strictDiagnosticSnapshotNames.has(snapshot.name); + if (!isStrictSnapshot && trackedUnrelatedSnapshotNameCount >= unrelatedSnapshotNameLimit) { + continue; + } + if (snapshots.size >= TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error( + `Bombadil trace exceeds ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES)} named snapshots`, + ); + } + entry = { + changeAfterActionKind: new Map(), + changeAfterNonWaitCount: 0, + lastObservationIndex: null, + lastValueSha256: null, + observationCount: 0, + values: new Set(), + }; + snapshots.set(snapshot.name, entry); + if (!isStrictSnapshot) trackedUnrelatedSnapshotNameCount += 1; + } + if ( + !entry.values.has(snapshot.valueSha256) + && entry.values.size >= TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME + ) { + if ( + snapshot.name === "direct" + || strictDiagnosticSnapshotNames.has(snapshot.name) + ) { + throw new Error( + `Bombadil trace named snapshot ${snapshot.name} exceeds ${String(TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME)} distinct values`, + ); + } + continue; + } + const changedAfterRecordedAction = + recordedActionKind !== null + && entry.lastObservationIndex === policyObservationCount - 1 + && entry.lastValueSha256 !== null + && entry.lastValueSha256 !== snapshot.valueSha256; + if (changedAfterRecordedAction) { + entry.changeAfterActionKind.set( + recordedActionKind, + (entry.changeAfterActionKind.get(recordedActionKind) ?? 0) + 1, + ); + } + if (changedAfterRecordedAction && recordedActionKind !== "Wait") { + entry.changeAfterNonWaitCount += 1; + } + entry.lastObservationIndex = policyObservationCount; + entry.lastValueSha256 = snapshot.valueSha256; + entry.observationCount += 1; + entry.values.add(snapshot.valueSha256); + } + previousObservationWasExact = true; + } + } finally { + lines.close(); + stream.destroy(); + } + if (lineCount === 0) throw new Error("Bombadil did not produce a nonempty trace.jsonl"); + + const policyFailures: string[] = []; + if (policy !== null) { + if (nonWaitCount < policy.minNonWaitActions) { + policyFailures.push("minimum non-Wait action count was not reached"); + } + for (const kind of policy.requiredActionKinds) { + if ((actionCounts.get(kind) ?? 0) === 0) { + policyFailures.push(`required action kind ${kind} was not observed`); + } + } + for (const name of policy.requiredNamedSnapshots) { + if (!snapshots.has(name)) { + policyFailures.push(`required named snapshot ${name} was not observed`); + } + } + for (const [name, minimum] of Object.entries(policy.minDistinctNamedSnapshotValues)) { + if ((snapshots.get(name)?.values.size ?? 0) < minimum) { + policyFailures.push(`named snapshot ${name} did not reach its distinct-value minimum`); + } + } + for (const [name, minimum] of Object.entries( + policy.minNamedSnapshotChangesAfterNonWait, + )) { + if ((snapshots.get(name)?.changeAfterNonWaitCount ?? 0) < minimum) { + policyFailures.push( + `named snapshot ${name} did not reach its post-non-Wait change minimum`, + ); + } + } + for (const [name, minimumByKind] of Object.entries( + policy.minNamedSnapshotChangesAfterActionKind, + )) { + for (const [kind, minimum] of Object.entries(minimumByKind) as [ + DirectBombadilActionKind, + number, + ][]) { + if ((snapshots.get(name)?.changeAfterActionKind.get(kind) ?? 0) < minimum) { + policyFailures.push( + `named snapshot ${name} did not reach its post-${kind} change minimum`, + ); + } + } + } + if (policy.requireStableTargetUrl && !stableTarget) { + policyFailures.push("the browser did not remain on the exact target URL"); + } + } + const traceBytes = await readFile(options.tracePath); + return Object.freeze({ + schema: "direct.bombadil-exploration-summary/v2", + trace: Object.freeze({ + bytes: metadata.size, + lineCount, + sha256: sha256(traceBytes), + }), + actions: Object.freeze({ + byKind: sortedCountRecord(actionCounts), + maxWaitStreak, + nonWaitCount, + targetTags: sortedCountRecord(targetTags) as Readonly>, + total: totalActions, + }), + urls: Object.freeze({ + distinctFingerprintCount: urlFingerprints.size, + fingerprintSha256: Object.freeze([...urlFingerprints].sort(compareCodeUnits)), + observationCount: policyObservationCount, + rawDistinctFingerprintCount: rawUrlFingerprints.size, + rawFingerprintSha256: Object.freeze( + [...rawUrlFingerprints].sort(compareCodeUnits), + ), + rawObservationCount: lineCount, + stableTarget, + }), + transitions: Object.freeze({ + distinctNonNullHashCount: transitionHashes.size, + nonNullHashCount, + rawDistinctNonNullHashCount: rawTransitionHashes.size, + rawNonNullHashCount, + }), + namedSnapshots: Object.freeze([...snapshots.entries()] + .sort(([left], [right]) => compareCodeUnits(left, right)) + .map(([name, entry]) => Object.freeze({ + changeAfterActionKind: sortedCountRecord(entry.changeAfterActionKind), + changeAfterNonWaitCount: entry.changeAfterNonWaitCount, + distinctValueCount: entry.values.size, + distinctValueSha256: Object.freeze([...entry.values].sort(compareCodeUnits)), + name, + observationCount: entry.observationCount, + }))), + propertyViolations: Object.freeze({ + byName: sortedCountRecord(propertyViolations) as Readonly>, + total: [...propertyViolations.values()].reduce((total, value) => total + value, 0), + }), + resourceHighWaterMarks: Object.freeze(resources), + policy: Object.freeze({ + configured: policy !== null, + failures: Object.freeze(policyFailures), + satisfied: policyFailures.length === 0, + }), + }); +} + export function parseDirectBombadilFuzzArguments( arguments_: readonly string[], defaultBaseUrl: string, @@ -689,7 +1661,7 @@ function validateTargetQuery(value: unknown): Readonly> { } const validated: Record = {}; for (const [name, queryValue] of [...entries].sort(([left], [right]) => - left.localeCompare(right) + compareCodeUnits(left, right) )) { if ( name.length === 0 @@ -714,6 +1686,227 @@ function validateTargetQuery(value: unknown): Readonly> { return Object.freeze(validated); } +function validateSnapshotName(value: unknown, label: string): string { + if ( + typeof value !== "string" + || value.length === 0 + || value.length > 128 + || !SNAPSHOT_NAME_PATTERN.test(value) + || PROTOTYPE_PROPERTY_NAMES.has(value) + || hasControlCharacters(value) + ) { + throw new Error(`${label} must be a safe bounded snapshot name`); + } + return value; +} + +function validateViewport(value: unknown): ValidatedViewport { + if (value === undefined) { + return Object.freeze({ + deviceScaleFactor: DEFAULT_DEVICE_SCALE_FACTOR, + height: DEFAULT_VIEWPORT_HEIGHT, + width: DEFAULT_VIEWPORT_WIDTH, + }); + } + if (!isRecord(value) || !Object.keys(value).every((key) => VIEWPORT_KEYS.has(key))) { + throw new Error("viewport must contain only width, height, and deviceScaleFactor"); + } + const validateDimension = (name: "height" | "width", input: unknown): number => { + if ( + typeof input !== "number" + || !Number.isSafeInteger(input) + || input < 1 + || input > 65_535 + ) { + throw new Error(`viewport.${name} must be an integer between 1 and 65535`); + } + return input; + }; + const width = validateDimension("width", value.width ?? DEFAULT_VIEWPORT_WIDTH); + const height = validateDimension("height", value.height ?? DEFAULT_VIEWPORT_HEIGHT); + const deviceScaleFactor = value.deviceScaleFactor ?? DEFAULT_DEVICE_SCALE_FACTOR; + if ( + typeof deviceScaleFactor !== "number" + || !Number.isFinite(deviceScaleFactor) + || deviceScaleFactor < 0.1 + || deviceScaleFactor > 10 + ) { + throw new Error("viewport.deviceScaleFactor must be a finite number between 0.1 and 10"); + } + return Object.freeze({ deviceScaleFactor, height, width }); +} + +function validateSnapshotMinimumMap(options: { + readonly label: string; + readonly maximum: number; + readonly value: unknown; +}): Readonly> { + if (!isRecord(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated: Record = {}; + for (const [rawName, minimum] of Object.entries(options.value).sort(([left], [right]) => + compareCodeUnits(left, right) + )) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if ( + typeof minimum !== "number" + || !Number.isSafeInteger(minimum) + || minimum < 1 + || minimum > options.maximum + ) { + throw new Error( + `${options.label} ${name} must be an integer between 1 and ${String(options.maximum)}`, + ); + } + validated[name] = minimum; + } + return Object.freeze(validated); +} + +function validateSnapshotActionMinimumMap(options: { + readonly label: string; + readonly value: unknown; +}): Readonly>> +>> { + if (!isRecord(options.value) || Object.keys(options.value).length > 32) { + throw new Error(`${options.label} must be a bounded object`); + } + const validated: Record< + string, + Readonly>> + > = {}; + for (const [rawName, rawMinimumByKind] of Object.entries(options.value) + .sort(([left], [right]) => compareCodeUnits(left, right))) { + const name = validateSnapshotName(rawName, `${options.label} key`); + if ( + !isRecord(rawMinimumByKind) + || Object.keys(rawMinimumByKind).length === 0 + || Object.keys(rawMinimumByKind).length > ACTION_KINDS.length + ) { + throw new Error(`${options.label} ${name} must be a bounded action map`); + } + const minimumByKind: Partial> = {}; + for (const [rawKind, minimum] of Object.entries(rawMinimumByKind) + .sort(([left], [right]) => compareCodeUnits(left, right))) { + if (!ACTION_KIND_SET.has(rawKind)) { + throw new Error(`${options.label} ${name} contains an unknown action kind`); + } + if ( + typeof minimum !== "number" + || !Number.isSafeInteger(minimum) + || minimum < 1 + || minimum > TRACE_MAX_LINES + ) { + throw new Error( + `${options.label} ${name}.${rawKind} must be an integer between 1 and ${String(TRACE_MAX_LINES)}`, + ); + } + minimumByKind[rawKind as DirectBombadilActionKind] = minimum; + } + validated[name] = Object.freeze(minimumByKind); + } + return Object.freeze(validated); +} + +function explorationPolicySnapshotNames( + policy: ValidatedExplorationPolicy | null, +): ReadonlySet { + const names = new Set(["direct"]); + if (policy === null) return names; + for (const name of policy.requiredNamedSnapshots) names.add(name); + for (const name of Object.keys(policy.minDistinctNamedSnapshotValues)) names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterNonWait)) names.add(name); + for (const name of Object.keys(policy.minNamedSnapshotChangesAfterActionKind)) names.add(name); + return names; +} + +function validateExplorationPolicy( + value: unknown, +): ValidatedExplorationPolicy | null { + if (value === undefined) return null; + if ( + !isRecord(value) + || !Object.keys(value).every((key) => EXPLORATION_POLICY_KEYS.has(key)) + ) { + throw new Error("explorationPolicy contains an unknown field"); + } + const minNonWaitActions = value.minNonWaitActions ?? 0; + if ( + typeof minNonWaitActions !== "number" + || !Number.isSafeInteger(minNonWaitActions) + || minNonWaitActions < 0 + || minNonWaitActions > TRACE_MAX_LINES + ) { + throw new Error( + `explorationPolicy.minNonWaitActions must be an integer between 0 and ${String(TRACE_MAX_LINES)}`, + ); + } + const requiredActionKindsInput = value.requiredActionKinds ?? []; + if (!Array.isArray(requiredActionKindsInput) || requiredActionKindsInput.length > ACTION_KINDS.length) { + throw new Error("explorationPolicy.requiredActionKinds must be a bounded array"); + } + const requiredActionKinds = [...requiredActionKindsInput]; + if ( + !requiredActionKinds.every((kind): kind is DirectBombadilActionKind => + typeof kind === "string" && ACTION_KIND_SET.has(kind) + ) + || new Set(requiredActionKinds).size !== requiredActionKinds.length + ) { + throw new Error("explorationPolicy.requiredActionKinds contains an unknown or duplicate kind"); + } + requiredActionKinds.sort(compareCodeUnits); + + const requiredNamedSnapshotsInput = value.requiredNamedSnapshots ?? []; + if (!Array.isArray(requiredNamedSnapshotsInput) || requiredNamedSnapshotsInput.length > 32) { + throw new Error("explorationPolicy.requiredNamedSnapshots must be a bounded array"); + } + const requiredNamedSnapshots = requiredNamedSnapshotsInput.map((name) => + validateSnapshotName(name, "explorationPolicy.requiredNamedSnapshots entry") + ); + if (new Set(requiredNamedSnapshots).size !== requiredNamedSnapshots.length) { + throw new Error("explorationPolicy.requiredNamedSnapshots contains a duplicate name"); + } + requiredNamedSnapshots.sort(compareCodeUnits); + + const minDistinctNamedSnapshotValues = validateSnapshotMinimumMap({ + label: "explorationPolicy.minDistinctNamedSnapshotValues", + maximum: TRACE_MAX_DISTINCT_SNAPSHOT_VALUES_PER_NAME, + value: value.minDistinctNamedSnapshotValues ?? {}, + }); + const minNamedSnapshotChangesAfterActionKind = + validateSnapshotActionMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterActionKind", + value: value.minNamedSnapshotChangesAfterActionKind ?? {}, + }); + const minNamedSnapshotChangesAfterNonWait = validateSnapshotMinimumMap({ + label: "explorationPolicy.minNamedSnapshotChangesAfterNonWait", + maximum: TRACE_MAX_LINES, + value: value.minNamedSnapshotChangesAfterNonWait ?? {}, + }); + const requireStableTargetUrl = value.requireStableTargetUrl ?? false; + if (typeof requireStableTargetUrl !== "boolean") { + throw new Error("explorationPolicy.requireStableTargetUrl must be a boolean"); + } + const validated: ValidatedExplorationPolicy = Object.freeze({ + minDistinctNamedSnapshotValues, + minNamedSnapshotChangesAfterActionKind, + minNamedSnapshotChangesAfterNonWait, + minNonWaitActions, + requireStableTargetUrl, + requiredActionKinds: Object.freeze(requiredActionKinds), + requiredNamedSnapshots: Object.freeze(requiredNamedSnapshots), + }); + if (explorationPolicySnapshotNames(validated).size > TRACE_MAX_NAMED_SNAPSHOT_NAMES) { + throw new Error( + `explorationPolicy may reference at most ${String(TRACE_MAX_NAMED_SNAPSHOT_NAMES - 1)} distinct non-Direct snapshots`, + ); + } + return validated; +} + export function validateDirectBombadilFuzzConfig( config: DirectBombadilFuzzConfig, baseUrlOverride?: string, @@ -782,6 +1975,8 @@ export function validateDirectBombadilFuzzConfig( const entryPath = config.entryPath ?? "/"; validateEntryPath(entryPath); const targetQuery = validateTargetQuery(config.targetQuery ?? {}); + const viewport = validateViewport(config.viewport); + const explorationPolicy = validateExplorationPolicy(config.explorationPolicy); const startupTimeoutMs = config.server.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS; if ( !Number.isSafeInteger(startupTimeoutMs) @@ -803,8 +1998,10 @@ export function validateDirectBombadilFuzzConfig( artifactRoot: join(repositoryRoot, "artifacts", "direct-bombadil", config.artifactName), bombadilExecutable: bombadilNativeBinary(repositoryRoot), entryPath, + explorationPolicy, port, targetQuery, + viewport, server: { ...config.server, cwd: serverCwd, @@ -834,7 +2031,9 @@ export function createDirectBombadilInvocation(options: { readonly specificationPath: string; readonly targetQuery?: Readonly>; readonly timeLimitSeconds: number; + readonly viewport?: DirectBombadilViewportConfig; }): DirectBombadilInvocation { + const viewport = validateViewport(options.viewport); const target = new URL(options.entryPath ?? "/", `${options.baseUrl}/`); target.searchParams.set(SCENARIO_QUERY_KEY, options.scenario); for (const [name, value] of Object.entries(options.targetQuery ?? {})) { @@ -850,6 +2049,12 @@ export function createDirectBombadilInvocation(options: { options.outputPath, "--headless", "--instrument-javascript=", + "--width", + String(viewport.width), + "--height", + String(viewport.height), + "--device-scale-factor", + String(viewport.deviceScaleFactor), ]; if (options.replayPath === null) { command.push( @@ -1007,6 +2212,7 @@ export async function runBombadilNativeProcess( const defaultDependencies: DirectBombadilRunnerDependencies = { acquireServer: acquireVerificationServer, + createAbortController: () => new AbortController(), now: () => new Date(), runBombadil: runBombadilNativeProcess, serverOutputTimeoutMs: SERVER_OUTPUT_TIMEOUT_MS, @@ -1181,6 +2387,113 @@ function helpText(defaultBaseUrl: string): string { ].join("\n"); } +function parseMatrixCampaignArgument(arguments_: readonly string[]): { + readonly arguments: readonly string[]; + readonly campaignId: string | null; + readonly help: boolean; +} { + const forwarded: string[] = []; + let campaignId: string | null = null; + let help = false; + for (let index = 0; index < arguments_.length; index += 1) { + const argument = arguments_[index]; + if (argument === undefined) continue; + if (argument === "--help" || argument === "-h") help = true; + if (argument === "--campaign" || argument.startsWith("--campaign=")) { + if (campaignId !== null) throw new Error("--campaign may be provided only once"); + if (argument === "--campaign") { + const next = readOptionValue(arguments_, index, "--campaign"); + campaignId = next.value; + index = next.index; + } else { + campaignId = argument.slice("--campaign=".length); + } + if (campaignId.length === 0) throw new Error("--campaign requires a value"); + continue; + } + forwarded.push(argument); + } + return { arguments: Object.freeze(forwarded), campaignId, help }; +} + +function validateCampaignMatrix( + campaigns: readonly DirectBombadilFuzzCampaign[], +): readonly DirectBombadilFuzzCampaign[] { + if (campaigns.length === 0 || campaigns.length > 32) { + throw new Error("Bombadil campaign matrix must contain 1-32 campaigns"); + } + const ids = new Set(); + for (const campaign of campaigns) { + if (!ARTIFACT_NAME_PATTERN.test(campaign.id) || ids.has(campaign.id)) { + throw new Error("Bombadil campaign IDs must be unique lowercase kebab identifiers"); + } + ids.add(campaign.id); + } + return campaigns; +} + +/** Runs a bounded product-owned campaign matrix serially. */ +export async function runDirectBombadilFuzzMatrix( + campaignsInput: readonly DirectBombadilFuzzCampaign[], + arguments_: readonly string[] = process.argv.slice(2), + dependencyOverrides: Partial = {}, +): Promise { + const campaigns = validateCampaignMatrix(campaignsInput); + const parsed = parseMatrixCampaignArgument(arguments_); + if (parsed.help) { + process.stdout.write(`${[ + helpText(campaigns[0]?.config.baseUrl ?? ""), + " --campaign Run one campaign; required with --replay", + "", + `Campaigns: ${campaigns.map((campaign) => campaign.id).join(", ")}`, + ].join("\n")}\n`); + return { kind: "help" }; + } + const selected = parsed.campaignId === null + ? campaigns + : campaigns.filter((campaign) => campaign.id === parsed.campaignId); + if (selected.length === 0) { + throw new Error(`Unknown Bombadil campaign ${parsed.campaignId ?? ""}`); + } + if ( + parsed.campaignId === null + && parsed.arguments.some((argument) => + argument === "--replay" || argument.startsWith("--replay=") + ) + ) { + throw new Error("--replay requires exactly one --campaign in matrix mode"); + } + const results: Array<{ + readonly campaignId: string; + readonly result: Extract; + }> = []; + for (const campaign of selected) { + const result = await runDirectBombadilFuzz( + campaign.config, + parsed.arguments, + dependencyOverrides, + ); + if (result.kind !== "run") { + throw new Error("Bombadil campaign unexpectedly returned help during matrix execution"); + } + results.push({ campaignId: campaign.id, result }); + } + return { kind: "matrix", results: Object.freeze(results) }; +} + +function throwIfBombadilRunAborted(signal: AbortSignal): void { + if (signal.aborted) throw new Error("Bombadil fuzzing was interrupted"); +} + +function terminateAbortedOwnedServer( + signal: AbortSignal, + server: ManagedVerificationServer, +): void { + if (!signal.aborted) return; + if (server.exitCode() === null) server.terminate(); + throwIfBombadilRunAborted(signal); +} + /** Runs one bounded diagnostic Bombadil campaign and always releases its server lease. */ export async function runDirectBombadilFuzz( config: DirectBombadilFuzzConfig, @@ -1212,7 +2525,7 @@ export async function runDirectBombadilFuzz( }); const outputPath = join(artifactRun.runDirectory, "bombadil"); const tracePath = join(outputPath, "trace.jsonl"); - const abortController = new AbortController(); + const abortController = dependencies.createAbortController?.() ?? new AbortController(); const invocation = createDirectBombadilInvocation({ baseUrl: validated.baseUrl, bombadilExecutable: validated.bombadilExecutable, @@ -1224,6 +2537,7 @@ export async function runDirectBombadilFuzz( specificationPath: validated.specificationPath, targetQuery: validated.targetQuery, timeLimitSeconds: parsed.timeLimitSeconds, + viewport: validated.viewport, }); const abortableInvocation = { ...invocation, abortSignal: abortController.signal }; const serverCommand = validated.server.command.map((argument) => @@ -1236,6 +2550,8 @@ export async function runDirectBombadilFuzz( let processResult: BombadilProcessResult | null = null; let attestation: DirectBombadilTraceAttestation | null = null; let attestationFailure: unknown = null; + let explorationSummary: DirectBombadilExplorationSummary | null = null; + let explorationSummaryFailure: unknown = null; let rawTracePath: string | null = null; let serverOutput = ""; let serverOutputFailure: unknown = null; @@ -1255,23 +2571,36 @@ export async function runDirectBombadilFuzz( try { await requireRegularFile(validated.bombadilExecutable, "The root Bombadil executable"); bombadilVersion = await readExactBombadilVersion(validated.repositoryRoot); - if (abortController.signal.aborted) throw new Error("Bombadil fuzzing was interrupted"); - - lease = await dependencies.acquireServer({ - baseUrl: validated.baseUrl, - label: validated.label, - readinessPath: validated.server.readinessPath, - reuseExistingLocalServer: false, - startupTimeoutMs: validated.server.startupTimeoutMs, - startServer: () => { - ownedServer = dependencies.spawnServer({ - command: serverCommand, - cwd: validated.server.cwd, - ...(validated.server.env === undefined ? {} : { env: validated.server.env }), - }); - return ownedServer; - }, - }); + throwIfBombadilRunAborted(abortController.signal); + + try { + lease = await dependencies.acquireServer({ + abortSignal: abortController.signal, + baseUrl: validated.baseUrl, + label: validated.label, + readinessPath: validated.server.readinessPath, + reuseExistingLocalServer: false, + startupTimeoutMs: validated.server.startupTimeoutMs, + startServer: () => { + throwIfBombadilRunAborted(abortController.signal); + ownedServer = dependencies.spawnServer({ + command: serverCommand, + cwd: validated.server.cwd, + ...(validated.server.env === undefined ? {} : { env: validated.server.env }), + }); + terminateAbortedOwnedServer(abortController.signal, ownedServer); + return ownedServer; + }, + }); + } catch (error) { + if (abortController.signal.aborted) throwIfBombadilRunAborted(abortController.signal); + throw error; + } + if (abortController.signal.aborted) { + const acquiredOwnedServer = ownedServer as ManagedVerificationServer | null; + if (acquiredOwnedServer?.exitCode() === null) acquiredOwnedServer.terminate(); + throwIfBombadilRunAborted(abortController.signal); + } let processFailure: unknown = null; try { processResult = await dependencies.runBombadil(abortableInvocation); @@ -1291,6 +2620,17 @@ export async function runDirectBombadilFuzz( } catch (error) { attestationFailure = error; } + try { + explorationSummary = await summarizeDirectBombadilTrace({ + ...(validated.explorationPolicy === null + ? {} + : { explorationPolicy: validated.explorationPolicy }), + targetUrl: invocation.targetUrl, + tracePath, + }); + } catch (error) { + explorationSummaryFailure = error; + } if (processFailure !== null) { throw processFailure instanceof Error ? processFailure @@ -1313,6 +2653,16 @@ export async function runDirectBombadilFuzz( ? attestationFailure : new Error(renderUnknown(attestationFailure)); } + if (explorationSummaryFailure !== null) { + throw explorationSummaryFailure instanceof Error + ? explorationSummaryFailure + : new Error(renderUnknown(explorationSummaryFailure)); + } + if (explorationSummary?.policy.satisfied !== true) { + throw new Error( + `Bombadil exploration policy was not satisfied: ${explorationSummary?.policy.failures.join("; ") ?? "summary unavailable"}`, + ); + } } catch (error) { failure = error; } @@ -1351,6 +2701,10 @@ export async function runDirectBombadilFuzz( const status = failure === null ? "passed" : "failed"; const logPath = join(artifactRun.runDirectory, "bombadil.log"); const serverLogPath = join(artifactRun.runDirectory, "server.log"); + const explorationSummaryPath = join( + artifactRun.runDirectory, + "exploration-summary.json", + ); const record = { schema: ARTIFACT_SCHEMA, evidenceClass: "diagnostic-fuzz", @@ -1366,6 +2720,8 @@ export async function runDirectBombadilFuzz( entryPath: validated.entryPath, targetQuery: validated.targetQuery, targetUrl: invocation.targetUrl, + viewport: validated.viewport, + explorationPolicy: validated.explorationPolicy, specificationPath: validated.specificationPath, replayPath, timeLimitSeconds: replayPath === null ? parsed.timeLimitSeconds : null, @@ -1387,6 +2743,11 @@ export async function runDirectBombadilFuzz( }, attestation, attestationFailure: attestationFailure === null ? null : renderUnknown(attestationFailure), + explorationSummary, + explorationSummaryPath: explorationSummary === null ? null : explorationSummaryPath, + explorationSummaryFailure: explorationSummaryFailure === null + ? null + : renderUnknown(explorationSummaryFailure), initialDirect: attestation?.initial ?? null, interruptedSignal: capturedSignal, failure: failure === null ? null : renderUnknown(failure), @@ -1401,10 +2762,28 @@ export async function runDirectBombadilFuzz( `${serverOutput}${serverOutput.length > 0 ? "\n" : ""}`, "utf8", ); + if (explorationSummary !== null) { + await writeJsonAtomically(explorationSummaryPath, explorationSummary); + } await writeJsonAtomically(join(artifactRun.runDirectory, "run.json"), record); await writeJsonAtomically(artifactRun.manifestPath, record); - const summary = `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}; artifacts: ${artifactRun.runDirectory}; log: ${logPath}`; + const exploration = explorationSummary === null + ? "exploration=unavailable" + : [ + `nonWait=${String(explorationSummary.actions.nonWaitCount)}`, + `maxWaitStreak=${String(explorationSummary.actions.maxWaitStreak)}`, + `namedChanges=${explorationSummary.namedSnapshots + .map((snapshot) => `${snapshot.name}:${String(snapshot.changeAfterNonWaitCount)}`) + .join(",") || "none"}`, + `policy=${explorationSummary.policy.satisfied ? "satisfied" : "failed"}`, + ].join("; "); + const summary = [ + `${status === "passed" ? "PASS" : "FAIL"} ${validated.label}`, + exploration, + `artifacts: ${artifactRun.runDirectory}`, + `log: ${logPath}`, + ].join("; "); (status === "passed" ? process.stdout : process.stderr).write(`${summary}\n`); if (failure !== null) { diff --git a/src/tooling/bombadil.ts b/src/tooling/bombadil.ts index 96f2b64..ec72ce2 100644 --- a/src/tooling/bombadil.ts +++ b/src/tooling/bombadil.ts @@ -1,15 +1,22 @@ import { attestDirectBombadilTrace as attestTrace, runDirectBombadilFuzz as runFuzz, + runDirectBombadilFuzzMatrix as runMatrix, + summarizeDirectBombadilTrace as summarizeTrace, } from "./bombadil-runner.js"; import type { DirectBombadilFuzzConfig, + DirectBombadilFuzzCampaign, + DirectBombadilFuzzMatrixResult, DirectBombadilFuzzResult, } from "./bombadil-runner.js"; /** Host-side exact attestation for one bounded Bombadil 0.7.2 JSONL trace. */ export const attestDirectBombadilTrace: typeof attestTrace = attestTrace; +/** Derives bounded diagnostic navigation metadata without replacing the raw trace. */ +export const summarizeDirectBombadilTrace: typeof summarizeTrace = summarizeTrace; + /** Runs one bounded local Bombadil campaign and preserves diagnostic artifacts. */ export function runDirectBombadilFuzz( config: DirectBombadilFuzzConfig, @@ -18,10 +25,24 @@ export function runDirectBombadilFuzz( return arguments_ === undefined ? runFuzz(config) : runFuzz(config, arguments_); } +/** Runs a bounded product campaign matrix serially and selects one for replay. */ +export function runDirectBombadilFuzzMatrix( + campaigns: readonly DirectBombadilFuzzCampaign[], + arguments_?: readonly string[], +): Promise { + return arguments_ === undefined ? runMatrix(campaigns) : runMatrix(campaigns, arguments_); +} + export type { + DirectBombadilActionKind, + DirectBombadilExplorationPolicy, + DirectBombadilExplorationSummary, + DirectBombadilFuzzCampaign, DirectBombadilFuzzConfig, + DirectBombadilFuzzMatrixResult, DirectBombadilFuzzResult, DirectBombadilServerConfig, DirectBombadilTraceAttestation, DirectBombadilTraceBinding, + DirectBombadilViewportConfig, } from "./bombadil-runner.js"; diff --git a/src/tooling/browser-verification.test.ts b/src/tooling/browser-verification.test.ts index 2e84b3d..13e36e6 100644 --- a/src/tooling/browser-verification.test.ts +++ b/src/tooling/browser-verification.test.ts @@ -730,6 +730,68 @@ describe("server leases", () => { expect(fixture.calls).toEqual(["terminate"]); }); + test("aborts a pending readiness probe and terminates the owned server", async () => { + const fixture = fakeServer(); + const controller = new AbortController(); + let markPendingProbe!: () => void; + const pendingProbe = new Promise((resolve) => { + markPendingProbe = resolve; + }); + const neverReachable = new Promise(() => undefined); + let probes = 0; + const acquisition = acquireVerificationServer({ + abortSignal: controller.signal, + baseUrl: "http://localhost:8080", + label: "Fixture server", + startupTimeoutMs: 120_000, + startServer: () => fixture.server, + isReachable: () => { + probes += 1; + if (probes === 1) return false; + markPendingProbe(); + return neverReachable; + }, + }); + await pendingProbe; + controller.abort(); + const failure = await rejection(acquisition); + expect(failure.message).toBe("Verification server acquisition was aborted"); + expect(fixture.calls).toEqual(["terminate"]); + }, 1_000); + + test("does not return a lease when cancellation follows readiness", async () => { + const fixture = fakeServer(); + const controller = new AbortController(); + let markPendingProbe!: () => void; + const pendingProbe = new Promise((resolve) => { + markPendingProbe = resolve; + }); + let resolveReachability!: (reachable: boolean) => void; + const reachability = new Promise((resolve) => { + resolveReachability = resolve; + }); + let probes = 0; + const acquisition = acquireVerificationServer({ + abortSignal: controller.signal, + baseUrl: "http://localhost:8080", + label: "Fixture server", + startupTimeoutMs: 120_000, + startServer: () => fixture.server, + isReachable: () => { + probes += 1; + if (probes === 1) return false; + markPendingProbe(); + return reachability; + }, + }); + await pendingProbe; + resolveReachability(true); + queueMicrotask(() => controller.abort()); + const failure = await rejection(acquisition); + expect(failure.message).toBe("Verification server acquisition was aborted"); + expect(fixture.calls).toEqual(["terminate"]); + }); + test("bounds cleanup when a server never exits after SIGKILL", async () => { const calls: string[] = []; const never = new Promise(() => undefined); diff --git a/src/tooling/browser-verification.ts b/src/tooling/browser-verification.ts index 95da4e9..46ca7f0 100644 --- a/src/tooling/browser-verification.ts +++ b/src/tooling/browser-verification.ts @@ -813,7 +813,38 @@ export async function stopVerificationServer( await stopVerificationServerWithOutput(server, stopTimeoutMs); } +function verificationServerAcquisitionAbortError(): Error { + return new Error("Verification server acquisition was aborted"); +} + +function throwIfVerificationServerAcquisitionAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted === true) throw verificationServerAcquisitionAbortError(); +} + +async function waitForVerificationServerAcquisitionStep( + promise: Promise, + signal: AbortSignal | undefined, +): Promise { + if (signal === undefined) return await promise; + throwIfVerificationServerAcquisitionAborted(signal); + let abortListener: (() => void) | undefined; + const aborted = new Promise((_resolve, reject) => { + abortListener = () => reject(verificationServerAcquisitionAbortError()); + signal.addEventListener("abort", abortListener, { once: true }); + if (signal.aborted) abortListener(); + }); + let value: Value; + try { + value = await Promise.race([promise, aborted]); + } finally { + if (abortListener !== undefined) signal.removeEventListener("abort", abortListener); + } + throwIfVerificationServerAcquisitionAborted(signal); + return value; +} + export async function acquireVerificationServer(options: { + readonly abortSignal?: AbortSignal; readonly baseUrl: string; readonly label: string; readonly localHosts?: ReadonlySet; @@ -837,7 +868,11 @@ export async function acquireVerificationServer(options: { options.baseUrl, options.localHosts, ); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); + if (await waitForVerificationServerAcquisitionStep( + Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), + options.abortSignal, + )) { if (canStartLocally && options.reuseExistingLocalServer === false) { throw new Error( `A local server is already reachable at ${options.baseUrl}; ` @@ -847,8 +882,15 @@ export async function acquireVerificationServer(options: { // A verifier-owned command can exit before its child listener has finished // shutting down. Require the listener to survive a bounded interval before // another verifier trusts it as independently managed infrastructure. - await Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS); - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + await waitForVerificationServerAcquisitionStep( + Bun.sleep(options.reuseProbeIntervalMs ?? DEFAULT_REUSE_PROBE_INTERVAL_MS), + options.abortSignal, + ); + if (await waitForVerificationServerAcquisitionStep( + Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), + options.abortSignal, + )) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "reused" }; } } @@ -858,6 +900,7 @@ export async function acquireVerificationServer(options: { ); } + throwIfVerificationServerAcquisitionAborted(options.abortSignal); const server = options.startServer(); let exitedWithCode: number | null = null; try { @@ -868,10 +911,17 @@ export async function acquireVerificationServer(options: { exitedWithCode = exitCode; break; } - if (await isReachable(options.baseUrl, probeTimeoutMs, readinessPath)) { + if (await waitForVerificationServerAcquisitionStep( + Promise.resolve(isReachable(options.baseUrl, probeTimeoutMs, readinessPath)), + options.abortSignal, + )) { + throwIfVerificationServerAcquisitionAborted(options.abortSignal); return { source: "started", server }; } - await Bun.sleep(options.pollIntervalMs ?? 200); + await waitForVerificationServerAcquisitionStep( + Bun.sleep(options.pollIntervalMs ?? 200), + options.abortSignal, + ); } } catch (error) { await stopVerificationServer(server);