From ac8db2b4628ed94340c022187bb9ba65cefc82ae Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Thu, 6 Aug 2026 14:46:43 -0400 Subject: [PATCH] feat(cli): preflight bounded Node heaps (#2226) AI assistance: OpenAI GPT-5.6 Sol via OpenCode general coding subagent implemented the bounded host Node heap profile, preflight, diagnostics, tests, and docs. Chris Huber reviewed and remains responsible for every line. --- README.md | 16 ++++ package.json | 1 + packages/cli/src/commands/recipe-run-types.ts | 11 ++- packages/cli/src/commands/recipe-run.ts | 18 ++++ packages/cli/src/host-node-heap.ts | 82 +++++++++++++++++++ packages/cli/src/recipe-validation.ts | 2 + packages/runtime-core/src/recipe-schema.ts | 9 ++ .../runtime-core/src/runtime-contracts.ts | 7 ++ tests/host-node-heap.test.ts | 36 ++++++++ 9 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/host-node-heap.ts create mode 100644 tests/host-node-heap.test.ts diff --git a/README.md b/README.md index 21c7bef7..cc7b246e 100644 --- a/README.md +++ b/README.md @@ -733,6 +733,22 @@ Supported runtime commands today: `wordpress.core-phpunit` **requires the mounted `wordpress-develop` checkout to already have its Composer dev dependencies installed** before you mount it. WordPress core's `tests/phpunit/includes/bootstrap.php` hard-requires the test toolchain (PHPUnit plus the Yoast PHPUnit Polyfills at `vendor/yoast/phpunit-polyfills/phpunitpolyfills-autoload.php`) and `die()`s if it is absent — a freshly cloned `wordpress-develop` tree has **no `vendor/`**. Run `composer install` (or `composer update -W`) inside the checkout first, or mount a checkout that already has `vendor/`. WP Codebox does **not** silently fetch these dependencies for you (sandbox network downloads remain gated behind `WP_CODEBOX_ALLOW_NETWORK_DOWNLOADS=1`). When the toolchain is missing, the command now fails with a clear, structured error naming the missing paths instead of crashing with an opaque "crashed before producing a structured response" — the pre-flight check runs before core's bootstrap, and a mid-`require` `die()` is captured via output buffering + a shutdown handler so diagnostics always reach `files/core-phpunit/.pg-test-result.txt`. +### Bounded Host Node Heap Profiles + +Memory-heavy recipes can declare a bounded V8 old-space requirement under `runtime.hostNodeHeap`: + +```json +"hostNodeHeap": { "minimumMiB": 12288, "maximumMiB": 16384 } +``` + +`recipe-run` compares this profile with Node's effective V8 heap limit before boot. When insufficient, it prints the supported replay option. Replay through WP Codebox: + +```sh +wp-codebox recipe-run --recipe recipe.json --host-node-heap-mb=12288 +``` + +The option must remain within the profile bounds; WP Codebox never selects an unbounded heap. Runtime failure evidence distinguishes host V8 heap exhaustion from PHP.wasm memory exhaustion. + `wordpress.browser-probe` accepts `wait-for=domcontentloaded|load|networkidle|selector:|duration`, `duration=s`, `viewport=x` (for example `viewport=390x844`), `pre-page-script=`, repeated `assert=` arguments, and `capture=console,errors,html,network,performance,memory,screenshot`. Use `pre-page-script` for controlled capability mocks that page scripts must observe during startup, such as `ApplePaySession`, `PaymentRequest`, wallet availability probes, or other browser/payment feature state. The script is installed with Playwright before navigation and before application scripts run; artifact summaries preserve only its SHA-256 and byte length, not the source. Assertions support `exists:`, `not-exists:`, `visible:`, `hidden:`, `count:`, `text: contains `, `attr:[name][=value]`, `no-console-errors`, `no-page-errors`, and `no-errors`; prefix with `advisory:` to record a failing assertion without failing the probe. Assertion results are included in the command JSON and `summary.json`, and non-advisory failures fail the command after artifacts are written. It records machine-readable evidence refs such as `files/browser/console.jsonl`, `files/browser/errors.jsonl`, `files/browser/network.jsonl`, `files/browser/performance.json`, `files/browser/memory.json`, `files/browser/checkpoints.jsonl`, `files/browser/snapshot.html`, `files/browser/screenshot.png`, and `files/browser/summary.json` when those captures are enabled. The summary includes requested/final URLs, effective viewport/device metadata, optional pre-page script metadata, HTML and screenshot hashes, assertion results, network event counts, optional final/peak browser memory and performance summaries, and a generic `artifact-backed|partial|diagnostic-only` replayability classification. Performance and memory captures use generic browser/CDP data only: JS heap when available, CDP `Performance.getMetrics`, CDP DOM counters, DOM/resource counts and byte totals, and long task counts/duration. Probe scripts may call `window.__wpCodeboxProbeCheckpoint(name, metadata)` when `performance` or `memory` capture is enabled to record named generic checkpoint snapshots. WP Codebox intentionally keeps these browser evidence fields generic; consumers such as eval harnesses may interpret them without WP Codebox adding scoring, grading, or benchmark semantics. `wordpress.visual-compare` URL captures default to `reduced-motion=true`, `animations=freeze`, and `block-external-requests=true`. Use `frozen-time=` to make page wall-clock time deterministic. Accepted inputs use `YYYY-MM-DDTHH:mm:ss(.sss)(Z|+HH:MM)` and evidence records the canonical UTC millisecond value, such as `2020-01-01T00:00:00.000Z`. Use `capture-style=` for a bounded 16 KiB capture-only stylesheet. The summary records the effective capture contract, actual viewport, blocked and failed request outcomes, readiness duration, and font readiness. Layout diagnostics classify an identical y-offset across at least two unchanged in-flow anchors as an `anchor-proven global origin offset`; height, gap, added/removed, and non-uniform offset changes remain reflow evidence. diff --git a/package.json b/package.json index 315cf314..7997aca0 100644 --- a/package.json +++ b/package.json @@ -137,6 +137,7 @@ "test:playground-phpunit-bootstrap-failure-integration": "tsx tests/playground-phpunit-bootstrap-failure.integration.test.ts", "test:playground-custom-archive-cache": "tsx tests/playground-custom-archive-cache.test.ts && tsx tests/playground-custom-archive-cache-process.test.ts && tsx tests/playground-custom-archive-cache.integration.test.ts", "test:phpunit-runtime-failure-diagnostics": "tsx tests/phpunit-runtime-failure-diagnostics.test.ts", + "test:host-node-heap": "tsx tests/host-node-heap.test.ts", "test:phpunit-structured-evidence": "tsx tests/phpunit-structured-evidence.test.ts", "test:phpunit-runtime-rejection": "tsx tests/phpunit-runtime-rejection.test.ts", "test:playground-worker-runtime-rejection": "tsx tests/playground-worker-runtime-rejection.test.ts", diff --git a/packages/cli/src/commands/recipe-run-types.ts b/packages/cli/src/commands/recipe-run-types.ts index 87921fce..86cdec8a 100644 --- a/packages/cli/src/commands/recipe-run-types.ts +++ b/packages/cli/src/commands/recipe-run-types.ts @@ -29,6 +29,7 @@ export interface RecipeRunOptions { json: boolean summary: boolean dryRun: boolean + hostNodeHeapMiB?: number } export interface RecipeValidateOptions { @@ -331,6 +332,14 @@ export interface RecipePhpWasmRuntimeDiagnostic { repair?: string } +export interface RecipeMemoryRuntimeDiagnostic { + schema: "wp-codebox/runtime-memory-diagnostic/v1" + severity: "error" + kind: "host-v8-oom" | "php-wasm-oom" + message: string + replay?: string +} + export interface RecipePhaseDiagnostic { schema: "wp-codebox/recipe-phase-diagnostic/v1" severity: "error" @@ -342,7 +351,7 @@ export interface RecipePhaseDiagnostic { executionIndex?: number } -export type RecipeRuntimeDiagnostic = RecipePluginRuntimeDiagnostic | RecipePhaseDiagnostic | RecipePhpWasmRuntimeDiagnostic +export type RecipeRuntimeDiagnostic = RecipePluginRuntimeDiagnostic | RecipePhaseDiagnostic | RecipePhpWasmRuntimeDiagnostic | RecipeMemoryRuntimeDiagnostic export interface RecipeRunSiteSeed extends Omit { action: "imported" | "skipped" diff --git a/packages/cli/src/commands/recipe-run.ts b/packages/cli/src/commands/recipe-run.ts index ad8d404f..a2d414f5 100644 --- a/packages/cli/src/commands/recipe-run.ts +++ b/packages/cli/src/commands/recipe-run.ts @@ -32,6 +32,7 @@ import { applyRecipeRuntimeSetup, cleanupInputMountBaselines, prepareRecipeRunti import { provisionRuntimeServices, provisionRuntimeServicesForRecipe, runtimeServiceEvidenceFromError, type RuntimeServiceEvidence } from "../runtime-services.js" import { distributionStartupProbeFailure, executeRecipeCollectWorkloadResult, executeRecipeWorkflowStep, recipeAdvisoryFailure, recipeBrowserEvidence, recipeStepFailure, recipeWorkflowArgsEvidence, recipeWorkflowStepIsAdvisory, runDistributionSetupArtifacts, runDistributionStartupProbes, runRecipeProbes, withRecipeExecutionPhase } from "./recipe-run-workflow-evidence.js" import { recipeAdversarialCampaignFailure, runRecipeAdversarialCampaigns, writeRecipeAdversarialEvidence, type RecipeAdversarialCampaignOutput } from "../adversarial-recipe.js" +import { classifyRuntimeMemoryFailure, replayWithHostNodeHeap } from "../host-node-heap.js" import type { RecipeAdvisoryFailure, RecipeBrowserEvidence, RecipeDiagnosticArtifactRef, RecipeEffectiveRecipeArtifact, RecipeExecutionResult, RecipeFuzzCaseCommandRef, RecipeFuzzCaseResult, RecipeFuzzCaseStatus, RecipeFuzzRunResult, RecipeInterruptionController, RecipePhaseEvidence, RecipePhaseName, RecipePhpWasmRuntimeDiagnostic, RecipeRunCommandOutput, RecipeRunComponentContract, RecipeRunDeclaredArtifact, RecipeRunDistributionSetupArtifact, RecipeRunDistributionStartupProbe, RecipeRunFixtureDatabase, RecipeRunOptions, RecipeRunOutput, RecipeRunPreparedExtraPlugin, RecipeRunProbe, RecipeRunProvenance, RecipeRunStagedFile, RecipeRuntimeDiagnostic, RecipeStepFailure, RecipeValidateOptions, RecipeValidateOutput } from "./recipe-run-types.js" const DEFAULT_RECIPE_RUN_TIMEOUT_MS = 25 * 60 * 1000 @@ -39,6 +40,8 @@ const SUCCESSFUL_RECIPE_RUNTIME_SNAPSHOT_TIMEOUT_MS = 120 * 1000 const packageRequire = createRequire(import.meta.url) export async function runRecipeRunCommand(args: string[]): Promise { const options = parseRecipeRunOptions(args) + const replayExitCode = await replayWithHostNodeHeap(args, options.hostNodeHeapMiB, (await loadWorkspaceRecipe(options.recipePath)).runtime?.hostNodeHeap) + if (replayExitCode !== undefined) return replayExitCode if (options.previewLeaseRequested && !options.previewLeaseChild) { return startPreviewLeaseRecipeRun({ args, json: options.json, recipePath: options.recipePath, artifactsDirectory: options.artifactsDirectory, runRegistryDirectory: options.runRegistryDirectory, previewHoldSeconds: options.previewHoldSeconds }) } @@ -837,6 +840,9 @@ function parseRecipeRunOptions(args: string[]): RecipeRunOptions { case "--adversarial-replay": options.adversarialReplayPath = value break + case "--host-node-heap-mb": + options.hostNodeHeapMiB = Number(value) + break default: throw new Error(`Unknown option: ${name}`) } @@ -1260,6 +1266,18 @@ function recipeRuntimeDiagnostics(recipe: WorkspaceRecipe, executions: RecipeExe diagnostics.push(phpWasmDiagnostic) } + const memoryFailure = classifyRuntimeMemoryFailure(error) + if (memoryFailure) { + const requirement = recipe.runtime?.hostNodeHeap + diagnostics.push({ + schema: "wp-codebox/runtime-memory-diagnostic/v1", + severity: "error", + kind: memoryFailure, + message: memoryFailure === "host-v8-oom" ? "Node V8 exhausted its old-space heap while the runtime was active." : "PHP.wasm exhausted WebAssembly memory while the runtime was active.", + ...(memoryFailure === "host-v8-oom" && requirement ? { replay: `wp-codebox recipe-run --recipe --host-node-heap-mb=${requirement.minimumMiB}` } : {}), + }) + } + const message = error instanceof Error ? error.message : String(error) if (error instanceof RecipePhaseError && diagnostics.length === 0) { diagnostics.push({ diff --git a/packages/cli/src/host-node-heap.ts b/packages/cli/src/host-node-heap.ts new file mode 100644 index 00000000..3a75d4cc --- /dev/null +++ b/packages/cli/src/host-node-heap.ts @@ -0,0 +1,82 @@ +import { spawn } from "node:child_process" +import v8 from "node:v8" +import type { WorkspaceRecipeHostNodeHeap } from "@automattic/wp-codebox-core" + +const MIB = 1024 * 1024 + +export interface HostNodeHeapPreflight { + status: "ready" | "insufficient" + effectiveMiB: number + minimumMiB: number + maximumMiB: number + replayOption: string +} + +export class HostNodeHeapPreflightError extends Error { + readonly code = "wp-codebox-host-node-heap-insufficient" + + constructor(readonly preflight: HostNodeHeapPreflight) { + super(`The effective Node V8 heap limit is ${preflight.effectiveMiB} MiB, but this runtime profile requires at least ${preflight.minimumMiB} MiB. Replay with ${preflight.replayOption}. The profile caps the host heap at ${preflight.maximumMiB} MiB.`) + this.name = "HostNodeHeapPreflightError" + } +} + +export function preflightHostNodeHeap(requirement: WorkspaceRecipeHostNodeHeap | undefined, effectiveBytes = v8.getHeapStatistics().heap_size_limit): HostNodeHeapPreflight | undefined { + if (!requirement) return undefined + assertHostNodeHeapRequirement(requirement) + const effectiveMiB = Math.floor(effectiveBytes / MIB) + return { + status: effectiveMiB >= requirement.minimumMiB ? "ready" : "insufficient", + effectiveMiB, + minimumMiB: requirement.minimumMiB, + maximumMiB: requirement.maximumMiB, + replayOption: `--host-node-heap-mb=${requirement.minimumMiB}`, + } +} + +export function assertHostNodeHeapRequirement(requirement: WorkspaceRecipeHostNodeHeap): void { + for (const [name, value] of Object.entries(requirement)) { + if (!Number.isInteger(value) || value < 256 || value > 16_384) { + throw new Error(`runtime.hostNodeHeap.${name} must be an integer from 256 to 16384 MiB`) + } + } + if (requirement.minimumMiB > requirement.maximumMiB) { + throw new Error("runtime.hostNodeHeap.minimumMiB must not exceed runtime.hostNodeHeap.maximumMiB") + } +} + +export async function replayWithHostNodeHeap(args: string[], requestedMiB: number | undefined, requirement: WorkspaceRecipeHostNodeHeap | undefined, effectiveBytes = v8.getHeapStatistics().heap_size_limit, spawnProcess = spawn): Promise { + const preflight = preflightHostNodeHeap(requirement, effectiveBytes) + if (!preflight || preflight.status === "ready") return undefined + if (requestedMiB === undefined) throw new HostNodeHeapPreflightError(preflight) + if (!Number.isInteger(requestedMiB) || requestedMiB < preflight.minimumMiB || requestedMiB > preflight.maximumMiB) { + throw new Error(`--host-node-heap-mb must be an integer from ${preflight.minimumMiB} to ${preflight.maximumMiB} MiB for this runtime profile`) + } + + return await new Promise((resolve, reject) => { + const child = spawnProcess(process.execPath, hostNodeHeapReplayArgs(args, requestedMiB), { stdio: "inherit" }) + child.once("error", reject) + child.once("exit", (code, signal) => resolve(code ?? (signal ? 1 : 0))) + }) +} + +export function hostNodeHeapReplayArgs(args: string[], heapMiB: number): string[] { + const forwarded: string[] = [] + for (let index = 0; index < args.length; index += 1) { + if (args[index] === "--host-node-heap-mb") { + index += 1 + continue + } + if (!args[index].startsWith("--host-node-heap-mb=")) forwarded.push(args[index]) + } + return [`--max-old-space-size=${heapMiB}`, process.argv[1], ...forwarded] +} + +export type RuntimeMemoryFailureKind = "host-v8-oom" | "php-wasm-oom" + +export function classifyRuntimeMemoryFailure(error: unknown): RuntimeMemoryFailureKind | undefined { + const message = error instanceof Error ? `${error.message}\n${error.stack ?? ""}` : String(error) + if (/FATAL ERROR:.*(?:heap out of memory|Ineffective mark-compacts)|JavaScript heap out of memory/i.test(message)) return "host-v8-oom" + if (/(?:php\.wasm|WebAssembly\.Memory).*?(?:out of memory|memory access out of bounds)|(?:out of memory|cannot enlarge memory).*?(?:php\.wasm|wasm)/is.test(message)) return "php-wasm-oom" + return undefined +} diff --git a/packages/cli/src/recipe-validation.ts b/packages/cli/src/recipe-validation.ts index 7fe951fd..51f632ea 100644 --- a/packages/cli/src/recipe-validation.ts +++ b/packages/cli/src/recipe-validation.ts @@ -5,6 +5,7 @@ import { BROWSER_PROBE_CHROMIUM_PROFILE_IDS, RUNTIME_BACKED_FUZZ_SUITE_RUNNER_CA import { commandValidationDescriptorFor, effectivePolicyCommandsFor, type CommandArgValidationDescriptor } from "@automattic/wp-codebox-core/contracts" import { composerPackageVendorPath, evaluateRecipeSourcePolicy, isComposerPackageName, pluginTarget, recipeExtraPluginSlug, recipeExtraPluginSource, recipeExtraPluginSourceRoot, recipeExtraPluginSourceSubpath, recipeExtraPlugins, recipeSource, resolveRecipeExtraPluginFile } from "./recipe-sources.js" import { loadConfiguredRuntimeOverlayDescriptors, registeredRuntimeOverlayDescriptors, runtimeOverlayDescriptor, runtimeOverlayTarget } from "./runtime-overlay-registry.js" +import { assertHostNodeHeapRequirement } from "./host-node-heap.js" import { cliRuntimeBackendRecipePolicy, listCliRecipeCommandIds, listCliRuntimeBackendKinds } from "./runtime-backends.js" import { evaluateZipSourcePolicy } from "./source-policy.js" @@ -154,6 +155,7 @@ export function validateWorkspaceRecipeShape(recipe: WorkspaceRecipe, recipePath validateRecipeRuntimeBundledExtensions(recipe.runtime?.bundledExtensions, recipePath) validateRecipeRuntimeWordPressInstallMode(recipe.runtime?.wordpressInstallMode, recipePath) validateRecipeRuntimePreview(recipe.runtime?.preview, recipePath) + if (recipe.runtime?.hostNodeHeap) assertHostNodeHeapRequirement(recipe.runtime.hostNodeHeap) validateRecipeMounts(recipe.inputs?.mounts, "mounts", recipePath) validateRecipeDependencyOverlays(recipe.inputs?.dependency_overlays, recipePath) diff --git a/packages/runtime-core/src/recipe-schema.ts b/packages/runtime-core/src/recipe-schema.ts index a48c2bd2..72074cbf 100644 --- a/packages/runtime-core/src/recipe-schema.ts +++ b/packages/runtime-core/src/recipe-schema.ts @@ -145,6 +145,15 @@ export function createWorkspaceRecipeJsonSchema(options: WorkspaceRecipeJsonSche }, backendPackage: { $ref: "#/$defs/runtimeBackendPackage" }, stack: { $ref: "#/$defs/runtimeStack" }, + hostNodeHeap: { + type: "object", + additionalProperties: false, + required: ["minimumMiB", "maximumMiB"], + properties: { + minimumMiB: { type: "integer", minimum: 256, maximum: 16384 }, + maximumMiB: { type: "integer", minimum: 256, maximum: 16384 }, + }, + }, overlays: { type: "array", description: "Typed runtime overlays prepared by WP Codebox before mounting into Playground.", diff --git a/packages/runtime-core/src/runtime-contracts.ts b/packages/runtime-core/src/runtime-contracts.ts index cd581a28..67fea403 100644 --- a/packages/runtime-core/src/runtime-contracts.ts +++ b/packages/runtime-core/src/runtime-contracts.ts @@ -163,6 +163,12 @@ export interface WorkspaceRecipeRuntimeStack { mounts?: WorkspaceRecipeMount[] } +/** A bounded host V8 old-space budget for memory-heavy runtime profiles. */ +export interface WorkspaceRecipeHostNodeHeap { + minimumMiB: number + maximumMiB: number +} + export type WorkspaceRecipeRuntimeOverlayKind = string export type WorkspaceRecipeRuntimeOverlayLibrary = string export type WorkspaceRecipeRuntimeOverlayStrategy = string @@ -660,6 +666,7 @@ export interface WorkspaceRecipe { backendPackage?: WorkspaceRecipeRuntimeBackendPackage stack?: WorkspaceRecipeRuntimeStack overlays?: WorkspaceRecipeRuntimeOverlay[] + hostNodeHeap?: WorkspaceRecipeHostNodeHeap } inputs?: { workspaces?: WorkspaceRecipeWorkspace[] diff --git a/tests/host-node-heap.test.ts b/tests/host-node-heap.test.ts new file mode 100644 index 00000000..f265b671 --- /dev/null +++ b/tests/host-node-heap.test.ts @@ -0,0 +1,36 @@ +import assert from "node:assert/strict" +import { HostNodeHeapPreflightError, assertHostNodeHeapRequirement, classifyRuntimeMemoryFailure, hostNodeHeapReplayArgs, preflightHostNodeHeap } from "../packages/cli/src/host-node-heap.js" +import { validateWorkspaceRecipeJsonSchema } from "../packages/runtime-core/src/index.js" + +const requirement = { minimumMiB: 12288, maximumMiB: 16384 } +const preflight = preflightHostNodeHeap(requirement, 4096 * 1024 * 1024) +assert.deepEqual(preflight, { + status: "insufficient", + effectiveMiB: 4096, + minimumMiB: 12288, + maximumMiB: 16384, + replayOption: "--host-node-heap-mb=12288", +}) +assert.match(new HostNodeHeapPreflightError(preflight!).message, /--host-node-heap-mb=12288/) + +assert.deepEqual(hostNodeHeapReplayArgs(["recipe-run", "--recipe", "memory.json", "--host-node-heap-mb=12288"], 12288).slice(0, 2), ["--max-old-space-size=12288", process.argv[1]]) +assert.doesNotMatch(hostNodeHeapReplayArgs(["recipe-run", "--host-node-heap-mb=12288"], 12288).join(" "), /--host-node-heap-mb/) +assert.doesNotMatch(hostNodeHeapReplayArgs(["recipe-run", "--host-node-heap-mb", "12288"], 12288).join(" "), /12288$/) +assert.throws(() => assertHostNodeHeapRequirement({ minimumMiB: 16384, maximumMiB: 12288 }), /must not exceed/) + +assert.equal(classifyRuntimeMemoryFailure(new Error("FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory")), "host-v8-oom") +assert.equal(classifyRuntimeMemoryFailure(new Error("RuntimeError: WebAssembly.Memory(): out of memory at php.wasm")), "php-wasm-oom") +assert.equal(classifyRuntimeMemoryFailure(new Error("PHP Fatal error")), undefined) + +assert.equal(validateWorkspaceRecipeJsonSchema({ + schema: "wp-codebox/workspace-recipe/v1", + runtime: { hostNodeHeap: requirement }, + workflow: { steps: [{ command: "wordpress.phpunit" }] }, +}).valid, true) +assert.equal(validateWorkspaceRecipeJsonSchema({ + schema: "wp-codebox/workspace-recipe/v1", + runtime: { hostNodeHeap: { minimumMiB: 12288 } }, + workflow: { steps: [{ command: "wordpress.phpunit" }] }, +}).valid, false) + +console.log("host node heap contract ok")