From 1f748de7f0d7f856b8a24828244a7ff70ff6901e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 30 Jul 2026 20:16:44 -0400 Subject: [PATCH 01/25] chore(policy): checkpoint Shields MCP reconciliation --- ci/source-architecture-budget.json | 2 +- .../checks/openshell-policy-mutation-read.mts | 2 +- src/lib/actions/sandbox/mcp-bridge-policy.ts | 181 ++++++++++++++ src/lib/shields/flow.test.ts | 201 +++++++++++++++- src/lib/shields/index.ts | 183 +++++++++++--- src/lib/shields/mcp-policy-transition.test.ts | 227 ++++++++++++++++++ src/lib/shields/mcp-policy-transition.ts | 75 ++++++ src/lib/shields/permissive-runtime.ts | 105 +++++++- src/lib/shields/timer.test.ts | 42 +--- src/lib/shields/timer.ts | 6 +- test/e2e/live/mcp-bridge.test.ts | 16 +- test/e2e/support/mcp-bridge-sandbox.test.ts | 36 ++- test/permissive-runtime.test.ts | 69 +++++- 13 files changed, 1057 insertions(+), 88 deletions(-) create mode 100644 src/lib/shields/mcp-policy-transition.test.ts create mode 100644 src/lib/shields/mcp-policy-transition.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 3168eb519ba..8245db9694e 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -24,7 +24,7 @@ "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 25, "src/lib/onboard/gateway-binding.ts": 47, - "src/lib/runner.ts": 89, + "src/lib/runner.ts": 88, "src/lib/security/redact.ts": 51, "src/lib/state/onboard-session.ts": 34, "src/lib/state/registry.ts": 99, diff --git a/scripts/checks/openshell-policy-mutation-read.mts b/scripts/checks/openshell-policy-mutation-read.mts index 133a981dc62..2c0b4088676 100644 --- a/scripts/checks/openshell-policy-mutation-read.mts +++ b/scripts/checks/openshell-policy-mutation-read.mts @@ -58,7 +58,7 @@ export const MUTATION_READS: readonly AuditedMutationRead[] = [ }, { relativePath: "src/lib/shields/index.ts", - expectedReadCalls: 1, + expectedReadCalls: 2, baseCommand: "runCapture(buildPolicyGetCommand(sandboxName))", unsafeBaseCommand: "runCapture(buildPolicyGetCommand(sandboxName), {", fullCommand: "runCapture(buildPolicyGetFullCommand(sandboxName))", diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.ts b/src/lib/actions/sandbox/mcp-bridge-policy.ts index 88a5e024655..2584fdad555 100644 --- a/src/lib/actions/sandbox/mcp-bridge-policy.ts +++ b/src/lib/actions/sandbox/mcp-bridge-policy.ts @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { isDeepStrictEqual } from "node:util"; +import YAML from "yaml"; + import type { AgentMcpAdapter } from "../../agent/defs"; import * as policies from "../../policy"; import type { McpBridgeEntry } from "../../state/registry"; @@ -23,6 +26,184 @@ export { MCP_BRIDGE_ALLOWED_METHODS, MCP_BRIDGE_POLICY_MAX_BODY_BYTES, } from "./mcp-bridge-policy-render"; +export { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts"; + +export interface ExactManagedMcpPolicy { + key: string; + networkPolicy: unknown; + policyName: string; + server: string; +} + +type ManagedMcpPolicyInspectionDeps = { + getSandbox: typeof registry.getSandbox; +}; + +const managedMcpPolicyInspectionDeps: ManagedMcpPolicyInspectionDeps = { + getSandbox: registry.getSandbox, +}; + +function parseManagedPolicyDocument(source: string, label: string): Record { + let parsed: unknown; + try { + parsed = YAML.parse(source); + } catch { + throw new Error(`${label} is not valid YAML`); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error(`${label} must be a YAML mapping`); + } + return parsed as Record; +} + +function readManagedNetworkPolicies( + document: Record, + label: string, +): Record { + const networkPolicies = document.network_policies; + if (networkPolicies === undefined || networkPolicies === null) return {}; + if (typeof networkPolicies !== "object" || Array.isArray(networkPolicies)) { + throw new Error(`${label} network_policies must be a mapping`); + } + return networkPolicies as Record; +} + +function requireCanonicalManagedPolicy( + sandbox: registry.SandboxEntry, + server: string, + livePolicies: Record, +): ExactManagedMcpPolicy { + const bridge = sandbox.mcp?.bridges[server]; + if (!bridge || bridge.addState || bridge.server !== server) { + throw new Error(`Managed MCP bridge '${server}' has an incomplete lifecycle transition`); + } + + const policyName = buildMcpBridgePolicyName(server); + const policyKey = buildMcpBridgePolicyKey(server); + if (bridge.policyName !== policyName) { + throw new Error(`Managed MCP bridge '${server}' has a non-canonical policy name`); + } + + const registrations = (sandbox.customPolicies ?? []).filter( + (policy) => policy.name === policyName, + ); + if (registrations.length !== 1) { + throw new Error( + `Managed MCP bridge '${server}' does not have one exact policy ownership record`, + ); + } + const [registration] = registrations; + if (registration?.sourcePath !== MCP_BRIDGE_POLICY_SOURCE) { + throw new Error(`Managed MCP bridge '${server}' has no NemoClaw-owned policy record`); + } + if (registration.pendingContent !== undefined) { + throw new Error(`Managed MCP bridge '${server}' has an incomplete policy transition`); + } + + const registeredDocument = parseManagedPolicyDocument( + registration.content, + `Managed MCP policy '${policyName}'`, + ); + const preset = registeredDocument.preset; + if ( + !preset || + typeof preset !== "object" || + Array.isArray(preset) || + (preset as Record).name !== policyName + ) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical preset metadata`); + } + const registeredPolicies = readManagedNetworkPolicies( + registeredDocument, + `Managed MCP policy '${policyName}'`, + ); + const registeredKeys = Object.keys(registeredPolicies); + if (registeredKeys.length !== 1 || registeredKeys[0] !== policyKey) { + throw new Error(`Managed MCP policy '${policyName}' has a non-canonical network policy key`); + } + + if (!Object.hasOwn(livePolicies, policyKey)) { + throw new Error(`Managed MCP policy '${policyName}' is absent from the live gateway policy`); + } + if (!isDeepStrictEqual(livePolicies[policyKey], registeredPolicies[policyKey])) { + throw new Error(`Managed MCP policy '${policyName}' has drifted from its ownership record`); + } + + return { + key: policyKey, + networkPolicy: registeredPolicies[policyKey], + policyName, + server, + }; +} + +/** + * Resolve the exact generated MCP entries that NemoClaw currently owns. + * + * The registry is an ownership claim, not sufficient authority to overwrite + * the gateway. Every committed bridge must have one canonical, fully + * committed custom-policy record whose sole network entry exactly matches the + * live base policy. + */ +export function inspectExactManagedMcpPolicies( + sandboxName: string, + livePolicyYaml: string, + deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, +): ExactManagedMcpPolicy[] { + const sandbox = deps.getSandbox(sandboxName); + if (!sandbox?.mcp) return []; + if (sandbox.mcp.destroyPreparedAt || sandbox.mcp.destroyPendingAt) { + throw new Error("Managed MCP sandbox destruction is incomplete"); + } + + const bridgeEntries = Object.entries(sandbox.mcp.bridges); + if (bridgeEntries.some(([, bridge]) => bridge.addState !== undefined)) { + throw new Error("A managed MCP bridge lifecycle transition is incomplete"); + } + const generatedRegistrations = (sandbox.customPolicies ?? []).filter( + (policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE, + ); + if (bridgeEntries.length === 0 && generatedRegistrations.length === 0) return []; + + const liveDocument = parseManagedPolicyDocument(livePolicyYaml, "Live gateway policy"); + const livePolicies = readManagedNetworkPolicies(liveDocument, "Live gateway policy"); + const exact = bridgeEntries.map(([server]) => + requireCanonicalManagedPolicy(sandbox, server, livePolicies), + ); + + const committedPolicyNames = new Set(exact.map((entry) => entry.policyName)); + const orphaned = generatedRegistrations.find( + (registration) => !committedPolicyNames.has(registration.name), + ); + if (orphaned) { + throw new Error( + `Generated MCP policy '${orphaned.name}' has no committed managed bridge ownership`, + ); + } + + const keys = new Set(); + for (const entry of exact) { + if (keys.has(entry.key)) { + throw new Error(`Managed MCP policy key '${entry.key}' has ambiguous bridge ownership`); + } + keys.add(entry.key); + } + return exact.sort((left, right) => left.key.localeCompare(right.key)); +} + +export function hasManagedMcpPolicyClaims( + sandboxName: string, + deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, +): boolean { + const sandbox = deps.getSandbox(sandboxName); + if (!sandbox?.mcp) return false; + return ( + Object.keys(sandbox.mcp.bridges).length > 0 || + Boolean(sandbox.mcp.destroyPreparedAt) || + Boolean(sandbox.mcp.destroyPendingAt) || + (sandbox.customPolicies ?? []).some((policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE) + ); +} type GeneratedPolicyRegistrationState = { policy: registry.CustomPolicyEntry; diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 7c9be44f48a..06eae9d80a7 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -8,6 +8,8 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; +import YAML from "yaml"; +import type { SandboxEntry } from "../state/registry"; const requireDist = createRequire(import.meta.url); const shieldsModulePath = "./index.js"; @@ -26,9 +28,11 @@ setInterval(() => {}, 60000); `; type ShieldsHarness = { + applyShieldsPolicySnapshot: typeof import("./index.js").applyShieldsPolicySnapshot; auditSpy: MockInstance; errorSpy: MockInstance; logSpy: MockInstance; + policySetBodies: string[]; runSpy: MockInstance; shieldsDown: typeof import("./index.js").shieldsDown; shieldsStatus: typeof import("./index.js").shieldsStatus; @@ -61,7 +65,9 @@ type HarnessOptions = { send: () => boolean; kill: () => boolean; }; + livePolicy?: string; run?: (cmd: unknown) => { status: number }; + sandboxEntry?: SandboxEntry; }; function throwHarnessError(error: Error): never { @@ -87,16 +93,24 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { const dockerExec = requireDist("../adapters/docker/exec.js"); const audit = requireDist("./audit.js"); const childProcess = requireDist("node:child_process"); + const policySetBodies: string[] = []; let openClawPosture: "locked" | "mutable" = "mutable"; vi.spyOn(runner, "validateName").mockImplementation((name: unknown) => String(name)); - vi.spyOn(runner, "runCapture").mockReturnValue("version: 1\nnetwork_policies:\n test: {}\n"); + vi.spyOn(runner, "runCapture").mockReturnValue( + options.livePolicy ?? "version: 1\nnetwork_policies:\n test: {}\n", + ); const runSpy = vi.spyOn(runner, "run").mockImplementation((cmd: unknown) => { return options.run ? options.run(cmd) : { status: 0 }; }); options.fork && vi.spyOn(childProcess, "fork").mockImplementation(options.fork); vi.spyOn(policy, "buildPolicyGetCommand").mockReturnValue(["openshell", "policy", "get"]); - vi.spyOn(policy, "buildPolicySetCommand").mockReturnValue(["openshell", "policy", "set"]); + vi.spyOn(policy, "buildPolicySetCommand").mockImplementation((file: string) => { + if (fs.existsSync(file)) { + policySetBodies.push(fs.readFileSync(file, "utf-8")); + } + return ["openshell", "policy", "set"]; + }); vi.spyOn(policy, "parseCurrentPolicy").mockImplementation((raw: unknown) => String(raw)); vi.spyOn(policy, "resolvePermissivePolicyPath").mockReturnValue( path.join(tmpDir, "permissive.yaml"), @@ -109,7 +123,9 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { configPath: "/sandbox/.openclaw/openclaw.json", format: "json", }); - vi.spyOn(registry, "getSandbox").mockReturnValue({ name: "openclaw", openshellDriver: "docker" }); + vi.spyOn(registry, "getSandbox").mockReturnValue( + options.sandboxEntry ?? { name: "openclaw", openshellDriver: "docker" }, + ); vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [{ name: "openclaw" }] }); const directSandboxUnavailableError = new Error( "No running direct OpenShell sandbox container found for 'openclaw' (driver: docker). Expected a running container named openshell-openclaw or openshell-openclaw-*. Is the sandbox running?", @@ -212,9 +228,11 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { errorSpy.mockClear(); auditSpy.mockClear(); return { + applyShieldsPolicySnapshot: shields.applyShieldsPolicySnapshot, auditSpy, errorSpy, logSpy, + policySetBodies, runSpy, shieldsDown: shields.shieldsDown, shieldsStatus: shields.shieldsStatus, @@ -283,6 +301,183 @@ describe("shields command flow", () => { ); }); + it("shieldsDown preserves an exact managed MCP policy and records its snapshot key (#7952)", { + timeout: 15_000, + }, () => { + const managedPolicy = YAML.stringify({ + preset: { + name: "mcp-bridge-alpha", + description: "Generated MCP policy for alpha", + }, + network_policies: { + mcp_bridge_alpha: { + name: "mcp_bridge_alpha", + endpoints: [ + { + host: "alpha.example.com", + port: 443, + path: "/mcp", + protocol: "mcp", + }, + ], + binaries: [{ path: "/opt/hermes/.venv/bin/python*" }], + }, + }, + }); + const managedNetworkPolicy = YAML.parse(managedPolicy).network_policies.mcp_bridge_alpha; + const harness = createHarness({ + livePolicy: YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + mcp_bridge_alpha: managedNetworkPolicy, + }, + }), + sandboxEntry: { + name: "openclaw", + openshellDriver: "docker", + customPolicies: [ + { + name: "mcp-bridge-alpha", + content: managedPolicy, + sourcePath: "generated:nemoclaw-mcp-bridge", + }, + ], + mcp: { + bridges: { + alpha: { + server: "alpha", + agent: "hermes", + adapter: "hermes-config", + url: "https://alpha.example.com/mcp", + env: ["MCP_SECRET"], + policyName: "mcp-bridge-alpha", + addedAt: "2026-07-30T00:00:00.000Z", + }, + }, + }, + }, + }); + + harness.shieldsDown("openclaw", { + timeout: "5m", + reason: "managed MCP transition coverage", + skipTimer: true, + throwOnError: true, + }); + + const state = JSON.parse( + fs.readFileSync(path.join(tmpDir, ".nemoclaw", "state", "shields-openclaw.json"), "utf-8"), + ); + expect(state.shieldsManagedMcpPolicyKeys).toEqual(["mcp_bridge_alpha"]); + const applied = YAML.parse(harness.policySetBodies.at(-1)!); + expect(applied.network_policies.mcp_bridge_alpha).toEqual(managedNetworkPolicy); + expect(applied.network_policies).not.toHaveProperty("restrictive_baseline"); + }); + + it("shared snapshot restore keeps managed MCP additions made while Shields are down (#7952)", () => { + const policyFor = (server: string, address: string) => + YAML.stringify({ + preset: { + name: `mcp-bridge-${server}`, + description: `Generated MCP policy for ${server}`, + }, + network_policies: { + [`mcp_bridge_${server}`]: { + name: `mcp_bridge_${server}`, + endpoints: [ + { + host: `${server}.example.com`, + port: 443, + path: "/mcp", + protocol: "mcp", + allowed_ips: [address], + }, + ], + binaries: [{ path: "/opt/hermes/.venv/bin/python*" }], + }, + }, + }); + const alphaPolicy = policyFor("alpha", "8.8.8.8"); + const betaPolicy = policyFor("beta", "1.1.1.1"); + const alphaEntry = YAML.parse(alphaPolicy).network_policies.mcp_bridge_alpha; + const betaEntry = YAML.parse(betaPolicy).network_policies.mcp_bridge_beta; + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const snapshotPath = path.join(stateDir, "policy-snapshot-managed-restore.yaml"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + snapshotPath, + YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + mcp_bridge_alpha: alphaEntry, + }, + }), + ); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: ["mcp_bridge_alpha"], + }), + ); + const harness = createHarness({ + livePolicy: YAML.stringify({ + version: 1, + network_policies: { + permissive_baseline: { endpoints: [{ host: "*" }] }, + mcp_bridge_alpha: alphaEntry, + mcp_bridge_beta: betaEntry, + }, + }), + sandboxEntry: { + name: "openclaw", + openshellDriver: "docker", + customPolicies: [ + { + name: "mcp-bridge-alpha", + content: alphaPolicy, + sourcePath: "generated:nemoclaw-mcp-bridge", + }, + { + name: "mcp-bridge-beta", + content: betaPolicy, + sourcePath: "generated:nemoclaw-mcp-bridge", + }, + ], + mcp: { + bridges: Object.fromEntries( + ["alpha", "beta"].map((server) => [ + server, + { + server, + agent: "hermes", + adapter: "hermes-config", + url: `https://${server}.example.com/mcp`, + env: ["MCP_SECRET"], + policyName: `mcp-bridge-${server}`, + addedAt: "2026-07-30T00:00:00.000Z", + }, + ]), + ), + }, + }, + }); + + const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath); + + expect(result.status).toBe(0); + const restored = YAML.parse(harness.policySetBodies.at(-1)!); + expect(Object.keys(restored.network_policies).sort()).toEqual([ + "mcp_bridge_alpha", + "mcp_bridge_beta", + "restrictive_baseline", + ]); + expect(restored.network_policies.mcp_bridge_beta).toEqual(betaEntry); + }); + it("binds manual shields-up to the active auto-restore timer generation", () => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); const sandboxName = "openclaw"; diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 3202c998986..f9942127fe3 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -58,7 +58,11 @@ const { resolveNemoclawStateDir } = require("../state/paths"); const { appendAuditEntry } = require("./audit"); const { resolveAgentConfig } = require("../sandbox/config"); const { + buildRuntimeManagedMcpPolicy, buildRuntimePermissivePolicy, + hasManagedMcpPolicyClaims, + inspectExactManagedMcpPolicies, + isManagedMcpPolicyKey, }: typeof import("./permissive-runtime") = require("./permissive-runtime"); const { cleanupTempDir } = require("../onboard/temp-files"); const { verifyShieldsLockState }: typeof import("./verify-lock") = require("./verify-lock"); @@ -716,6 +720,8 @@ interface ShieldsState { shieldsDownReason?: string | null; shieldsDownPolicy?: string | null; shieldsPolicySnapshotPath?: string | null; + /** Exact generated MCP keys owned in the restrictive snapshot. */ + shieldsManagedMcpPolicyKeys?: string[]; chattrApplied?: boolean; // SHA-256 seal of each locked file, captured by `shields up` after the // lock verification passes. `shields status` re-hashes the same files @@ -946,6 +952,17 @@ function isOptionalHashMap(value: unknown): value is { [path: string]: string } return true; } +function isOptionalManagedMcpPolicyKeys(value: unknown): value is string[] | undefined { + if (value === undefined) return true; + if (!Array.isArray(value) || value.length > 256) return false; + const keys = new Set(); + for (const key of value) { + if (!isManagedMcpPolicyKey(key) || keys.has(key)) return false; + keys.add(key); + } + return true; +} + function isShieldsState(value: unknown): value is ShieldsState { return ( isObjectRecord(value) && @@ -955,6 +972,7 @@ function isShieldsState(value: unknown): value is ShieldsState { isOptionalNullableString(value.shieldsDownReason) && isOptionalNullableString(value.shieldsDownPolicy) && isOptionalNullableString(value.shieldsPolicySnapshotPath) && + isOptionalManagedMcpPolicyKeys(value.shieldsManagedMcpPolicyKeys) && isOptionalBoolean(value.chattrApplied) && isOptionalHashMap(value.fileHashes) && isOptionalString(value.updatedAt) @@ -2226,9 +2244,7 @@ function synchronizeAutoRestoreTransition( // above waits until the forward path has either committed its last weakening // mutation or its owner has died; restore the restrictive snapshot again at // that stable boundary before locking config. - const restoreResult = run(buildPolicySetCommand(transition.snapshotPath, sandboxName), { - ignoreError: true, - }); + const restoreResult = applyShieldsPolicySnapshot(sandboxName, transition.snapshotPath); const status = typeof restoreResult.status === "number" ? restoreResult.status : 1; if (status !== 0) { throw new Error( @@ -2330,6 +2346,73 @@ function lockAgentConfig( }); } +function resolveExactManagedMcpPolicies( + sandboxName: string, + livePolicyYaml?: string, +): ReturnType { + if (!hasManagedMcpPolicyClaims(sandboxName)) return []; + + let effectiveLivePolicy = livePolicyYaml; + if (!effectiveLivePolicy) { + let rawPolicy: string; + try { + rawPolicy = runCapture(buildPolicyGetCommand(sandboxName)); + } catch (error) { + throw new Error("Cannot read the live gateway policy for managed MCP reconciliation", { + cause: error, + }); + } + effectiveLivePolicy = parseCurrentPolicy(rawPolicy); + } + if (!effectiveLivePolicy) { + throw new Error("Cannot parse the live gateway policy for managed MCP reconciliation"); + } + return inspectExactManagedMcpPolicies(sandboxName, effectiveLivePolicy); +} + +/** + * Restore a saved complete policy while reconciling only exact generated MCP + * entries. Snapshot-time keys are removed before current owned entries are + * overlaid, so changes made during the Shields-down window survive both manual + * and timer restoration. + */ +function applyShieldsPolicySnapshot( + sandboxName: string, + snapshotPath: string, +): ReturnType { + const state = loadShieldsState(sandboxName); + const snapshotManagedPolicyKeys = state.shieldsManagedMcpPolicyKeys; + // A timer or manual restore created by an older NemoClaw build has no exact + // snapshot-time ownership manifest. Preserve its prior raw-snapshot behavior + // instead of guessing from lifetime MCP tombstones or deleting an unowned + // same-prefix key. + if (snapshotManagedPolicyKeys === undefined) { + return run(buildPolicySetCommand(snapshotPath, sandboxName), { + ignoreError: true, + }); + } + if (state.shieldsPolicySnapshotPath !== snapshotPath) { + throw new Error("Saved managed MCP ownership does not match the Shields policy snapshot"); + } + + const managedMcpPolicies = resolveExactManagedMcpPolicies(sandboxName); + const runtimePolicyPath = buildRuntimeManagedMcpPolicy(snapshotPath, { + managedMcpPolicies, + snapshotManagedPolicyKeys, + readBasePolicy: () => fs.readFileSync(snapshotPath, "utf-8"), + }); + const runtimePolicyIsTemp = runtimePolicyPath !== snapshotPath; + try { + return run(buildPolicySetCommand(runtimePolicyPath, sandboxName), { + ignoreError: true, + }); + } finally { + if (runtimePolicyIsTemp) { + cleanupTempDir(runtimePolicyPath, "nemoclaw-permissive-runtime"); + } + } +} + function rollbackShieldsDown( sandboxName: string, target: AgentConfigTarget, @@ -2338,12 +2421,16 @@ function rollbackShieldsDown( cachedProtocol?: HermesShieldsProtocol, ): void { console.error(" Rolling back — restoring policy from snapshot..."); - const rollbackResult = run(buildPolicySetCommand(snapshotPath, sandboxName), { - ignoreError: true, - }); + let rollbackResult: ReturnType | null = null; + try { + rollbackResult = applyShieldsPolicySnapshot(sandboxName, snapshotPath); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(` Warning: Policy restore preparation failed during rollback: ${message}`); + } let rollbackChattrApplied: boolean | null = null; let rollbackFileHashes: { [path: string]: string } | null = null; - if (rollbackResult.status === 0) { + if (rollbackResult?.status === 0) { // Re-confirm after the settle window so a reconciler revert cannot leave // the rolled-back config DRIFTED — same fail-closed treatment as the // auto-restore path. Leaves the hashes null (→ "manual intervention" @@ -2397,9 +2484,17 @@ function activateLockdownFromSnapshot( return { ok: false, error: "saved snapshot is missing" }; } - const restoreResult = run(buildPolicySetCommand(snapshotPath, sandboxName), { - ignoreError: true, - }); + let restoreResult: ReturnType; + try { + restoreResult = applyShieldsPolicySnapshot(sandboxName, snapshotPath); + } catch (error) { + return { + ok: false, + error: `policy restore preparation failed: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } const restoreStatus = typeof restoreResult.status === "number" ? restoreResult.status : 1; if (restoreStatus !== 0) { return { @@ -2644,6 +2739,19 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = return failShieldsCommand("Cannot capture current policy", opts.throwOnError); } + let managedMcpPolicies: ReturnType; + try { + managedMcpPolicies = resolveExactManagedMcpPolicies(sandboxName, policyYaml); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(` Cannot preserve managed MCP policy state: ${message}`); + return failShieldsCommand( + `Cannot preserve managed MCP policy state: ${message}`, + opts.throwOnError, + ); + } + const snapshotManagedMcpPolicyKeys = managedMcpPolicies.map((policy) => policy.key); + const ts = Date.now(); const snapshotPath = path.join(STATE_DIR, `policy-snapshot-${ts}.yaml`); fs.mkdirSync(STATE_DIR, { recursive: true, mode: 0o700 }); @@ -2653,25 +2761,40 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = // 2. Determine and apply relaxed policy let policyFile: string; let policyFileIsTemp = false; - if (policyName === "permissive") { - const basePath = resolvePermissivePolicyPath(sandboxName); - // Union the live sandbox's filesystem_policy.read_only/read_write into - // the static permissive baseline. OpenShell rejects removal of those - // paths on a live sandbox, and runtime-injected entries (/proc on - // GPU, /opt/hermes on Hermes, /home/linuxbrew on post-#3913 OpenClaw, - // etc.) are not present in the static YAML. See #3942, #3957, #3168. - // policyYaml is the pre-parsed body we already captured for the - // snapshot above — reuse it instead of re-fetching. - policyFile = buildRuntimePermissivePolicy(basePath, { - livePolicyYaml: policyYaml, - readBasePolicy: () => fs.readFileSync(basePath, "utf-8"), - }); - policyFileIsTemp = policyFile !== basePath; - } else if (fs.existsSync(policyName)) { - policyFile = path.resolve(policyName); - } else { - console.error(` Unknown policy "${policyName}". Use "permissive" or a path to a YAML file.`); - return failShieldsCommand(`Unknown policy "${policyName}"`, opts.throwOnError); + try { + if (policyName === "permissive") { + const basePath = resolvePermissivePolicyPath(sandboxName); + // Union the live sandbox's filesystem_policy.read_only/read_write into + // the static permissive baseline. OpenShell rejects removal of those + // paths on a live sandbox, and runtime-injected entries (/proc on + // GPU, /opt/hermes on Hermes, /home/linuxbrew on post-#3913 OpenClaw, + // etc.) are not present in the static YAML. See #3942, #3957, #3168. + // policyYaml is the pre-parsed body we already captured for the + // snapshot above — reuse it instead of re-fetching. Exact generated MCP + // entries are overlaid without copying any unrelated live egress. + policyFile = buildRuntimePermissivePolicy(basePath, { + livePolicyYaml: policyYaml, + managedMcpPolicies, + readBasePolicy: () => fs.readFileSync(basePath, "utf-8"), + }); + policyFileIsTemp = policyFile !== basePath; + } else if (fs.existsSync(policyName)) { + const basePath = path.resolve(policyName); + policyFile = buildRuntimeManagedMcpPolicy(basePath, { + managedMcpPolicies, + readBasePolicy: () => fs.readFileSync(basePath, "utf-8"), + }); + policyFileIsTemp = policyFile !== basePath; + } else { + console.error(` Unknown policy "${policyName}". Use "permissive" or a path to a YAML file.`); + fs.rmSync(snapshotPath, { force: true }); + return failShieldsCommand(`Unknown policy "${policyName}"`, opts.throwOnError); + } + } catch (error) { + fs.rmSync(snapshotPath, { force: true }); + const message = error instanceof Error ? error.message : String(error); + console.error(` Cannot compose Shields-down policy: ${message}`); + return failShieldsCommand(`Cannot compose Shields-down policy: ${message}`, opts.throwOnError); } const now = new Date().toISOString(); @@ -2776,6 +2899,7 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = shieldsDownReason: reason, shieldsDownPolicy: policyName, shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: snapshotManagedMcpPolicyKeys, }); } catch (error) { if (transition) { @@ -3402,6 +3526,7 @@ function clearShieldsState(sandboxName: string): void { // --------------------------------------------------------------------------- export { + applyShieldsPolicySnapshot, clearShieldsState, DEFAULT_TIMEOUT_SECONDS, deriveShieldsMode, diff --git a/src/lib/shields/mcp-policy-transition.test.ts b/src/lib/shields/mcp-policy-transition.test.ts new file mode 100644 index 00000000000..458e7a777fc --- /dev/null +++ b/src/lib/shields/mcp-policy-transition.test.ts @@ -0,0 +1,227 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import YAML from "yaml"; +import { describe, expect, it } from "vitest"; + +import { + inspectExactManagedMcpPolicies as inspectRegisteredManagedMcpPolicies, + MCP_BRIDGE_POLICY_SOURCE, +} from "../actions/sandbox/mcp-bridge-policy"; +import { + buildMcpBridgePolicyKey, + buildMcpBridgePolicyName, + buildMcpBridgePolicyYaml, +} from "../actions/sandbox/mcp-bridge-policy-render"; +import type { SandboxEntry } from "../state/registry"; +import { composeManagedMcpPolicies } from "./mcp-policy-transition"; + +const ADAPTER = "hermes-config"; + +function registeredPolicy( + server: string, + address: string, +): NonNullable[number] { + return { + name: buildMcpBridgePolicyName(server), + content: buildMcpBridgePolicyYaml(server, `https://${server}.example.com/mcp`, ADAPTER, [ + address, + ]), + sourcePath: MCP_BRIDGE_POLICY_SOURCE, + }; +} + +function bridge(server: string): NonNullable["bridges"]>[string] { + return { + server, + agent: "hermes", + adapter: ADAPTER, + url: `https://${server}.example.com/mcp`, + env: ["MCP_SECRET"], + providerName: `sandbox-mcp-${server}`, + providerId: `provider-${server}`, + policyName: buildMcpBridgePolicyName(server), + addedAt: "2026-07-30T00:00:00.000Z", + }; +} + +function sandboxWithPolicies( + policies: Array>, + bridgeServers = policies.map((policy) => policy.name.replace(/^mcp-bridge-/, "")), +): SandboxEntry { + return { + name: "alpha", + agent: "hermes", + customPolicies: policies, + mcp: { + bridges: Object.fromEntries(bridgeServers.map((server) => [server, bridge(server)])), + }, + }; +} + +function networkEntry(content: string, server: string): unknown { + return YAML.parse(content).network_policies[buildMcpBridgePolicyKey(server)]; +} + +function livePolicy( + entries: Array<{ content: string; server: string }>, + extra: Record = {}, +): string { + return YAML.stringify({ + version: 1, + network_policies: { + ...extra, + ...Object.fromEntries( + entries.map(({ content, server }) => [ + buildMcpBridgePolicyKey(server), + networkEntry(content, server), + ]), + ), + }, + }); +} + +function inspectExactManagedMcpPolicies(sandbox: SandboxEntry, livePolicyYaml: string) { + return inspectRegisteredManagedMcpPolicies("alpha", livePolicyYaml, { + getSandbox: () => sandbox, + }); +} + +describe("managed MCP Shields policy transitions (#7952)", () => { + it("admits only canonical committed registrations that exactly match the live policy", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const sandbox = sandboxWithPolicies([alpha]); + + const exact = inspectExactManagedMcpPolicies( + sandbox, + livePolicy([{ content: alpha.content, server: "alpha" }], { + unrelated_live_entry: { endpoints: [{ host: "unrelated.example.com" }] }, + }), + ); + + expect(exact).toEqual([ + expect.objectContaining({ + key: "mcp_bridge_alpha", + policyName: "mcp-bridge-alpha", + server: "alpha", + }), + ]); + }); + + it.each([ + { + label: "pending policy content", + mutate: (sandbox: SandboxEntry) => { + sandbox.customPolicies![0]!.pendingContent = sandbox.customPolicies![0]!.content; + }, + expected: /incomplete policy transition/, + }, + { + label: "an orphaned generated registration", + mutate: (sandbox: SandboxEntry) => { + sandbox.customPolicies!.push(registeredPolicy("orphan", "1.1.1.1")); + }, + expected: /no committed managed bridge ownership/, + }, + { + label: "an incomplete bridge add", + mutate: (sandbox: SandboxEntry) => { + sandbox.mcp!.bridges.alpha!.addState = "prepared"; + }, + expected: /lifecycle transition is incomplete/, + }, + ])("fails closed on $label", ({ mutate, expected }) => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const sandbox = sandboxWithPolicies([alpha]); + mutate(sandbox); + + expect(() => + inspectExactManagedMcpPolicies( + sandbox, + livePolicy( + (sandbox.customPolicies ?? []).map((policy) => ({ + content: policy.content, + server: policy.name.replace(/^mcp-bridge-/, ""), + })), + ), + ), + ).toThrow(expected); + }); + + it("fails closed when the live policy differs from the ownership record", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const drifted = registeredPolicy("alpha", "1.1.1.1"); + + expect(() => + inspectExactManagedMcpPolicies( + sandboxWithPolicies([alpha]), + livePolicy([{ content: drifted.content, server: "alpha" }]), + ), + ).toThrow(/drifted from its ownership record/); + }); + + it("retains additions while restoring the restrictive snapshot", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const beta = registeredPolicy("beta", "1.1.1.1"); + const current = inspectExactManagedMcpPolicies( + sandboxWithPolicies([alpha, beta]), + livePolicy([ + { content: alpha.content, server: "alpha" }, + { content: beta.content, server: "beta" }, + ]), + ); + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + mcp_bridge_alpha: networkEntry(alpha.content, "alpha"), + }, + }); + + const restored = YAML.parse(composeManagedMcpPolicies(snapshot, current, ["mcp_bridge_alpha"])); + + expect(Object.keys(restored.network_policies).sort()).toEqual([ + "mcp_bridge_alpha", + "mcp_bridge_beta", + "restrictive_baseline", + ]); + }); + + it("does not resurrect a managed MCP policy removed while Shields are down", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + mcp_bridge_alpha: networkEntry(alpha.content, "alpha"), + }, + }); + + const restored = YAML.parse(composeManagedMcpPolicies(snapshot, [], ["mcp_bridge_alpha"])); + + expect(restored.network_policies).toEqual({ + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + }); + }); + + it("replaces a stale snapshot entry with the current exact registration", () => { + const oldAlpha = registeredPolicy("alpha", "8.8.8.8"); + const currentAlpha = registeredPolicy("alpha", "1.1.1.1"); + const current = inspectExactManagedMcpPolicies( + sandboxWithPolicies([currentAlpha]), + livePolicy([{ content: currentAlpha.content, server: "alpha" }]), + ); + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + mcp_bridge_alpha: networkEntry(oldAlpha.content, "alpha"), + }, + }); + + const restored = YAML.parse(composeManagedMcpPolicies(snapshot, current, ["mcp_bridge_alpha"])); + + expect(restored.network_policies.mcp_bridge_alpha).toEqual( + networkEntry(currentAlpha.content, "alpha"), + ); + }); +}); diff --git a/src/lib/shields/mcp-policy-transition.ts b/src/lib/shields/mcp-policy-transition.ts new file mode 100644 index 00000000000..c06bc0efdcc --- /dev/null +++ b/src/lib/shields/mcp-policy-transition.ts @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import YAML from "yaml"; + +import type { ExactManagedMcpPolicy } from "../actions/sandbox/mcp-bridge-policy"; + +const MANAGED_MCP_POLICY_KEY_RE = /^mcp_bridge_[a-z][a-z0-9_]{0,63}$/; + +function parsePolicyDocument(source: string, label: string): Record { + let parsed: unknown; + try { + parsed = YAML.parse(source); + } catch { + throw new Error(`${label} is not valid YAML`); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error(`${label} must be a YAML mapping`); + } + return parsed as Record; +} + +function readNetworkPolicies( + document: Record, + label: string, +): Record { + const policies = document.network_policies; + if (policies === undefined || policies === null) return {}; + if (typeof policies !== "object" || Array.isArray(policies)) { + throw new Error(`${label} network_policies must be a mapping`); + } + return policies as Record; +} + +/** + * Reconcile generated MCP entries into a complete target policy. + * + * Snapshot-time keys are removed first so an MCP server deleted while Shields + * are down cannot be resurrected. The current exact entries are then overlaid, + * retaining additions and replacing stale pins. Every non-MCP target entry + * remains authoritative; unrelated live entries are never copied. + */ +export function composeManagedMcpPolicies( + targetPolicyYaml: string, + currentPolicies: readonly ExactManagedMcpPolicy[], + snapshotManagedPolicyKeys: readonly string[] = [], +): string { + const target = parsePolicyDocument(targetPolicyYaml, "Target Shields policy"); + const targetPolicies = readNetworkPolicies(target, "Target Shields policy"); + + const snapshotKeys = new Set(); + for (const key of snapshotManagedPolicyKeys) { + if (!MANAGED_MCP_POLICY_KEY_RE.test(key) || snapshotKeys.has(key)) { + throw new Error("Saved Shields MCP policy ownership is invalid"); + } + snapshotKeys.add(key); + delete targetPolicies[key]; + } + + const currentKeys = new Set(); + for (const policy of currentPolicies) { + if (!MANAGED_MCP_POLICY_KEY_RE.test(policy.key) || currentKeys.has(policy.key)) { + throw new Error(`Managed MCP policy key '${policy.key}' has ambiguous ownership`); + } + currentKeys.add(policy.key); + targetPolicies[policy.key] = policy.networkPolicy; + } + + target.network_policies = targetPolicies; + return YAML.stringify(target); +} + +export function isManagedMcpPolicyKey(value: unknown): value is string { + return typeof value === "string" && MANAGED_MCP_POLICY_KEY_RE.test(value); +} diff --git a/src/lib/shields/permissive-runtime.ts b/src/lib/shields/permissive-runtime.ts index 46f523f60be..10232613a28 100644 --- a/src/lib/shields/permissive-runtime.ts +++ b/src/lib/shields/permissive-runtime.ts @@ -4,7 +4,15 @@ import fs from "node:fs"; import YAML from "yaml"; +export { + hasManagedMcpPolicyClaims, + inspectExactManagedMcpPolicies, + type ExactManagedMcpPolicy, +} from "../actions/sandbox/mcp-bridge-policy"; +import type { ExactManagedMcpPolicy } from "../actions/sandbox/mcp-bridge-policy"; import { cleanupTempDir, secureTempFile } from "../onboard/temp-files"; +export { isManagedMcpPolicyKey } from "./mcp-policy-transition"; +import { composeManagedMcpPolicies } from "./mcp-policy-transition"; const TEMP_FILE_PREFIX = "nemoclaw-permissive-runtime"; @@ -60,6 +68,10 @@ export interface PermissiveRuntimeDeps { // secureTempFile when omitted. Exposed so tests can drive the // write-failure fallback path without monkey-patching node:fs. writeTempPolicy?: (yaml: string) => string; + // Exact, live-matching generated MCP policies resolved by the Shields + // coordinator. These entries remain active while the static policy replaces + // the rest of the complete gateway policy. + managedMcpPolicies?: readonly ExactManagedMcpPolicy[]; } export function buildRuntimePermissivePolicy( @@ -69,21 +81,31 @@ export function buildRuntimePermissivePolicy( const live = deps.livePolicyYaml ? safeYamlObject(deps.livePolicyYaml) : null; const liveRw = readStringList(live, "read_write"); const liveRo = readStringList(live, "read_only"); + const managedMcpPolicies = deps.managedMcpPolicies ?? []; // No live filesystem section to merge — keep the static path so the - // caller's apply path is unchanged. - if (liveRw.length === 0 && liveRo.length === 0) { + // caller's apply path is unchanged unless exact managed MCP entries must + // survive the complete-policy replacement. + if (liveRw.length === 0 && liveRo.length === 0 && managedMcpPolicies.length === 0) { return basePermissivePath; } let baseYaml: string; try { baseYaml = deps.readBasePolicy(); - } catch { + } catch (error) { + if (managedMcpPolicies.length > 0) { + throw new Error("Cannot read the Shields-down policy while managed MCP policies are active", { + cause: error, + }); + } return basePermissivePath; } const base = safeYamlObject(baseYaml); if (!base) { + if (managedMcpPolicies.length > 0) { + throw new Error("Cannot parse the Shields-down policy while managed MCP policies are active"); + } return basePermissivePath; } const fsPolicy = @@ -108,11 +130,17 @@ export function buildRuntimePermissivePolicy( fsPolicy.read_write = [...baseRw]; fsPolicy.read_only = [...baseRo]; - const yaml = YAML.stringify(base); + const yaml = composeManagedMcpPolicies(YAML.stringify(base), managedMcpPolicies); if (deps.writeTempPolicy) { try { return deps.writeTempPolicy(yaml); - } catch { + } catch (error) { + if (managedMcpPolicies.length > 0) { + throw new Error( + "Cannot stage the Shields-down policy while managed MCP policies are active", + { cause: error }, + ); + } return basePermissivePath; } } @@ -121,15 +149,80 @@ export function buildRuntimePermissivePolicy( tmpPath = secureTempFile(TEMP_FILE_PREFIX, ".yaml"); fs.writeFileSync(tmpPath, yaml, { mode: 0o600 }); return tmpPath; - } catch { + } catch (error) { // secureTempFile may have created an mkdtemp directory before // writeFileSync failed. Clean it up so we do not leak a 0700 dir // on /tmp every time the write path errors. if (tmpPath) cleanupTempDir(tmpPath, TEMP_FILE_PREFIX); + if (managedMcpPolicies.length > 0) { + throw new Error( + "Cannot stage the Shields-down policy while managed MCP policies are active", + { cause: error }, + ); + } return basePermissivePath; } } +export interface ManagedMcpRuntimePolicyDeps { + managedMcpPolicies: readonly ExactManagedMcpPolicy[]; + readBasePolicy: () => string; + snapshotManagedPolicyKeys?: readonly string[]; + writeTempPolicy?: (yaml: string) => string; +} + +/** + * Reconcile current generated MCP policies into a custom Shields-down policy + * or a saved restrictive snapshot. Unlike the legacy filesystem-only fallback, + * this path must fail closed: returning the unmodified base could silently + * discard a managed entry or resurrect one that was removed while Shields were + * down. + */ +export function buildRuntimeManagedMcpPolicy( + basePolicyPath: string, + deps: ManagedMcpRuntimePolicyDeps, +): string { + const snapshotManagedPolicyKeys = deps.snapshotManagedPolicyKeys ?? []; + if (deps.managedMcpPolicies.length === 0 && snapshotManagedPolicyKeys.length === 0) { + return basePolicyPath; + } + + let baseYaml: string; + try { + baseYaml = deps.readBasePolicy(); + } catch (error) { + throw new Error("Cannot read the Shields policy for managed MCP reconciliation", { + cause: error, + }); + } + const yaml = composeManagedMcpPolicies( + baseYaml, + deps.managedMcpPolicies, + snapshotManagedPolicyKeys, + ); + if (deps.writeTempPolicy) { + try { + return deps.writeTempPolicy(yaml); + } catch (error) { + throw new Error("Cannot stage the Shields policy for managed MCP reconciliation", { + cause: error, + }); + } + } + + let tmpPath: string | null = null; + try { + tmpPath = secureTempFile(TEMP_FILE_PREFIX, ".yaml"); + fs.writeFileSync(tmpPath, yaml, { mode: 0o600 }); + return tmpPath; + } catch (error) { + if (tmpPath) cleanupTempDir(tmpPath, TEMP_FILE_PREFIX); + throw new Error("Cannot stage the Shields policy for managed MCP reconciliation", { + cause: error, + }); + } +} + function safeYamlObject(text: string): Record | null { try { const parsed = YAML.parse(text); diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index e244dd8104a..26f126e1dbd 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -9,31 +9,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { getMcpLifecycleLockPath } from "../state/mcp-lifecycle-lock"; const shieldsIndexMock = vi.hoisted(() => ({ + applyShieldsPolicySnapshot: vi.fn(() => ({ status: 0 })), lockAgentConfig: vi.fn() as unknown, prepareAutoRestoreTransitionTakeover: vi.fn(), })); const PROCESS_TOKEN = "a".repeat(32); -const runMock = vi.fn(() => ({ status: 0 })); - -vi.mock("../runner", async (importOriginal) => ({ - ...(await importOriginal()), - run: runMock, -})); - -vi.mock("../policy", () => ({ - buildPolicySetCommand: vi.fn((file: string, name: string) => [ - "openshell", - "policy", - "set", - "--policy", - file, - "--wait", - name, - ]), -})); - vi.mock("../sandbox/config", () => ({ DEFAULT_AGENT_CONFIG: Symbol("DEFAULT_AGENT_CONFIG"), resolveAgentConfig: vi.fn(() => ({ @@ -43,6 +25,7 @@ vi.mock("../sandbox/config", () => ({ })); vi.mock("./index", () => ({ + applyShieldsPolicySnapshot: shieldsIndexMock.applyShieldsPolicySnapshot, get lockAgentConfig() { return shieldsIndexMock.lockAgentConfig; }, @@ -55,6 +38,7 @@ describe("shields timer authorization", () => { beforeEach(() => { tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "shields-timer-")); vi.stubEnv("HOME", tmpHome); + shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementation(() => ({ status: 0 })); shieldsIndexMock.lockAgentConfig = vi.fn(); vi.resetModules(); vi.clearAllMocks(); @@ -124,7 +108,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); }); @@ -166,7 +150,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); expect(fs.existsSync(markerPath)).toBe(true); }); @@ -263,7 +247,7 @@ describe("shields timer authorization", () => { await timer.runRestoreTimer(args!); - expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); expect(exitSpy).not.toHaveBeenCalled(); expect(vi.getTimerCount()).toBe(1); expect(fs.existsSync(markerPath)).toBe(true); @@ -300,7 +284,7 @@ describe("shields timer authorization", () => { leaseOwnerStartIdentity: "proc:dead-owner", }), ); - runMock.mockImplementationOnce(() => { + shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementationOnce(() => { expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); return { status: 17 }; }); @@ -319,7 +303,7 @@ describe("shields timer authorization", () => { await timer.runRestoreTimer(args!); - expect(runMock).toHaveBeenCalledTimes(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); expect(exitSpy).not.toHaveBeenCalled(); expect(vi.getTimerCount()).toBe(1); expect(fs.existsSync(markerPath)).toBe(true); @@ -368,7 +352,7 @@ describe("shields timer authorization", () => { const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); expect(exitCode).toBe(0); - expect(runMock).not.toHaveBeenCalled(); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); expect(fs.existsSync(markerPath)).toBe(true); }); @@ -453,7 +437,7 @@ describe("shields timer authorization", () => { expect(args).not.toBeNull(); const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); - runMock.mockImplementationOnce(() => { + shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementationOnce(() => { expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); expect(JSON.parse(fs.readFileSync(lockPath, "utf-8"))).toMatchObject({ sandboxName, @@ -468,7 +452,7 @@ describe("shields timer authorization", () => { const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); expect(exitCode).toBe(0); - expect(runMock).toHaveBeenCalledTimes(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); expect(updatedState.shieldsDown).toBe(false); expect(updatedState.shieldsDownAt).toBeNull(); expect(fs.existsSync(markerPath)).toBe(false); @@ -587,7 +571,7 @@ describe("shields timer authorization", () => { const updatedState = JSON.parse(fs.readFileSync(stateFile, "utf-8")); expect(exitCode).toBe(0); - expect(runMock).toHaveBeenCalledTimes(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); // #4663: relockAndReconfirm applies then re-confirms after the settle // window (0ms under test), so lockAgentConfig is invoked twice for a clean // lock. @@ -653,7 +637,7 @@ describe("shields timer authorization", () => { .split("\n") .map((line) => JSON.parse(line)); - expect(runMock).toHaveBeenCalledTimes(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); expect(updatedState.shieldsDown).toBe(true); expect(auditEntries).toContainEqual( expect.objectContaining({ diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index ded68a60723..8b8a8820209 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -11,8 +11,6 @@ import fs from "node:fs"; import path from "node:path"; import { isObjectRecord, type UnknownRecord } from "../core/json-types"; -import { buildPolicySetCommand } from "../policy"; -import { run } from "../runner"; import { resolveAgentConfig } from "../sandbox/config"; import { withSandboxMutationLock } from "../state/mcp-lifecycle-lock"; import { resolveNemoclawStateDir } from "../state/paths"; @@ -291,9 +289,7 @@ async function runRestoreTimer(args: TimerArgs): Promise { } // Restore policy (slow — openshell policy set --wait blocks) - const result = run(buildPolicySetCommand(args.snapshotPath, args.sandboxName), { - ignoreError: true, - }); + const result = shields.applyShieldsPolicySnapshot(args.sandboxName, args.snapshotPath); const status = typeof result.status === "number" ? result.status : 1; if (status !== 0) { diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 177165258c1..b0001d63b98 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -1170,6 +1170,13 @@ mcpBridgeShardTest("hermes")( challenge: TOOL_CHALLENGE, resultToken: hermesResult, }); + const assertHermesToolCall = (artifactName: string) => + assertRealAdapterToolCall(sandbox, fakeMcp, { + agent: "hermes", + sandboxName: HERMES_SANDBOX_NAME, + resultToken: hermesResult, + artifactName, + }); cleanup.add("stop fake Hermes MCP HTTPS server", () => fakeMcp.close()); const fakeMcpTunnel = await startPublicMcpHttpsTunnel({ cleanup, @@ -1231,6 +1238,7 @@ mcpBridgeShardTest("hermes")( HERMES_SANDBOX_NAME, mcpUrl, ); + await assertHermesToolCall("hermes-real-mcp-tool-call-immediately-after-shields-down"); await assertSecretAbsentFromSandbox( sandbox, HERMES_SANDBOX_NAME, @@ -1250,14 +1258,10 @@ mcpBridgeShardTest("hermes")( sandboxName: HERMES_SANDBOX_NAME, secretPaths: ["/sandbox/.hermes"], }); + await assertHermesToolCall("hermes-real-mcp-tool-call-after-dns-rebinding-remove"); const survivingDiscoveryOffset = fakeMcp.requests.length; await restartBridgeWithoutHostSecret(host, HERMES_SANDBOX_NAME, "hermes"); - await assertRealAdapterToolCall(sandbox, fakeMcp, { - agent: "hermes", - sandboxName: HERMES_SANDBOX_NAME, - resultToken: hermesResult, - artifactName: "hermes-real-mcp-tool-call-after-rediscovery-restart", - }); + await assertHermesToolCall("hermes-real-mcp-tool-call-after-rediscovery-restart"); await assertAuthenticatedMcpRediscovery(survivingMcp, survivingDiscoveryOffset); fakeMcp.setSecret(ROTATED_HOST_SECRET); await rotateBridgeCredential(host, HERMES_SANDBOX_NAME, "hermes"); diff --git a/test/e2e/support/mcp-bridge-sandbox.test.ts b/test/e2e/support/mcp-bridge-sandbox.test.ts index 4346ae4b62b..9688afaa7fe 100644 --- a/test/e2e/support/mcp-bridge-sandbox.test.ts +++ b/test/e2e/support/mcp-bridge-sandbox.test.ts @@ -317,29 +317,51 @@ network_policies: expect(source).toContain(").toHaveLength(0);"); }); - it("captures the Hermes rediscovery offset after route removal and before restart", () => { + it("proves the surviving Hermes route before and after the unrelated route lifecycle", () => { const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); const denialProof = source.indexOf("rebound request must not reach the upstream MCP server"); const restore = source.indexOf("await restoreDnsRebindingHostsFixture", denialProof); const remove = source.indexOf("const remove = await host.nemoclaw", denialProof); const hermesTest = source.indexOf('mcpBridgeShardTest("hermes")'); + const shieldsDown = source.indexOf( + "assertHermesManagedAddSurvivesLockedGatewayRestartAndStateLayout", + hermesTest, + ); + const afterShieldsDownToolCall = source.indexOf( + "hermes-real-mcp-tool-call-immediately-after-shields-down", + shieldsDown, + ); const rebinding = source.indexOf("await assertAdapterDnsRebindingDenied", hermesTest); + const afterRemoveToolCall = source.indexOf( + "hermes-real-mcp-tool-call-after-dns-rebinding-remove", + rebinding, + ); const offset = source.indexOf( "const survivingDiscoveryOffset = fakeMcp.requests.length", - rebinding, + afterRemoveToolCall, ); const restart = source.indexOf("await restartBridgeWithoutHostSecret", offset); - const toolCall = source.indexOf("await assertRealAdapterToolCall", restart); - const rediscovery = source.indexOf("await assertAuthenticatedMcpRediscovery", toolCall); + const afterRestartToolCall = source.indexOf( + "hermes-real-mcp-tool-call-after-rediscovery-restart", + restart, + ); + const rediscovery = source.indexOf( + "await assertAuthenticatedMcpRediscovery", + afterRestartToolCall, + ); expect(denialProof).toBeGreaterThanOrEqual(0); expect(restore).toBeGreaterThan(denialProof); expect(remove).toBeGreaterThan(restore); + expect(shieldsDown).toBeGreaterThan(hermesTest); + expect(afterShieldsDownToolCall).toBeGreaterThan(shieldsDown); expect(rebinding).toBeGreaterThan(hermesTest); - expect(offset).toBeGreaterThan(rebinding); + expect(rebinding).toBeGreaterThan(afterShieldsDownToolCall); + expect(afterRemoveToolCall).toBeGreaterThan(rebinding); + expect(offset).toBeGreaterThan(afterRemoveToolCall); expect(restart).toBeGreaterThan(offset); - expect(toolCall).toBeGreaterThan(restart); - expect(rediscovery).toBeGreaterThan(toolCall); + expect(afterRestartToolCall).toBeGreaterThan(restart); + expect(rediscovery).toBeGreaterThan(afterRestartToolCall); expect(source).toContain("Hermes MCP rediscovery after explicit restart"); }); diff --git a/test/permissive-runtime.test.ts b/test/permissive-runtime.test.ts index 504d9a2898d..01b3f91212b 100644 --- a/test/permissive-runtime.test.ts +++ b/test/permissive-runtime.test.ts @@ -7,7 +7,10 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import YAML from "yaml"; -import { buildRuntimePermissivePolicy } from "../src/lib/shields/permissive-runtime.js"; +import { + buildRuntimePermissivePolicy, + type ExactManagedMcpPolicy, +} from "../src/lib/shields/permissive-runtime.js"; const BASE_PERMISSIVE = YAML.stringify({ filesystem_policy: { @@ -46,6 +49,51 @@ afterEach(() => { }); describe("buildRuntimePermissivePolicy (#3942)", () => { + it("preserves exact managed MCP entries without copying unrelated live egress (#7952)", () => { + const managedPolicy: ExactManagedMcpPolicy = { + key: "mcp_bridge_alpha", + networkPolicy: { + endpoints: [{ host: "alpha.example.com", port: 443, protocol: "mcp" }], + binaries: [{ path: "/opt/hermes/.venv/bin/python*" }], + }, + policyName: "mcp-bridge-alpha", + server: "alpha", + }; + const liveYaml = YAML.stringify({ + filesystem_policy: { read_write: ["/proc"] }, + network_policies: { + mcp_bridge_alpha: managedPolicy.networkPolicy, + unrelated_live_entry: { + endpoints: [{ host: "unrelated.example.com", port: 443 }], + }, + }, + }); + + const out = buildRuntimePermissivePolicy("/unused-base.yaml", { + livePolicyYaml: liveYaml, + managedMcpPolicies: [managedPolicy], + readBasePolicy: () => + YAML.stringify({ + ...YAML.parse(BASE_PERMISSIVE), + network_policies: { + permissive_baseline: { + endpoints: [{ host: "*", port: 443 }], + }, + }, + }), + }); + trackTempForCleanup(out, "/unused-base.yaml"); + + const result = YAML.parse(fs.readFileSync(out, "utf-8")); + expect(result.network_policies).toMatchObject({ + mcp_bridge_alpha: managedPolicy.networkPolicy, + permissive_baseline: { + endpoints: [{ host: "*", port: 443 }], + }, + }); + expect(result.network_policies).not.toHaveProperty("unrelated_live_entry"); + }); + it("preserves /proc when the live GPU sandbox has it in read_write", () => { const liveYaml = YAML.stringify({ filesystem_policy: { @@ -167,6 +215,25 @@ describe("buildRuntimePermissivePolicy (#3942)", () => { expect(out).toBe(basePath); }); + it("fails closed when the base cannot be read with managed MCP policies active (#7952)", () => { + expect(() => + buildRuntimePermissivePolicy("/path/to/static.yaml", { + livePolicyYaml: "version: 1\nnetwork_policies: {}\n", + managedMcpPolicies: [ + { + key: "mcp_bridge_alpha", + networkPolicy: {}, + policyName: "mcp-bridge-alpha", + server: "alpha", + }, + ], + readBasePolicy: () => { + throw new Error("ENOENT"); + }, + }), + ).toThrow(/Cannot read the Shields-down policy/); + }); + it("returns the static base path when base YAML is unparseable", () => { const basePath = "/path/to/static.yaml"; const liveYaml = YAML.stringify({ From 467259e598dd67609beff75d47d160fe352ff8cb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 00:08:24 -0400 Subject: [PATCH 02/25] fix(shields): preserve managed MCP policies --- docs/manage-sandboxes/backup-restore.mdx | 6 +- docs/manage-sandboxes/runtime-controls.mdx | 12 +- docs/reference/commands.mdx | 16 +- .../checks/openshell-policy-mutation-read.mts | 2 +- src/lib/actions/maintenance.test.ts | 52 +- src/lib/actions/maintenance.ts | 19 +- src/lib/actions/sandbox/mcp-bridge-policy.ts | 312 ++++- src/lib/shields/flow.test.ts | 741 ++++-------- src/lib/shields/index.test.ts | 73 +- src/lib/shields/index.ts | 762 ++++++++---- src/lib/shields/mcp-policy-transition.test.ts | 410 ++++++- src/lib/shields/mcp-policy-transition.ts | 107 +- src/lib/shields/permissive-runtime.ts | 63 +- src/lib/shields/timer-bound-lock.ts | 68 +- src/lib/shields/timer-control.ts | 157 +-- src/lib/shields/timer.test.ts | 273 ++++- src/lib/shields/timer.ts | 129 +- src/lib/shields/transition-lock.test.ts | 126 +- src/lib/shields/transition-lock.ts | 42 +- .../state/mcp-lifecycle-lock-acquisition.ts | 1065 ++++++++++++++++- src/lib/state/mcp-lifecycle-lock-identity.ts | 7 + src/lib/state/mcp-lifecycle-lock-storage.ts | 127 ++ src/lib/state/mcp-lifecycle-lock.ts | 9 + .../shields-timer-authority.ts | 89 ++ test/e2e/live/mcp-bridge-sandbox.ts | 76 ++ test/e2e/live/mcp-bridge.test.ts | 58 +- test/e2e/support/mcp-bridge-sandbox.test.ts | 20 +- test/mcp-lifecycle-lock.test.ts | 574 +++++++-- 28 files changed, 4152 insertions(+), 1243 deletions(-) create mode 100644 src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index 63546c38306..b0d1c7de1fb 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -83,8 +83,10 @@ NemoClaw computes versions (`v1`, `v2`, through `vN`) from timestamp order, so ` `snapshot create` requires shields to be down. Snapshot creation and restore share the per-sandbox transition lock with the shields auto-restore timer. -If a timed shields-down window expires during snapshot work, auto-restore can interrupt the operation and restore lockdown instead of allowing state or policy changes to continue past the deadline. -Retry the snapshot in a new shields-down window if the deadline interrupts it. +If a timed shields-down window expires during snapshot work, auto-restore closes the per-sandbox lifecycle deadline gate and waits for the recorded live owner to finish. +New mutations remain blocked until the snapshot owner releases its exact lock generation and auto-restore restores lockdown. +If the owner exits before releasing that generation, NemoClaw leaves Shields down, records permanent containment, and reports operator recovery guidance. +Stop all NemoClaw processes for that sandbox, then follow the exact lock-generation guidance before retrying the snapshot. Tag a snapshot with a human-readable label: diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 4aca95e49cf..82db54f47f6 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -108,10 +108,18 @@ Run `$$nemoclaw shields down` before the change, then restore lockdown wi NemoClaw serializes host-side config and inference writes, snapshot mutation, sandbox destruction, and shields transitions for each sandbox. When `shields down --timeout` is active, each mutation binds to that exact timer generation so a replaced or expired timer cannot race a later command or a new sandbox that reuses the same name. -If the timeout expires while a mutation is still changing sandbox state, auto-restore can stop that exact process tree, reclaim the transition, and restore the restrictive policy and config posture. +If the timeout expires while a mutation is still changing sandbox state, auto-restore closes the per-sandbox lifecycle deadline gate. +The gate blocks new mutations and lets the recorded live owner finish without sending it a signal. +After that owner releases its exact lock generation, auto-restore restores the restrictive policy and config posture. The ownership check includes both the process ID and process start identity so PID reuse does not grant control over an unrelated process. -Retry a command that the auto-restore deadline interrupts after you open a new shields-down window. +During restoration, NemoClaw reconciles the current exact managed MCP entries into the saved policy. +A removed MCP server stays removed, while an unrelated surviving server keeps its recorded endpoint and address pins. + +If the recorded owner exits before releasing its lock generation, NemoClaw leaves Shields down and records permanent containment. +New mutations remain blocked so an untracked descendant cannot change the sandbox after restoration. +Stop all NemoClaw processes for that sandbox, then follow the exact lock-generation recovery guidance in the reported error or audit entry. +Do not remove a recorded lifecycle lock while any NemoClaw process for that sandbox is running. ## Related Topics diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index fc7a16bc883..beee1d048bd 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1149,8 +1149,14 @@ If `shields up` reports that the config remains unlocked or drifted, confirm tha If the retry still fails, rebuild a known-good baseline with `$$nemoclaw rebuild --yes`. Host-side config and inference writes, snapshot mutation, sandbox destruction, and shields transitions serialize per sandbox. -When a timed shields-down window reaches its deadline, auto-restore can interrupt the exact process tree holding that transition and restore lockdown. -Retry an interrupted command in a new shields-down window. +When a timed shields-down window reaches its deadline, auto-restore closes the per-sandbox lifecycle deadline gate. +The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation before auto-restore restores lockdown. +NemoClaw does not signal that process because portable process inspection cannot prove that all descendants are contained. +If the owner exits before releasing the generation, NemoClaw leaves Shields down, records permanent containment, and reports operator recovery guidance. +Stop all NemoClaw processes for that sandbox, then follow the exact lock-generation guidance before retrying. + +Policy restoration reconciles the current exact managed MCP entries into the saved restrictive policy. +An MCP server removed during the shields-down window stays removed, while an unrelated surviving server keeps its recorded endpoint and address pins. @@ -2890,7 +2896,8 @@ A skipped sandbox's uncommitted state is not included in its last successful bac Create a timestamped snapshot of sandbox state. Snapshots are stored in `~/.nemoclaw/rebuild-backups//`. The command requires shields to be down and keeps the shields check and backup under one per-sandbox transition. -An expired auto-restore timer can interrupt a long-running backup and restore lockdown. +When the timer expires during a long-running backup, the lifecycle deadline gate blocks new mutations and waits for the recorded backup owner to release its exact lock generation before restoring lockdown. +If the owner exits first, NemoClaw leaves Shields down and reports the permanent-containment recovery guidance. When the sandbox has active baseline exclusions, successful output lists their keys and repeats that excluded egress leaves dependent agent features unsupported for that sandbox. ```bash @@ -2924,7 +2931,8 @@ If no selector is provided, the latest snapshot is used. Restore removes files added after the snapshot only from state directories selected for cleanup. It preserves directories that exist only in the target manifest or whose backup failed. The state replacement, mutable-config permission repair, and policy reconciliation run under the same per-sandbox transition. -An expired auto-restore timer can interrupt that work and restore lockdown. +When the timer expires during that work, the lifecycle deadline gate blocks new mutations and waits for the recorded restore owner to release its exact lock generation before restoring lockdown. +If the owner exits first, NemoClaw leaves Shields down and reports the permanent-containment recovery guidance. The selector accepts any of: diff --git a/scripts/checks/openshell-policy-mutation-read.mts b/scripts/checks/openshell-policy-mutation-read.mts index 2c0b4088676..63fc1e5db7b 100644 --- a/scripts/checks/openshell-policy-mutation-read.mts +++ b/scripts/checks/openshell-policy-mutation-read.mts @@ -58,7 +58,7 @@ export const MUTATION_READS: readonly AuditedMutationRead[] = [ }, { relativePath: "src/lib/shields/index.ts", - expectedReadCalls: 2, + expectedReadCalls: 3, baseCommand: "runCapture(buildPolicyGetCommand(sandboxName))", unsafeBaseCommand: "runCapture(buildPolicyGetCommand(sandboxName), {", fullCommand: "runCapture(buildPolicyGetFullCommand(sandboxName))", diff --git a/src/lib/actions/maintenance.test.ts b/src/lib/actions/maintenance.test.ts index 3e4e8288b03..f9cc0b05b2d 100644 --- a/src/lib/actions/maintenance.test.ts +++ b/src/lib/actions/maintenance.test.ts @@ -18,6 +18,10 @@ const mocks = vi.hoisted(() => ({ isSandboxContainerDefinitivelyAbsent: vi.fn(), openBackupShieldsWindow: vi.fn(), relockBackupShieldsWindow: vi.fn(), + withSandboxMutationLock: vi.fn( + async (_sandboxName: string, action: () => unknown, _options?: { timeoutMs?: number }) => + action(), + ), })); vi.mock("../state/registry", () => ({ @@ -29,6 +33,9 @@ vi.mock("../state/sandbox", () => ({ backupSandboxState: mocks.backupSandboxState, BackupResult: {}, })); +vi.mock("../state/mcp-lifecycle-lock", () => ({ + withSandboxMutationLock: mocks.withSandboxMutationLock, +})); vi.mock("../openshell-sandbox-list", () => ({ captureSandboxListWithGatewayPreflightOrExit: mocks.captureSandboxListWithGatewayPreflightOrExit, })); @@ -82,6 +89,10 @@ describe("backupAll", () => { beforeEach(() => { vi.clearAllMocks(); mocks.backupStartedSandboxState.mockReset(); + mocks.withSandboxMutationLock.mockImplementation( + async (_sandboxName: string, action: () => unknown, _options?: { timeoutMs?: number }) => + action(), + ); delete process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS; mocks.captureSandboxListWithGatewayPreflightOrExit.mockResolvedValue({ status: 0, @@ -275,13 +286,23 @@ describe("backupAll", () => { logSpy.mockRestore(); }); - it("closes each shields window before backing up the next sandbox (#6455)", async () => { + it("serializes each Shields transition without holding the lock during backup (#7952)", async () => { mocks.listSandboxes.mockReturnValue({ sandboxes: [{ name: "alpha" }, { name: "beta" }], defaultSandbox: "alpha", }); mocks.parseReadySandboxNames.mockReturnValue(new Set(["alpha", "beta"])); const events: string[] = []; + mocks.withSandboxMutationLock.mockImplementation( + async (name: string, action: () => unknown) => { + events.push(`lock:start:${name}`); + try { + return await action(); + } finally { + events.push(`lock:end:${name}`); + } + }, + ); mocks.openBackupShieldsWindow.mockImplementation( ( name: string, @@ -321,13 +342,27 @@ describe("backupAll", () => { await backupAll(); expect(events).toEqual([ + "lock:start:alpha", "open:alpha", + "lock:end:alpha", "backup:alpha", + "lock:start:alpha", "relock:alpha", + "lock:end:alpha", + "lock:start:beta", "open:beta", + "lock:end:beta", "backup:beta", + "lock:start:beta", "relock:beta", + "lock:end:beta", ]); + expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith( + 2, + "alpha", + expect.any(Function), + { timeoutMs: 30_000 }, + ); }); it("relocks shields after a credential permission failure and keeps the failure hard (#6455)", async () => { @@ -431,7 +466,18 @@ describe("backupAll", () => { mocks.backupSandboxState.mockImplementation(() => { throw backupError; }); - mocks.relockBackupShieldsWindow.mockReturnValue(false); + const relockLockError = new Error("mutation lock timed out"); + let lockAttempt = 0; + mocks.withSandboxMutationLock.mockImplementation( + async (_sandboxName: string, action: () => unknown, options?: { timeoutMs?: number }) => { + lockAttempt += 1; + if (lockAttempt === 2) { + expect(options).toEqual({ timeoutMs: 30_000 }); + throw relockLockError; + } + return action(); + }, + ); vi.spyOn(console, "log").mockImplementation(() => undefined); const failure = await backupAll().catch((error: unknown) => error); @@ -443,11 +489,13 @@ describe("backupAll", () => { expect((failure as AggregateError).errors).toEqual([ backupError, expect.objectContaining({ + cause: relockLockError, message: expect.stringContaining( "Shields lockdown could not be restored for 'alpha' after backup-all", ), }), ]); + expect(mocks.relockBackupShieldsWindow).not.toHaveBeenCalled(); }); it("preserves an orphan-manifest error when shields restoration also fails (#6455)", async () => { diff --git a/src/lib/actions/maintenance.ts b/src/lib/actions/maintenance.ts index 9792f2be1c7..aa83bd28f35 100644 --- a/src/lib/actions/maintenance.ts +++ b/src/lib/actions/maintenance.ts @@ -22,6 +22,7 @@ import { SANDBOX_IMAGE_REPOS } from "../domain/sandbox/image-tag"; import { resolveGatewayName, resolveSandboxGatewayName } from "../onboard/gateway-binding"; import { captureSandboxListWithGatewayPreflightOrExit } from "../openshell-sandbox-list"; import { parseLiveSandboxNames, parseReadySandboxNames } from "../runtime-recovery"; +import { withSandboxMutationLock } from "../state/mcp-lifecycle-lock"; import * as registry from "../state/registry"; import * as sandboxState from "../state/sandbox"; import { nemoclawStateRoot, resolveHome } from "../state/state-root"; @@ -79,7 +80,9 @@ async function backupSandboxWithinShieldsWindow( backup: () => sandboxState.BackupResult | Promise, ): Promise { const shieldsWindowOptions = backupAllShieldsWindowOptions(sandboxName); - const window = openBackupShieldsWindow(sandboxName, shieldsWindowOptions); + const window = await withSandboxMutationLock(sandboxName, () => + openBackupShieldsWindow(sandboxName, shieldsWindowOptions), + ); if (!window) { return { result: null, @@ -107,9 +110,21 @@ async function backupSandboxWithinShieldsWindow( hasBackupError = true; } } finally { - if (!relockBackupShieldsWindow(sandboxName, window, true, shieldsWindowOptions)) { + try { + const relocked = await withSandboxMutationLock( + sandboxName, + () => relockBackupShieldsWindow(sandboxName, window, true, shieldsWindowOptions), + { timeoutMs: 30_000 }, + ); + if (!relocked) { + relockError = new Error( + `Shields lockdown could not be restored for '${sandboxName}' after backup-all; aborting remaining backups.`, + ); + } + } catch (error) { relockError = new Error( `Shields lockdown could not be restored for '${sandboxName}' after backup-all; aborting remaining backups.`, + { cause: error }, ); } } diff --git a/src/lib/actions/sandbox/mcp-bridge-policy.ts b/src/lib/actions/sandbox/mcp-bridge-policy.ts index 2584fdad555..b951485b690 100644 --- a/src/lib/actions/sandbox/mcp-bridge-policy.ts +++ b/src/lib/actions/sandbox/mcp-bridge-policy.ts @@ -1,11 +1,13 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { isIP } from "node:net"; import { isDeepStrictEqual } from "node:util"; import YAML from "yaml"; import type { AgentMcpAdapter } from "../../agent/defs"; import * as policies from "../../policy"; +import { isBlockedMcpUrlTargetHost } from "../../security/mcp-url-target"; import type { McpBridgeEntry } from "../../state/registry"; import * as registry from "../../state/registry"; import { @@ -19,6 +21,7 @@ import { buildMcpBridgePolicyYaml, } from "./mcp-bridge-policy-render"; +export { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts"; export { buildMcpBridgePolicyKey, buildMcpBridgePolicyName, @@ -26,7 +29,6 @@ export { MCP_BRIDGE_ALLOWED_METHODS, MCP_BRIDGE_POLICY_MAX_BODY_BYTES, } from "./mcp-bridge-policy-render"; -export { MCP_BRIDGE_POLICY_SOURCE } from "./mcp-bridge-contracts"; export interface ExactManagedMcpPolicy { key: string; @@ -35,6 +37,18 @@ export interface ExactManagedMcpPolicy { server: string; } +export interface ManagedMcpPolicyOmission { + key?: string; + policyName?: string; + server?: string; + reason: string; +} + +export interface ProvableManagedMcpPolicies { + policies: ExactManagedMcpPolicy[]; + omissions: ManagedMcpPolicyOmission[]; +} + type ManagedMcpPolicyInspectionDeps = { getSandbox: typeof registry.getSandbox; }; @@ -68,6 +82,58 @@ function readManagedNetworkPolicies( return networkPolicies as Record; } +function requireCanonicalAllowedIps(networkPolicy: unknown, policyName: string): readonly string[] { + if (!networkPolicy || typeof networkPolicy !== "object" || Array.isArray(networkPolicy)) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); + } + const endpoints = (networkPolicy as Record).endpoints; + if (!Array.isArray(endpoints) || endpoints.length !== 1) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); + } + const endpoint = endpoints[0]; + if (!endpoint || typeof endpoint !== "object" || Array.isArray(endpoint)) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); + } + const allowedIps = (endpoint as Record).allowed_ips; + if (!Array.isArray(allowedIps) || allowedIps.length === 0) { + throw new Error(`Managed MCP policy '${policyName}' has no exact public address pins`); + } + if ( + allowedIps.some( + (address) => + typeof address !== "string" || + address !== address.toLowerCase() || + address.includes("%") || + isIP(address) === 0 || + isBlockedMcpUrlTargetHost(address), + ) + ) { + throw new Error(`Managed MCP policy '${policyName}' has invalid public address pins`); + } + const pins = allowedIps as string[]; + if (new Set(pins).size !== pins.length || !isDeepStrictEqual(pins, [...pins].sort())) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical public address pins`); + } + return pins; +} + +function resolveCanonicalManagedMcpAdapter( + sandbox: registry.SandboxEntry, + bridge: McpBridgeEntry, +): AgentMcpAdapter { + if (isAgentMcpAdapter(bridge.adapter)) return bridge.adapter; + switch (sandbox.agent || "openclaw") { + case "openclaw": + return "mcporter"; + case "hermes": + return "hermes-config"; + case "langchain-deepagents-code": + return "deepagents-config"; + default: + throw new Error("Managed MCP bridge has no canonical adapter"); + } +} + function requireCanonicalManagedPolicy( sandbox: registry.SandboxEntry, server: string, @@ -122,16 +188,36 @@ function requireCanonicalManagedPolicy( throw new Error(`Managed MCP policy '${policyName}' has a non-canonical network policy key`); } + const registeredNetworkPolicy = registeredPolicies[policyKey]; + const allowedIps = requireCanonicalAllowedIps(registeredNetworkPolicy, policyName); + let expectedDocument: Record; + try { + expectedDocument = parseManagedPolicyDocument( + buildMcpBridgePolicyYaml( + bridge.server, + bridge.url, + resolveCanonicalManagedMcpAdapter(sandbox, bridge), + allowedIps, + ), + `Canonical managed MCP policy '${policyName}'`, + ); + } catch { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); + } + if (!isDeepStrictEqual(registeredDocument, expectedDocument)) { + throw new Error(`Managed MCP policy '${policyName}' has non-canonical generated content`); + } + if (!Object.hasOwn(livePolicies, policyKey)) { throw new Error(`Managed MCP policy '${policyName}' is absent from the live gateway policy`); } - if (!isDeepStrictEqual(livePolicies[policyKey], registeredPolicies[policyKey])) { + if (!isDeepStrictEqual(livePolicies[policyKey], registeredNetworkPolicy)) { throw new Error(`Managed MCP policy '${policyName}' has drifted from its ownership record`); } return { key: policyKey, - networkPolicy: registeredPolicies[policyKey], + networkPolicy: registeredNetworkPolicy, policyName, server, }; @@ -150,8 +236,36 @@ export function inspectExactManagedMcpPolicies( livePolicyYaml: string, deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, ): ExactManagedMcpPolicy[] { + const liveDocument = parseManagedPolicyDocument(livePolicyYaml, "Live gateway policy"); + const livePolicies = readManagedNetworkPolicies(liveDocument, "Live gateway policy"); const sandbox = deps.getSandbox(sandboxName); - if (!sandbox?.mcp) return []; + if (!sandbox) { + const unclassifiedKey = Object.keys(livePolicies).find((key) => key.startsWith("mcp_bridge_")); + if (unclassifiedKey) { + throw new Error( + `Reserved MCP policy key '${unclassifiedKey}' has no committed managed bridge ownership`, + ); + } + return []; + } + const generatedRegistrations = (sandbox.customPolicies ?? []).filter( + (policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE, + ); + if (!sandbox.mcp) { + const orphaned = generatedRegistrations[0]; + if (orphaned) { + throw new Error( + `Generated MCP policy '${orphaned.name}' has no committed managed bridge ownership`, + ); + } + const unclassifiedKey = Object.keys(livePolicies).find((key) => key.startsWith("mcp_bridge_")); + if (unclassifiedKey) { + throw new Error( + `Reserved MCP policy key '${unclassifiedKey}' has no committed managed bridge ownership`, + ); + } + return []; + } if (sandbox.mcp.destroyPreparedAt || sandbox.mcp.destroyPendingAt) { throw new Error("Managed MCP sandbox destruction is incomplete"); } @@ -160,13 +274,6 @@ export function inspectExactManagedMcpPolicies( if (bridgeEntries.some(([, bridge]) => bridge.addState !== undefined)) { throw new Error("A managed MCP bridge lifecycle transition is incomplete"); } - const generatedRegistrations = (sandbox.customPolicies ?? []).filter( - (policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE, - ); - if (bridgeEntries.length === 0 && generatedRegistrations.length === 0) return []; - - const liveDocument = parseManagedPolicyDocument(livePolicyYaml, "Live gateway policy"); - const livePolicies = readManagedNetworkPolicies(liveDocument, "Live gateway policy"); const exact = bridgeEntries.map(([server]) => requireCanonicalManagedPolicy(sandbox, server, livePolicies), ); @@ -188,19 +295,194 @@ export function inspectExactManagedMcpPolicies( } keys.add(entry.key); } + const unclassifiedKey = Object.keys(livePolicies).find( + (key) => key.startsWith("mcp_bridge_") && !keys.has(key), + ); + if (unclassifiedKey) { + throw new Error( + `Reserved MCP policy key '${unclassifiedKey}' has no committed managed bridge ownership`, + ); + } return exact.sort((left, right) => left.key.localeCompare(right.key)); } +/** + * Deadline-only inspection for automatic Shields restoration. + * + * Each entry is admitted independently through the same exact committed/live + * proof as the strict path. Incomplete, drifted, orphaned, or ambiguous claims + * are omitted instead of extending the mutable window; registry state is never + * reconciled or rewritten here. + */ +export function inspectProvableManagedMcpPoliciesForDeadline( + sandboxName: string, + livePolicyYaml: string, + deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, +): ProvableManagedMcpPolicies { + const derivedIdentity = (server: string): { key?: string; policyName?: string } => { + try { + return { + key: buildMcpBridgePolicyKey(server), + policyName: buildMcpBridgePolicyName(server), + }; + } catch { + return {}; + } + }; + const omit = (reason: string, server?: string, policyName?: string): ManagedMcpPolicyOmission => { + const identity = server ? derivedIdentity(server) : {}; + return { + ...(server ? { server } : {}), + ...identity, + ...(policyName ? { policyName } : {}), + reason, + }; + }; + const sandbox = deps.getSandbox(sandboxName); + const generatedRegistrations = (sandbox?.customPolicies ?? []).filter( + (policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE, + ); + const bridgeEntries = Object.entries(sandbox?.mcp?.bridges ?? {}); + + if (sandbox?.mcp?.destroyPreparedAt || sandbox?.mcp?.destroyPendingAt) { + const reason = "Managed MCP sandbox destruction is incomplete"; + const omissions = bridgeEntries.map(([server]) => omit(reason, server)); + for (const registration of generatedRegistrations) { + if (!omissions.some((entry) => entry.policyName === registration.name)) { + omissions.push(omit(reason, undefined, registration.name)); + } + } + if (omissions.length === 0) omissions.push({ reason }); + return { policies: [], omissions }; + } + + let livePolicies: Record; + try { + livePolicies = readManagedNetworkPolicies( + parseManagedPolicyDocument(livePolicyYaml, "Live gateway policy"), + "Live gateway policy", + ); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + const omissions = bridgeEntries.map(([server]) => omit(reason, server)); + for (const registration of generatedRegistrations) { + if (!omissions.some((entry) => entry.policyName === registration.name)) { + omissions.push(omit(reason, undefined, registration.name)); + } + } + return { policies: [], omissions }; + } + + const policies: ExactManagedMcpPolicy[] = []; + const omissions: ManagedMcpPolicyOmission[] = []; + if (!sandbox) { + for (const key of Object.keys(livePolicies).filter((candidate) => + candidate.startsWith("mcp_bridge_"), + )) { + omissions.push({ + key, + reason: `Reserved MCP policy key '${key}' has no committed managed bridge ownership`, + }); + } + return { policies, omissions }; + } + const claimedServersByKey = new Map(); + const claimedServersByPolicyName = new Map(); + for (const [server] of bridgeEntries) { + const identity = derivedIdentity(server); + if (identity.key) { + const servers = claimedServersByKey.get(identity.key) ?? []; + servers.push(server); + claimedServersByKey.set(identity.key, servers); + } + if (identity.policyName) { + const servers = claimedServersByPolicyName.get(identity.policyName) ?? []; + servers.push(server); + claimedServersByPolicyName.set(identity.policyName, servers); + } + } + const ambiguousServers = new Set(); + for (const servers of [...claimedServersByKey.values(), ...claimedServersByPolicyName.values()]) { + if (servers.length <= 1) continue; + for (const server of servers) ambiguousServers.add(server); + } + for (const [server] of bridgeEntries) { + if (ambiguousServers.has(server)) { + omissions.push(omit("Managed MCP policy identity has ambiguous bridge ownership", server)); + continue; + } + try { + policies.push(requireCanonicalManagedPolicy(sandbox, server, livePolicies)); + } catch (error) { + omissions.push(omit(error instanceof Error ? error.message : String(error), server)); + } + } + + const bridgePolicyNames = new Set( + bridgeEntries + .map(([server]) => derivedIdentity(server).policyName) + .filter((name): name is string => name !== undefined), + ); + for (const registration of generatedRegistrations) { + if (!bridgePolicyNames.has(registration.name)) { + omissions.push( + omit( + `Generated MCP policy '${registration.name}' has no committed managed bridge ownership`, + undefined, + registration.name, + ), + ); + } + } + + const policiesByKey = new Map(); + for (const policy of policies) { + const entries = policiesByKey.get(policy.key) ?? []; + entries.push(policy); + policiesByKey.set(policy.key, entries); + } + const exact: ExactManagedMcpPolicy[] = []; + for (const entries of policiesByKey.values()) { + if (entries.length === 1) { + exact.push(entries[0]!); + continue; + } + for (const entry of entries) { + omissions.push( + omit(`Managed MCP policy key '${entry.key}' has ambiguous ownership`, entry.server), + ); + } + } + const exactKeys = new Set(exact.map((entry) => entry.key)); + for (const key of Object.keys(livePolicies).filter( + (candidate) => candidate.startsWith("mcp_bridge_") && !exactKeys.has(candidate), + )) { + if (omissions.some((entry) => entry.key === key)) continue; + omissions.push({ + key, + reason: `Reserved MCP policy key '${key}' has no exact committed managed bridge ownership`, + }); + } + return { + policies: exact.sort((left, right) => left.key.localeCompare(right.key)), + omissions, + }; +} + export function hasManagedMcpPolicyClaims( sandboxName: string, deps: ManagedMcpPolicyInspectionDeps = managedMcpPolicyInspectionDeps, ): boolean { const sandbox = deps.getSandbox(sandboxName); - if (!sandbox?.mcp) return false; + if (!sandbox) return false; return ( - Object.keys(sandbox.mcp.bridges).length > 0 || - Boolean(sandbox.mcp.destroyPreparedAt) || - Boolean(sandbox.mcp.destroyPendingAt) || + Boolean( + sandbox.mcp && + (Object.keys(sandbox.mcp.bridges).length > 0 || + (sandbox.mcp.managedServerNames?.length ?? 0) > 0 || + sandbox.mcp.destroyPreparedAt || + sandbox.mcp.destroyPendingAt), + ) || (sandbox.customPolicies ?? []).some((policy) => policy.sourcePath === MCP_BRIDGE_POLICY_SOURCE) ); } diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 06eae9d80a7..068c237ee70 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -9,23 +9,11 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, type MockInstance, vi } from "vitest"; import YAML from "yaml"; +import { buildMcpBridgePolicyYaml } from "../actions/sandbox/mcp-bridge-policy-render"; import type { SandboxEntry } from "../state/registry"; const requireDist = createRequire(import.meta.url); const shieldsModulePath = "./index.js"; -const HUNG_FORWARD_OWNER_SOURCE = ` -const { spawn } = require("node:child_process"); -const childScriptPath = process.argv[2]; -const childReadyPath = process.argv[3]; -spawn(process.execPath, [childScriptPath, childReadyPath], { stdio: "ignore" }); -setInterval(() => {}, 60000); -`; -const WEAKENING_CHILD_SOURCE = ` -const fs = require("node:fs"); -const childReadyPath = process.argv[2]; -fs.writeFileSync(childReadyPath, String(process.pid)); -setInterval(() => {}, 60000); -`; type ShieldsHarness = { applyShieldsPolicySnapshot: typeof import("./index.js").applyShieldsPolicySnapshot; @@ -70,6 +58,46 @@ type HarnessOptions = { sandboxEntry?: SandboxEntry; }; +function managedMcpPolicy(server: string, address = "8.8.8.8") { + const key = `mcp_bridge_${server}`; + const content = buildMcpBridgePolicyYaml( + server, + `https://${server}.example.com/mcp`, + "hermes-config", + [address], + ); + const networkPolicy = YAML.parse(content).network_policies[key]; + return { content, networkPolicy, server }; +} + +function managedMcpSandbox(policies: Array>): SandboxEntry { + return { + name: "openclaw", + openshellDriver: "docker", + customPolicies: policies.map(({ content, server }) => ({ + name: `mcp-bridge-${server}`, + content, + sourcePath: "generated:nemoclaw-mcp-bridge", + })), + mcp: { + bridges: Object.fromEntries( + policies.map(({ server }) => [ + server, + { + server, + agent: "hermes", + adapter: "hermes-config", + url: `https://${server}.example.com/mcp`, + env: ["MCP_SECRET"], + policyName: `mcp-bridge-${server}`, + addedAt: "2026-07-30T00:00:00.000Z", + }, + ]), + ), + }, + }; +} + function throwHarnessError(error: Error): never { throw error; } @@ -79,6 +107,8 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { delete require.cache[requireDist.resolve(shieldsModulePath)]; delete require.cache[requireDist.resolve("./timer-bound-lock.js")]; delete require.cache[requireDist.resolve("./transition-lock.js")]; + delete require.cache[requireDist.resolve("./permissive-runtime.js")]; + delete require.cache[requireDist.resolve("../actions/sandbox/mcp-bridge-policy.js")]; delete require.cache[requireDist.resolve("../sandbox/privileged-exec.js")]; delete require.cache[requireDist.resolve("../cli/branding.js")]; const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); @@ -105,9 +135,10 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { }); options.fork && vi.spyOn(childProcess, "fork").mockImplementation(options.fork); vi.spyOn(policy, "buildPolicyGetCommand").mockReturnValue(["openshell", "policy", "get"]); - vi.spyOn(policy, "buildPolicySetCommand").mockImplementation((file: string) => { - if (fs.existsSync(file)) { - policySetBodies.push(fs.readFileSync(file, "utf-8")); + vi.spyOn(policy, "buildPolicySetCommand").mockImplementation((file: unknown) => { + const policyFile = String(file); + if (fs.existsSync(policyFile)) { + policySetBodies.push(fs.readFileSync(policyFile, "utf-8")); } return ["openshell", "policy", "set"]; }); @@ -271,6 +302,8 @@ describe("shields command flow", () => { delete require.cache[requireDist.resolve(shieldsModulePath)]; delete require.cache[requireDist.resolve("./timer-bound-lock.js")]; delete require.cache[requireDist.resolve("./transition-lock.js")]; + delete require.cache[requireDist.resolve("./permissive-runtime.js")]; + delete require.cache[requireDist.resolve("../actions/sandbox/mcp-bridge-policy.js")]; delete require.cache[requireDist.resolve("../cli/branding.js")]; }); @@ -304,59 +337,16 @@ describe("shields command flow", () => { it("shieldsDown preserves an exact managed MCP policy and records its snapshot key (#7952)", { timeout: 15_000, }, () => { - const managedPolicy = YAML.stringify({ - preset: { - name: "mcp-bridge-alpha", - description: "Generated MCP policy for alpha", - }, - network_policies: { - mcp_bridge_alpha: { - name: "mcp_bridge_alpha", - endpoints: [ - { - host: "alpha.example.com", - port: 443, - path: "/mcp", - protocol: "mcp", - }, - ], - binaries: [{ path: "/opt/hermes/.venv/bin/python*" }], - }, - }, - }); - const managedNetworkPolicy = YAML.parse(managedPolicy).network_policies.mcp_bridge_alpha; + const alpha = managedMcpPolicy("alpha"); const harness = createHarness({ livePolicy: YAML.stringify({ version: 1, network_policies: { restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, - mcp_bridge_alpha: managedNetworkPolicy, + mcp_bridge_alpha: alpha.networkPolicy, }, }), - sandboxEntry: { - name: "openclaw", - openshellDriver: "docker", - customPolicies: [ - { - name: "mcp-bridge-alpha", - content: managedPolicy, - sourcePath: "generated:nemoclaw-mcp-bridge", - }, - ], - mcp: { - bridges: { - alpha: { - server: "alpha", - agent: "hermes", - adapter: "hermes-config", - url: "https://alpha.example.com/mcp", - env: ["MCP_SECRET"], - policyName: "mcp-bridge-alpha", - addedAt: "2026-07-30T00:00:00.000Z", - }, - }, - }, - }, + sandboxEntry: managedMcpSandbox([alpha]), }); harness.shieldsDown("openclaw", { @@ -371,37 +361,13 @@ describe("shields command flow", () => { ); expect(state.shieldsManagedMcpPolicyKeys).toEqual(["mcp_bridge_alpha"]); const applied = YAML.parse(harness.policySetBodies.at(-1)!); - expect(applied.network_policies.mcp_bridge_alpha).toEqual(managedNetworkPolicy); + expect(applied.network_policies.mcp_bridge_alpha).toEqual(alpha.networkPolicy); expect(applied.network_policies).not.toHaveProperty("restrictive_baseline"); }); - it("shared snapshot restore keeps managed MCP additions made while Shields are down (#7952)", () => { - const policyFor = (server: string, address: string) => - YAML.stringify({ - preset: { - name: `mcp-bridge-${server}`, - description: `Generated MCP policy for ${server}`, - }, - network_policies: { - [`mcp_bridge_${server}`]: { - name: `mcp_bridge_${server}`, - endpoints: [ - { - host: `${server}.example.com`, - port: 443, - path: "/mcp", - protocol: "mcp", - allowed_ips: [address], - }, - ], - binaries: [{ path: "/opt/hermes/.venv/bin/python*" }], - }, - }, - }); - const alphaPolicy = policyFor("alpha", "8.8.8.8"); - const betaPolicy = policyFor("beta", "1.1.1.1"); - const alphaEntry = YAML.parse(alphaPolicy).network_policies.mcp_bridge_alpha; - const betaEntry = YAML.parse(betaPolicy).network_policies.mcp_bridge_beta; + it("timer restore uses persisted MCP ownership after its transition marker clears (#7952)", () => { + const alpha = managedMcpPolicy("alpha", "8.8.8.8"); + const beta = managedMcpPolicy("beta", "1.1.1.1"); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); const snapshotPath = path.join(stateDir, "policy-snapshot-managed-restore.yaml"); fs.mkdirSync(stateDir, { recursive: true }); @@ -411,7 +377,7 @@ describe("shields command flow", () => { version: 1, network_policies: { restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, - mcp_bridge_alpha: alphaEntry, + mcp_bridge_alpha: alpha.networkPolicy, }, }), ); @@ -428,45 +394,16 @@ describe("shields command flow", () => { version: 1, network_policies: { permissive_baseline: { endpoints: [{ host: "*" }] }, - mcp_bridge_alpha: alphaEntry, - mcp_bridge_beta: betaEntry, + mcp_bridge_alpha: alpha.networkPolicy, + mcp_bridge_beta: beta.networkPolicy, }, }), - sandboxEntry: { - name: "openclaw", - openshellDriver: "docker", - customPolicies: [ - { - name: "mcp-bridge-alpha", - content: alphaPolicy, - sourcePath: "generated:nemoclaw-mcp-bridge", - }, - { - name: "mcp-bridge-beta", - content: betaPolicy, - sourcePath: "generated:nemoclaw-mcp-bridge", - }, - ], - mcp: { - bridges: Object.fromEntries( - ["alpha", "beta"].map((server) => [ - server, - { - server, - agent: "hermes", - adapter: "hermes-config", - url: `https://${server}.example.com/mcp`, - env: ["MCP_SECRET"], - policyName: `mcp-bridge-${server}`, - addedAt: "2026-07-30T00:00:00.000Z", - }, - ]), - ), - }, - }, + sandboxEntry: managedMcpSandbox([alpha, beta]), }); - const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath); + const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { + transitionProcessToken: "6".repeat(32), + }); expect(result.status).toBe(0); const restored = YAML.parse(harness.policySetBodies.at(-1)!); @@ -475,7 +412,126 @@ describe("shields command flow", () => { "mcp_bridge_beta", "restrictive_baseline", ]); - expect(restored.network_policies.mcp_bridge_beta).toEqual(betaEntry); + expect(restored.network_policies.mcp_bridge_beta).toEqual(beta.networkPolicy); + }); + + it("refuses managed snapshot restoration when persisted Shields state is corrupt (#7952)", () => { + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const snapshotPath = path.join(stateDir, "policy-snapshot-corrupt-state.yaml"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: ["../not-a-managed-key"], + }), + ); + const harness = createHarness(); + + expect(() => harness.applyShieldsPolicySnapshot("openclaw", snapshotPath)).toThrow( + /persisted state is corrupt/, + ); + expect(harness.policySetBodies).toHaveLength(0); + }); + + it("refuses a legacy restore whose persisted state names a different snapshot (#7952)", () => { + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const expectedSnapshotPath = path.join(stateDir, "policy-snapshot-expected.yaml"); + const requestedSnapshotPath = path.join(stateDir, "policy-snapshot-requested.yaml"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(expectedSnapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync(requestedSnapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsPolicySnapshotPath: expectedSnapshotPath, + }), + ); + const harness = createHarness(); + + expect(() => harness.applyShieldsPolicySnapshot("openclaw", requestedSnapshotPath)).toThrow( + /does not match the policy snapshot/, + ); + expect(harness.policySetBodies).toHaveLength(0); + }); + + it("uses token-bound transition ownership when the forward owner dies before state commit (#7952)", () => { + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const processToken = "8".repeat(32); + const snapshotPath = path.join(stateDir, "policy-snapshot-new-cycle.yaml"); + const oldSnapshotPath = path.join(stateDir, "policy-snapshot-old-cycle.yaml"); + const alpha = managedMcpPolicy("alpha", "8.8.8.8"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + snapshotPath, + YAML.stringify({ + version: 1, + network_policies: { mcp_bridge_alpha: alpha.networkPolicy }, + }), + ); + fs.writeFileSync(oldSnapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: false, + shieldsPolicySnapshotPath: oldSnapshotPath, + shieldsManagedMcpPolicyKeys: [], + }), + ); + fs.writeFileSync( + path.join(stateDir, `shields-transition-openclaw-${processToken}.json`), + JSON.stringify({ + version: 1, + phase: "preparing", + ownerPid: process.pid, + ownerStartIdentity: "forward-owner", + processToken, + sandboxName: "openclaw", + snapshotPath, + managedMcpPolicyKeys: ["mcp_bridge_alpha"], + }), + ); + const harness = createHarness({ + livePolicy: YAML.stringify({ + version: 1, + network_policies: { mcp_bridge_alpha: alpha.networkPolicy }, + }), + sandboxEntry: managedMcpSandbox([alpha]), + }); + + const result = harness.applyShieldsPolicySnapshot("openclaw", snapshotPath, { + transitionProcessToken: processToken, + }); + + expect(result.status).toBe(0); + expect(YAML.parse(harness.policySetBodies.at(-1)!).network_policies.mcp_bridge_alpha).toEqual( + alpha.networkPolicy, + ); + }); + + it("loads 257 managed keys recorded by Shields down (#7952)", () => { + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const snapshotPath = path.join(stateDir, "policy-snapshot-many-managed-keys.yaml"); + const keys = Array.from({ length: 257 }, (_, index) => `mcp_bridge_server_${index}`); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync( + snapshotPath, + YAML.stringify({ network_policies: Object.fromEntries(keys.map((key) => [key, {}])) }), + ); + fs.writeFileSync( + path.join(stateDir, "shields-openclaw.json"), + JSON.stringify({ + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: keys, + }), + ); + const harness = createHarness(); + + expect(harness.applyShieldsPolicySnapshot("openclaw", snapshotPath).status).toBe(0); }); it("binds manual shields-up to the active auto-restore timer generation", () => { @@ -541,42 +597,6 @@ describe("shields command flow", () => { }); }); - it("never selects the detached recovery timer or its children for owner-tree takeover", () => { - const shields = requireDist(shieldsModulePath) as { - excludeRecoveryProcessTree: ( - descendants: Array<{ pid: number; startIdentity: string; depth: number }>, - recovery: { pid: number; startIdentity: string }, - recoveryDescendants: Array<{ pid: number; startIdentity: string; depth: number }>, - ) => Array<{ pid: number; startIdentity: string; depth: number }>; - }; - const recovery = { pid: 200, startIdentity: "timer", depth: 1 }; - const recoveryChild = { pid: 201, startIdentity: "timer-child", depth: 2 }; - const weakeningChild = { pid: 300, startIdentity: "policy-set", depth: 1 }; - - expect( - shields.excludeRecoveryProcessTree([recovery, recoveryChild, weakeningChild], recovery, [ - recoveryChild, - ]), - ).toEqual([weakeningChild]); - }); - - it("does not exclude a weakening child that reused a recovery PID", () => { - const shields = requireDist(shieldsModulePath) as { - excludeRecoveryProcessTree: ( - descendants: Array<{ pid: number; startIdentity: string; depth: number }>, - recovery: { pid: number; startIdentity: string }, - recoveryDescendants: Array<{ pid: number; startIdentity: string; depth: number }>, - ) => Array<{ pid: number; startIdentity: string; depth: number }>; - }; - const recovery = { pid: 200, startIdentity: "timer", depth: 1 }; - const sampledRecoveryChild = { pid: 201, startIdentity: "timer-child", depth: 2 }; - const reusedPidChild = { pid: 201, startIdentity: "policy-set", depth: 1 }; - - expect( - shields.excludeRecoveryProcessTree([reusedPidChild], recovery, [sampledRecoveryChild]), - ).toEqual([reusedPidChild]); - }); - it("auto-restore waits for the forward shields-down commit before reclaiming policy", () => { const harness = createHarness(); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); @@ -646,154 +666,19 @@ describe("shields command flow", () => { } expect(Date.now() - startedAt).toBeGreaterThanOrEqual(100); - expect(fs.existsSync(transitionPath)).toBe(false); + expect(fs.existsSync(transitionPath)).toBe(true); expect(harness.runSpy).toHaveBeenCalledWith( ["openshell", "policy", "set"], expect.objectContaining({ ignoreError: true }), ); }); - it("preempts a hung forward owner and its weakening subprocess before restoring", { - timeout: 20_000, - }, async () => { + it("preserves a live transition owner instead of attempting portable process-tree takeover", () => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); - const sandboxName = "openclaw"; + const sandboxName = "live-transition-owner"; const processToken = "b".repeat(32); - const snapshotPath = path.join(stateDir, "policy-snapshot-hung.yaml"); - const childReadyPath = path.join(stateDir, "weakening-child-ready"); - const transitionPath = path.join( - stateDir, - `shields-transition-${sandboxName}-${processToken}.json`, - ); - const ownerScriptPath = path.join(stateDir, "hung-forward-owner.cjs"); - const childScriptPath = path.join(stateDir, "weakening-child.cjs"); - fs.writeFileSync(ownerScriptPath, HUNG_FORWARD_OWNER_SOURCE, { mode: 0o600 }); - fs.writeFileSync(childScriptPath, WEAKENING_CHILD_SOURCE, { mode: 0o600 }); - fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}\n"); - fs.writeFileSync( - path.join(stateDir, `shields-timer-${sandboxName}.json`), - JSON.stringify({ - pid: process.pid, - sandboxName, - snapshotPath, - restoreAt: new Date(Date.now() - 1_000).toISOString(), - processToken, - }), - ); - - const owner = spawn(process.execPath, [ownerScriptPath, childScriptPath, childReadyPath], { - stdio: "ignore", - }); - expect(owner.pid).toBeTypeOf("number"); - await vi.waitFor(() => expect(fs.existsSync(childReadyPath)).toBe(true), { - timeout: 5_000, - interval: 10, - }); - const childPid = Number(fs.readFileSync(childReadyPath, "utf-8")); - expect(Number.isInteger(childPid) && childPid > 0).toBe(true); - const timerControl = requireDist("./timer-control.js"); - const ownerStartIdentity = timerControl.readProcessStartIdentity(owner.pid); - expect(ownerStartIdentity).toBeTypeOf("string"); - const childStartIdentity = timerControl.readProcessStartIdentity(childPid); - expect(childStartIdentity).toBeTypeOf("string"); - const initialDescendants = timerControl.listDescendantProcessIdentities(owner.pid); - expect(initialDescendants).not.toBeNull(); - expect(initialDescendants.some(({ pid }: { pid: number }) => pid === childPid)).toBe(true); - const takeoverEvents: string[] = []; - const readProcessStartIdentity = timerControl.readProcessStartIdentity; - let unreadableOwnerIdentityReads = 2; - vi.spyOn(timerControl, "readProcessStartIdentity").mockImplementation((...args: unknown[]) => { - const [pid, deadline] = args as [number, number?]; - const unreadable = pid === owner.pid && unreadableOwnerIdentityReads > 0; - unreadableOwnerIdentityReads -= unreadable ? 1 : 0; - return unreadable ? null : readProcessStartIdentity(pid, deadline); - }); - const readProcessState = timerControl.readProcessState; - vi.spyOn(timerControl, "readProcessState").mockImplementation((...args: unknown[]) => { - const [pid, deadline] = args as [number, number?]; - const state = readProcessState(pid, deadline); - pid === owner.pid && /^[Tt]/.test(state ?? "") && takeoverEvents.push("owner-stopped"); - return state; - }); - const listDescendantProcessIdentities = timerControl.listDescendantProcessIdentities; - vi.spyOn(timerControl, "listDescendantProcessIdentities").mockImplementation( - (...args: unknown[]) => { - const [rootPid, deadline] = args as [number, number?]; - rootPid === owner.pid && takeoverEvents.push("owner-enumerated"); - return listDescendantProcessIdentities(rootPid, deadline); - }, - ); - const harness = createHarness({ - run: (cmd) => { - expect(cmd).toEqual(["openshell", "policy", "set"]); - const observedChildIdentity = readProcessStartIdentity(childPid); - const observedChildState = readProcessState(childPid); - let childCanBeSignaled = true; - try { - process.kill(childPid, 0); - } catch (error) { - childCanBeSignaled = (error as NodeJS.ErrnoException).code === "EPERM"; - } - const exactChildIsGone = - !childCanBeSignaled || - (observedChildIdentity !== null && observedChildIdentity !== childStartIdentity); - const childIsZombie = observedChildState?.startsWith("Z") === true; - expect(exactChildIsGone || childIsZombie).toBe(true); - takeoverEvents.push("policy-restored"); - return { status: 0 }; - }, - }); - fs.writeFileSync( - transitionPath, - JSON.stringify({ - version: 1, - phase: "preparing", - ownerPid: owner.pid, - ownerStartIdentity, - processToken, - sandboxName, - snapshotPath, - }), - { mode: 0o600 }, - ); - - try { - harness.synchronizeAutoRestoreWithShieldsDown(sandboxName); - } finally { - owner.kill("SIGKILL"); - try { - timerControl.readProcessStartIdentity(childPid) === childStartIdentity && - process.kill(childPid, "SIGKILL"); - } catch { - // The takeover already killed the exact child. - } - } - - expect(fs.existsSync(transitionPath)).toBe(false); - expect(takeoverEvents.indexOf("owner-stopped")).toBeGreaterThanOrEqual(0); - expect(takeoverEvents.indexOf("owner-enumerated")).toBeGreaterThan( - takeoverEvents.indexOf("owner-stopped"), - ); - expect(takeoverEvents).toContain("owner-enumerated"); - expect(takeoverEvents.indexOf("policy-restored")).toBeGreaterThan( - takeoverEvents.indexOf("owner-enumerated"), - ); - expect(harness.runSpy).toHaveBeenCalledWith( - ["openshell", "policy", "set"], - expect.objectContaining({ ignoreError: true }), - ); - }); - - it("fails closed when the weakening subprocess set never reaches quiescence", { - timeout: 10_000, - }, () => { - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - fs.mkdirSync(stateDir, { recursive: true }); - const sandboxName = "non-quiescent"; - const processToken = "c".repeat(32); const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); - const owner = spawn(process.execPath, ["-e", "setTimeout(() => {}, 5000)"], { stdio: "ignore", }); @@ -814,38 +699,6 @@ describe("shields command flow", () => { }), { mode: 0o600 }, ); - - const syntheticPidBase = 2_000_000_000; - const readProcessStartIdentity = timerControl.readProcessStartIdentity; - vi.spyOn(timerControl, "readProcessStartIdentity").mockImplementation((...args: unknown[]) => { - const [pid, deadline] = args as [number, number?]; - return pid === owner.pid - ? ownerStartIdentity - : pid >= syntheticPidBase - ? `synthetic:${String(pid)}` - : readProcessStartIdentity(pid, deadline); - }); - const readProcessState = timerControl.readProcessState; - vi.spyOn(timerControl, "readProcessState").mockImplementation((...args: unknown[]) => { - const [pid, deadline] = args as [number, number?]; - return pid === owner.pid ? "T" : readProcessState(pid, deadline); - }); - const listDescendantProcessIdentities = timerControl.listDescendantProcessIdentities; - let ownerEnumerationPass = 0; - vi.spyOn(timerControl, "listDescendantProcessIdentities").mockImplementation( - (...args: unknown[]) => { - const [rootPid, deadline] = args as [number, number?]; - const ownerEnumeration = rootPid === owner.pid; - ownerEnumerationPass += ownerEnumeration ? 1 : 0; - const syntheticPid = syntheticPidBase + ownerEnumerationPass; - return ownerEnumeration - ? [{ pid: syntheticPid, startIdentity: `synthetic:${String(syntheticPid)}`, depth: 1 }] - : rootPid === process.pid - ? [] - : listDescendantProcessIdentities(rootPid, deadline); - }, - ); - vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); const processKillSpy = vi.spyOn(process, "kill"); createHarness(); const shields = requireDist(shieldsModulePath) as { @@ -863,88 +716,46 @@ describe("shields command flow", () => { processToken, path.join(stateDir, "unused-snapshot.yaml"), ), - ).toThrow("Timed-out shields-down process tree could not be frozen safely"); - expect(processKillSpy).toHaveBeenCalledWith(owner.pid, "SIGSTOP"); + ).toThrow("still active"); + expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); + expect(fs.existsSync(lockPath)).toBe(true); } finally { - owner.kill("SIGCONT"); owner.kill("SIGKILL"); } - - expect(ownerEnumerationPass).toBe(8); - expect(fs.existsSync(lockPath)).toBe(true); }); - it("does not signal a replacement that reuses the owner PID during final verification", () => { + it.each([ + ["matching", "c".repeat(32)], + ["different", "d".repeat(32)], + ])("permanently contains a %s-token transition whose owner exited in the recovery gap", (_tokenRelationship, transitionOwnerToken) => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); - const sandboxName = "reused-owner"; - const processToken = "d".repeat(32); - const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); - const owner = spawn(process.execPath, ["-e", "setTimeout(() => {}, 5000)"], { - stdio: "ignore", - }); - expect(owner.pid).toBeTypeOf("number"); - const timerControl = requireDist("./timer-control.js"); - const ownerStartIdentity = timerControl.readProcessStartIdentity(owner.pid); - expect(ownerStartIdentity).toBeTypeOf("string"); + const sandboxName = "dead-transition-owner"; + const processToken = "c".repeat(32); + const transitionLockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); fs.writeFileSync( - lockPath, + transitionLockPath, JSON.stringify({ version: 1, sandboxName, - pid: owner.pid, - processStartIdentity: ownerStartIdentity, + pid: 2_147_483_647, + processStartIdentity: "dead-owner", command: "config set write", acquiredAtMs: Date.now(), - takeoverToken: processToken, + takeoverToken: transitionOwnerToken, }), { mode: 0o600 }, ); - - const processKill = process.kill; - let ownerLivenessChecks = 0; - let replacementVisible = false; - const processKillSpy = vi.spyOn(process, "kill").mockImplementation((...args: unknown[]) => { - const [pid, signal] = args as [number, NodeJS.Signals | 0 | undefined]; - const ownerLivenessCheck = pid === owner.pid && signal === 0; - ownerLivenessChecks += ownerLivenessCheck ? 1 : 0; - replacementVisible ||= ownerLivenessCheck && ownerLivenessChecks === 2; - return processKill(pid, signal); - }); - const readProcessStartIdentity = timerControl.readProcessStartIdentity; - vi.spyOn(timerControl, "readProcessStartIdentity").mockImplementation((...args: unknown[]) => { - const [pid, deadline] = args as [number, number?]; - return pid === owner.pid && replacementVisible - ? "replacement-process-start" - : readProcessStartIdentity(pid, deadline); - }); createHarness(); - const shields = requireDist(shieldsModulePath) as { - prepareAutoRestoreTransitionTakeover: ( + const transitionLock = requireDist("./transition-lock.js") as { + withShieldsTransitionLock: ( sandboxName: string, - processToken: string, - snapshotPath: string, + command: string, + fn: () => void, + options: { recoverStaleOwner: boolean; waitTimeoutMs: number }, ) => void; }; - - try { - shields.prepareAutoRestoreTransitionTakeover( - sandboxName, - processToken, - path.join(stateDir, "unused-snapshot.yaml"), - ); - expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGSTOP"); - expect(processKillSpy).not.toHaveBeenCalledWith(owner.pid, "SIGKILL"); - } finally { - owner.kill("SIGCONT"); - owner.kill("SIGKILL"); - } - - expect(ownerLivenessChecks).toBeGreaterThanOrEqual(2); - }); - - it("preempts timer-token config and inference mutations at the restore deadline", async () => { const shields = requireDist(shieldsModulePath) as { prepareAutoRestoreTransitionTakeover: ( sandboxName: string, @@ -952,54 +763,37 @@ describe("shields command flow", () => { snapshotPath: string, ) => void; }; - const transitionLockPath = path.join(import.meta.dirname, "transition-lock.ts"); - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - fs.mkdirSync(stateDir, { recursive: true }); - - for (const [index, command] of ["config set write", "inference set"].entries()) { - const sandboxName = `deadline-${String(index)}`; - const processToken = String(index + 1).repeat(32); - const readyPath = path.join(stateDir, `${sandboxName}.ready`); - const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); - const owner = spawn( - process.execPath, - [ - "--import", - "tsx", - "-e", - [ - `const {withShieldsTransitionLock}=require(${JSON.stringify(transitionLockPath)})`, - "const fs=require('fs')", - "const [name,command,token,ready]=process.argv.slice(1)", - "withShieldsTransitionLock(name,command,()=>{fs.writeFileSync(ready,'ready');Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,10000)},{takeoverToken:token})", - ].join(";"), - sandboxName, - command, - processToken, - readyPath, - ], - { env: { ...process.env, HOME: tmpDir }, stdio: "ignore" }, - ); - - try { - const deadline = Date.now() + 5_000; - while ((!fs.existsSync(readyPath) || !fs.existsSync(lockPath)) && Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, 10)); - } - expect(fs.existsSync(readyPath)).toBe(true); - expect(fs.existsSync(lockPath)).toBe(true); + const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js") as { + getMcpLifecycleLockPath: (sandboxName: string, stateDir: string) => string; + }; + const containmentPath = `${lifecycleLock.getMcpLifecycleLockPath( + sandboxName, + stateDir, + )}.containment`; - shields.prepareAutoRestoreTransitionTakeover( - sandboxName, - processToken, - path.join(stateDir, `${sandboxName}.snapshot.yaml`), - ); + expect(() => + transitionLock.withShieldsTransitionLock( + sandboxName, + "shields auto-restore contender", + () => undefined, + { + recoverStaleOwner: false, + waitTimeoutMs: 0, + }, + ), + ).toThrow("recorded owner PID"); + expect(fs.existsSync(transitionLockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(false); - expect(fs.existsSync(lockPath)).toBe(false); - } finally { - owner.kill("SIGKILL"); - } - } + expect(() => + shields.prepareAutoRestoreTransitionTakeover( + sandboxName, + processToken, + path.join(stateDir, "unused-snapshot.yaml"), + ), + ).toThrow("permanent containment"); + expect(fs.existsSync(transitionLockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }); it("publishes preparing recovery ownership before weakening and active only after unlock", () => { @@ -1065,6 +859,7 @@ describe("shields command flow", () => { ownerPid: process.pid, sandboxName: "openclaw", snapshotPath: expect.stringContaining("policy-snapshot-"), + managedMcpPolicyKeys: [], }); expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(true); }); @@ -1322,41 +1117,15 @@ describe("shields command flow", () => { ).toBe(true); }); - it("shieldsStatus restores an expired dead timer under the shared sandbox lock", async () => { - const configPath = "/sandbox/.openclaw/openclaw.json"; - const configDir = "/sandbox/.openclaw"; - const hashPath = `${configDir}/.config-hash`; - const configHash = "a".repeat(64); - const hashHash = "b".repeat(64); + it("shieldsStatus contains an expired timer whose transition owner exited", async () => { const processToken = "7".repeat(32); - const execCalls: string[] = []; - const execResponses = new Map([ - [` stat -c %a %U:%G ${hashPath}`, "444 root:root\n"], - [` stat -c %a %U:%G ${configPath}`, "444 root:root\n"], - [` stat -c %a %U:%G ${configDir}`, "755 root:root\n"], - [" stat -c %a %U:%G /sandbox", "1775 root:sandbox\n"], - [` lsattr -d ${hashPath}`, `----i---------e----- ${hashPath}\n`], - [` lsattr -d ${configPath}`, `----i---------e----- ${configPath}\n`], - [` sha256sum ${hashPath}`, `${hashHash} ${hashPath}\n`], - [` sha256sum ${configPath}`, `${configHash} ${configPath}\n`], - ]); const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); const sandboxMutationLockPath = lifecycleLock.getMcpLifecycleLockPath("openclaw"); - let policySetSawSandboxLock = false; - const harness = createHarness({ - run: () => { - policySetSawSandboxLock = fs.existsSync(sandboxMutationLockPath); - return { status: 0 }; - }, - dockerExecFileSync: (argv: unknown) => { - const args = Array.isArray(argv) ? argv.map(String) : []; - const cmd = args.join(" "); - execCalls.push(cmd); - return [...execResponses].find(([needle]) => cmd.includes(needle))?.[1] ?? ""; - }, - }); + const containmentPath = `${sandboxMutationLockPath}.containment`; + const harness = createHarness(); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); const lockPath = path.join(stateDir, "shields-transition-lock-openclaw.json"); + const timerMarkerPath = path.join(stateDir, "shields-timer-openclaw.json"); fs.mkdirSync(stateDir, { recursive: true }); const snapshotPath = path.join(stateDir, "policy-snapshot-expired.yaml"); fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}\n"); @@ -1372,7 +1141,7 @@ describe("shields command flow", () => { }), ); fs.writeFileSync( - path.join(stateDir, "shields-timer-openclaw.json"), + timerMarkerPath, JSON.stringify({ pid: 4242, sandboxName: "openclaw", @@ -1404,31 +1173,21 @@ describe("shields command flow", () => { return true; }); - await lifecycleLock.withSandboxMutationLock("openclaw", () => - harness.shieldsStatus("openclaw"), - ); + await expect( + lifecycleLock.withSandboxMutationLock("openclaw", () => harness.shieldsStatus("openclaw")), + ).rejects.toThrow("permanent containment"); const state = JSON.parse( fs.readFileSync(path.join(stateDir, "shields-openclaw.json"), "utf-8"), ); - expect(harness.logSpy).toHaveBeenCalledWith(" Shields: UP (lockdown active)"); - expect(state.shieldsDown).toBe(false); - expect(state.fileHashes).toMatchObject({ - [configPath]: configHash, - [hashPath]: hashHash, - }); - expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(false); - expect(fs.existsSync(lockPath)).toBe(false); - expect(policySetSawSandboxLock).toBe(true); + expect(state.shieldsDown).toBe(true); + expect(fs.existsSync(timerMarkerPath)).toBe(true); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); - expect(harness.auditSpy).toHaveBeenCalledWith( - expect.objectContaining({ - action: "shields_auto_restore", - policy_snapshot: snapshotPath, - restored_by: "auto_timer", - sandbox: "openclaw", - }), + expect(harness.runSpy).not.toHaveBeenCalledWith( + ["openshell", "policy", "set"], + expect.anything(), ); - expect(execCalls.some((cmd) => cmd.includes(` sha256sum ${hashPath}`))).toBe(true); }); }); diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index 5a25b5a3743..4670556b9f1 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -483,6 +483,73 @@ describe("shields — unit logic", () => { expect(logSpy).toHaveBeenCalledWith(" Shields: DOWN (temporarily unlocked)"); }); + it("deadline composition removes an unproven MCP add from the restrictive policy", async () => { + const snapshot = + "version: 1\nnetwork_policies:\n restrictive_baseline: {}\n mcp_bridge_beta: {}\n"; + const { composeDeadlineManagedMcpPolicies } = await import("./mcp-policy-transition"); + const composition = composeDeadlineManagedMcpPolicies(snapshot, [], ["mcp_bridge_beta"]); + + expect(composition.yaml).toContain("restrictive_baseline"); + expect(composition.yaml).not.toContain("mcp_bridge_beta"); + }); + + it("deadline restore removes saved MCP keys when the registry cannot be read", async () => { + const sandboxName = "openclaw"; + const processToken = "b".repeat(32); + const snapshotPath = path.join(stateDir(), "policy-snapshot-unreadable-registry.yaml"); + fs.mkdirSync(stateDir(), { recursive: true }); + fs.writeFileSync( + snapshotPath, + "version: 1\nnetwork_policies:\n restrictive_baseline: {}\n mcp_bridge_alpha: {}\n", + ); + writeState(sandboxName, { + shieldsDown: true, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: ["mcp_bridge_alpha"], + }); + writeMarker(sandboxName, { + pid: 2_147_483_647, + sandboxName, + snapshotPath, + restoreAt: new Date(Date.now() - 1_000).toISOString(), + processToken, + }); + const originalReadFileSync = fs.readFileSync.bind(fs); + vi.spyOn(fs, "readFileSync").mockImplementation( + (file: fs.PathOrFileDescriptor, options?: unknown) => { + if (String(file).endsWith(`${path.sep}sandboxes.json`)) { + throw Object.assign(new Error("registry permission denied"), { code: "EACCES" }); + } + return originalReadFileSync(file, options as never) as never; + }, + ); + vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { + if (pid === 2_147_483_647 && signal === 0) { + throw Object.assign(new Error("not running"), { code: "ESRCH" }); + } + return true; + }); + const { applyShieldsPolicySnapshot } = await loadShieldsModule(); + + const result = applyShieldsPolicySnapshot(sandboxName, snapshotPath, { + transitionProcessToken: processToken, + deadlineAuthoritative: true, + expiredTimerRecovery: true, + }); + + expect(result.managedMcpOmissions).toEqual([ + expect.objectContaining({ reason: expect.stringMatching(/Cannot read config file:/) }), + ]); + const { composeDeadlineManagedMcpPolicies } = await import("./mcp-policy-transition"); + const composition = composeDeadlineManagedMcpPolicies( + fs.readFileSync(snapshotPath, "utf-8"), + [], + ["mcp_bridge_alpha"], + ); + expect(composition.yaml).toContain("restrictive_baseline"); + expect(composition.yaml).not.toContain("mcp_bridge_alpha"); + }); + it("shieldsStatus warns and stays DOWN when inline recovery fails", async () => { const sandboxName = "openclaw"; const missingSnapshotPath = path.join(stateDir(), "missing-snapshot.yaml"); @@ -958,7 +1025,7 @@ describe("NC-2227-05: shields timer marker behavior", () => { expect(readTimerMarker("openclaw")).toBeNull(); }); - it("killTimer terminates verified live timer process and clears marker", async () => { + it("killTimer cooperatively revokes a verified live timer without signaling it", async () => { const sourceModulePath = path.join(process.cwd(), "src", "lib", "shields", "timer-control.ts"); const { killTimer } = await import(sourceModulePath); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); @@ -1004,11 +1071,11 @@ describe("NC-2227-05: shields timer marker behavior", () => { markerFound: true, markerPid: 7331, wasAlive: true, - terminated: true, + terminated: false, warnings: [], }); expect(processKillSpy).toHaveBeenCalledWith(7331, 0); - expect(processKillSpy).toHaveBeenCalledWith(7331, "SIGTERM"); + expect(processKillSpy).toHaveBeenCalledTimes(1); expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(false); }); diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index f9942127fe3..dd5465896e7 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -46,9 +46,7 @@ const { readTimerMarker, clearTimerMarker, isProcessAlive, - readProcessState, readProcessStartIdentity, - listDescendantProcessIdentities, processInspectionDeadlineAfter, processInspectionDeadlineReached, verifyTimerMarkerIdentity, @@ -58,21 +56,30 @@ const { resolveNemoclawStateDir } = require("../state/paths"); const { appendAuditEntry } = require("./audit"); const { resolveAgentConfig } = require("../sandbox/config"); const { + assertLegacyMcpPolicyRestoreSafe, + buildDeadlineRuntimeManagedMcpPolicy, buildRuntimeManagedMcpPolicy, buildRuntimePermissivePolicy, hasManagedMcpPolicyClaims, inspectExactManagedMcpPolicies, + inspectProvableManagedMcpPoliciesForDeadline, isManagedMcpPolicyKey, }: typeof import("./permissive-runtime") = require("./permissive-runtime"); const { cleanupTempDir } = require("../onboard/temp-files"); const { verifyShieldsLockState }: typeof import("./verify-lock") = require("./verify-lock"); const { relockAndReconfirm }: typeof import("./relock-reconfirm") = require("./relock-reconfirm"); const { - inspectShieldsTransitionLockOwner, - takeoverShieldsTransitionLock, + inspectAnyShieldsTransitionLockOwner, withShieldsTransitionLock, }: typeof import("./transition-lock") = require("./transition-lock"); const { + beginCommittedMcpLifecycleContainmentSync, + getMcpLifecycleLockPath, + isMcpLifecycleLockHeld, + readMcpLockProcessIdentity, + withMcpLifecycleDeadlineFenceSync, + withMcpLifecycleLockSync, + withTimerBoundAutoRestoreLock, withTimerBoundShieldsMutationLock, }: typeof import("./timer-bound-lock") = require("./timer-bound-lock"); const { @@ -100,8 +107,8 @@ const { }: typeof import("./mutable-config-repair") = require("./mutable-config-repair"); type MutableConfigPermsInspection = import("./mutable-config-perms").MutableConfigPermsInspection; type MutableConfigRepairResult = import("./mutable-config-perms").MutableConfigRepairResult; -type ProcessIdentity = import("./timer-control").ProcessIdentity; - +type ManagedMcpPolicyOmission = import("./permissive-runtime").ManagedMcpPolicyOmission; +type TimerMarker = import("./timer-control").TimerMarker; const STATE_DIR = resolveNemoclawStateDir(); const SHIELDS_TRANSITION_POLL_MS = 50; const SHIELDS_TRANSITION_HANDOFF_GRACE_MS = 500; @@ -120,13 +127,43 @@ type ShieldsDownTransition = { phase: "preparing" | "active"; ownerPid: number; ownerStartIdentity: string; + ownerMcpProcessIdentity?: string; processToken: string; sandboxName: string; snapshotPath: string; + /** Exact generated MCP keys owned when snapshotPath was captured. */ + managedMcpPolicyKeys?: string[]; }; const transitionPollBuffer = new Int32Array(new SharedArrayBuffer(4)); +function sameTimerMarkerGeneration(current: TimerMarker | null, expected: TimerMarker): boolean { + return ( + current?.pid === expected.pid && + current.sandboxName === expected.sandboxName && + current.snapshotPath === expected.snapshotPath && + current.restoreAt === expected.restoreAt && + current.processToken === expected.processToken && + current.allowLegacyHermesProtocol === expected.allowLegacyHermesProtocol && + current.leaseOwnerPid === expected.leaseOwnerPid && + current.leaseOwnerStartIdentity === expected.leaseOwnerStartIdentity + ); +} + +function assertTimerMarkerGeneration(sandboxName: string, expected: TimerMarker): void { + if (!sameTimerMarkerGeneration(readTimerMarker(sandboxName), expected)) { + throw new Error("Auto-restore authority changed before Shields transition takeover"); + } +} + +function appendAuditEntryBestEffort(entry: Parameters[0]): void { + try { + appendAuditEntry(entry); + } catch { + // A failed diagnostic write must not release an active recovery gate. + } +} + function shieldsDownTransitionPath(sandboxName: string, processToken: string): string { return path.join(STATE_DIR, `shields-transition-${sandboxName}-${processToken}.json`); } @@ -141,13 +178,25 @@ function isShieldsDownTransition(value: unknown): value is ShieldsDownTransition value.ownerPid > 0 && typeof value.ownerStartIdentity === "string" && value.ownerStartIdentity.length > 0 && + (value.ownerMcpProcessIdentity === undefined || + (typeof value.ownerMcpProcessIdentity === "string" && + value.ownerMcpProcessIdentity.length > 0)) && typeof value.processToken === "string" && /^[0-9a-f]{32}$/.test(value.processToken) && typeof value.sandboxName === "string" && - typeof value.snapshotPath === "string" + typeof value.snapshotPath === "string" && + isOptionalManagedMcpPolicyKeys(value.managedMcpPolicyKeys) ); } +function sameManagedMcpPolicyKeys( + left: readonly string[] | undefined, + right: readonly string[] | undefined, +): boolean { + if (left === undefined || right === undefined) return left === right; + return left.length === right.length && left.every((key, index) => key === right[index]); +} + function readShieldsDownTransition( sandboxName: string, processToken: string, @@ -175,7 +224,9 @@ function writeShieldsDownTransition( !current || current.phase !== expectedPhase || current.ownerPid !== transition.ownerPid || - current.snapshotPath !== transition.snapshotPath + current.snapshotPath !== transition.snapshotPath || + current.ownerMcpProcessIdentity !== transition.ownerMcpProcessIdentity || + !sameManagedMcpPolicyKeys(current.managedMcpPolicyKeys, transition.managedMcpPolicyKeys) ) { throw new Error("Shields-down recovery ownership changed during the transition"); } @@ -226,9 +277,30 @@ function readExactProcessStatus( return alive ? "current" : "gone"; } +function persistUnresolvedShieldsContainment( + sandboxName: string, + processToken: string, + ownerPid: number, +): void { + const containmentPath = `${getMcpLifecycleLockPath(sandboxName, STATE_DIR)}.containment`; + if (fs.existsSync(containmentPath)) return; + try { + beginCommittedMcpLifecycleContainmentSync( + sandboxName, + processToken, + `Shields recovery owner PID ${String(ownerPid)} exited without descendant-containment proof`, + STATE_DIR, + ); + } catch (error) { + if (fs.existsSync(containmentPath)) return; + throw error; + } +} + function waitForShieldsDownForwardCommit( sandboxName: string, processToken: string, + assertTakeoverAuthority?: () => void, ): ShieldsDownTransition | null { let observed = readShieldsDownTransition(sandboxName, processToken); if (!observed) return null; @@ -248,8 +320,10 @@ function waitForShieldsDownForwardCommit( if ( next.ownerPid !== observed.ownerPid || next.ownerStartIdentity !== observed.ownerStartIdentity || + next.ownerMcpProcessIdentity !== observed.ownerMcpProcessIdentity || next.snapshotPath !== observed.snapshotPath || - next.processToken !== observed.processToken + next.processToken !== observed.processToken || + !sameManagedMcpPolicyKeys(next.managedMcpPolicyKeys, observed.managedMcpPolicyKeys) ) { throw new Error("Shields-down recovery ownership changed while waiting for forward commit"); } @@ -257,150 +331,25 @@ function waitForShieldsDownForwardCommit( } if (observed.phase === "preparing") { - // The absolute shields-down deadline has expired while the forward owner - // is still able to weaken policy/config. Preempt that exact process - // instance, then restore from the captured snapshot. Waiting forever would - // turn the requested timeout into an unbounded mutable window. - stopTimedOutShieldsDownTree(observed.ownerPid, observed.ownerStartIdentity); - } - return observed; -} - -function excludeRecoveryProcessTree( - descendants: ProcessIdentity[], - recovery: Pick, - recoveryDescendants: ProcessIdentity[], -): ProcessIdentity[] { - const identityKey = ({ pid, startIdentity }: Pick) => - `${String(pid)}\0${startIdentity}`; - const excludedIdentities = new Set([recovery, ...recoveryDescendants].map(identityKey)); - return descendants.filter((descendant) => !excludedIdentities.has(identityKey(descendant))); -} - -function stopTimedOutShieldsDownTree(ownerPid: number, ownerStartIdentity: string): void { - let freezeDeadline = processInspectionDeadlineAfter(SHIELDS_TRANSITION_TERMINATE_GRACE_MS); - const waitForKnownExactProcess = ( - pid: number, - startIdentity: string, - deadline: number, - ): Exclude => { - while (true) { - const status = readExactProcessStatus(pid, startIdentity, deadline); - if (status !== "unknown") return status; - if (processInspectionDeadlineReached(deadline)) { - throw new Error("Timed-out shields-down process identity could not be verified safely"); - } - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); - } - }; - const signalExact = ( - pid: number, - startIdentity: string, - signal: NodeJS.Signals, - deadline: number, - ): void => { - if (waitForKnownExactProcess(pid, startIdentity, deadline) === "gone") return; - try { - process.kill(pid, signal); - } catch (error) { - const errno = error as NodeJS.ErrnoException; - if (errno.code !== "ESRCH") throw error; - } - }; - const waitForExactStop = (pid: number, startIdentity: string): "gone" | "stopped" => { - while (true) { - const state = readProcessState(pid, freezeDeadline); - const status = readExactProcessStatus(pid, startIdentity, freezeDeadline); - if (status === "gone" || state?.startsWith("Z")) return "gone"; - if (status === "current" && /^[Tt]/.test(state ?? "")) return "stopped"; - if (processInspectionDeadlineReached(freezeDeadline)) { - throw new Error("Timed-out shields-down process tree could not be frozen safely"); - } - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); - } - }; - if (waitForKnownExactProcess(ownerPid, ownerStartIdentity, freezeDeadline) === "gone") return; - // Stop the exact owner before enumerating its descendants so it cannot launch - // another weakening subprocess while takeover is being established. - signalExact(ownerPid, ownerStartIdentity, "SIGSTOP", freezeDeadline); - if (waitForExactStop(ownerPid, ownerStartIdentity) === "gone") return; - freezeDeadline = processInspectionDeadlineAfter(SHIELDS_TRANSITION_TERMINATE_GRACE_MS); - const recoveryStartIdentity = readProcessStartIdentity(process.pid, freezeDeadline); - if (recoveryStartIdentity === null) { - throw new Error("Cannot identify the auto-restore recovery process safely"); - } - const recoveryTree = listDescendantProcessIdentities(process.pid, freezeDeadline); - if (recoveryTree === null) { - throw new Error("Cannot identify the auto-restore recovery process tree safely"); - } - const tracked = new Map(); - let observedQuiescentPass = false; - for (let pass = 0; pass < 8; pass += 1) { - if (waitForExactStop(ownerPid, ownerStartIdentity) === "gone") { - throw new Error("Timed-out shields-down process tree could not be frozen safely"); - } - const descendants = listDescendantProcessIdentities(ownerPid, freezeDeadline); - if (descendants === null) { - throw new Error("Cannot enumerate timed-out shields-down subprocesses safely"); - } - let added = false; - const recoveryIsInsideOwnerTree = descendants.some( - ({ pid }: { pid: number }) => pid === process.pid, - ); - const passDescendants = excludeRecoveryProcessTree( - descendants, - { pid: process.pid, startIdentity: recoveryStartIdentity }, - recoveryIsInsideOwnerTree ? recoveryTree : [], + const ownerStatus = readExactProcessStatus( + observed.ownerPid, + observed.ownerStartIdentity, + processInspectionDeadlineAfter(SHIELDS_TRANSITION_TERMINATE_GRACE_MS), ); - for (const descendant of passDescendants) { - const previous = tracked.get(descendant.pid); - if (!previous || previous.startIdentity !== descendant.startIdentity) added = true; - tracked.set(descendant.pid, { - startIdentity: descendant.startIdentity, - depth: descendant.depth, - }); - signalExact(descendant.pid, descendant.startIdentity, "SIGSTOP", freezeDeadline); - } - for (const descendant of passDescendants) { - waitForExactStop(descendant.pid, descendant.startIdentity); - } - if (!added) { - observedQuiescentPass = true; - break; + if (ownerStatus === "gone") { + assertTakeoverAuthority?.(); + persistUnresolvedShieldsContainment(sandboxName, processToken, observed.ownerPid); + throw new Error( + "Shields-down forward owner exited before committing its final mutation; permanent containment requires operator resolution", + ); } - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); - } - if (!observedQuiescentPass) { - throw new Error("Timed-out shields-down process tree could not be frozen safely"); - } - - const deepestFirst = [...tracked.entries()].sort((a, b) => b[1].depth - a[1].depth); - const killDeadline = processInspectionDeadlineAfter(SHIELDS_TRANSITION_TERMINATE_GRACE_MS); - for (const [pid, identity] of deepestFirst) { - signalExact(pid, identity.startIdentity, "SIGKILL", killDeadline); - } - signalExact(ownerPid, ownerStartIdentity, "SIGKILL", killDeadline); - - const exactProcessIsGone = (pid: number, startIdentity: string): boolean => { - const state = readProcessState(pid, killDeadline); - return ( - state?.startsWith("Z") === true || - readExactProcessStatus(pid, startIdentity, killDeadline) === "gone" - ); - }; - while (!processInspectionDeadlineReached(killDeadline)) { - const survivor = deepestFirst.some( - ([pid, identity]) => !exactProcessIsGone(pid, identity.startIdentity), + throw new Error( + "Shields-down forward owner is still active; automatic recovery is waiting behind the deadline gate", ); - if (!survivor && exactProcessIsGone(ownerPid, ownerStartIdentity)) { - return; - } - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); } - throw new Error("Timed-out shields-down process tree could not be stopped safely"); + return observed; } -// --------------------------------------------------------------------------- // privileged sandbox exec — bypasses the sandbox's Landlock context // // openshell sandbox exec runs commands INSIDE the Landlock domain, so it @@ -873,30 +822,164 @@ function getShieldsPostureWithoutHostLock( return { ...describeShieldsMode(mode), state }; } -function prepareExpiredAutoRestoreHostLockTakeover(sandboxName: string): void { +type ExpiredAutoRestoreTakeover = { + marker: TimerMarker & { processToken: string }; +}; + +function inspectExpiredAutoRestoreMarker(sandboxName: string): TimerMarker | null { const state = loadShieldsState(sandboxName); - if (state._isCorrupt || state.shieldsDown !== true) return; + if (state._isCorrupt || state.shieldsDown !== true) return null; const marker = readTimerMarker(sandboxName); - if (!marker?.processToken || !/^[0-9a-f]{32}$/.test(marker.processToken)) return; + if (!marker) return null; const restoreAtMs = new Date(marker.restoreAt).getTime(); const now = Date.now(); - if (!Number.isFinite(restoreAtMs) || restoreAtMs > now) return; + if (!Number.isFinite(restoreAtMs) || restoreAtMs > now) return null; if ( isProcessAlive(marker.pid) && verifyTimerMarkerIdentity(marker).verified && now <= restoreAtMs + AUTO_RESTORE_COMPLETION_GRACE_MS ) { - return; + return null; + } + return marker; +} + +function inspectExpiredAutoRestoreTakeover( + sandboxName: string, + marker = inspectExpiredAutoRestoreMarker(sandboxName), +): ExpiredAutoRestoreTakeover | null { + if (!marker?.processToken || !/^[0-9a-f]{32}$/.test(marker.processToken)) return null; + return { + marker: marker as TimerMarker & { processToken: string }, + }; +} + +function retryInlineAutoRestore(sandboxName: string, marker: TimerMarker): void { + let notifiedError: string | null = null; + for (;;) { + try { + const recoveredState = recoverExpiredAutoRestoreGate(sandboxName, true); + if (!recoveredState._isCorrupt && recoveredState.shieldsDown !== true) { + return; + } + assertTimerMarkerGeneration(sandboxName, marker); + const message = "Inline auto-restore did not complete; retrying under the lifecycle gate"; + if (message !== notifiedError) { + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: message, + }); + notifiedError = message; + } + } catch (error) { + assertTimerMarkerGeneration(sandboxName, marker); + const message = error instanceof Error ? error.message : String(error); + if (message !== notifiedError) { + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: message, + }); + notifiedError = message; + } + } + Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); } - prepareAutoRestoreTransitionTakeover(sandboxName, marker.processToken, marker.snapshotPath); +} + +function withExpiredAutoRestoreDeadlineFence( + sandboxName: string, + command: string, + operation: (allowInlineRecovery: boolean) => T, +): T { + const expiredMarker = inspectExpiredAutoRestoreMarker(sandboxName); + const takeover = inspectExpiredAutoRestoreTakeover(sandboxName, expiredMarker); + const runWithHostLock = (callback: () => T) => + withTimerBoundShieldsMutationLock(sandboxName, command, callback); + const recoverThenRun = () => + withTimerBoundAutoRestoreLock(sandboxName, command, () => { + if (expiredMarker) retryInlineAutoRestore(sandboxName, expiredMarker); + return operation(false); + }); + if (isMcpLifecycleLockHeld(sandboxName, STATE_DIR)) { + if (!expiredMarker || !takeover) { + return runWithHostLock(() => operation(true)); + } + const { marker } = takeover; + prepareAutoRestoreTransitionTakeover( + sandboxName, + marker.processToken, + marker.snapshotPath, + () => assertTimerMarkerGeneration(sandboxName, marker), + ); + return recoverThenRun(); + } + if (!takeover) { + return withMcpLifecycleLockSync(sandboxName, () => runWithHostLock(() => operation(true)), { + stateDir: STATE_DIR, + }); + } + + const { marker } = takeover; + const assertTakeoverAuthority = () => assertTimerMarkerGeneration(sandboxName, marker); + return withMcpLifecycleDeadlineFenceSync( + sandboxName, + marker.processToken, + () => { + for (;;) { + try { + prepareAutoRestoreTransitionTakeover( + sandboxName, + marker.processToken, + marker.snapshotPath, + assertTakeoverAuthority, + ); + break; + } catch (error) { + assertTakeoverAuthority(); + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: error instanceof Error ? error.message : String(error), + }); + Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); + } + } + return recoverThenRun(); + }, + { + stateDir: STATE_DIR, + onContainment: ({ ownerPid, reason }) => { + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: `${reason}${ownerPid ? ` Contained owner PID: ${String(ownerPid)}.` : ""}`, + }); + }, + }, + ); } function getShieldsPosture(sandboxName: string, allowInlineRecovery = false): ShieldsPosture { if (!allowInlineRecovery) return getShieldsPostureWithoutHostLock(sandboxName, false); validateName(sandboxName, "sandbox name"); - prepareExpiredAutoRestoreHostLockTakeover(sandboxName); - return withTimerBoundShieldsMutationLock(sandboxName, "recover expired shields posture", () => - getShieldsPostureWithoutHostLock(sandboxName, true), + return withExpiredAutoRestoreDeadlineFence( + sandboxName, + "recover expired shields posture", + (allowInlineRecovery) => getShieldsPostureWithoutHostLock(sandboxName, allowInlineRecovery), ); } @@ -954,7 +1037,7 @@ function isOptionalHashMap(value: unknown): value is { [path: string]: string } function isOptionalManagedMcpPolicyKeys(value: unknown): value is string[] | undefined { if (value === undefined) return true; - if (!Array.isArray(value) || value.length > 256) return false; + if (!Array.isArray(value)) return false; const keys = new Set(); for (const key of value) { if (!isManagedMcpPolicyKey(key) || keys.has(key)) return false; @@ -1928,15 +2011,14 @@ function unlockAgentConfig( function inspectMutableConfigPerms(sandboxName: string): MutableConfigPermsInspection { validateName(sandboxName, "sandbox name"); - prepareExpiredAutoRestoreHostLockTakeover(sandboxName); - return withTimerBoundShieldsMutationLock( + return withExpiredAutoRestoreDeadlineFence( sandboxName, "inspect mutable config permissions", - () => { + (allowInlineRecovery) => { const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); return inspectMutableConfigPermsCore( target, - getShieldsPostureWithoutHostLock(sandboxName, true).mode, + getShieldsPostureWithoutHostLock(sandboxName, allowInlineRecovery).mode, (p) => privilegedSandboxExecCapture(sandboxName, ["stat", "-c", "%a %U:%G", p]), ); }, @@ -1945,15 +2027,18 @@ function inspectMutableConfigPerms(sandboxName: string): MutableConfigPermsInspe function repairMutableConfigPerms(sandboxName: string): MutableConfigRepairResult { validateName(sandboxName, "sandbox name"); - prepareExpiredAutoRestoreHostLockTakeover(sandboxName); - return withTimerBoundShieldsMutationLock(sandboxName, "repair mutable config permissions", () => { - const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); - return repairMutableConfigPermsCore( - target, - getShieldsPostureWithoutHostLock(sandboxName, true).mode, - () => normalizeMutableOpenClawConfig(sandboxName, target.configDir), - ); - }); + return withExpiredAutoRestoreDeadlineFence( + sandboxName, + "repair mutable config permissions", + (allowInlineRecovery) => { + const target = ensureConfigHashSensitiveFile(resolveAgentConfig(sandboxName)); + return repairMutableConfigPermsCore( + target, + getShieldsPostureWithoutHostLock(sandboxName, allowInlineRecovery).mode, + () => normalizeMutableOpenClawConfig(sandboxName, target.configDir), + ); + }, + ); } // --------------------------------------------------------------------------- @@ -2231,8 +2316,17 @@ function synchronizeAutoRestoreTransition( sandboxName: string, processToken: string, snapshotPath: string, + options: { + expiredTimerRecovery?: boolean; + retainTransition?: boolean; + assertTakeoverAuthority?: () => void; + } = {}, ): void { - const transition = waitForShieldsDownForwardCommit(sandboxName, processToken); + const transition = waitForShieldsDownForwardCommit( + sandboxName, + processToken, + options.assertTakeoverAuthority, + ); if (!transition) return; if (transition.snapshotPath !== snapshotPath) { throw new Error("Auto-restore snapshot does not match shields-down transition ownership"); @@ -2244,56 +2338,81 @@ function synchronizeAutoRestoreTransition( // above waits until the forward path has either committed its last weakening // mutation or its owner has died; restore the restrictive snapshot again at // that stable boundary before locking config. - const restoreResult = applyShieldsPolicySnapshot(sandboxName, transition.snapshotPath); + const marker = readTimerMarker(sandboxName); + const timerOwnsRecovery = + marker?.pid === process.pid && + marker.processToken === processToken && + marker.snapshotPath === transition.snapshotPath; + const deadlineAuthoritative = timerOwnsRecovery || options.expiredTimerRecovery === true; + const restoreResult = applyShieldsPolicySnapshot(sandboxName, transition.snapshotPath, { + transitionProcessToken: processToken, + ...(deadlineAuthoritative ? { deadlineAuthoritative: true } : {}), + ...(options.expiredTimerRecovery ? { expiredTimerRecovery: true } : {}), + }); const status = typeof restoreResult.status === "number" ? restoreResult.status : 1; if (status !== 0) { throw new Error( `Policy restore after shields-down handoff exited with status ${String(status)}`, ); } - clearShieldsDownTransition(sandboxName, processToken); + if (!options.retainTransition) { + clearShieldsDownTransition(sandboxName, processToken); + } } -function prepareAutoRestoreTransitionTakeover( +function inspectAutoRestoreTransitionTakeoverOwner( sandboxName: string, processToken: string, snapshotPath: string, -): void { +): { pid: number; processIdentity: string } | null { if (!/^[0-9a-f]{32}$/.test(processToken)) { throw new Error("Invalid auto-restore transition takeover token"); } + const transition = readShieldsDownTransition(sandboxName, processToken); + if (transition && transition.snapshotPath !== snapshotPath) { + throw new Error("Auto-restore snapshot does not match shields-down transition ownership"); + } + return transition?.ownerMcpProcessIdentity !== undefined + ? { pid: transition.ownerPid, processIdentity: transition.ownerMcpProcessIdentity } + : null; +} +function prepareAutoRestoreTransitionTakeover( + sandboxName: string, + processToken: string, + snapshotPath: string, + assertTakeoverAuthority?: () => void, +): { pid: number; processIdentity: string } | null { + const initialTransitionOwner = inspectAutoRestoreTransitionTakeoverOwner( + sandboxName, + processToken, + snapshotPath, + ); const transition = readShieldsDownTransition(sandboxName, processToken); if (transition && transition.snapshotPath !== snapshotPath) { throw new Error("Auto-restore snapshot does not match shields-down transition ownership"); } if (transition) { - // This waits briefly for the forward commit and stops its exact process - // tree if the deadline fired while it was still weakening the sandbox. - waitForShieldsDownForwardCommit(sandboxName, processToken); - } - - const owner = inspectShieldsTransitionLockOwner(sandboxName, processToken); - if (!owner) return; - // The same timer token is also propagated to config/inference/restart - // mutations made during the mutable window. At expiry those operations - // are weaker than restoring lockdown and may be preempted safely. The stop - // helper pins the exact identity and fails closed if it cannot be read. - stopTimedOutShieldsDownTree(owner.pid, owner.processStartIdentity); - const takeover = takeoverShieldsTransitionLock( - sandboxName, + waitForShieldsDownForwardCommit(sandboxName, processToken, assertTakeoverAuthority); + } + + const owner = inspectAnyShieldsTransitionLockOwner(sandboxName); + if (!owner) return initialTransitionOwner; + const ownerStatus = readExactProcessStatus( owner.pid, owner.processStartIdentity, - processToken, + processInspectionDeadlineAfter(SHIELDS_TRANSITION_TERMINATE_GRACE_MS), ); - if ( - !takeover.removed && - takeover.reason !== "missing" && - takeover.reason !== "path-changed" && - takeover.reason !== "owner-mismatch" - ) { - throw new Error(`Cannot take over expired shields transition lock: ${takeover.reason}`); + if (ownerStatus === "gone") { + assertTakeoverAuthority?.(); + persistUnresolvedShieldsContainment(sandboxName, processToken, owner.pid); + throw new Error( + "Shields transition owner exited without descendant-containment proof; permanent containment requires operator resolution", + ); } + throw new Error( + "Shields transition owner is still active; automatic recovery is waiting behind the deadline gate", + ); } function synchronizeAutoRestoreWithShieldsDown(sandboxName: string): void { @@ -2306,7 +2425,36 @@ function synchronizeAutoRestoreWithShieldsDown(sandboxName: string): void { ) { return; } - synchronizeAutoRestoreTransition(sandboxName, timerMarker.processToken, timerMarker.snapshotPath); + synchronizeAutoRestoreTransition( + sandboxName, + timerMarker.processToken, + timerMarker.snapshotPath, + { + retainTransition: true, + assertTakeoverAuthority: () => assertTimerMarkerGeneration(sandboxName, timerMarker), + }, + ); +} + +function completeAutoRestoreTransition( + sandboxName: string, + processToken: string, + snapshotPath: string, +): boolean { + const marker = readTimerMarker(sandboxName); + if ( + marker?.pid !== process.pid || + marker.processToken !== processToken || + marker.snapshotPath !== snapshotPath + ) { + return false; + } + const transition = readShieldsDownTransition(sandboxName, processToken); + if (transition && transition.snapshotPath !== snapshotPath) { + throw new Error("Auto-restore completion does not match shields-down transition ownership"); + } + clearShieldsDownTransition(sandboxName, processToken); + return true; } function lockAgentConfigWithoutHostLock( @@ -2350,8 +2498,6 @@ function resolveExactManagedMcpPolicies( sandboxName: string, livePolicyYaml?: string, ): ReturnType { - if (!hasManagedMcpPolicyClaims(sandboxName)) return []; - let effectiveLivePolicy = livePolicyYaml; if (!effectiveLivePolicy) { let rawPolicy: string; @@ -2370,42 +2516,154 @@ function resolveExactManagedMcpPolicies( return inspectExactManagedMcpPolicies(sandboxName, effectiveLivePolicy); } +function resolveProvableManagedMcpPoliciesForDeadline( + sandboxName: string, +): ReturnType { + try { + let effectiveLivePolicy = ""; + try { + effectiveLivePolicy = parseCurrentPolicy(runCapture(buildPolicyGetCommand(sandboxName))); + } catch { + // The tolerant deadline inspector records exact omissions for every claim + // when the live policy cannot be parsed or read. + } + return inspectProvableManagedMcpPoliciesForDeadline(sandboxName, effectiveLivePolicy); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return { + policies: [], + omissions: [ + { + reason: `Managed MCP registry inspection failed at the auto-restore deadline: ${message}`, + }, + ], + }; + } +} + /** * Restore a saved complete policy while reconciling only exact generated MCP - * entries. Snapshot-time keys are removed before current owned entries are - * overlaid, so changes made during the Shields-down window survive both manual + * entries. Snapshot-time keys are removed before currently owned entries are + * overlaid, so changes made during the shields-down window survive both manual * and timer restoration. */ +interface ShieldsPolicySnapshotRestoreOptions { + transitionProcessToken?: string; + deadlineAuthoritative?: boolean; + expiredTimerRecovery?: boolean; +} + +type ShieldsPolicySnapshotRestoreResult = ReturnType & { + managedMcpOmissions?: ManagedMcpPolicyOmission[]; +}; + function applyShieldsPolicySnapshot( sandboxName: string, snapshotPath: string, -): ReturnType { + options: ShieldsPolicySnapshotRestoreOptions = {}, +): ShieldsPolicySnapshotRestoreResult { const state = loadShieldsState(sandboxName); - const snapshotManagedPolicyKeys = state.shieldsManagedMcpPolicyKeys; + let transition: ShieldsDownTransition | null = null; + if (options.transitionProcessToken !== undefined) { + if (!/^[0-9a-f]{32}$/.test(options.transitionProcessToken)) { + throw new Error("Invalid Shields transition recovery token"); + } + transition = readShieldsDownTransition(sandboxName, options.transitionProcessToken); + if ( + !transition && + fs.existsSync(shieldsDownTransitionPath(sandboxName, options.transitionProcessToken)) + ) { + throw new Error("Shields transition recovery authority is invalid"); + } + if (transition && transition.snapshotPath !== snapshotPath) { + throw new Error("Shields transition does not authorize the policy snapshot being restored"); + } + } + if (options.deadlineAuthoritative) { + const marker = readTimerMarker(sandboxName); + const markerMatchesRecovery = + marker?.sandboxName === sandboxName && + marker.snapshotPath === snapshotPath && + marker.processToken === options.transitionProcessToken; + const restoreAtMs = marker ? new Date(marker.restoreAt).getTime() : Number.NaN; + const expiredTimerIsInactive = + options.expiredTimerRecovery === true && + markerMatchesRecovery && + Number.isFinite(restoreAtMs) && + restoreAtMs <= Date.now() && + (!isProcessAlive(marker!.pid) || !verifyTimerMarkerIdentity(marker!).verified); + if ( + options.transitionProcessToken === undefined || + !markerMatchesRecovery || + (marker!.pid !== process.pid && !expiredTimerIsInactive) + ) { + throw new Error("The active auto-restore timer does not authorize deadline restoration"); + } + } + + if (state._isCorrupt && !transition) { + throw new Error( + `Cannot restore a Shields policy while persisted state is corrupt: ${ + state._corruptError ?? "invalid state" + }`, + ); + } + // A preparing transition can outlive its owner before Shields state is + // committed; its token-bound marker is then the recovery authority. + // Every ordinary restore remains bound to the exact persisted snapshot. + if (!transition && state.shieldsPolicySnapshotPath !== snapshotPath) { + throw new Error("Shields state does not match the policy snapshot being restored"); + } + const persistedSnapshotMatches = state.shieldsPolicySnapshotPath === snapshotPath; + if ( + transition?.managedMcpPolicyKeys !== undefined && + persistedSnapshotMatches && + state.shieldsManagedMcpPolicyKeys !== undefined && + !sameManagedMcpPolicyKeys(transition.managedMcpPolicyKeys, state.shieldsManagedMcpPolicyKeys) + ) { + throw new Error("Shields transition ownership does not match persisted policy ownership"); + } + const snapshotManagedPolicyKeys = + transition?.managedMcpPolicyKeys ?? + (persistedSnapshotMatches ? state.shieldsManagedMcpPolicyKeys : undefined); // A timer or manual restore created by an older NemoClaw build has no exact - // snapshot-time ownership manifest. Preserve its prior raw-snapshot behavior - // instead of guessing from lifetime MCP tombstones or deleting an unowned - // same-prefix key. + // snapshot-time ownership manifest. Preserve raw-snapshot behavior only + // when neither current state nor the snapshot can involve managed MCP; + // otherwise ownership cannot be reconciled without guessing. if (snapshotManagedPolicyKeys === undefined) { + assertLegacyMcpPolicyRestoreSafe( + fs.readFileSync(snapshotPath, "utf-8"), + hasManagedMcpPolicyClaims(sandboxName), + ); return run(buildPolicySetCommand(snapshotPath, sandboxName), { ignoreError: true, }); } - if (state.shieldsPolicySnapshotPath !== snapshotPath) { - throw new Error("Saved managed MCP ownership does not match the Shields policy snapshot"); + let managedMcpOmissions: ManagedMcpPolicyOmission[] = []; + let runtimePolicyPath: string; + if (options.deadlineAuthoritative) { + const inspection = resolveProvableManagedMcpPoliciesForDeadline(sandboxName); + const runtime = buildDeadlineRuntimeManagedMcpPolicy(snapshotPath, { + managedMcpPolicies: inspection.policies, + snapshotManagedPolicyKeys, + readBasePolicy: () => fs.readFileSync(snapshotPath, "utf-8"), + }); + runtimePolicyPath = runtime.path; + managedMcpOmissions = [...inspection.omissions, ...runtime.omissions]; + } else { + const managedMcpPolicies = resolveExactManagedMcpPolicies(sandboxName); + runtimePolicyPath = buildRuntimeManagedMcpPolicy(snapshotPath, { + managedMcpPolicies, + snapshotManagedPolicyKeys, + readBasePolicy: () => fs.readFileSync(snapshotPath, "utf-8"), + }); } - - const managedMcpPolicies = resolveExactManagedMcpPolicies(sandboxName); - const runtimePolicyPath = buildRuntimeManagedMcpPolicy(snapshotPath, { - managedMcpPolicies, - snapshotManagedPolicyKeys, - readBasePolicy: () => fs.readFileSync(snapshotPath, "utf-8"), - }); const runtimePolicyIsTemp = runtimePolicyPath !== snapshotPath; try { - return run(buildPolicySetCommand(runtimePolicyPath, sandboxName), { + const result = run(buildPolicySetCommand(runtimePolicyPath, sandboxName), { ignoreError: true, }); + return managedMcpOmissions.length > 0 ? { ...result, managedMcpOmissions } : result; } finally { if (runtimePolicyIsTemp) { cleanupTempDir(runtimePolicyPath, "nemoclaw-permissive-runtime"); @@ -2471,6 +2729,7 @@ interface LockdownActivationResult { error?: string; chattrApplied?: boolean; fileHashes?: { [path: string]: string }; + managedMcpOmissions?: ManagedMcpPolicyOmission[]; } function activateLockdownFromSnapshot( @@ -2479,14 +2738,15 @@ function activateLockdownFromSnapshot( allowLegacyHermesProtocol = false, cachedTarget?: AgentConfigTarget, cachedProtocol?: HermesShieldsProtocol, + restoreOptions: ShieldsPolicySnapshotRestoreOptions = {}, ): LockdownActivationResult { if (!snapshotPath || !fs.existsSync(snapshotPath)) { return { ok: false, error: "saved snapshot is missing" }; } - let restoreResult: ReturnType; + let restoreResult: ShieldsPolicySnapshotRestoreResult; try { - restoreResult = applyShieldsPolicySnapshot(sandboxName, snapshotPath); + restoreResult = applyShieldsPolicySnapshot(sandboxName, snapshotPath, restoreOptions); } catch (error) { return { ok: false, @@ -2536,6 +2796,9 @@ function activateLockdownFromSnapshot( ok: true, chattrApplied: relock.lastResult.chattrApplied, fileHashes: relock.lastResult.fileHashes, + ...(restoreResult.managedMcpOmissions + ? { managedMcpOmissions: restoreResult.managedMcpOmissions } + : {}), }; } @@ -2562,20 +2825,10 @@ function recoverExpiredAutoRestoreInline( if (Date.now() <= restoreAtMs + AUTO_RESTORE_COMPLETION_GRACE_MS) { return { attempted: false, restored: false }; } - const timerStartIdentity = readProcessStartIdentity(marker.pid); - if (!timerStartIdentity) { - console.error( - " Recovery warning: expired auto-restore timer identity cannot be pinned safely.", - ); - return { attempted: true, restored: false }; - } - try { - stopTimedOutShieldsDownTree(marker.pid, timerStartIdentity); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(` Recovery warning: ${message}`); - return { attempted: true, restored: false }; - } + console.error( + " Recovery warning: the expired auto-restore timer is still active; refusing portable process-tree preemption and waiting for it to exit.", + ); + return { attempted: true, restored: false }; } console.error( @@ -2584,7 +2837,11 @@ function recoverExpiredAutoRestoreInline( if (marker.processToken && /^[0-9a-f]{32}$/.test(marker.processToken)) { try { - synchronizeAutoRestoreTransition(sandboxName, marker.processToken, marker.snapshotPath); + synchronizeAutoRestoreTransition(sandboxName, marker.processToken, marker.snapshotPath, { + expiredTimerRecovery: true, + retainTransition: true, + assertTakeoverAuthority: () => assertTimerMarkerGeneration(sandboxName, marker), + }); } catch (error) { const message = error instanceof Error ? error.message : String(error); appendAuditEntry({ @@ -2604,6 +2861,15 @@ function recoverExpiredAutoRestoreInline( sandboxName, marker.snapshotPath, marker.allowLegacyHermesProtocol === true, + undefined, + undefined, + marker.processToken && /^[0-9a-f]{32}$/.test(marker.processToken) + ? { + transitionProcessToken: marker.processToken, + deadlineAuthoritative: true, + expiredTimerRecovery: true, + } + : {}, ); const nowIso = new Date().toISOString(); if (!activation.ok) { @@ -2644,6 +2910,13 @@ function recoverExpiredAutoRestoreInline( restored_by: "auto_timer", policy_snapshot: marker.snapshotPath, restored_at: nowIso, + ...(activation.managedMcpOmissions?.length + ? { + warning: `Inline auto-restore omitted ${String( + activation.managedMcpOmissions.length, + )} unproven managed MCP policy entries`, + } + : {}), }); return { attempted: true, restored: true }; } @@ -2822,9 +3095,15 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = (() => { throw new Error("Cannot identify shields-down owner process"); })(), + ownerMcpProcessIdentity: + readMcpLockProcessIdentity(process.pid, true) ?? + (() => { + throw new Error("Cannot identify shields-down lifecycle owner process"); + })(), processToken, sandboxName, snapshotPath, + managedMcpPolicyKeys: snapshotManagedMcpPolicyKeys, }; const leaseOwnerPid = opts.deferAutoRestoreWhileOwnerAlive ? transition.ownerPid : null; const leaseOwnerStartIdentity = opts.deferAutoRestoreWhileOwnerAlive @@ -2878,11 +3157,6 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = timerChild.disconnect(); timerChild.unref(); } catch (err) { - try { - timerChild?.kill("SIGTERM"); - } catch { - // Best effort; without a matching marker the child has no authority. - } clearTimerMarker(sandboxName); clearShieldsDownTransition(sandboxName, processToken); const message = err instanceof Error ? err.message : String(err); @@ -3464,9 +3738,10 @@ function shieldsStatus( shieldsStatusWithoutHostLock(sandboxName, false, deps), ); } - prepareExpiredAutoRestoreHostLockTakeover(sandboxName); - return withTimerBoundShieldsMutationLock(sandboxName, "shields status", () => - shieldsStatusWithoutHostLock(sandboxName, true, deps), + return withExpiredAutoRestoreDeadlineFence( + sandboxName, + "shields status", + (allowInlineRecovery) => shieldsStatusWithoutHostLock(sandboxName, allowInlineRecovery, deps), ); } catch (error) { return completeDeferredShieldsExit(error); @@ -3528,10 +3803,11 @@ function clearShieldsState(sandboxName: string): void { export { applyShieldsPolicySnapshot, clearShieldsState, + completeAutoRestoreTransition, DEFAULT_TIMEOUT_SECONDS, deriveShieldsMode, - excludeRecoveryProcessTree, getShieldsPosture, + inspectAutoRestoreTransitionTakeoverOwner, inspectMutableConfigPerms, isShieldsDown, killTimer, diff --git a/src/lib/shields/mcp-policy-transition.test.ts b/src/lib/shields/mcp-policy-transition.test.ts index 458e7a777fc..ff60502e00b 100644 --- a/src/lib/shields/mcp-policy-transition.test.ts +++ b/src/lib/shields/mcp-policy-transition.test.ts @@ -1,10 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import YAML from "yaml"; import { describe, expect, it } from "vitest"; +import YAML from "yaml"; import { + hasManagedMcpPolicyClaims, + inspectProvableManagedMcpPoliciesForDeadline, inspectExactManagedMcpPolicies as inspectRegisteredManagedMcpPolicies, MCP_BRIDGE_POLICY_SOURCE, } from "../actions/sandbox/mcp-bridge-policy"; @@ -14,7 +16,11 @@ import { buildMcpBridgePolicyYaml, } from "../actions/sandbox/mcp-bridge-policy-render"; import type { SandboxEntry } from "../state/registry"; -import { composeManagedMcpPolicies } from "./mcp-policy-transition"; +import { + assertLegacyMcpPolicyRestoreSafe, + composeDeadlineManagedMcpPolicies, + composeManagedMcpPolicies, +} from "./mcp-policy-transition"; const ADAPTER = "hermes-config"; @@ -63,6 +69,18 @@ function networkEntry(content: string, server: string): unknown { return YAML.parse(content).network_policies[buildMcpBridgePolicyKey(server)]; } +function mutateRegisteredNetworkPolicy( + policy: ReturnType, + server: string, + mutate: (entry: Record) => void, +): void { + const document = YAML.parse(policy.content) as { + network_policies: Record>; + }; + mutate(document.network_policies[buildMcpBridgePolicyKey(server)]!); + policy.content = YAML.stringify(document); +} + function livePolicy( entries: Array<{ content: string; server: string }>, extra: Record = {}, @@ -160,6 +178,129 @@ describe("managed MCP Shields policy transitions (#7952)", () => { ).toThrow(/drifted from its ownership record/); }); + it("rejects matching registry and live documents with weakened generated semantics", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + mutateRegisteredNetworkPolicy(alpha, "alpha", (entry) => { + const endpoint = (entry.endpoints as Array>)[0]!; + endpoint.enforcement = "observe"; + }); + const sandbox = sandboxWithPolicies([alpha]); + const live = livePolicy([{ content: alpha.content, server: "alpha" }]); + + expect(() => inspectExactManagedMcpPolicies(sandbox, live)).toThrow( + /non-canonical generated content/, + ); + expect( + inspectProvableManagedMcpPoliciesForDeadline("alpha", live, { + getSandbox: () => sandbox, + }), + ).toEqual({ + policies: [], + omissions: [ + expect.objectContaining({ + server: "alpha", + reason: expect.stringMatching(/non-canonical generated content/), + }), + ], + }); + }); + + it.each([ + { + label: "a private literal", + pins: ["127.0.0.1"], + expected: /invalid public address pins/, + }, + { + label: "a scoped public IPv6 literal", + pins: ["2001:4860:4860::8888%lo0"], + expected: /invalid public address pins/, + }, + { + label: "duplicate literals", + pins: ["8.8.8.8", "8.8.8.8"], + expected: /non-canonical public address pins/, + }, + { + label: "unsorted literals", + pins: ["8.8.8.8", "1.1.1.1"], + expected: /non-canonical public address pins/, + }, + ])("rejects matching registry and live documents with $label", ({ pins, expected }) => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + mutateRegisteredNetworkPolicy(alpha, "alpha", (entry) => { + const endpoint = (entry.endpoints as Array>)[0]!; + endpoint.allowed_ips = pins; + }); + + expect(() => + inspectExactManagedMcpPolicies( + sandboxWithPolicies([alpha]), + livePolicy([{ content: alpha.content, server: "alpha" }]), + ), + ).toThrow(expected); + }); + + it("fails closed on a generated policy record without managed MCP state", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const sandbox: SandboxEntry = { + name: "alpha", + agent: "hermes", + customPolicies: [alpha], + }; + const deps = { getSandbox: () => sandbox }; + + expect(hasManagedMcpPolicyClaims("alpha", deps)).toBe(true); + expect(() => + inspectRegisteredManagedMcpPolicies( + "alpha", + livePolicy([{ content: alpha.content, server: "alpha" }]), + deps, + ), + ).toThrow(/no committed managed bridge ownership/); + }); + + it("treats residual managed server history as an ownership claim", () => { + const sandbox: SandboxEntry = { + name: "alpha", + agent: "hermes", + mcp: { bridges: {}, managedServerNames: ["retired"] }, + }; + const deps = { getSandbox: () => sandbox }; + + expect(hasManagedMcpPolicyClaims("alpha", deps)).toBe(true); + expect( + inspectRegisteredManagedMcpPolicies( + "alpha", + livePolicy([], { unrelated_live_entry: {} }), + deps, + ), + ).toEqual([]); + }); + + it.each([ + { + label: "no sandbox registry entry", + sandbox: undefined, + }, + { + label: "only residual ownership history", + sandbox: { + name: "alpha", + agent: "hermes", + mcp: { bridges: {}, managedServerNames: ["retired"] }, + } satisfies SandboxEntry, + }, + ])("rejects an unclassified reserved live key with $label", ({ sandbox }) => { + expect(() => + inspectRegisteredManagedMcpPolicies("alpha", livePolicy([], { mcp_bridge_retired: {} }), { + getSandbox: () => sandbox ?? null, + }), + ).toThrow( + /Reserved MCP policy key 'mcp_bridge_retired'.*no committed managed bridge ownership/, + ); + }); + it("retains additions while restoring the restrictive snapshot", () => { const alpha = registeredPolicy("alpha", "8.8.8.8"); const beta = registeredPolicy("beta", "1.1.1.1"); @@ -187,7 +328,7 @@ describe("managed MCP Shields policy transitions (#7952)", () => { ]); }); - it("does not resurrect a managed MCP policy removed while Shields are down", () => { + it("does not restore a managed MCP policy removed during the shields-down window", () => { const alpha = registeredPolicy("alpha", "8.8.8.8"); const snapshot = YAML.stringify({ version: 1, @@ -224,4 +365,267 @@ describe("managed MCP Shields policy transitions (#7952)", () => { networkEntry(currentAlpha.content, "alpha"), ); }); + + it("rejects an unclassified reserved key in the restrictive snapshot", () => { + const currentAlpha = registeredPolicy("alpha", "1.1.1.1"); + const current = inspectExactManagedMcpPolicies( + sandboxWithPolicies([currentAlpha]), + livePolicy([{ content: currentAlpha.content, server: "alpha" }]), + ); + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + mcp_bridge_alpha: { + name: "operator-owned-alpha", + endpoints: [{ host: "operator.example.com" }], + }, + }, + }); + + expect(() => composeManagedMcpPolicies(snapshot, current, [])).toThrow( + /Reserved MCP policy key 'mcp_bridge_alpha'.*absent from the saved ownership manifest/, + ); + }); + + it("accepts an empty ownership manifest when the snapshot has no reserved keys", () => { + const snapshot = YAML.stringify({ + version: 1, + network_policies: { restrictive_baseline: {} }, + }); + + expect(YAML.parse(composeManagedMcpPolicies(snapshot, [], [])).network_policies).toEqual({ + restrictive_baseline: {}, + }); + }); + + it("rejects a saved managed key that is absent from its policy snapshot", () => { + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + }, + }); + + expect(() => composeManagedMcpPolicies(snapshot, [], ["mcp_bridge_alpha"])).toThrow( + /absent from its policy snapshot/, + ); + }); + + it.each([ + { + label: "current managed MCP ownership", + hasCurrentManagedClaims: true, + networkPolicies: { restrictive_baseline: {} }, + }, + { + label: "a managed-shaped key in the snapshot", + hasCurrentManagedClaims: false, + networkPolicies: { mcp_bridge_alpha: {} }, + }, + ])("refuses legacy restore with $label", ({ hasCurrentManagedClaims, networkPolicies }) => { + expect(() => + assertLegacyMcpPolicyRestoreSafe( + YAML.stringify({ version: 1, network_policies: networkPolicies }), + hasCurrentManagedClaims, + ), + ).toThrow(/no managed MCP ownership manifest/); + }); + + it("allows a legacy restore with no current or snapshot MCP ownership", () => { + expect(() => + assertLegacyMcpPolicyRestoreSafe( + YAML.stringify({ + version: 1, + network_policies: { restrictive_baseline: {} }, + }), + false, + ), + ).not.toThrow(); + }); + + it("proves committed bridges independently while omitting an incomplete add at the deadline", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const beta = registeredPolicy("beta", "1.1.1.1"); + const sandbox = sandboxWithPolicies([alpha, beta]); + sandbox.mcp!.bridges.beta!.addState = "prepared"; + + const result = inspectProvableManagedMcpPoliciesForDeadline( + "alpha", + livePolicy([ + { content: alpha.content, server: "alpha" }, + { content: beta.content, server: "beta" }, + ]), + { getSandbox: () => sandbox }, + ); + + expect(result.policies.map((policy) => policy.server)).toEqual(["alpha"]); + expect(result.omissions).toEqual([ + expect.objectContaining({ server: "beta", reason: expect.stringMatching(/incomplete/) }), + ]); + }); + + it("omits every deadline claimant whose canonical policy identity collides", () => { + const collidingPolicy = registeredPolicy("foo-bar", "8.8.8.8"); + const sandbox = sandboxWithPolicies([collidingPolicy], ["foo-bar", "foo_bar"]); + + const result = inspectProvableManagedMcpPoliciesForDeadline( + "alpha", + livePolicy([{ content: collidingPolicy.content, server: "foo-bar" }]), + { getSandbox: () => sandbox }, + ); + + expect(result.policies).toEqual([]); + expect(result.omissions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + server: "foo-bar", + reason: expect.stringMatching(/ambiguous bridge ownership/), + }), + expect.objectContaining({ + server: "foo_bar", + reason: expect.stringMatching(/ambiguous bridge ownership/), + }), + ]), + ); + }); + + it.each([ + "destroyPreparedAt", + "destroyPendingAt", + ] as const)("omits every generated policy while %s is present", (marker) => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const sandbox = sandboxWithPolicies([alpha]); + sandbox.mcp![marker] = "2026-07-30T01:00:00.000Z"; + + const result = inspectProvableManagedMcpPoliciesForDeadline( + "alpha", + livePolicy([{ content: alpha.content, server: "alpha" }]), + { getSandbox: () => sandbox }, + ); + + expect(result.policies).toEqual([]); + expect(result.omissions).toEqual([ + expect.objectContaining({ server: "alpha", reason: expect.stringMatching(/destruction/) }), + ]); + }); + + it("omits drift and orphan claims without discarding another exact bridge", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const beta = registeredPolicy("beta", "1.1.1.1"); + const driftedBeta = registeredPolicy("beta", "9.9.9.9"); + const orphan = registeredPolicy("orphan", "4.4.4.4"); + const sandbox = sandboxWithPolicies([alpha, beta, orphan], ["alpha", "beta"]); + + const result = inspectProvableManagedMcpPoliciesForDeadline( + "alpha", + livePolicy([ + { content: alpha.content, server: "alpha" }, + { content: driftedBeta.content, server: "beta" }, + { content: orphan.content, server: "orphan" }, + ]), + { getSandbox: () => sandbox }, + ); + + expect(result.policies.map((policy) => policy.server)).toEqual(["alpha"]); + expect(result.omissions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ server: "beta", reason: expect.stringMatching(/drifted/) }), + expect.objectContaining({ + policyName: "mcp-bridge-orphan", + reason: expect.stringMatching(/no committed managed bridge ownership/), + }), + ]), + ); + }); + + it("deadline inspection reports an unclassified reserved live key", () => { + const result = inspectProvableManagedMcpPoliciesForDeadline( + "alpha", + livePolicy([], { mcp_bridge_residual: {} }), + { getSandbox: () => null }, + ); + + expect(result).toEqual({ + policies: [], + omissions: [ + expect.objectContaining({ + key: "mcp_bridge_residual", + reason: expect.stringMatching(/no committed managed bridge ownership/), + }), + ], + }); + }); + + it("deadline composition strips unclassified reserved keys before overlaying proven entries", () => { + const alpha = registeredPolicy("alpha", "8.8.8.8"); + const beta = registeredPolicy("beta", "1.1.1.1"); + const current = inspectExactManagedMcpPolicies( + sandboxWithPolicies([alpha, beta]), + livePolicy([ + { content: alpha.content, server: "alpha" }, + { content: beta.content, server: "beta" }, + ]), + ); + const operatorEntry = { endpoints: [{ host: "operator.example.com" }] }; + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + mcp_bridge_alpha: networkEntry(alpha.content, "alpha"), + mcp_bridge_beta: operatorEntry, + restrictive_baseline: {}, + }, + }); + + const result = composeDeadlineManagedMcpPolicies(snapshot, current, ["mcp_bridge_alpha"]); + const restored = YAML.parse(result.yaml); + + expect(restored.network_policies.mcp_bridge_alpha).toEqual( + networkEntry(alpha.content, "alpha"), + ); + expect(restored.network_policies.mcp_bridge_beta).toEqual(networkEntry(beta.content, "beta")); + expect(result.omissions).toEqual([ + expect.objectContaining({ + key: "mcp_bridge_beta", + reason: expect.stringMatching(/absent from the saved ownership manifest/), + }), + ]); + }); + + it("deadline composition strips every reserved shape with an empty manifest", () => { + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + mcp_bridge_: {}, + mcp_bridge_legacy_invalid_name: {}, + restrictive_baseline: {}, + }, + }); + + const result = composeDeadlineManagedMcpPolicies(snapshot, [], []); + + expect(YAML.parse(result.yaml).network_policies).toEqual({ restrictive_baseline: {} }); + expect(result.omissions.map((entry) => entry.key)).toEqual([ + "mcp_bridge_", + "mcp_bridge_legacy_invalid_name", + ]); + }); + + it("deadline composition restores the restrictive baseline when a saved key is absent", () => { + const snapshot = YAML.stringify({ + version: 1, + network_policies: { + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + }, + }); + + const result = composeDeadlineManagedMcpPolicies(snapshot, [], ["mcp_bridge_alpha"]); + const restored = YAML.parse(result.yaml); + + expect(restored.network_policies).toEqual({ + restrictive_baseline: { endpoints: [{ host: "baseline.example.com" }] }, + }); + expect(result.omissions).toEqual([ + expect.objectContaining({ reason: expect.stringMatching(/already absent/) }), + ]); + }); }); diff --git a/src/lib/shields/mcp-policy-transition.ts b/src/lib/shields/mcp-policy-transition.ts index c06bc0efdcc..984d7cde2be 100644 --- a/src/lib/shields/mcp-policy-transition.ts +++ b/src/lib/shields/mcp-policy-transition.ts @@ -3,9 +3,13 @@ import YAML from "yaml"; -import type { ExactManagedMcpPolicy } from "../actions/sandbox/mcp-bridge-policy"; +import type { + ExactManagedMcpPolicy, + ManagedMcpPolicyOmission, +} from "../actions/sandbox/mcp-bridge-policy"; -const MANAGED_MCP_POLICY_KEY_RE = /^mcp_bridge_[a-z][a-z0-9_]{0,63}$/; +const CANONICAL_MANAGED_MCP_POLICY_KEY_RE = /^mcp_bridge_[a-z][a-z0-9_]{0,63}$/; +const RESERVED_MANAGED_MCP_POLICY_KEY_RE = /^mcp_bridge_/; function parsePolicyDocument(source: string, label: string): Record { let parsed: unknown; @@ -35,8 +39,8 @@ function readNetworkPolicies( /** * Reconcile generated MCP entries into a complete target policy. * - * Snapshot-time keys are removed first so an MCP server deleted while Shields - * are down cannot be resurrected. The current exact entries are then overlaid, + * Snapshot-time keys are removed first so an MCP server deleted during the + * shields-down window cannot be restored. The current exact entries are then overlaid, * retaining additions and replacing stale pins. Every non-MCP target entry * remains authoritative; unrelated live entries are never copied. */ @@ -50,16 +54,27 @@ export function composeManagedMcpPolicies( const snapshotKeys = new Set(); for (const key of snapshotManagedPolicyKeys) { - if (!MANAGED_MCP_POLICY_KEY_RE.test(key) || snapshotKeys.has(key)) { + if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(key) || snapshotKeys.has(key)) { throw new Error("Saved Shields MCP policy ownership is invalid"); } + if (!Object.hasOwn(targetPolicies, key)) { + throw new Error(`Saved Shields MCP policy '${key}' is absent from its policy snapshot`); + } snapshotKeys.add(key); delete targetPolicies[key]; } + const unclassifiedKey = Object.keys(targetPolicies).find((key) => + RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(key), + ); + if (unclassifiedKey) { + throw new Error( + `Reserved MCP policy key '${unclassifiedKey}' is absent from the saved ownership manifest`, + ); + } const currentKeys = new Set(); for (const policy of currentPolicies) { - if (!MANAGED_MCP_POLICY_KEY_RE.test(policy.key) || currentKeys.has(policy.key)) { + if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(policy.key) || currentKeys.has(policy.key)) { throw new Error(`Managed MCP policy key '${policy.key}' has ambiguous ownership`); } currentKeys.add(policy.key); @@ -70,6 +85,84 @@ export function composeManagedMcpPolicies( return YAML.stringify(target); } +export interface DeadlineManagedMcpPolicyComposition { + yaml: string; + omissions: ManagedMcpPolicyOmission[]; +} + +/** + * Security-authoritative deadline composition. + * + * Every reserved key is removed from the snapshot, including keys missing from + * an incomplete manifest. Only independently proven current entries are then + * overlaid. + */ +export function composeDeadlineManagedMcpPolicies( + targetPolicyYaml: string, + currentPolicies: readonly ExactManagedMcpPolicy[], + snapshotManagedPolicyKeys: readonly string[], +): DeadlineManagedMcpPolicyComposition { + const target = parsePolicyDocument(targetPolicyYaml, "Target Shields policy"); + const targetPolicies = readNetworkPolicies(target, "Target Shields policy"); + + const snapshotKeys = new Set(); + const omissions: ManagedMcpPolicyOmission[] = []; + for (const key of snapshotManagedPolicyKeys) { + if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(key) || snapshotKeys.has(key)) { + throw new Error("Saved Shields MCP policy ownership is invalid"); + } + if (!Object.hasOwn(targetPolicies, key)) { + omissions.push({ + reason: `Saved Shields MCP policy '${key}' was already absent from its policy snapshot`, + }); + } + snapshotKeys.add(key); + delete targetPolicies[key]; + } + for (const key of Object.keys(targetPolicies)) { + if (!RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(key)) continue; + delete targetPolicies[key]; + omissions.push({ + key, + reason: `Reserved MCP policy key '${key}' was absent from the saved ownership manifest`, + }); + } + + const currentKeys = new Set(); + for (const policy of currentPolicies) { + if (!CANONICAL_MANAGED_MCP_POLICY_KEY_RE.test(policy.key) || currentKeys.has(policy.key)) { + throw new Error(`Managed MCP policy key '${policy.key}' has ambiguous ownership`); + } + currentKeys.add(policy.key); + targetPolicies[policy.key] = policy.networkPolicy; + } + + target.network_policies = targetPolicies; + return { yaml: YAML.stringify(target), omissions }; +} + export function isManagedMcpPolicyKey(value: unknown): value is string { - return typeof value === "string" && MANAGED_MCP_POLICY_KEY_RE.test(value); + return typeof value === "string" && RESERVED_MANAGED_MCP_POLICY_KEY_RE.test(value); +} + +/** + * Refuse to guess managed ownership for a Shields snapshot captured before the + * ownership manifest existed. Current claims prove reconciliation is needed; + * a managed-shaped snapshot key may be a removed bridge or an operator entry. + * Either case requires explicit recovery instead of a destructive raw apply. + */ +export function assertLegacyMcpPolicyRestoreSafe( + snapshotPolicyYaml: string, + hasCurrentManagedClaims: boolean, +): void { + const snapshot = parsePolicyDocument(snapshotPolicyYaml, "Legacy Shields policy snapshot"); + const snapshotPolicies = readNetworkPolicies(snapshot, "Legacy Shields policy snapshot"); + if ( + hasCurrentManagedClaims || + Object.keys(snapshotPolicies).some((key) => isManagedMcpPolicyKey(key)) + ) { + throw new Error( + "Legacy Shields state has no managed MCP ownership manifest; refusing policy restore", + ); + } } diff --git a/src/lib/shields/permissive-runtime.ts b/src/lib/shields/permissive-runtime.ts index 10232613a28..ddc53900a76 100644 --- a/src/lib/shields/permissive-runtime.ts +++ b/src/lib/shields/permissive-runtime.ts @@ -5,14 +5,28 @@ import fs from "node:fs"; import YAML from "yaml"; export { + type ExactManagedMcpPolicy, hasManagedMcpPolicyClaims, inspectExactManagedMcpPolicies, - type ExactManagedMcpPolicy, + inspectProvableManagedMcpPoliciesForDeadline, + type ManagedMcpPolicyOmission, +} from "../actions/sandbox/mcp-bridge-policy"; + +import type { + ExactManagedMcpPolicy, + ManagedMcpPolicyOmission, } from "../actions/sandbox/mcp-bridge-policy"; -import type { ExactManagedMcpPolicy } from "../actions/sandbox/mcp-bridge-policy"; import { cleanupTempDir, secureTempFile } from "../onboard/temp-files"; -export { isManagedMcpPolicyKey } from "./mcp-policy-transition"; -import { composeManagedMcpPolicies } from "./mcp-policy-transition"; + +export { + assertLegacyMcpPolicyRestoreSafe, + isManagedMcpPolicyKey, +} from "./mcp-policy-transition"; + +import { + composeDeadlineManagedMcpPolicies, + composeManagedMcpPolicies, +} from "./mcp-policy-transition"; const TEMP_FILE_PREFIX = "nemoclaw-permissive-runtime"; @@ -175,17 +189,14 @@ export interface ManagedMcpRuntimePolicyDeps { * Reconcile current generated MCP policies into a custom Shields-down policy * or a saved restrictive snapshot. Unlike the legacy filesystem-only fallback, * this path must fail closed: returning the unmodified base could silently - * discard a managed entry or resurrect one that was removed while Shields were - * down. + * discard a managed entry or restore one that was removed during the + * shields-down window. */ export function buildRuntimeManagedMcpPolicy( basePolicyPath: string, deps: ManagedMcpRuntimePolicyDeps, ): string { const snapshotManagedPolicyKeys = deps.snapshotManagedPolicyKeys ?? []; - if (deps.managedMcpPolicies.length === 0 && snapshotManagedPolicyKeys.length === 0) { - return basePolicyPath; - } let baseYaml: string; try { @@ -223,6 +234,40 @@ export function buildRuntimeManagedMcpPolicy( } } +export interface DeadlineManagedMcpRuntimePolicy { + path: string; + omissions: ManagedMcpPolicyOmission[]; +} + +export function buildDeadlineRuntimeManagedMcpPolicy( + basePolicyPath: string, + deps: ManagedMcpRuntimePolicyDeps, +): DeadlineManagedMcpRuntimePolicy { + const baseYaml = deps.readBasePolicy(); + const composition = composeDeadlineManagedMcpPolicies( + baseYaml, + deps.managedMcpPolicies, + deps.snapshotManagedPolicyKeys ?? [], + ); + let runtimePath: string | null = null; + try { + runtimePath = deps.writeTempPolicy + ? deps.writeTempPolicy(composition.yaml) + : secureTempFile(TEMP_FILE_PREFIX, ".yaml"); + if (!deps.writeTempPolicy) { + fs.writeFileSync(runtimePath, composition.yaml, { mode: 0o600 }); + } + return { path: runtimePath, omissions: composition.omissions }; + } catch (error) { + if (runtimePath && !deps.writeTempPolicy) { + cleanupTempDir(runtimePath, TEMP_FILE_PREFIX); + } + throw new Error("Cannot stage the deadline Shields policy for managed MCP reconciliation", { + cause: error, + }); + } +} + function safeYamlObject(text: string): Record | null { try { const parsed = YAML.parse(text); diff --git a/src/lib/shields/timer-bound-lock.ts b/src/lib/shields/timer-bound-lock.ts index 4514cefe55f..7a94aa0fe4c 100644 --- a/src/lib/shields/timer-bound-lock.ts +++ b/src/lib/shields/timer-bound-lock.ts @@ -2,7 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 import { readAutoRestoreTakeoverToken } from "./timer-control"; -import { withShieldsTransitionLock, withShieldsTransitionLockAsync } from "./transition-lock"; +import { + type ShieldsTransitionLockOptions, + withShieldsTransitionLock, + withShieldsTransitionLockAsync, +} from "./transition-lock"; + +export { + beginCommittedMcpLifecycleContainmentSync, + getMcpLifecycleLockPath, + isMcpLifecycleLockHeld, + readMcpLockProcessIdentity, + withMcpLifecycleDeadlineFenceSync, + withMcpLifecycleLockSync, +} from "../state/mcp-lifecycle-lock"; const MAX_TIMER_GENERATION_RETRIES = 3; @@ -20,18 +33,12 @@ const defaultDeps: TimerBoundLockDeps = { type Attempt = { retry: true } | { retry: false; value: T }; -/** - * Serialize a mutation and bind its lock owner to the exact active restore - * timer generation. If a timer is replaced while this operation waits for the - * lock, release without mutating and retry with the new token. This prevents a - * command that observed timer A from becoming non-preemptible inside timer B's - * mutable window. - */ -export function withTimerBoundShieldsMutationLock( +function withTimerBoundShieldsMutationLockOptions( sandboxName: string, command: string, fn: () => T, - deps: TimerBoundLockDeps = defaultDeps, + lockOptions: ShieldsTransitionLockOptions, + deps: TimerBoundLockDeps, ): T { for (let attempt = 0; attempt < MAX_TIMER_GENERATION_RETRIES; attempt += 1) { const token = deps.readToken(sandboxName); @@ -42,13 +49,52 @@ export function withTimerBoundShieldsMutationLock( if (deps.readToken(sandboxName) !== token) return { retry: true }; return { retry: false, value: fn() }; }, - token ? { takeoverToken: token } : {}, + { + ...lockOptions, + ...(token ? { takeoverToken: token } : {}), + }, ); if (!result.retry) return result.value; } throw new Error(`Auto-restore timer generation kept changing while acquiring '${command}'`); } +/** + * Serialize a mutation and bind its lock owner to the exact active restore + * timer generation. If a timer is replaced while this operation waits for the + * lock, release without mutating and retry with the new token. This prevents a + * command that observed timer A from becoming non-preemptible inside timer B's + * mutable window. + */ +export function withTimerBoundShieldsMutationLock( + sandboxName: string, + command: string, + fn: () => T, + deps: TimerBoundLockDeps = defaultDeps, +): T { + return withTimerBoundShieldsMutationLockOptions(sandboxName, command, fn, {}, deps); +} + +/** + * Auto-restore uses a stronger stale-owner protocol than ordinary commands. + * A stale transition owner is preserved so the recovery coordinator can + * publish permanent containment instead of deleting a generation whose + * descendants cannot be ruled out. + */ +export function withTimerBoundAutoRestoreLock( + sandboxName: string, + command: string, + fn: () => T, +): T { + return withTimerBoundShieldsMutationLockOptions( + sandboxName, + command, + fn, + { recoverStaleOwner: false, waitTimeoutMs: 0 }, + defaultDeps, + ); +} + export async function withTimerBoundShieldsMutationLockAsync( sandboxName: string, command: string, diff --git a/src/lib/shields/timer-control.ts b/src/lib/shields/timer-control.ts index 727a89b7347..9055eab296a 100644 --- a/src/lib/shields/timer-control.ts +++ b/src/lib/shields/timer-control.ts @@ -3,11 +3,14 @@ import { execFileSync } from "node:child_process"; import fs from "node:fs"; -import path from "node:path"; import { performance } from "node:perf_hooks"; -import { isObjectRecord } from "../core/json-types"; -import { resolveNemoclawStateDir } from "../state/paths"; +import { + readShieldsTimerMarker, + readShieldsTimerTakeoverToken, + type ShieldsTimerMarker, + shieldsTimerMarkerPath, +} from "../state/mcp-lifecycle-lock/shields-timer-authority"; const DEFAULT_PROCESS_INSPECTION_TIMEOUT_MS = 5_000; @@ -28,68 +31,16 @@ function processInspectionDeadlineReached(deadline: number): boolean { return performance.now() >= deadline; } -interface TimerMarker { - pid: number; - sandboxName: string; - snapshotPath: string; - restoreAt: string; - processToken?: string; - allowLegacyHermesProtocol?: boolean; - leaseOwnerPid?: number; - leaseOwnerStartIdentity?: string; -} - -function isTimerMarker(value: unknown): value is TimerMarker { - if (!isObjectRecord(value)) return false; - const pid = value.pid; - return ( - typeof pid === "number" && - Number.isInteger(pid) && - pid > 0 && - typeof value.sandboxName === "string" && - typeof value.snapshotPath === "string" && - typeof value.restoreAt === "string" && - (value.processToken === undefined || typeof value.processToken === "string") && - (value.allowLegacyHermesProtocol === undefined || - typeof value.allowLegacyHermesProtocol === "boolean") && - (value.leaseOwnerPid === undefined || - (typeof value.leaseOwnerPid === "number" && - Number.isInteger(value.leaseOwnerPid) && - value.leaseOwnerPid > 0)) && - (value.leaseOwnerStartIdentity === undefined || - typeof value.leaseOwnerStartIdentity === "string") && - ((value.leaseOwnerPid === undefined && value.leaseOwnerStartIdentity === undefined) || - (typeof value.leaseOwnerPid === "number" && - typeof value.leaseOwnerStartIdentity === "string" && - value.leaseOwnerStartIdentity.length > 0)) - ); -} - function timerMarkerPath(sandboxName: string): string { - return path.join(resolveNemoclawStateDir(), `shields-timer-${sandboxName}.json`); + return shieldsTimerMarkerPath(sandboxName); } -function readTimerMarker(sandboxName: string): TimerMarker | null { - const p = timerMarkerPath(sandboxName); - if (!fs.existsSync(p)) return null; - try { - const parsed = JSON.parse(fs.readFileSync(p, "utf-8")); - return isTimerMarker(parsed) ? parsed : null; - } catch { - return null; - } +function readTimerMarker(sandboxName: string): ShieldsTimerMarker | null { + return readShieldsTimerMarker(sandboxName); } function readAutoRestoreTakeoverToken(sandboxName: string): string | undefined { - const marker = readTimerMarker(sandboxName); - if ( - marker?.sandboxName !== sandboxName || - typeof marker.processToken !== "string" || - !/^[0-9a-f]{32}$/.test(marker.processToken) - ) { - return undefined; - } - return marker.processToken; + return readShieldsTimerTakeoverToken(sandboxName); } interface ClearTimerMarkerResult { @@ -193,66 +144,6 @@ function readProcessStartIdentity( } } -interface ProcessIdentity { - pid: number; - startIdentity: string; - depth: number; -} - -function listDescendantProcessIdentities( - rootPid: number, - deadline = processInspectionDeadline(), -): ProcessIdentity[] | null { - if (!Number.isInteger(rootPid) || rootPid <= 0) return null; - let rows: Array<{ pid: number; ppid: number }> = []; - try { - const timeout = remainingProcessInspectionTimeout(deadline); - if (timeout === null) return null; - rows = execFileSync("ps", ["-e", "-o", "pid=,ppid="], { - stdio: ["ignore", "pipe", "ignore"], - timeout, - }) - .toString() - .split("\n") - .map((line) => line.trim().split(/\s+/)) - .filter((parts) => parts.length >= 2) - .map(([pid, ppid]) => ({ pid: Number(pid), ppid: Number(ppid) })) - .filter((row) => Number.isInteger(row.pid) && Number.isInteger(row.ppid)); - } catch { - return null; - } - - const descendants: Array<{ pid: number; depth: number }> = []; - let frontier = [{ pid: rootPid, depth: 0 }]; - const seen = new Set([rootPid]); - while (frontier.length > 0) { - const next: Array<{ pid: number; depth: number }> = []; - for (const parent of frontier) { - for (const row of rows) { - if (row.ppid !== parent.pid || seen.has(row.pid)) continue; - seen.add(row.pid); - const child = { pid: row.pid, depth: parent.depth + 1 }; - descendants.push(child); - next.push(child); - } - } - frontier = next; - } - - const identities: ProcessIdentity[] = []; - for (const { pid, depth } of descendants) { - const startIdentity = readProcessStartIdentity(pid, deadline); - if (startIdentity) { - identities.push({ pid, startIdentity, depth }); - } else if (isProcessAlive(pid, deadline)) { - // A live descendant that cannot be identity-pinned must not be signaled; - // callers fail closed instead of risking PID-reuse collateral damage. - return null; - } - } - return identities.sort((a, b) => b.depth - a.depth); -} - function readProcessCommandLine( pid: number, deadline = processInspectionDeadline(), @@ -284,7 +175,10 @@ function readProcessCommandLine( } } -function verifyTimerMarkerIdentity(marker: TimerMarker): { verified: boolean; warning?: string } { +function verifyTimerMarkerIdentity(marker: ShieldsTimerMarker): { + verified: boolean; + warning?: string; +} { const commandLine = readProcessCommandLine(marker.pid); if (!commandLine) { return { @@ -325,7 +219,6 @@ interface KillTimerResult { function killTimer(sandboxName: string): KillTimerResult { const marker = readTimerMarker(sandboxName); let wasAlive = false; - let terminated = false; const warnings: string[] = []; if (marker) { @@ -336,22 +229,15 @@ function killTimer(sandboxName: string): KillTimerResult { if (verification.warning) { warnings.push(verification.warning); } - } else { - try { - process.kill(marker.pid, "SIGTERM"); - terminated = true; - } catch (error) { - const errno = error as NodeJS.ErrnoException; - if (errno.code !== "ESRCH") { - warnings.push( - `Failed to terminate shields timer PID ${String(marker.pid)} for sandbox '${sandboxName}': ${errno.message}`, - ); - } - } } } } + // Marker removal is cooperative cancellation and revokes the timer's exact + // recovery generation. Do not signal a verified live timer: it may own the + // lifecycle deadline fence, and an unhandled signal could bypass its finally + // cleanup and strand the fence. Recovery loops re-check marker authority and + // unwind their locks after this revocation. const markerClear = clearTimerMarker(sandboxName); if (markerClear.warning) { warnings.push(markerClear.warning); @@ -361,17 +247,16 @@ function killTimer(sandboxName: string): KillTimerResult { markerFound: marker !== null, markerPid: marker?.pid ?? null, wasAlive, - terminated, + terminated: false, warnings, }; } -export type { ClearTimerMarkerResult, KillTimerResult, ProcessIdentity, TimerMarker }; +export type { ClearTimerMarkerResult, KillTimerResult, ShieldsTimerMarker as TimerMarker }; export { clearTimerMarker, isProcessAlive, killTimer, - listDescendantProcessIdentities, processInspectionDeadlineAfter, processInspectionDeadlineReached, readAutoRestoreTakeoverToken, diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index 26f126e1dbd..1eb4741ff46 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -6,10 +6,11 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { getMcpLifecycleLockPath } from "../state/mcp-lifecycle-lock"; +import { getMcpLifecycleLockPath, withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; const shieldsIndexMock = vi.hoisted(() => ({ applyShieldsPolicySnapshot: vi.fn(() => ({ status: 0 })), + completeAutoRestoreTransition: vi.fn(() => true), lockAgentConfig: vi.fn() as unknown, prepareAutoRestoreTransitionTakeover: vi.fn(), })); @@ -26,6 +27,7 @@ vi.mock("../sandbox/config", () => ({ vi.mock("./index", () => ({ applyShieldsPolicySnapshot: shieldsIndexMock.applyShieldsPolicySnapshot, + completeAutoRestoreTransition: shieldsIndexMock.completeAutoRestoreTransition, get lockAgentConfig() { return shieldsIndexMock.lockAgentConfig; }, @@ -50,7 +52,7 @@ describe("shields timer authorization", () => { }); async function invokeTimerAndCaptureExit( - runRestoreTimer: (args: any) => Promise, + runRestoreTimer: (args: any, options?: { retryDelayMs?: number }) => Promise, args: unknown, ): Promise { const exitSpy = vi.spyOn(process, "exit").mockImplementation((code?: any) => { @@ -58,7 +60,7 @@ describe("shields timer authorization", () => { }); try { - await runRestoreTimer(args); + await runRestoreTimer(args, { retryDelayMs: 1 }); throw new Error("Expected runRestoreTimer to exit"); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -71,20 +73,45 @@ describe("shields timer authorization", () => { } async function invokeTimerAndExpectRetry( - runRestoreTimer: (args: any) => Promise, + runRestoreTimer: (args: any, options?: { retryDelayMs?: number }) => Promise, args: unknown, ): Promise { - vi.useFakeTimers(); const exitSpy = vi .spyOn(process, "exit") .mockImplementation((() => undefined) as typeof process.exit); + const markerPath = (args as { markerPath?: string }).markerPath; + const markerContents = + markerPath && fs.existsSync(markerPath) ? fs.readFileSync(markerPath) : null; try { - await runRestoreTimer(args); + const pending = runRestoreTimer(args, { retryDelayMs: 50 }); + const sandboxName = (args as { sandboxName?: string }).sandboxName; + const deadlinePath = sandboxName + ? `${getMcpLifecycleLockPath( + sandboxName, + path.join(tmpHome, ".nemoclaw", "state"), + )}.deadline` + : null; + const auditPath = markerPath + ? path.join(path.dirname(markerPath), "shields-audit.jsonl") + : null; + for (let attempt = 0; attempt < 200; attempt += 1) { + if ( + (!deadlinePath || fs.existsSync(deadlinePath)) && + (!auditPath || fs.existsSync(auditPath)) + ) { + break; + } + await new Promise((resolve) => setTimeout(resolve, 1)); + } expect(exitSpy).not.toHaveBeenCalled(); - expect(vi.getTimerCount()).toBe(1); + if (deadlinePath) expect(fs.existsSync(deadlinePath)).toBe(true); + if (markerPath) fs.rmSync(markerPath, { force: true }); + await pending; } finally { + if (markerPath && markerContents) { + fs.writeFileSync(markerPath, markerContents); + } exitSpy.mockRestore(); - vi.useRealTimers(); } } @@ -258,60 +285,51 @@ describe("shields timer authorization", () => { }); it("retains a dead rebuild owner's timer and retries a transient restore failure", async () => { - vi.useFakeTimers(); - const exitSpy = vi - .spyOn(process, "exit") - .mockImplementation((() => undefined) as typeof process.exit); - try { - const timer = await import("./timer"); - const stateDir = path.join(tmpHome, ".nemoclaw", "state"); - fs.mkdirSync(stateDir, { recursive: true }); - const sandboxName = "rebuild-dead"; - const snapshotPath = path.join(stateDir, "snapshot.yaml"); - const restoreAtIso = new Date().toISOString(); - const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); - const sandboxMutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); - fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); - fs.writeFileSync( - markerPath, - JSON.stringify({ - pid: process.pid, - sandboxName, - snapshotPath, - restoreAt: restoreAtIso, - processToken: PROCESS_TOKEN, - leaseOwnerPid: 2_147_483_000, - leaseOwnerStartIdentity: "proc:dead-owner", - }), - ); - shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementationOnce(() => { - expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); - return { status: 17 }; - }); - const args = timer.parseTimerArgs([ + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const sandboxName = "rebuild-dead"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date().toISOString(); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const sandboxMutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid, sandboxName, snapshotPath, - restoreAtIso, - "", - "", - PROCESS_TOKEN, - "0", - "2147483000", - "proc:dead-owner", - ]); - expect(args).not.toBeNull(); + restoreAt: restoreAtIso, + processToken: PROCESS_TOKEN, + leaseOwnerPid: 2_147_483_000, + leaseOwnerStartIdentity: "proc:dead-owner", + }), + ); + shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementationOnce(() => { + expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); + expect(fs.existsSync(`${sandboxMutationLockPath}.deadline`)).toBe(true); + return { status: 17 }; + }); + const args = timer.parseTimerArgs([ + sandboxName, + snapshotPath, + restoreAtIso, + "", + "", + PROCESS_TOKEN, + "0", + "2147483000", + "proc:dead-owner", + ]); + expect(args).not.toBeNull(); - await timer.runRestoreTimer(args!); + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); - expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); - expect(exitSpy).not.toHaveBeenCalled(); - expect(vi.getTimerCount()).toBe(1); - expect(fs.existsSync(markerPath)).toBe(true); - expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); - } finally { - exitSpy.mockRestore(); - vi.useRealTimers(); - } + expect(exitCode).toBe(0); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(2); + expect(fs.existsSync(markerPath)).toBe(false); + expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); }); it("does not restore or rewrite state when marker pid mismatches", async () => { @@ -403,6 +421,54 @@ describe("shields timer authorization", () => { expect(JSON.parse(fs.readFileSync(markerPath, "utf-8"))).toEqual(replacementMarker); }); + it("does not preempt a transition owner after timer authority is revoked", async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const sandboxName = "revoked-takeover"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date().toISOString(); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const deadlinePath = `${getMcpLifecycleLockPath(sandboxName, stateDir)}.deadline`; + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies: {}\n"); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: PROCESS_TOKEN, + }), + ); + const args = timer.parseTimerArgs([ + sandboxName, + snapshotPath, + restoreAtIso, + "", + "", + PROCESS_TOKEN, + ]); + expect(args).not.toBeNull(); + shieldsIndexMock.prepareAutoRestoreTransitionTakeover.mockImplementationOnce( + (_sandboxName, _processToken, _snapshotPath, assertTakeoverAuthority) => { + expect(fs.existsSync(deadlinePath)).toBe(true); + fs.rmSync(markerPath); + expect(assertTakeoverAuthority).toBeTypeOf("function"); + assertTakeoverAuthority(); + }, + ); + + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + + expect(exitCode).toBe(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + const mutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + expect(fs.existsSync(mutationLockPath)).toBe(false); + expect(fs.existsSync(deadlinePath)).toBe(false); + expect(fs.existsSync(`${mutationLockPath}.containment`)).toBe(false); + }); + it("restores and updates state when marker matches current timer invocation", async () => { const timer = await import("./timer"); const stateDir = path.join(tmpHome, ".nemoclaw", "state"); @@ -439,12 +505,16 @@ describe("shields timer authorization", () => { const lockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementationOnce(() => { expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); + expect(fs.existsSync(`${sandboxMutationLockPath}.deadline`)).toBe(true); expect(JSON.parse(fs.readFileSync(lockPath, "utf-8"))).toMatchObject({ sandboxName, command: "shields auto-restore", takeoverToken: PROCESS_TOKEN, }); - return { status: 0 }; + return { + status: 0, + managedMcpOmissions: [{ server: "beta", reason: "incomplete add" }], + }; }); const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); @@ -457,6 +527,86 @@ describe("shields timer authorization", () => { expect(updatedState.shieldsDownAt).toBeNull(); expect(fs.existsSync(markerPath)).toBe(false); expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); + expect( + fs + .readFileSync(path.join(stateDir, "shields-audit.jsonl"), "utf-8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)), + ).toContainEqual( + expect.objectContaining({ + action: "shields_auto_restore", + warning: "Auto-restore omitted 1 unproven managed MCP policy entries", + }), + ); + }); + + it("keeps the deadline gate closed while a failed restore retries", async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + const sandboxName = "retry-gate"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date().toISOString(); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const mutationLockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); + fs.writeFileSync( + markerPath, + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: PROCESS_TOKEN, + }), + ); + shieldsIndexMock.applyShieldsPolicySnapshot + .mockReturnValueOnce({ status: 1 }) + .mockReturnValue({ status: 0 }); + const args = timer.parseTimerArgs([ + sandboxName, + snapshotPath, + restoreAtIso, + "", + "", + PROCESS_TOKEN, + ]); + expect(args).not.toBeNull(); + const exitSpy = vi + .spyOn(process, "exit") + .mockImplementation((() => undefined) as typeof process.exit); + let contenderEntered = false; + + try { + const restore = timer.runRestoreTimer(args!, { retryDelayMs: 100 }); + for ( + let attempt = 0; + attempt < 200 && shieldsIndexMock.applyShieldsPolicySnapshot.mock.calls.length === 0; + attempt += 1 + ) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); + expect(fs.existsSync(`${mutationLockPath}.deadline`)).toBe(true); + + const contender = withMcpLifecycleLock( + sandboxName, + () => { + contenderEntered = true; + }, + { stateDir, pollIntervalMs: 5, timeoutMs: 2_000 }, + ); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(contenderEntered).toBe(false); + expect(fs.existsSync(`${mutationLockPath}.deadline`)).toBe(true); + + await Promise.all([restore, contender]); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(2); + expect(contenderEntered).toBe(true); + } finally { + exitSpy.mockRestore(); + } }); it("retains recovery authority when the locked-state commit cannot be persisted", async () => { @@ -572,6 +722,11 @@ describe("shields timer authorization", () => { expect(exitCode).toBe(0); expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(1); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledWith( + sandboxName, + snapshotPath, + { deadlineAuthoritative: true, transitionProcessToken: PROCESS_TOKEN }, + ); // #4663: relockAndReconfirm applies then re-confirms after the settle // window (0ms under test), so lockAgentConfig is invoked twice for a clean // lock. diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index 8b8a8820209..ddb6c908ec4 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -12,7 +12,7 @@ import fs from "node:fs"; import path from "node:path"; import { isObjectRecord, type UnknownRecord } from "../core/json-types"; import { resolveAgentConfig } from "../sandbox/config"; -import { withSandboxMutationLock } from "../state/mcp-lifecycle-lock"; +import { withMcpLifecycleDeadlineFence } from "../state/mcp-lifecycle-lock"; import { resolveNemoclawStateDir } from "../state/paths"; import { appendAuditEntry, type ShieldsAuditEntry } from "./audit"; import * as shields from "./index"; @@ -46,7 +46,12 @@ interface TimerArgs { leaseOwnerStartIdentity?: string; } +interface TimerRuntimeOptions { + retryDelayMs?: number; +} + type LockAgentConfig = typeof shields.lockAgentConfig; +type RestoreAttemptOutcome = "complete" | "retry" | "revoked"; const STATE_DIR = resolveNemoclawStateDir(); const AUTO_RESTORE_RETRY_MS = 5_000; @@ -225,16 +230,24 @@ function rebuildLeaseOwnerIsCurrent(args: TimerArgs): boolean { ); } -async function runRestoreTimer(args: TimerArgs): Promise { +async function runRestoreTimer( + args: TimerArgs, + runtimeOptions: TimerRuntimeOptions = {}, +): Promise { const now = new Date().toISOString(); + const retryDelayMs = + Number.isFinite(runtimeOptions.retryDelayMs) && (runtimeOptions.retryDelayMs ?? 0) >= 0 + ? Math.floor(runtimeOptions.retryDelayMs!) + : AUTO_RESTORE_RETRY_MS; let exitCode = 0; let retryScheduled = false; + let managedMcpWarning: string | undefined; const scheduleRetry = (): boolean => { if (!markerMatchesCurrentTimer(args)) return false; retryScheduled = true; setTimeout(() => { - void runRestoreTimer(args); - }, AUTO_RESTORE_RETRY_MS); + void runRestoreTimer(args, runtimeOptions); + }, retryDelayMs); return true; }; @@ -259,13 +272,12 @@ async function runRestoreTimer(args: TimerArgs): Promise { if (!args.processToken || !/^[0-9a-f]{32}$/.test(args.processToken)) { throw new Error("Auto-restore timer has no valid transition takeover token"); } - shields.prepareAutoRestoreTransitionTakeover( - args.sandboxName, - args.processToken, - args.snapshotPath, - ); - - await withSandboxMutationLock(args.sandboxName, () => + const assertTakeoverAuthority = (): void => { + if (!markerMatchesCurrentTimer(args)) { + throw new Error("Auto-restore authority changed before Shields transition takeover"); + } + }; + const restoreUnderDeadlineFence = (): RestoreAttemptOutcome => withShieldsTransitionLock( args.sandboxName, "shields auto-restore", @@ -273,7 +285,7 @@ async function runRestoreTimer(args: TimerArgs): Promise { // A manual hardening command may have completed while this timer waited // for the host mutation lock. The marker is the timer's authority, so // re-check it only after serialization is established. - if (!markerMatchesCurrentTimer(args)) return; + if (!markerMatchesCurrentTimer(args)) return "revoked"; if (!fs.existsSync(args.snapshotPath)) { appendAudit({ @@ -284,13 +296,20 @@ async function runRestoreTimer(args: TimerArgs): Promise { error: "Policy snapshot file missing", }); exitCode = 1; - scheduleRetry(); - return; + return "retry"; } // Restore policy (slow — openshell policy set --wait blocks) - const result = shields.applyShieldsPolicySnapshot(args.sandboxName, args.snapshotPath); + const result = shields.applyShieldsPolicySnapshot(args.sandboxName, args.snapshotPath, { + transitionProcessToken: args.processToken, + deadlineAuthoritative: true, + }); const status = typeof result.status === "number" ? result.status : 1; + if (result.managedMcpOmissions?.length) { + managedMcpWarning = `Auto-restore omitted ${String( + result.managedMcpOmissions.length, + )} unproven managed MCP policy entries`; + } if (status !== 0) { appendAudit({ @@ -301,14 +320,13 @@ async function runRestoreTimer(args: TimerArgs): Promise { error: `Policy restore exited with status ${String(status)}`, }); exitCode = 1; - scheduleRetry(); - return; + return "retry"; } // Destroy and force-restore can revoke this marker while a slow // policy restore is already in flight. Stop before the next sandbox // mutation if this timer generation no longer owns recovery. - if (!markerMatchesCurrentTimer(args)) return; + if (!markerMatchesCurrentTimer(args)) return "revoked"; // Re-lock config file using the shared lockAgentConfig from shields.ts. // lockAgentConfig runs each operation independently and verifies the @@ -359,7 +377,7 @@ async function runRestoreTimer(args: TimerArgs): Promise { } if (lockTarget) { try { - if (!markerMatchesCurrentTimer(args)) return; + if (!markerMatchesCurrentTimer(args)) return "revoked"; const lockAgentConfig = resolveLockAgentConfig(); // #4663: a single instantaneous lock+verify cannot prove an // in-sandbox reconciler didn't re-permission .config-hash after the @@ -406,7 +424,7 @@ async function runRestoreTimer(args: TimerArgs): Promise { // Re-lock verification includes a settle window. Do not rewrite state // or remove a replacement marker if authority changed while it ran. - if (!markerMatchesCurrentTimer(args)) return; + if (!markerMatchesCurrentTimer(args)) return "revoked"; // Only mark shields as UP if the lock was verified (or no config path). if (lockVerified) { @@ -420,6 +438,15 @@ async function runRestoreTimer(args: TimerArgs): Promise { if (lockedChattr !== null) patch.chattrApplied = lockedChattr; if (lockedHashes !== null) patch.fileHashes = lockedHashes; updateState(args.stateFile, patch); + if ( + !shields.completeAutoRestoreTransition( + args.sandboxName, + args.processToken!, + args.snapshotPath, + ) + ) { + return "revoked"; + } appendAudit({ action: "shields_auto_restore", @@ -428,9 +455,11 @@ async function runRestoreTimer(args: TimerArgs): Promise { restored_by: "auto_timer", policy_snapshot: args.snapshotPath, scheduled_restore_at: args.restoreAtIso, + ...(managedMcpWarning ? { warning: managedMcpWarning } : {}), }); cleanupOwnedTimerMarker(args); - return; + exitCode = 0; + return "complete"; } // Explicitly ensure state reflects shields are still DOWN. @@ -445,10 +474,62 @@ async function runRestoreTimer(args: TimerArgs): Promise { error: "Config re-lock verification failed — shields remain DOWN", }); exitCode = 1; - scheduleRetry(); + return "retry"; + }, + { + takeoverToken: args.processToken, + recoverStaleOwner: false, + waitTimeoutMs: 0, }, - { takeoverToken: args.processToken }, - ), + ); + const restoreWhileDeadlineOwned = async (): Promise => { + for (;;) { + let outcome: RestoreAttemptOutcome; + try { + shields.prepareAutoRestoreTransitionTakeover( + args.sandboxName, + args.processToken!, + args.snapshotPath, + assertTakeoverAuthority, + ); + outcome = restoreUnderDeadlineFence(); + } catch (error) { + appendAudit({ + action: "shields_up_failed", + sandbox: args.sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: args.snapshotPath, + error: error instanceof Error ? error.message : String(error), + }); + exitCode = 1; + outcome = "retry"; + } + if (outcome !== "retry") return; + if (!markerMatchesCurrentTimer(args)) return; + await new Promise((resolve) => setTimeout(resolve, retryDelayMs)); + if (!markerMatchesCurrentTimer(args)) return; + } + }; + await withMcpLifecycleDeadlineFence( + args.sandboxName, + args.processToken, + restoreWhileDeadlineOwned, + { + stateDir: STATE_DIR, + pollIntervalMs: 50, + timeoutMs: 5_000, + onContainment: ({ ownerPid, reason }) => { + appendAudit({ + action: "shields_up_failed", + sandbox: args.sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: args.snapshotPath, + error: `${reason}${ownerPid ? ` Contained owner PID: ${String(ownerPid)}.` : ""}`, + }); + }, + }, ); } catch (error: unknown) { appendAudit({ diff --git a/src/lib/shields/transition-lock.test.ts b/src/lib/shields/transition-lock.test.ts index 5d966815b98..1fbf625584f 100644 --- a/src/lib/shields/transition-lock.test.ts +++ b/src/lib/shields/transition-lock.test.ts @@ -289,6 +289,11 @@ describe("host shields transition lock", () => { processStartIdentity: "proc:holder", command: "shields down", }); + expect(locker.inspectAnyShieldsTransitionLockOwner("alpha")).toEqual({ + pid: 202, + processStartIdentity: "proc:holder", + command: "shields down", + }); }); it("returns no inspected owner when the canonical path changes identity before open", () => { @@ -491,6 +496,39 @@ describe("host shields transition lock", () => { expect(fs.readdirSync(stateDir)).toEqual([]); }); + it("preserves a stale owner when the containment protocol owns recovery", () => { + const recorded = owner("alpha", 202, "proc:dead-holder", "shields down", TAKEOVER_TOKEN); + const lockPath = writeOwner("alpha", recorded); + + expect(() => + manager().withShieldsTransitionLock("alpha", "timer restore", () => undefined, { + takeoverToken: TAKEOVER_TOKEN, + recoverStaleOwner: false, + waitTimeoutMs: 0, + }), + ).toThrow(/recorded owner PID 202 is not running/); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8"))).toEqual(recorded); + }); + + it("preserves a reused-PID owner when the containment protocol owns recovery", () => { + const holderPid = 202; + const recorded = owner("alpha", holderPid, "proc:original"); + const lockPath = writeOwner("alpha", recorded); + const locker = manager({ + isProcessAlive: (pid) => pid === holderPid || pid === SELF_PID, + readProcessStartIdentity: (pid) => + pid === SELF_PID ? SELF_IDENTITY : pid === holderPid ? "proc:reused" : null, + }); + + expect(() => + locker.withShieldsTransitionLock("alpha", "gateway restart", () => undefined, { + recoverStaleOwner: false, + waitTimeoutMs: 0, + }), + ).toThrow(/PID 202 now has process-start identity/); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8"))).toEqual(recorded); + }); + it("recovers a stale lock when a live PID has been reused", () => { const holderPid = 202; const recorded = owner("alpha", holderPid, "proc:original"); @@ -502,16 +540,21 @@ describe("host shields transition lock", () => { }); expect( - locker.withShieldsTransitionLock("alpha", "timer restore", () => { - const replacement = JSON.parse(fs.readFileSync(lockPath, "utf8")); - expect(replacement).toMatchObject({ - sandboxName: "alpha", - pid: SELF_PID, - processStartIdentity: SELF_IDENTITY, - command: "timer restore", - }); - return "acquired"; - }), + locker.withShieldsTransitionLock( + "alpha", + "timer restore", + () => { + const replacement = JSON.parse(fs.readFileSync(lockPath, "utf8")); + expect(replacement).toMatchObject({ + sandboxName: "alpha", + pid: SELF_PID, + processStartIdentity: SELF_IDENTITY, + command: "timer restore", + }); + return "acquired"; + }, + { recoverStaleOwner: true }, + ), ).toBe("acquired"); expect(fs.existsSync(lockPath)).toBe(false); }); @@ -535,6 +578,7 @@ describe("host shields transition lock", () => { expect(() => locker.withShieldsTransitionLock("alpha", "nemoclaw alpha shields up", () => undefined, { + recoverStaleOwner: true, waitTimeoutMs: 2, pollIntervalMs: 1, }), @@ -566,6 +610,7 @@ describe("host shields transition lock", () => { throw new Error("should not acquire after timeout"); }, { + recoverStaleOwner: true, waitTimeoutMs: 2, pollIntervalMs: 1, }, @@ -608,16 +653,21 @@ describe("host shields transition lock", () => { }); await expect( - locker.withShieldsTransitionLockAsync("alpha", "timer restore", async () => { - const replacement = JSON.parse(fs.readFileSync(lockPath, "utf8")); - expect(replacement).toMatchObject({ - sandboxName: "alpha", - pid: SELF_PID, - processStartIdentity: SELF_IDENTITY, - command: "timer restore", - }); - return "acquired"; - }), + locker.withShieldsTransitionLockAsync( + "alpha", + "timer restore", + async () => { + const replacement = JSON.parse(fs.readFileSync(lockPath, "utf8")); + expect(replacement).toMatchObject({ + sandboxName: "alpha", + pid: SELF_PID, + processStartIdentity: SELF_IDENTITY, + command: "timer restore", + }); + return "acquired"; + }, + { recoverStaleOwner: true }, + ), ).resolves.toBe("acquired"); expect(fs.existsSync(lockPath)).toBe(false); }); @@ -677,6 +727,7 @@ describe("host shields transition lock", () => { expect(() => locker.withShieldsTransitionLock("alpha", "nemoclaw alpha shields up", () => undefined, { + recoverStaleOwner: true, waitTimeoutMs: 2, pollIntervalMs: 1, }), @@ -728,7 +779,9 @@ describe("host shields transition lock", () => { const locker = manager(); expect( - locker.withShieldsTransitionLock("alpha", "nemoclaw alpha shields up", () => "acquired"), + locker.withShieldsTransitionLock("alpha", "nemoclaw alpha shields up", () => "acquired", { + recoverStaleOwner: true, + }), ).toBe("acquired"); expect(thirdAcquired).toBe(false); @@ -776,6 +829,7 @@ describe("host shields transition lock", () => { "alpha", "nemoclaw alpha shields up", async () => "acquired", + { recoverStaleOwner: true }, ), ).resolves.toBe("acquired"); @@ -793,11 +847,16 @@ describe("host shields transition lock", () => { const locker = manager(); expect( - locker.withShieldsTransitionLock("alpha", "nemoclaw alpha shields up", () => { - expect(fs.existsSync(lockPath)).toBe(true); - expect(fs.existsSync(guardPath)).toBe(false); - return "acquired"; - }), + locker.withShieldsTransitionLock( + "alpha", + "nemoclaw alpha shields up", + () => { + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(guardPath)).toBe(false); + return "acquired"; + }, + { recoverStaleOwner: true }, + ), ).toBe("acquired"); expect(fs.existsSync(lockPath)).toBe(false); @@ -813,11 +872,16 @@ describe("host shields transition lock", () => { const locker = manager(); await expect( - locker.withShieldsTransitionLockAsync("alpha", "nemoclaw alpha shields up", async () => { - expect(fs.existsSync(lockPath)).toBe(true); - expect(fs.existsSync(guardPath)).toBe(false); - return "acquired"; - }), + locker.withShieldsTransitionLockAsync( + "alpha", + "nemoclaw alpha shields up", + async () => { + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(guardPath)).toBe(false); + return "acquired"; + }, + { recoverStaleOwner: true }, + ), ).resolves.toBe("acquired"); expect(fs.existsSync(lockPath)).toBe(false); diff --git a/src/lib/shields/transition-lock.ts b/src/lib/shields/transition-lock.ts index b31eb7a635c..4575f87e18e 100644 --- a/src/lib/shields/transition-lock.ts +++ b/src/lib/shields/transition-lock.ts @@ -35,6 +35,8 @@ export interface ShieldsTransitionLockOptions { pollIntervalMs?: number; malformedStaleMs?: number; takeoverToken?: string; + /** Preserve stale owners for a caller that applies a stronger containment protocol. */ + recoverStaleOwner?: boolean; } export interface ShieldsTransitionLockDependencies { @@ -488,6 +490,26 @@ export class ShieldsTransitionLockManager { } } + inspectAnyShieldsTransitionLockOwner( + sandboxName: string, + ): InspectedShieldsTransitionOwner | null { + const validName = validateSandboxName(sandboxName); + const lockPath = shieldsTransitionLockPath(validName, this.stateDir); + const snapshot = readExistingLock(lockPath, validName); + if (!snapshot) return null; + try { + const owner = snapshot.owner; + if (!owner) return null; + return { + pid: owner.pid, + processStartIdentity: owner.processStartIdentity, + command: owner.command, + }; + } finally { + closeSnapshot(snapshot); + } + } + takeoverShieldsTransitionLock( sandboxName: string, expectedOwnerPid: number, @@ -816,7 +838,12 @@ export class ShieldsTransitionLockManager { ); if (!observed) continue; lastWaitReason = observed; - if (this.recoveredObservedStaleOwner(sandboxName, observed)) continue; + if ( + options.recoverStaleOwner !== false && + this.recoveredObservedStaleOwner(sandboxName, observed) + ) { + continue; + } } this.sleep(this.waitDuration(state, lastWaitReason)); } @@ -847,7 +874,12 @@ export class ShieldsTransitionLockManager { ); if (!observed) continue; lastWaitReason = observed; - if (this.recoveredObservedStaleOwner(sandboxName, observed)) continue; + if ( + options.recoverStaleOwner !== false && + this.recoveredObservedStaleOwner(sandboxName, observed) + ) { + continue; + } } await this.sleepAsync(this.waitDuration(state, lastWaitReason)); } @@ -1140,6 +1172,12 @@ export function inspectShieldsTransitionLockOwner( return defaultManager.inspectShieldsTransitionLockOwner(sandboxName, takeoverToken); } +export function inspectAnyShieldsTransitionLockOwner( + sandboxName: string, +): InspectedShieldsTransitionOwner | null { + return defaultManager.inspectAnyShieldsTransitionLockOwner(sandboxName); +} + export function takeoverShieldsTransitionLock( sandboxName: string, expectedOwnerPid: number, diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.ts index b1606c16667..86642eddc5c 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.ts @@ -7,19 +7,27 @@ import fs from "node:fs"; import path from "node:path"; import { performance } from "node:perf_hooks"; +import { readShieldsTimerTakeoverToken } from "./mcp-lifecycle-lock/shields-timer-authority"; import { classifyMcpLifecycleLock, createMcpLifecycleLockOwner, type LockObservation, type McpLifecycleLockDisposition, + type McpLifecycleLockOwner, + readMcpLockHostIdentity, + readMcpLockPidNamespaceIdentity, + readMcpLockProcessIdentity, } from "./mcp-lifecycle-lock-identity"; import { getMcpLifecycleLockPath, mcpLifecycleLockPathExists, + mcpLifecycleLockPathExistsSync, readMcpLifecycleLockObservation, - reclaimStaleMcpLifecycleLockGeneration, + readMcpLifecycleLockObservationSync, safelyReleaseMcpLifecycleLock, + safelyReleaseMcpLifecycleLockSync, writeMcpLifecycleLockCandidateAndLink, + writeMcpLifecycleLockCandidateAndLinkSync, } from "./mcp-lifecycle-lock-storage"; import { resolveNemoclawStateDir } from "./paths"; @@ -37,6 +45,21 @@ interface AcquiredMcpLifecycleLock { token: string; } +export interface McpLifecycleDeadlineFenceOptions extends McpLifecycleLockOptions { + /** Audit an owner that keeps the deadline gate closed while it exits naturally. */ + onContainment?: (details: McpLifecycleDeadlineContainment) => Promise | void; +} + +export interface McpLifecycleDeadlineFenceSyncOptions extends McpLifecycleLockOptions { + /** Audit an owner that keeps the deadline gate closed while it exits naturally. */ + onContainment?: (details: McpLifecycleDeadlineContainment) => void; +} + +export interface McpLifecycleDeadlineContainment { + ownerPid: number | null; + reason: string; +} + export interface McpLifecycleLockOptions { /** Override used by focused tests. Production callers use ~/.nemoclaw/state. */ stateDir?: string; @@ -61,6 +84,93 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +const sleepBuffer = new Int32Array(new SharedArrayBuffer(4)); + +function sleepSync(ms: number): void { + Atomics.wait(sleepBuffer, 0, 0, ms); +} + +function committedContainmentPath(lockPath: string): string { + return `${lockPath}.containment`; +} + +function fsyncLockDirectorySync(lockPath: string): void { + const directoryFd = fs.openSync(path.dirname(lockPath), fs.constants.O_RDONLY); + try { + fs.fsyncSync(directoryFd); + } finally { + fs.closeSync(directoryFd); + } +} + +function beginCommittedContainmentAtPathSync( + lockPath: string, + sandboxName: string, + takeoverToken: string | undefined, + reason: string, +): void { + const containmentPath = committedContainmentPath(lockPath); + const token = crypto.randomUUID(); + const owner = { + ...createMcpLifecycleLockOwner(sandboxName, token, takeoverToken), + containmentReason: reason, + }; + fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + if (!writeMcpLifecycleLockCandidateAndLinkSync(containmentPath, owner)) { + throw new Error( + `A committed process-tree containment already exists for sandbox '${sandboxName}'`, + ); + } + try { + fsyncLockDirectorySync(containmentPath); + } catch (error) { + safelyReleaseMcpLifecycleLockSync(containmentPath, token); + throw error; + } +} + +function ensurePermanentContainmentForStaleGenerationSync( + lockPath: string, + sandboxName: string, + stateDir: string, + observation: LockObservation, + reason: string, +): void { + const containmentPath = committedContainmentPath(lockPath); + if (mcpLifecycleLockPathExistsSync(containmentPath)) return; + const generation = `${String(observation.dev)}:${String(observation.ino)}:${ + observation.owner?.token ?? "invalid" + }`; + try { + beginCommittedContainmentAtPathSync( + lockPath, + sandboxName, + readShieldsTimerTakeoverToken(sandboxName, stateDir), + `${reason}; contained generation ${generation}`, + ); + } catch (error) { + if (mcpLifecycleLockPathExistsSync(containmentPath)) return; + throw error; + } +} + +export function beginCommittedMcpLifecycleContainmentSync( + sandboxName: string, + takeoverToken: string, + reason: string, + stateDir = resolveNemoclawStateDir(), +): void { + if (!/^[0-9a-f]{32}$/.test(takeoverToken)) { + throw new Error("Auto-restore takeover token must be 32 lowercase hexadecimal characters"); + } + beginCommittedContainmentAtPathSync( + getMcpLifecycleLockPath(sandboxName, stateDir), + sandboxName, + takeoverToken, + reason, + ); +} + function resetCorruptGenerationTracker(tracker: CorruptGenerationTracker): void { tracker.generation = null; tracker.firstSeenAt = 0; @@ -93,33 +203,6 @@ function classifyObservedMcpLifecycleLock( ); } -async function tryReapStaleLock( - lockPath: string, - sandboxName: string, - corruptLockGraceMs: number, - corruptTracker: CorruptGenerationTracker, -): Promise { - const reaperPath = `${lockPath}.reaper`; - const reaperToken = crypto.randomUUID(); - const reaperOwner = createMcpLifecycleLockOwner(sandboxName, reaperToken); - if (!(await writeMcpLifecycleLockCandidateAndLink(reaperPath, reaperOwner))) return false; - - try { - const latest = await readMcpLifecycleLockObservation(lockPath); - if (!latest) return true; - if ( - classifyObservedMcpLifecycleLock(latest, sandboxName, corruptLockGraceMs, corruptTracker) !== - "stale" - ) { - return false; - } - - return reclaimStaleMcpLifecycleLockGeneration(lockPath, latest); - } finally { - await safelyReleaseMcpLifecycleLock(reaperPath, reaperToken); - } -} - async function acquireMcpLifecycleLock( sandboxName: string, options: McpLifecycleLockOptions, @@ -130,7 +213,8 @@ async function acquireMcpLifecycleLock( options.corruptLockGraceMs, DEFAULT_CORRUPT_LOCK_GRACE_MS, ); - const lockPath = getMcpLifecycleLockPath(sandboxName, options.stateDir); + const stateDir = options.stateDir ?? resolveNemoclawStateDir(); + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); await fs.promises.mkdir(path.dirname(lockPath), { recursive: true, mode: 0o700, @@ -139,15 +223,53 @@ async function acquireMcpLifecycleLock( const startedAt = performance.now(); const corruptMainTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; const corruptReaperTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + const corruptDeadlineTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; let lastOwnerPid: number | null = null; for (;;) { if (performance.now() - startedAt >= timeoutMs) { + const containmentPath = committedContainmentPath(lockPath); + const containment = await readMcpLifecycleLockObservation(containmentPath); + if (containment) { + throw new Error( + `Sandbox mutation containment is active for '${sandboxName}' at '${containmentPath}' (generation token '${containment.owner?.token ?? "invalid"}'). A previous owner or stale-lock reaper exited without proof that every descendant stopped. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', and '${lockPath}.deadline'; record each target's file kind, device/inode, and owner token when present; verify those identities and this containment token are unchanged; remove only those exact stale owner generations first and this exact containment generation last before retrying.`, + ); + } const ownerSuffix = lastOwnerPid ? ` (owner pid ${lastOwnerPid})` : ""; throw new Error( `Timed out waiting for the sandbox mutation lock for '${sandboxName}'${ownerSuffix}. Another lifecycle, policy, channel, shields, or snapshot operation is still running.`, ); } + const containmentPath = committedContainmentPath(lockPath); + if (await mcpLifecycleLockPathExists(containmentPath)) { + await sleep(pollIntervalMs); + continue; + } + + const deadlinePath = `${lockPath}.deadline`; + const deadlineObservation = await readMcpLifecycleLockObservation(deadlinePath); + if (deadlineObservation) { + const deadlineDisposition = classifyObservedMcpLifecycleLock( + deadlineObservation, + sandboxName, + corruptLockGraceMs, + corruptDeadlineTracker, + ); + if (deadlineDisposition === "stale") { + ensurePermanentContainmentForStaleGenerationSync( + lockPath, + sandboxName, + stateDir, + deadlineObservation, + "An auto-restore deadline owner exited before its recovery operation completed", + ); + continue; + } + await sleep(pollIntervalMs); + continue; + } + resetCorruptGenerationTracker(corruptDeadlineTracker); + const reaperPath = `${lockPath}.reaper`; const reaperObservation = await readMcpLifecycleLockObservation(reaperPath); if (reaperObservation) { @@ -158,10 +280,13 @@ async function acquireMcpLifecycleLock( corruptReaperTracker, ); if (reaperDisposition === "stale") { - // The reaper has the same atomic, PID-identified owner format as the - // main lock. A SIGKILL at any point in stale-lock cleanup is therefore - // recoverable without age-expiring a legitimate long operation. - await reclaimStaleMcpLifecycleLockGeneration(reaperPath, reaperObservation); + ensurePermanentContainmentForStaleGenerationSync( + lockPath, + sandboxName, + stateDir, + reaperObservation, + "A stale-lock reaper exited before cleanup completed", + ); continue; } await sleep(pollIntervalMs); @@ -169,14 +294,25 @@ async function acquireMcpLifecycleLock( } resetCorruptGenerationTracker(corruptReaperTracker); - if (!(await mcpLifecycleLockPathExists(reaperPath))) { + if ( + !(await mcpLifecycleLockPathExists(deadlinePath)) && + !(await mcpLifecycleLockPathExists(reaperPath)) + ) { const token = crypto.randomUUID(); - const owner = createMcpLifecycleLockOwner(sandboxName, token); + const shieldsTakeoverToken = readShieldsTimerTakeoverToken(sandboxName, stateDir); + const owner = createMcpLifecycleLockOwner(sandboxName, token, shieldsTakeoverToken); if (await writeMcpLifecycleLockCandidateAndLink(lockPath, owner)) { // A stale-lock reaper may have appeared between our pre-check and the // atomic link. Do not enter the critical section until that generation // gate has gone away. - if (!(await mcpLifecycleLockPathExists(reaperPath))) return { lockPath, token }; + if ( + !(await mcpLifecycleLockPathExists(containmentPath)) && + !(await mcpLifecycleLockPathExists(deadlinePath)) && + !(await mcpLifecycleLockPathExists(reaperPath)) && + readShieldsTimerTakeoverToken(sandboxName, stateDir) === shieldsTakeoverToken + ) { + return { lockPath, token }; + } await safelyReleaseMcpLifecycleLock(lockPath, token); } } @@ -192,17 +328,870 @@ async function acquireMcpLifecycleLock( corruptMainTracker, ) === "stale" ) { - if (await tryReapStaleLock(lockPath, sandboxName, corruptLockGraceMs, corruptMainTracker)) { - continue; + ensurePermanentContainmentForStaleGenerationSync( + lockPath, + sandboxName, + stateDir, + observation, + "A sandbox mutation owner exited before its descendants could be proven contained", + ); + continue; + } + } else { + resetCorruptGenerationTracker(corruptMainTracker); + } + await sleep(pollIntervalMs); + } +} + +function acquireMcpLifecycleLockSync( + sandboxName: string, + options: McpLifecycleLockOptions & { stateDir: string }, +): AcquiredMcpLifecycleLock { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const timeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptLockGraceMs = positiveInteger( + options.corruptLockGraceMs, + DEFAULT_CORRUPT_LOCK_GRACE_MS, + ); + const lockPath = getMcpLifecycleLockPath(sandboxName, options.stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + + const startedAt = performance.now(); + const corruptMainTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + const corruptReaperTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + const corruptDeadlineTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + let lastOwnerPid: number | null = null; + for (;;) { + if (performance.now() - startedAt >= timeoutMs) { + const containmentPath = committedContainmentPath(lockPath); + const containment = readMcpLifecycleLockObservationSync(containmentPath); + if (containment) { + throw new Error( + `Sandbox mutation containment is active for '${sandboxName}' at '${containmentPath}' (generation token '${containment.owner?.token ?? "invalid"}'). A previous owner or stale-lock reaper exited without proof that every descendant stopped. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', and '${lockPath}.deadline'; record each target's file kind, device/inode, and owner token when present; verify those identities and this containment token are unchanged; remove only those exact stale owner generations first and this exact containment generation last before retrying.`, + ); + } + throw new Error( + `Timed out waiting for sandbox mutation lock for '${sandboxName}'${ + lastOwnerPid ? ` (owner PID ${lastOwnerPid})` : "" + }`, + ); + } + + const containmentPath = committedContainmentPath(lockPath); + if (mcpLifecycleLockPathExistsSync(containmentPath)) { + sleepSync(pollIntervalMs); + continue; + } + + const deadlinePath = `${lockPath}.deadline`; + const deadlineObservation = readMcpLifecycleLockObservationSync(deadlinePath); + if (deadlineObservation) { + const deadlineDisposition = classifyObservedMcpLifecycleLock( + deadlineObservation, + sandboxName, + corruptLockGraceMs, + corruptDeadlineTracker, + ); + if (deadlineDisposition === "stale") { + ensurePermanentContainmentForStaleGenerationSync( + lockPath, + sandboxName, + options.stateDir, + deadlineObservation, + "An auto-restore deadline owner exited before its recovery operation completed", + ); + continue; + } + sleepSync(pollIntervalMs); + continue; + } + resetCorruptGenerationTracker(corruptDeadlineTracker); + + const reaperPath = `${lockPath}.reaper`; + const reaperObservation = readMcpLifecycleLockObservationSync(reaperPath); + if (reaperObservation) { + const reaperDisposition = classifyObservedMcpLifecycleLock( + reaperObservation, + sandboxName, + corruptLockGraceMs, + corruptReaperTracker, + ); + if (reaperDisposition === "stale") { + ensurePermanentContainmentForStaleGenerationSync( + lockPath, + sandboxName, + options.stateDir, + reaperObservation, + "A stale-lock reaper exited before cleanup completed", + ); + continue; + } + sleepSync(pollIntervalMs); + continue; + } + resetCorruptGenerationTracker(corruptReaperTracker); + + if ( + !mcpLifecycleLockPathExistsSync(deadlinePath) && + !mcpLifecycleLockPathExistsSync(reaperPath) + ) { + const token = crypto.randomUUID(); + const shieldsTakeoverToken = readShieldsTimerTakeoverToken(sandboxName, options.stateDir); + const owner = createMcpLifecycleLockOwner(sandboxName, token, shieldsTakeoverToken); + if (writeMcpLifecycleLockCandidateAndLinkSync(lockPath, owner)) { + if ( + !mcpLifecycleLockPathExistsSync(containmentPath) && + !mcpLifecycleLockPathExistsSync(deadlinePath) && + !mcpLifecycleLockPathExistsSync(reaperPath) && + readShieldsTimerTakeoverToken(sandboxName, options.stateDir) === shieldsTakeoverToken + ) { + return { lockPath, token }; } + safelyReleaseMcpLifecycleLockSync(lockPath, token); + } + } + + const observation = readMcpLifecycleLockObservationSync(lockPath); + if (observation) { + lastOwnerPid = observation.owner?.pid ?? null; + if ( + classifyObservedMcpLifecycleLock( + observation, + sandboxName, + corruptLockGraceMs, + corruptMainTracker, + ) === "stale" + ) { + ensurePermanentContainmentForStaleGenerationSync( + lockPath, + sandboxName, + options.stateDir, + observation, + "A sandbox mutation owner exited before its descendants could be proven contained", + ); + continue; } } else { + lastOwnerPid = null; resetCorruptGenerationTracker(corruptMainTracker); } + sleepSync(pollIntervalMs); + } +} + +function sameLockGeneration(left: LockObservation, right: LockObservation | null): boolean { + if (!right || left.dev !== right.dev || left.ino !== right.ino) return false; + const leftToken = left.owner?.token; + const rightToken = right.owner?.token; + return leftToken === rightToken; +} + +function selfOwnedDeadlineMainToken( + observation: LockObservation | null, + sandboxName: string, + takeoverToken: string, + expectedToken: string, +): string | null { + const owner = observation?.owner; + const processIdentity = readMcpLockProcessIdentity(process.pid); + return owner?.sandboxName === sandboxName && + owner.pid === process.pid && + Boolean(processIdentity) && + owner.processIdentity === processIdentity && + owner.hostIdentity === readMcpLockHostIdentity() && + owner.pidNamespaceIdentity === readMcpLockPidNamespaceIdentity() && + owner.shieldsTakeoverToken === takeoverToken && + owner.token === expectedToken + ? owner.token + : null; +} + +function isPermanentContainmentError(error: unknown): boolean { + return ( + error instanceof Error && + (error as Error & { code?: string }).code === "NEMOCLAW_PERMANENT_CONTAINMENT" + ); +} + +function permanentContainmentFailure(error: unknown): Error & { code: string } { + const failure = new Error( + `Permanent sandbox mutation containment requires operator resolution: ${ + error instanceof Error ? error.message : String(error) + }`, + ) as Error & { code: string }; + failure.code = "NEMOCLAW_PERMANENT_CONTAINMENT"; + return failure; +} + +async function deadlineMainStillPresent(lockPath: string): Promise { + try { + return (await readMcpLifecycleLockObservation(lockPath)) !== null; + } catch { + return true; + } +} + +function deadlineMainStillPresentSync(lockPath: string): boolean { + try { + return readMcpLifecycleLockObservationSync(lockPath) !== null; + } catch { + return true; + } +} + +async function reportDeadlineContainment( + options: McpLifecycleDeadlineFenceOptions, + details: McpLifecycleDeadlineContainment, +): Promise { + try { + await options.onContainment?.(details); + } catch { + // Reporting must not release the security gate it is describing. + } +} + +function reportDeadlineContainmentSync( + options: McpLifecycleDeadlineFenceSyncOptions, + details: McpLifecycleDeadlineContainment, +): void { + try { + options.onContainment?.(details); + } catch { + // Reporting must not release the security gate it is describing. + } +} + +async function acquireDeadlineFence( + sandboxName: string, + takeoverToken: string, + options: McpLifecycleDeadlineFenceOptions, +): Promise { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const timeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptLockGraceMs = positiveInteger( + options.corruptLockGraceMs, + DEFAULT_CORRUPT_LOCK_GRACE_MS, + ); + const lockPath = getMcpLifecycleLockPath(sandboxName, options.stateDir); + const deadlinePath = `${lockPath}.deadline`; + await fs.promises.mkdir(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + + let blockedAt = performance.now(); + let notifiedGeneration: string | null = null; + const corruptTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + for (;;) { + if (readShieldsTimerTakeoverToken(sandboxName, options.stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + if (await mcpLifecycleLockPathExists(committedContainmentPath(lockPath))) { + if ( + notifiedGeneration !== "committed-containment" && + performance.now() - blockedAt >= timeoutMs + ) { + await reportDeadlineContainment(options, { + ownerPid: null, + reason: + "A committed process-tree containment requires operator resolution before auto-restore can continue.", + }); + notifiedGeneration = "committed-containment"; + } + await sleep(pollIntervalMs); + continue; + } + + const token = crypto.randomUUID(); + const owner = createMcpLifecycleLockOwner(sandboxName, token, takeoverToken); + if (await writeMcpLifecycleLockCandidateAndLink(deadlinePath, owner)) { + if (readShieldsTimerTakeoverToken(sandboxName, options.stateDir) === takeoverToken) { + return { lockPath: deadlinePath, token }; + } + await safelyReleaseMcpLifecycleLock(deadlinePath, token); + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + + const observation = await readMcpLifecycleLockObservation(deadlinePath); + if ( + observation && + classifyObservedMcpLifecycleLock( + observation, + sandboxName, + corruptLockGraceMs, + corruptTracker, + ) === "stale" + ) { + ensurePermanentContainmentForStaleGenerationSync( + lockPath, + sandboxName, + options.stateDir ?? resolveNemoclawStateDir(), + observation, + "An auto-restore deadline owner exited before its recovery operation completed", + ); + continue; + } + if (observation) { + const generation = `${String(observation.dev)}:${String(observation.ino)}:${ + observation.owner?.token ?? "invalid" + }`; + if (generation !== notifiedGeneration && performance.now() - blockedAt >= timeoutMs) { + await reportDeadlineContainment(options, { + ownerPid: observation.owner?.pid ?? null, + reason: + "Another auto-restore deadline owner is active or cannot be verified; the deadline gate remains closed pending operator or distributed-lease resolution.", + }); + notifiedGeneration = generation; + } + } else { + blockedAt = performance.now(); + notifiedGeneration = null; + } + await sleep(pollIntervalMs); + } +} + +function acquireDeadlineFenceSync( + sandboxName: string, + takeoverToken: string, + options: McpLifecycleDeadlineFenceSyncOptions & { stateDir: string }, +): AcquiredMcpLifecycleLock { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const timeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptLockGraceMs = positiveInteger( + options.corruptLockGraceMs, + DEFAULT_CORRUPT_LOCK_GRACE_MS, + ); + const lockPath = getMcpLifecycleLockPath(sandboxName, options.stateDir); + const deadlinePath = `${lockPath}.deadline`; + fs.mkdirSync(path.dirname(lockPath), { recursive: true, mode: 0o700 }); + + let blockedAt = performance.now(); + let notifiedGeneration: string | null = null; + const corruptTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + for (;;) { + if (readShieldsTimerTakeoverToken(sandboxName, options.stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + if (mcpLifecycleLockPathExistsSync(committedContainmentPath(lockPath))) { + if ( + notifiedGeneration !== "committed-containment" && + performance.now() - blockedAt >= timeoutMs + ) { + reportDeadlineContainmentSync(options, { + ownerPid: null, + reason: + "A committed process-tree containment requires operator resolution before auto-restore can continue.", + }); + notifiedGeneration = "committed-containment"; + } + sleepSync(pollIntervalMs); + continue; + } + + const token = crypto.randomUUID(); + const owner = createMcpLifecycleLockOwner(sandboxName, token, takeoverToken); + if (writeMcpLifecycleLockCandidateAndLinkSync(deadlinePath, owner)) { + if (readShieldsTimerTakeoverToken(sandboxName, options.stateDir) === takeoverToken) { + return { lockPath: deadlinePath, token }; + } + safelyReleaseMcpLifecycleLockSync(deadlinePath, token); + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + + const observation = readMcpLifecycleLockObservationSync(deadlinePath); + if ( + observation && + classifyObservedMcpLifecycleLock( + observation, + sandboxName, + corruptLockGraceMs, + corruptTracker, + ) === "stale" + ) { + ensurePermanentContainmentForStaleGenerationSync( + lockPath, + sandboxName, + options.stateDir, + observation, + "An auto-restore deadline owner exited before its recovery operation completed", + ); + continue; + } + if (observation) { + const generation = `${String(observation.dev)}:${String(observation.ino)}:${ + observation.owner?.token ?? "invalid" + }`; + if (generation !== notifiedGeneration && performance.now() - blockedAt >= timeoutMs) { + reportDeadlineContainmentSync(options, { + ownerPid: observation.owner?.pid ?? null, + reason: + "Another auto-restore deadline owner is active or cannot be verified; the deadline gate remains closed pending operator or distributed-lease resolution.", + }); + notifiedGeneration = generation; + } + } else { + blockedAt = performance.now(); + notifiedGeneration = null; + } + sleepSync(pollIntervalMs); + } +} + +async function clearDeadlineProtectedPath( + targetPath: string, + targetLabel: string, + sandboxName: string, + takeoverToken: string, + stateDir: string, + options: McpLifecycleDeadlineFenceOptions, +): Promise { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const containmentTimeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + let blockedAt = performance.now(); + let notifiedGeneration: string | null = null; + + for (;;) { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const observed = await readMcpLifecycleLockObservation(targetPath); + if (!observed) return; + + const disposition = classifyObservedMcpLifecycleLock(observed, sandboxName, 0, corruptTracker); + const owner = observed.owner; + const exactLocalOwner = + owner?.sandboxName === sandboxName && + Boolean(owner.processIdentity) && + owner.hostIdentity === readMcpLockHostIdentity() && + owner.pidNamespaceIdentity === readMcpLockPidNamespaceIdentity(); + if (disposition === "stale" && exactLocalOwner) { + const confirmed = await readMcpLifecycleLockObservation(targetPath); + if (!sameLockGeneration(observed, confirmed)) continue; + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const lifecyclePath = targetPath.endsWith(".reaper") + ? targetPath.slice(0, -".reaper".length) + : targetPath; + ensurePermanentContainmentForStaleGenerationSync( + lifecyclePath, + sandboxName, + stateDir, + observed, + `The ${targetLabel} owner PID ${String(owner?.pid)} was already gone before takeover`, + ); + throw permanentContainmentFailure( + new Error( + `The exact ${targetLabel} owner was already gone, so surviving descendants cannot be ruled out`, + ), + ); + } + + const generation = `${String(observed.dev)}:${String(observed.ino)}:${ + owner?.token ?? "invalid" + }`; + if ( + generation !== notifiedGeneration && + performance.now() - blockedAt >= containmentTimeoutMs + ) { + await reportDeadlineContainment(options, { + ownerPid: owner?.pid ?? null, + reason: `The active ${targetLabel} owner was preserved because portable process inspection cannot prove that forcibly terminating it would contain every descendant. Shields remain DOWN and the deadline gate is blocking new mutations. Wait for the owner to finish, or stop all NemoClaw processes for this sandbox before resolving the recorded lock generation.`, + }); + notifiedGeneration = generation; + } + await sleep(pollIntervalMs); + } +} + +function clearDeadlineProtectedPathSync( + targetPath: string, + targetLabel: string, + sandboxName: string, + takeoverToken: string, + stateDir: string, + options: McpLifecycleDeadlineFenceSyncOptions, +): void { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + const containmentTimeoutMs = positiveInteger(options.timeoutMs, DEFAULT_TIMEOUT_MS); + const corruptTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; + let blockedAt = performance.now(); + let notifiedGeneration: string | null = null; + + for (;;) { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const observed = readMcpLifecycleLockObservationSync(targetPath); + if (!observed) return; + + const disposition = classifyObservedMcpLifecycleLock(observed, sandboxName, 0, corruptTracker); + const owner = observed.owner; + const exactLocalOwner = + owner?.sandboxName === sandboxName && + Boolean(owner.processIdentity) && + owner.hostIdentity === readMcpLockHostIdentity() && + owner.pidNamespaceIdentity === readMcpLockPidNamespaceIdentity(); + if (exactLocalOwner && owner?.pid === process.pid) { + const error = new Error( + "Synchronous auto-restore cannot wait behind a sibling lifecycle operation in this process", + ) as Error & { code: string }; + error.code = "NEMOCLAW_SYNC_REENTRANT_OWNER"; + throw error; + } + if (disposition === "stale" && exactLocalOwner) { + const confirmed = readMcpLifecycleLockObservationSync(targetPath); + if (!sameLockGeneration(observed, confirmed)) continue; + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const lifecyclePath = targetPath.endsWith(".reaper") + ? targetPath.slice(0, -".reaper".length) + : targetPath; + ensurePermanentContainmentForStaleGenerationSync( + lifecyclePath, + sandboxName, + stateDir, + observed, + `The ${targetLabel} owner PID ${String(owner?.pid)} was already gone before takeover`, + ); + throw permanentContainmentFailure( + new Error( + `The exact ${targetLabel} owner was already gone, so surviving descendants cannot be ruled out`, + ), + ); + } + + const generation = `${String(observed.dev)}:${String(observed.ino)}:${ + owner?.token ?? "invalid" + }`; + if ( + generation !== notifiedGeneration && + performance.now() - blockedAt >= containmentTimeoutMs + ) { + reportDeadlineContainmentSync(options, { + ownerPid: owner?.pid ?? null, + reason: `The active ${targetLabel} owner was preserved because portable process inspection cannot prove that forcibly terminating it would contain every descendant. Shields remain DOWN and the deadline gate is blocking new mutations. Wait for the owner to finish, or stop all NemoClaw processes for this sandbox before resolving the recorded lock generation.`, + }); + notifiedGeneration = generation; + } + sleepSync(pollIntervalMs); + } +} + +async function publishDeadlineMainOwner( + lockPath: string, + sandboxName: string, + takeoverToken: string, + stateDir: string, + options: McpLifecycleDeadlineFenceOptions, +): Promise { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + let notifiedError: string | null = null; + let pendingCandidateToken: string | null = null; + for (;;) { + try { + if (pendingCandidateToken) { + const existingSelfToken = selfOwnedDeadlineMainToken( + await readMcpLifecycleLockObservation(lockPath), + sandboxName, + takeoverToken, + pendingCandidateToken, + ); + if (existingSelfToken) return existingSelfToken; + pendingCandidateToken = null; + } + await clearDeadlineProtectedPath( + `${lockPath}.reaper`, + "stale-lock reaper", + sandboxName, + takeoverToken, + stateDir, + options, + ); + await clearDeadlineProtectedPath( + lockPath, + "sandbox mutation", + sandboxName, + takeoverToken, + stateDir, + options, + ); + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const candidateToken = crypto.randomUUID(); + pendingCandidateToken = candidateToken; + const timerOwner = createMcpLifecycleLockOwner(sandboxName, candidateToken, takeoverToken); + if (await writeMcpLifecycleLockCandidateAndLink(lockPath, timerOwner)) { + return candidateToken; + } + pendingCandidateToken = null; + notifiedError = null; + } catch (error) { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const message = error instanceof Error ? error.message : String(error); + if (isPermanentContainmentError(error)) { + if (message !== notifiedError) { + await reportDeadlineContainment(options, { + ownerPid: null, + reason: `${message} The deadline gate remains closed. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${lockPath}.deadline', and '${committedContainmentPath(lockPath)}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`, + }); + notifiedError = message; + } + while ( + readShieldsTimerTakeoverToken(sandboxName, stateDir) === takeoverToken && + ((await mcpLifecycleLockPathExists(committedContainmentPath(lockPath))) || + (await deadlineMainStillPresent(lockPath))) + ) { + await sleep(pollIntervalMs); + } + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + continue; + } + if (message !== notifiedError) { + await reportDeadlineContainment(options, { + ownerPid: null, + reason: `Auto-restore deadline setup is retrying while the gate remains closed: ${message}`, + }); + notifiedError = message; + } + } await sleep(pollIntervalMs); } } +function publishDeadlineMainOwnerSync( + lockPath: string, + sandboxName: string, + takeoverToken: string, + stateDir: string, + options: McpLifecycleDeadlineFenceSyncOptions, +): string { + const pollIntervalMs = positiveInteger(options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS); + let notifiedError: string | null = null; + let pendingCandidateToken: string | null = null; + for (;;) { + try { + if (pendingCandidateToken) { + const existingSelfToken = selfOwnedDeadlineMainToken( + readMcpLifecycleLockObservationSync(lockPath), + sandboxName, + takeoverToken, + pendingCandidateToken, + ); + if (existingSelfToken) return existingSelfToken; + pendingCandidateToken = null; + } + clearDeadlineProtectedPathSync( + `${lockPath}.reaper`, + "stale-lock reaper", + sandboxName, + takeoverToken, + stateDir, + options, + ); + clearDeadlineProtectedPathSync( + lockPath, + "sandbox mutation", + sandboxName, + takeoverToken, + stateDir, + options, + ); + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + const candidateToken = crypto.randomUUID(); + pendingCandidateToken = candidateToken; + const timerOwner = createMcpLifecycleLockOwner(sandboxName, candidateToken, takeoverToken); + if (writeMcpLifecycleLockCandidateAndLinkSync(lockPath, timerOwner)) { + return candidateToken; + } + pendingCandidateToken = null; + notifiedError = null; + } catch (error) { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + if ( + error instanceof Error && + (error as Error & { code?: string }).code === "NEMOCLAW_SYNC_REENTRANT_OWNER" + ) { + throw error; + } + const message = error instanceof Error ? error.message : String(error); + if (isPermanentContainmentError(error)) { + if (message !== notifiedError) { + reportDeadlineContainmentSync(options, { + ownerPid: null, + reason: `${message} The deadline gate remains closed. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${lockPath}.deadline', and '${committedContainmentPath(lockPath)}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`, + }); + notifiedError = message; + } + while ( + readShieldsTimerTakeoverToken(sandboxName, stateDir) === takeoverToken && + (mcpLifecycleLockPathExistsSync(committedContainmentPath(lockPath)) || + deadlineMainStillPresentSync(lockPath)) + ) { + sleepSync(pollIntervalMs); + } + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + continue; + } + if (message !== notifiedError) { + reportDeadlineContainmentSync(options, { + ownerPid: null, + reason: `Auto-restore deadline setup is retrying while the gate remains closed: ${message}`, + }); + notifiedError = message; + } + } + sleepSync(pollIntervalMs); + } +} + +/** + * Establish the auto-restore deadline as a generation-pinned exclusion fence. + * + * Ordinary acquisitions refuse to enter while `.deadline` exists. + * The timer keeps that gate through policy restoration and configuration re-locking, + * and waits for an active owner to release naturally. Portable PID inspection + * cannot prove that force-killing a process also contained every descendant. + */ +export async function withMcpLifecycleDeadlineFence( + sandboxName: string, + takeoverToken: string, + operation: () => Promise | T, + options: McpLifecycleDeadlineFenceOptions, +): Promise { + if (!/^[0-9a-f]{32}$/.test(takeoverToken)) { + throw new Error("Auto-restore takeover token must be 32 lowercase hexadecimal characters"); + } + const stateDir = options.stateDir ?? resolveNemoclawStateDir(); + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + const inherited = heldLocks.getStore(); + if (inherited?.get(lockPath)?.active) return await operation(); + + const fence = await acquireDeadlineFence(sandboxName, takeoverToken, { + ...options, + stateDir, + }); + let mainToken: string | null = null; + try { + // An ordinary acquirer can pass its pre-publication deadline check before + // this fence exists, then link the main path after an earlier clear. Keep + // the fence and repeat takeover until this timer owns the main generation. + mainToken = await publishDeadlineMainOwner( + lockPath, + sandboxName, + takeoverToken, + stateDir, + options, + ); + + const lease: HeldLockLease = { active: true }; + const context = new Map(inherited ?? []); + context.set(lockPath, lease); + return await heldLocks.run(context, async () => { + try { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + return await operation(); + } finally { + lease.active = false; + } + }); + } finally { + if (mainToken) await safelyReleaseMcpLifecycleLock(lockPath, mainToken); + await safelyReleaseMcpLifecycleLock(fence.lockPath, fence.token); + } +} + +/** + * Synchronous deadline fence for inline recovery from synchronous status and + * permission-inspection paths. + */ +export function withMcpLifecycleDeadlineFenceSync( + sandboxName: string, + takeoverToken: string, + operation: () => T, + options: McpLifecycleDeadlineFenceSyncOptions, +): T { + if (!/^[0-9a-f]{32}$/.test(takeoverToken)) { + throw new Error("Auto-restore takeover token must be 32 lowercase hexadecimal characters"); + } + const stateDir = options.stateDir ?? resolveNemoclawStateDir(); + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + const inherited = heldLocks.getStore(); + if (inherited?.get(lockPath)?.active) return operation(); + const fence = acquireDeadlineFenceSync(sandboxName, takeoverToken, { + ...options, + stateDir, + }); + let mainToken: string | null = null; + try { + mainToken = publishDeadlineMainOwnerSync( + lockPath, + sandboxName, + takeoverToken, + stateDir, + options, + ); + + const lease: HeldLockLease = { active: true }; + const context = new Map(inherited ?? []); + context.set(lockPath, lease); + try { + if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { + throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); + } + return heldLocks.run(context, operation); + } finally { + lease.active = false; + } + } finally { + if (mainToken) safelyReleaseMcpLifecycleLockSync(lockPath, mainToken); + safelyReleaseMcpLifecycleLockSync(fence.lockPath, fence.token); + } +} + +export function isMcpLifecycleLockHeld( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), +): boolean { + return heldLocks.getStore()?.get(getMcpLifecycleLockPath(sandboxName, stateDir))?.active === true; +} + +export function withMcpLifecycleLockSync( + sandboxName: string, + operation: () => T, + options: McpLifecycleLockOptions = {}, +): T { + const stateDir = options.stateDir ?? resolveNemoclawStateDir(); + const lockPath = getMcpLifecycleLockPath(sandboxName, stateDir); + const inherited = heldLocks.getStore(); + if (inherited?.get(lockPath)?.active) return operation(); + + const acquired = acquireMcpLifecycleLockSync(sandboxName, { ...options, stateDir }); + const lease: HeldLockLease = { active: true }; + const context = new Map(inherited ?? []); + context.set(lockPath, lease); + try { + return heldLocks.run(context, operation); + } finally { + lease.active = false; + safelyReleaseMcpLifecycleLockSync(acquired.lockPath, acquired.token); + } +} + /** * Serializes the complete MCP lifecycle for one sandbox across processes. * AsyncLocalStorage makes nested calls in the same lifecycle operation diff --git a/src/lib/state/mcp-lifecycle-lock-identity.ts b/src/lib/state/mcp-lifecycle-lock-identity.ts index b8e864ea4aa..5b0690efcb5 100644 --- a/src/lib/state/mcp-lifecycle-lock-identity.ts +++ b/src/lib/state/mcp-lifecycle-lock-identity.ts @@ -21,6 +21,8 @@ export interface McpLifecycleLockOwner { hostIdentity?: string | null; /** Linux PID namespace identity. Cross-namespace owners fail closed. */ pidNamespaceIdentity?: string | null; + /** Exact Shields timer generation correlated with this mutable-window operation. */ + shieldsTakeoverToken?: string; token: string; acquiredAt: string; } @@ -59,6 +61,9 @@ export function isMcpLifecycleLockOwner(value: unknown): value is McpLifecycleLo (candidate.pidNamespaceIdentity === undefined || candidate.pidNamespaceIdentity === null || typeof candidate.pidNamespaceIdentity === "string") && + (candidate.shieldsTakeoverToken === undefined || + (typeof candidate.shieldsTakeoverToken === "string" && + /^[0-9a-f]{32}$/.test(candidate.shieldsTakeoverToken))) && typeof candidate.token === "string" && candidate.token.length > 0 && typeof candidate.acquiredAt === "string" @@ -175,6 +180,7 @@ const LOCAL_IDENTITY_PROBES: McpLifecycleLockIdentityProbes = { export function createMcpLifecycleLockOwner( sandboxName: string, token: string, + shieldsTakeoverToken?: string, ): McpLifecycleLockOwner { return { version: LOCK_SCHEMA_VERSION, @@ -183,6 +189,7 @@ export function createMcpLifecycleLockOwner( processIdentity: readMcpLockProcessIdentity(process.pid), hostIdentity: LOCAL_HOST_IDENTITY, pidNamespaceIdentity: LOCAL_PID_NAMESPACE_IDENTITY, + ...(shieldsTakeoverToken ? { shieldsTakeoverToken } : {}), token, acquiredAt: new Date().toISOString(), }; diff --git a/src/lib/state/mcp-lifecycle-lock-storage.ts b/src/lib/state/mcp-lifecycle-lock-storage.ts index ad5d7c0b5c7..d75caf4ac4e 100644 --- a/src/lib/state/mcp-lifecycle-lock-storage.ts +++ b/src/lib/state/mcp-lifecycle-lock-storage.ts @@ -76,6 +76,48 @@ export async function readMcpLifecycleLockObservation( } } +export function readMcpLifecycleLockObservationSync(lockPath: string): LockObservation | null { + let fd: number; + try { + fd = fs.openSync( + lockPath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK, + ); + } catch (error) { + if (isErrnoException(error) && error.code === "ENOENT") return null; + try { + const stat = fs.lstatSync(lockPath); + if (!stat.isFile() || stat.isSymbolicLink()) { + return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino }; + } + } catch (statError) { + if (isErrnoException(statError) && statError.code === "ENOENT") return null; + throw statError; + } + throw error; + } + + try { + const stat = fs.fstatSync(fd); + if (!stat.isFile()) { + return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino }; + } + try { + const parsed: unknown = JSON.parse(fs.readFileSync(fd, "utf8")); + return { + owner: isMcpLifecycleLockOwner(parsed) ? parsed : null, + mtimeMs: stat.mtimeMs, + dev: stat.dev, + ino: stat.ino, + }; + } catch { + return { owner: null, mtimeMs: stat.mtimeMs, dev: stat.dev, ino: stat.ino }; + } + } finally { + fs.closeSync(fd); + } +} + export async function mcpLifecycleLockPathExists(targetPath: string): Promise { try { await fs.promises.lstat(targetPath); @@ -86,6 +128,16 @@ export async function mcpLifecycleLockPathExists(targetPath: string): Promise= 2 && published?.owner?.token === owner.token) { + return true; + } + if (isErrnoException(error) && error.code === "EEXIST") return false; + throw error; + } + } finally { + try { + fs.rmSync(candidatePath, { force: true }); + } catch { + // Publication is decided only by LINK plus owner-token reconciliation. + } + } +} diff --git a/src/lib/state/mcp-lifecycle-lock.ts b/src/lib/state/mcp-lifecycle-lock.ts index 4d6bc2934c6..da7cf069840 100644 --- a/src/lib/state/mcp-lifecycle-lock.ts +++ b/src/lib/state/mcp-lifecycle-lock.ts @@ -2,13 +2,22 @@ // SPDX-License-Identifier: Apache-2.0 export { + beginCommittedMcpLifecycleContainmentSync, + isMcpLifecycleLockHeld, + type McpLifecycleDeadlineContainment, + type McpLifecycleDeadlineFenceOptions, + type McpLifecycleDeadlineFenceSyncOptions, type McpLifecycleLockOptions, + withMcpLifecycleDeadlineFence, + withMcpLifecycleDeadlineFenceSync, withMcpLifecycleLock, withMcpLifecycleLock as withSandboxMutationLock, + withMcpLifecycleLockSync, } from "./mcp-lifecycle-lock-acquisition"; export { classifyMcpLifecycleLock, type McpLifecycleLockDisposition, + type McpLifecycleLockOwner, readMcpLockHostIdentity, readMcpLockPidNamespaceIdentity, readMcpLockProcessIdentity, diff --git a/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts b/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts new file mode 100644 index 00000000000..e446b8c2055 --- /dev/null +++ b/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { isObjectRecord } from "../../core/json-types"; +import { NAME_MAX_LENGTH, NAME_VALID_PATTERN } from "../../name-validation"; +import { resolveNemoclawStateDir } from "../paths"; + +export interface ShieldsTimerMarker { + pid: number; + sandboxName: string; + snapshotPath: string; + restoreAt: string; + processToken?: string; + allowLegacyHermesProtocol?: boolean; + leaseOwnerPid?: number; + leaseOwnerStartIdentity?: string; +} + +function isShieldsTimerMarker(value: unknown): value is ShieldsTimerMarker { + if (!isObjectRecord(value)) return false; + const pid = value.pid; + return ( + typeof pid === "number" && + Number.isInteger(pid) && + pid > 0 && + typeof value.sandboxName === "string" && + typeof value.snapshotPath === "string" && + typeof value.restoreAt === "string" && + (value.processToken === undefined || typeof value.processToken === "string") && + (value.allowLegacyHermesProtocol === undefined || + typeof value.allowLegacyHermesProtocol === "boolean") && + (value.leaseOwnerPid === undefined || + (typeof value.leaseOwnerPid === "number" && + Number.isInteger(value.leaseOwnerPid) && + value.leaseOwnerPid > 0)) && + (value.leaseOwnerStartIdentity === undefined || + typeof value.leaseOwnerStartIdentity === "string") && + ((value.leaseOwnerPid === undefined && value.leaseOwnerStartIdentity === undefined) || + (typeof value.leaseOwnerPid === "number" && + typeof value.leaseOwnerStartIdentity === "string" && + value.leaseOwnerStartIdentity.length > 0)) + ); +} + +export function shieldsTimerMarkerPath( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), +): string { + if ( + sandboxName.length === 0 || + sandboxName.length > NAME_MAX_LENGTH || + !NAME_VALID_PATTERN.test(sandboxName) + ) { + throw new Error("Cannot resolve a Shields timer marker for an invalid sandbox name"); + } + return path.join(stateDir, `shields-timer-${sandboxName}.json`); +} + +export function readShieldsTimerMarker( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), +): ShieldsTimerMarker | null { + try { + const markerPath = shieldsTimerMarkerPath(sandboxName, stateDir); + if (!fs.existsSync(markerPath)) return null; + const parsed = JSON.parse(fs.readFileSync(markerPath, "utf-8")); + return isShieldsTimerMarker(parsed) ? parsed : null; + } catch { + return null; + } +} + +export function readShieldsTimerTakeoverToken( + sandboxName: string, + stateDir = resolveNemoclawStateDir(), +): string | undefined { + const marker = readShieldsTimerMarker(sandboxName, stateDir); + if ( + marker?.sandboxName !== sandboxName || + typeof marker.processToken !== "string" || + !/^[0-9a-f]{32}$/.test(marker.processToken) + ) { + return undefined; + } + return marker.processToken; +} diff --git a/test/e2e/live/mcp-bridge-sandbox.ts b/test/e2e/live/mcp-bridge-sandbox.ts index 5d9f5b8927e..aa801ef4c28 100644 --- a/test/e2e/live/mcp-bridge-sandbox.ts +++ b/test/e2e/live/mcp-bridge-sandbox.ts @@ -1,15 +1,91 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import assert from "node:assert/strict"; +import YAML from "yaml"; import { shellQuote } from "../../../src/lib/core/shell-quote"; +import { parseOpenShellPolicy } from "../../../src/lib/policy/merge"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { assertExitZero, resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; +import type { SandboxClient } from "../fixtures/clients/sandbox.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const MCP_CURL_HTTP_CODE_MARKER = "NEMOCLAW_MCP_CURL_HTTP_CODE="; export type McpDnsRebindingAdapter = "mcporter" | "hermes-config" | "deepagents-config"; +export type CapturedManagedMcpPolicy = { + networkPolicies: Record; + policy: McpNetworkPolicy; +}; + +type McpNetworkPolicy = { + endpoints?: Array<{ + host?: string; + allowed_ips?: string[]; + [key: string]: unknown; + }>; + [key: string]: unknown; +}; + +export async function captureManagedMcpPolicy( + sandbox: SandboxClient, + options: { + artifactName: string; + label: string; + policyKey: string; + sandboxName: string; + url: string; + }, +): Promise { + const result = await sandbox.openshell(["policy", "get", "--full", options.sandboxName], { + artifactName: options.artifactName, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + assertExitZero(result, options.label); + const document = YAML.parse(parseOpenShellPolicy(resultText(result)).yamlBody) as { + network_policies?: Record; + }; + const networkPolicies = document.network_policies ?? {}; + const policy = networkPolicies[options.policyKey]; + if (!policy) { + throw new Error(`${options.label}: managed MCP policy '${options.policyKey}' is absent`); + } + const endpoint = policy.endpoints?.[0]; + const expectedHost = new URL(options.url).hostname; + if (endpoint?.host !== expectedHost) { + throw new Error(`${options.label}: expected managed MCP host '${expectedHost}'`); + } + if ( + !Array.isArray(endpoint.allowed_ips) || + endpoint.allowed_ips.length === 0 || + endpoint.allowed_ips.some((address) => typeof address !== "string") + ) { + throw new Error(`${options.label}: expected at least one managed MCP address pin`); + } + return { networkPolicies, policy }; +} + +export function assertManagedMcpPolicySurvivedRemoval( + before: McpNetworkPolicy, + after: CapturedManagedMcpPolicy, + removedPolicyKey: string, +): void { + assert.deepStrictEqual(after.policy, before); + assert.equal(after.networkPolicies[removedPolicyKey], undefined); +} + +export function expectExitNonZero(result: ShellProbeResult, label: string, pattern: RegExp): void { + assert.notEqual( + result.exitCode, + 0, + `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ); + assert.match(resultText(result), pattern); +} + export async function hostAddressForSandbox(_host: HostCliClient): Promise { return "host.openshell.internal"; } diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index b0001d63b98..098ac564da0 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -4,14 +4,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import YAML from "yaml"; import { buildDeepAgentsMcpStatusCommand, buildHermesMcpStatusCommand, buildOpenClawMcporterInspectCommand, } from "../../../src/lib/actions/sandbox/mcp-bridge-adapter-status"; import { shellQuote } from "../../../src/lib/core/shell-quote"; -import { parseOpenShellPolicy } from "../../../src/lib/policy/merge"; import type { McpBridgeEntry } from "../../../src/lib/state/registry"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { CleanupRegistry } from "../fixtures/cleanup.ts"; @@ -33,7 +31,10 @@ import { buildMcpBridgeExactMainEnv, buildMcpBridgeOnboardEnv } from "./mcp-brid import { MCP_BRIDGE_PHASES } from "./mcp-bridge-phases.ts"; import { retryAfterHermesRestartTransportFailure } from "./mcp-bridge-reliability.ts"; import { + assertManagedMcpPolicySurvivedRemoval, buildMcpDnsRebindingProbeScript, + captureManagedMcpPolicy, + expectExitNonZero, hostAddressForSandbox, hostPrivateAddressForSandbox, isExpectedMcpCurlPolicyDenial, @@ -91,18 +92,6 @@ const MCP_MUTATION_TIMEOUT_MS: Record = { const MCP_BRIDGE_ALREADY_ABSENT = /No MCP servers are registered|No MCP server '.+' is registered|MCP server '.+' not found/iu; -function expectExitNonZero(result: ShellProbeResult, label: string, pattern: RegExp): void { - expect( - result.exitCode, - `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, - ).not.toBe(0); - expect(resultText(result)).toMatch(pattern); -} - -function parseCurrentPolicy(raw: string): string { - return parseOpenShellPolicy(raw).yamlBody; -} - async function cleanupMcpBridge( host: HostCliClient, sandboxName: string, @@ -189,6 +178,7 @@ async function assertAdapterDnsRebindingDenied( artifactPrefix: string; sandboxName: string; secretPaths: string[]; + survivingMcpUrl: string; }, ): Promise { const rebindMcp = await startFakeMcpHttpsServer({ secret: REBIND_HOST_SECRET }); @@ -207,6 +197,14 @@ async function assertAdapterDnsRebindingDenied( cleanup.add(`restore ${options.artifactPrefix} DNS rebinding hosts fixture`, () => restoreDnsRebindingHostsFixture(host, options.sandboxName, hostsFixture), ); + const survivingPolicyBeforeAddResult = await captureManagedMcpPolicy(sandbox, { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-surviving-policy-before-add`, + label: `${options.artifactPrefix} captures the surviving MCP policy before adding the rebinding route`, + policyKey: SERVER_POLICY_KEY, + sandboxName: options.sandboxName, + url: options.survivingMcpUrl, + }); + const survivingPolicyBeforeAdd = survivingPolicyBeforeAddResult.policy; await remapDnsRebindingHostname( host, options.sandboxName, @@ -261,19 +259,14 @@ async function assertAdapterDnsRebindingDenied( policy: { gatewayPresent: true }, adapter: { registered: true }, }); - const policy = await sandbox.openshell(["policy", "get", "--full", options.sandboxName], { + const rebindingPolicy = await captureManagedMcpPolicy(sandbox, { artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-policy-pinned-public-ip`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, + label: `${options.artifactPrefix} validates the add-time DNS pin`, + policyKey: REBIND_POLICY_KEY, + sandboxName: options.sandboxName, + url: rebindMcpUrl, }); - expectExitZero(policy, `${options.artifactPrefix} inspects add-time DNS pin`); - const policyJson = YAML.parse(parseCurrentPolicy(resultText(policy))) as { - network_policies?: Record< - string, - { endpoints?: Array<{ host?: string; allowed_ips?: string[] }> } - >; - }; - expect(policyJson.network_policies?.[REBIND_POLICY_KEY]?.endpoints?.[0]).toMatchObject({ + expect(rebindingPolicy.policy.endpoints?.[0]).toMatchObject({ host: REBIND_HOSTNAME, allowed_ips: [REBIND_PUBLIC_IP], }); @@ -322,6 +315,18 @@ async function assertAdapterDnsRebindingDenied( timeoutMs: MCP_MUTATION_TIMEOUT_MS[options.adapter], }); expectExitZero(remove, `${options.artifactPrefix} removes DNS rebinding route after proof`); + const survivingPolicyAfterRemoveResult = await captureManagedMcpPolicy(sandbox, { + artifactName: `${options.artifactPrefix}-mcp-dns-rebinding-surviving-policy-after-remove`, + label: `${options.artifactPrefix} inspects MCP policy after removing the rebinding route`, + policyKey: SERVER_POLICY_KEY, + sandboxName: options.sandboxName, + url: options.survivingMcpUrl, + }); + assertManagedMcpPolicySurvivedRemoval( + survivingPolicyBeforeAdd, + survivingPolicyAfterRemoveResult, + REBIND_POLICY_KEY, + ); } async function addBridgeAndReadStatus( host: HostCliClient, @@ -980,6 +985,7 @@ test("mcp-bridge", { artifactPrefix: "openclaw", sandboxName: OPENCLAW_SANDBOX_NAME, secretPaths: ["/sandbox/.openclaw", "/sandbox/.mcp.json"], + survivingMcpUrl: mcpUrl, }); const requestCountBeforeAllowedNodeProof = fakeMcp.requests.length; @@ -1257,6 +1263,7 @@ mcpBridgeShardTest("hermes")( artifactPrefix: "hermes", sandboxName: HERMES_SANDBOX_NAME, secretPaths: ["/sandbox/.hermes"], + survivingMcpUrl: mcpUrl, }); await assertHermesToolCall("hermes-real-mcp-tool-call-after-dns-rebinding-remove"); const survivingDiscoveryOffset = fakeMcp.requests.length; @@ -1419,6 +1426,7 @@ mcpBridgeShardTest("deepagents")( artifactPrefix: "deepagents", sandboxName: DEEPAGENTS_SANDBOX_NAME, secretPaths: ["/sandbox/.deepagents"], + survivingMcpUrl: mcpUrl, }); progress.phase("exercise lifecycle and confirm Deep Agents bridge removal"); await assertRealAdapterToolCall(sandbox, fakeMcp, { diff --git a/test/e2e/support/mcp-bridge-sandbox.test.ts b/test/e2e/support/mcp-bridge-sandbox.test.ts index 9688afaa7fe..5d504d05322 100644 --- a/test/e2e/support/mcp-bridge-sandbox.test.ts +++ b/test/e2e/support/mcp-bridge-sandbox.test.ts @@ -302,10 +302,19 @@ network_policies: expect(contractSource).not.toContain("assertAdapterDnsRebindingDenied"); }); - it("runs the zero-upstream rebinding proof for all three adapters", () => { + it("runs the zero-upstream rebinding proof and preserves the surviving policy for all three adapters", () => { const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); + const helper = source.indexOf("async function assertAdapterDnsRebindingDenied"); + const survivingPolicyBeforeAdd = source.indexOf("const survivingPolicyBeforeAddResult", helper); + const add = source.indexOf("const add = await host.nemoclaw", survivingPolicyBeforeAdd); + const remove = source.indexOf("const remove = await host.nemoclaw", add); + const survivingPolicyAfterRemove = source.indexOf( + "const survivingPolicyAfterRemoveResult", + remove, + ); expect(source.match(/await assertAdapterDnsRebindingDenied/g)).toHaveLength(3); + expect(source.match(/survivingMcpUrl: mcpUrl/g)).toHaveLength(3); for (const adapter of [ 'adapter: "mcporter"', 'adapter: "hermes-config"', @@ -315,9 +324,16 @@ network_policies: } expect(source).toContain("rebound request must not reach the upstream MCP server"); expect(source).toContain(").toHaveLength(0);"); + expect(survivingPolicyBeforeAdd).toBeGreaterThan(helper); + expect(add).toBeGreaterThan(survivingPolicyBeforeAdd); + expect(remove).toBeGreaterThan(add); + expect(survivingPolicyAfterRemove).toBeGreaterThan(remove); + expect(source).toContain( + "assertManagedMcpPolicySurvivedRemoval(\n survivingPolicyBeforeAdd,\n survivingPolicyAfterRemoveResult,\n REBIND_POLICY_KEY,", + ); }); - it("proves the surviving Hermes route before and after the unrelated route lifecycle", () => { + it("requires the Hermes E2E journey to call the surviving route before and after the unrelated route lifecycle", () => { const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); const denialProof = source.indexOf("rebound request must not reach the upstream MCP server"); const restore = source.indexOf("await restoreDnsRebindingHostsFixture", denialProof); diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts index 79599b94ed5..00f93a21d02 100644 --- a/test/mcp-lifecycle-lock.test.ts +++ b/test/mcp-lifecycle-lock.test.ts @@ -42,6 +42,19 @@ function deferred(): { promise: Promise; resolve: () => void } { return { promise, resolve }; } +function writeTimerMarker(sandboxName: string, processToken: string): void { + fs.writeFileSync( + path.join(stateDir, `shields-timer-${sandboxName}.json`), + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath: path.join(stateDir, "snapshot.yaml"), + restoreAt: new Date(Date.now() + 60_000).toISOString(), + processToken, + }), + ); +} + function waitForLine(child: ChildProcess, expected: string): Promise { return new Promise((resolve, reject) => { let output = ""; @@ -104,9 +117,10 @@ describe("MCP lifecycle lock", () => { }); it.skipIf(process.platform === "win32")( - "does not follow a symlink when observing lock ownership", + "contains a symlink generation without following or deleting it", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; const targetPath = path.join(stateDir, "operator-owned-target"); const target = `${JSON.stringify({ version: 1, @@ -121,18 +135,21 @@ describe("MCP lifecycle lock", () => { fs.symlinkSync(targetPath, lockPath); await expect( - lifecycleLock.withMcpLifecycleLock("alpha", () => "acquired", options()), - ).resolves.toBe("acquired"); + lifecycleLock.withMcpLifecycleLock("alpha", () => "acquired", options({ timeoutMs: 50 })), + ).rejects.toThrow(/containment is active/); expect(fs.readFileSync(targetPath, "utf8")).toBe(target); + expect(fs.lstatSync(lockPath).isSymbolicLink()).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }, ); it.skipIf(process.platform === "win32")( - "reaps a non-regular Unix socket found at the lock path", + "contains a non-regular Unix socket generation without deleting it", async () => { const shortStateDir = path.join("/tmp", `m${process.pid}`); fs.rmSync(shortStateDir, { recursive: true, force: true }); const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", shortStateDir); + const containmentPath = `${lockPath}.containment`; expect(Buffer.byteLength(lockPath)).toBeLessThan(104); fs.mkdirSync(path.dirname(lockPath), { recursive: true }); const server = createServer(); @@ -147,8 +164,11 @@ describe("MCP lifecycle lock", () => { lifecycleLock.withMcpLifecycleLock("alpha", () => "acquired", { ...options(), stateDir: shortStateDir, + timeoutMs: 50, }), - ).resolves.toBe("acquired"); + ).rejects.toThrow(/containment is active/); + expect(fs.lstatSync(lockPath).isSocket()).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); } finally { await new Promise((resolve) => server.close(() => resolve())); fs.rmSync(shortStateDir, { recursive: true, force: true }); @@ -277,8 +297,9 @@ const releasePath = process.argv[3]; children.delete(child); }); - it("recovers an atomic lock left by a dead owner", async () => { + it("permanently contains an atomic lock left by a dead owner", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; fs.mkdirSync(path.dirname(lockPath), { recursive: true }); fs.writeFileSync( lockPath, @@ -294,16 +315,72 @@ const releasePath = process.argv[3]; })}\n`, ); - let entered = false; - await lifecycleLock.withMcpLifecycleLock( - "alpha", - () => { - entered = true; - }, - options(), + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), + ).rejects.toThrow("Sandbox mutation containment is active"); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); + }); + + it("permanently contains a stale deadline generation before an ordinary mutation", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const deadlinePath = `${lockPath}.deadline`; + const containmentPath = `${lockPath}.containment`; + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + deadlinePath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-deadline", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "stale-deadline-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, ); - expect(entered).toBe(true); - expect(fs.existsSync(lockPath)).toBe(false); + + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), + ).rejects.toThrow("Sandbox mutation containment is active"); + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); + }); + + it("permanently contains a stale deadline generation before deadline recovery", async () => { + const processToken = "9".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const deadlinePath = `${lockPath}.deadline`; + const containmentPath = `${lockPath}.containment`; + writeTimerMarker("alpha", processToken); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + deadlinePath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-deadline", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + shieldsTakeoverToken: processToken, + token: "stale-deadline-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + setTimeout(() => writeTimerMarker("alpha", "8".repeat(32)), 40); + + await expect( + lifecycleLock.withMcpLifecycleDeadlineFence( + "alpha", + processToken, + () => undefined, + options({ timeoutMs: 40 }), + ), + ).rejects.toThrow("Auto-restore authority changed"); + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }); it("waits for a foreign-host owner instead of reaping it with local PID checks", async () => { @@ -421,8 +498,9 @@ const releasePath = process.argv[3]; expect(fs.existsSync(lockPath)).toBe(false); }); - it("waits for grace then recovers a stable truncated owner record", async () => { + it("waits for grace then permanently contains a stable truncated owner record", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; fs.mkdirSync(path.dirname(lockPath), { recursive: true }); fs.writeFileSync(lockPath, '{"version":1,"sandboxName":"alpha"'); @@ -441,16 +519,18 @@ const releasePath = process.argv[3]; await expect( lifecycleLock.withMcpLifecycleLock( "alpha", - () => "acquired", - options({ corruptLockGraceMs: 20 }), + () => undefined, + options({ timeoutMs: 50, corruptLockGraceMs: 20 }), ), - ).resolves.toBe("acquired"); - expect(fs.existsSync(lockPath)).toBe(false); + ).rejects.toThrow("Sandbox mutation containment is active"); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }); - it("recovers a reaper whose owner was killed during stale-lock cleanup", async () => { + it("permanently contains a reaper whose owner died during stale-lock cleanup", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); const reaperPath = `${lockPath}.reaper`; + const containmentPath = `${lockPath}.containment`; fs.mkdirSync(path.dirname(lockPath), { recursive: true }); fs.writeFileSync( reaperPath, @@ -466,122 +546,43 @@ const releasePath = process.argv[3]; })}\n`, ); - let entered = false; - await lifecycleLock.withMcpLifecycleLock( - "alpha", - () => { - entered = true; - }, - options(), - ); - expect(entered).toBe(true); - expect(fs.existsSync(lockPath)).toBe(false); - expect(fs.existsSync(reaperPath)).toBe(false); - }); - - it("does not unlink a replacement reaper published during stale recovery", async () => { - const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); - const reaperPath = `${lockPath}.reaper`; - fs.mkdirSync(path.dirname(lockPath), { recursive: true }); - fs.writeFileSync( - reaperPath, - `${JSON.stringify({ - version: 1, - sandboxName: "alpha", - pid: 2_147_483_647, - processIdentity: "dead-reaper", - hostIdentity: currentHostIdentity, - pidNamespaceIdentity: currentPidNamespaceIdentity, - token: "observed-stale-token", - acquiredAt: "2026-01-01T00:00:00.000Z", - })}\n`, - ); - const replacement = { - version: 1, - sandboxName: "alpha", - pid: process.pid, - processIdentity: lifecycleLock.readMcpLockProcessIdentity(process.pid), - hostIdentity: currentHostIdentity, - pidNamespaceIdentity: currentPidNamespaceIdentity, - token: "replacement-reaper-token", - acquiredAt: new Date().toISOString(), - }; - const rename = fs.promises.rename.bind(fs.promises); - let injectedReplacement = false; - const renameSpy = vi.spyOn(fs.promises, "rename").mockImplementation(async (from, to) => { - const shouldInject = !injectedReplacement && String(from) === reaperPath; - switch (shouldInject) { - case true: - injectedReplacement = true; - fs.unlinkSync(reaperPath); - fs.writeFileSync(reaperPath, `${JSON.stringify(replacement)}\n`); - } - return rename(from, to); - }); - - try { - await expect( - lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 50 })), - ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); - } finally { - renameSpy.mockRestore(); - } - expect(JSON.parse(fs.readFileSync(reaperPath, "utf8")).token).toBe("replacement-reaper-token"); + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), + ).rejects.toThrow("Sandbox mutation containment is active"); + expect(fs.existsSync(reaperPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }); - it("does not delete a replacement main lock during stale recovery", async () => { - const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); - fs.mkdirSync(path.dirname(lockPath), { recursive: true }); - fs.writeFileSync( - lockPath, - `${JSON.stringify({ - version: 1, - sandboxName: "alpha", - pid: 2_147_483_647, - processIdentity: "dead-process", - hostIdentity: currentHostIdentity, - pidNamespaceIdentity: currentPidNamespaceIdentity, - token: "observed-stale-token", - acquiredAt: "2026-01-01T00:00:00.000Z", - })}\n`, + it("never overwrites an existing permanent-containment generation", () => { + const processToken = "a".repeat(32); + const containmentPath = `${lifecycleLock.getMcpLifecycleLockPath( + "alpha", + stateDir, + )}.containment`; + lifecycleLock.beginCommittedMcpLifecycleContainmentSync( + "alpha", + processToken, + "first containment", + stateDir, ); - const replacement = { - version: 1, - sandboxName: "alpha", - pid: process.pid, - processIdentity: lifecycleLock.readMcpLockProcessIdentity(process.pid), - hostIdentity: currentHostIdentity, - pidNamespaceIdentity: currentPidNamespaceIdentity, - token: "replacement-main-token", - acquiredAt: new Date().toISOString(), - }; - const rename = fs.promises.rename.bind(fs.promises); - let injectedReplacement = false; - const renameSpy = vi.spyOn(fs.promises, "rename").mockImplementation(async (from, to) => { - const shouldInject = !injectedReplacement && String(from) === lockPath; - switch (shouldInject) { - case true: - injectedReplacement = true; - fs.unlinkSync(lockPath); - fs.writeFileSync(lockPath, `${JSON.stringify(replacement)}\n`); - } - return rename(from, to); - }); + const firstGeneration = fs.readFileSync(containmentPath, "utf8"); - try { - await expect( - lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 50 })), - ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); - } finally { - renameSpy.mockRestore(); - } - expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-main-token"); + expect(() => + lifecycleLock.beginCommittedMcpLifecycleContainmentSync( + "alpha", + processToken, + "replacement containment", + stateDir, + ), + ).toThrow("already exists"); + expect(fs.readFileSync(containmentPath, "utf8")).toBe(firstGeneration); }); it.skipIf(currentProcessIdentity === null)( - "recovers a recycled PID by comparing process-start identity", + "permanently contains a recycled PID because prior descendants are unknown", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; fs.mkdirSync(path.dirname(lockPath), { recursive: true }); fs.writeFileSync( lockPath, @@ -598,9 +599,10 @@ const releasePath = process.argv[3]; ); await expect( - lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options()), - ).resolves.toBeUndefined(); - expect(fs.existsSync(lockPath)).toBe(false); + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), + ).rejects.toThrow("Sandbox mutation containment is active"); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); }, ); @@ -642,4 +644,314 @@ const releasePath = process.argv[3]; expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-token"); }); + + it("binds an ordinary lifecycle owner to the active Shields timer generation", async () => { + const processToken = "a".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + writeTimerMarker("alpha", processToken); + + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).shieldsTakeoverToken).toBe( + processToken, + ); + }, + options(), + ); + }); + + it("does not read a timer marker through a traversal-shaped lifecycle key", async () => { + const sandboxName = `a/../../escaped-${path.basename(stateDir)}`; + const escapedMarkerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const processToken = "a".repeat(32); + fs.writeFileSync( + escapedMarkerPath, + JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath: path.join(stateDir, "snapshot.yaml"), + restoreAt: new Date(Date.now() + 60_000).toISOString(), + processToken, + }), + ); + + try { + const lockPath = lifecycleLock.getMcpLifecycleLockPath(sandboxName, stateDir); + await lifecycleLock.withMcpLifecycleLock( + sandboxName, + () => { + expect( + JSON.parse(fs.readFileSync(lockPath, "utf8")).shieldsTakeoverToken, + ).toBeUndefined(); + }, + options(), + ); + } finally { + fs.rmSync(escapedMarkerPath, { force: true }); + } + }); + + it("retries without entering when the timer generation changes during lock publication", async () => { + const firstProcessToken = "a".repeat(32); + const replacementProcessToken = "b".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + writeTimerMarker("alpha", firstProcessToken); + const link = fs.promises.link.bind(fs.promises); + let replaced = false; + const linkSpy = vi.spyOn(fs.promises, "link").mockImplementation(async (from, to) => { + await link(from, to); + if (!replaced && String(to) === lockPath) { + replaced = true; + writeTimerMarker("alpha", replacementProcessToken); + } + }); + + try { + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).shieldsTakeoverToken).toBe( + replacementProcessToken, + ); + }, + options(), + ); + } finally { + linkSpy.mockRestore(); + } + expect(replaced).toBe(true); + }); + + it("keeps the deadline fence closed through restore and configuration relock", async () => { + const processToken = "c".repeat(32); + writeTimerMarker("alpha", processToken); + const entered = deferred(); + const release = deferred(); + let contenderEntered = false; + + const deadline = lifecycleLock.withMcpLifecycleDeadlineFence( + "alpha", + processToken, + async () => { + entered.resolve(); + await release.promise; + }, + options(), + ); + await entered.promise; + const contender = lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + contenderEntered = true; + }, + options({ timeoutMs: 2_000 }), + ); + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(contenderEntered).toBe(false); + + release.resolve(); + await Promise.all([deadline, contender]); + expect(contenderEntered).toBe(true); + }); + + it("keeps the deadline fence when an in-flight ordinary publication wins the main link", async () => { + const processToken = "0".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const deadlinePath = `${lockPath}.deadline`; + writeTimerMarker("alpha", processToken); + + const ordinaryLinkStarted = deferred(); + const allowOrdinaryPublication = deferred(); + const ordinaryPublished = deferred(); + const allowOrdinaryLinkReturn = deferred(); + const timerMainLinkStarted = deferred(); + const timerEntered = deferred(); + const releaseTimer = deferred(); + const link = fs.promises.link.bind(fs.promises); + let mainLinkCalls = 0; + let ordinaryEntered = false; + const linkSpy = vi.spyOn(fs.promises, "link").mockImplementation(async (from, to) => { + if (String(to) !== lockPath) { + await link(from, to); + return; + } + + mainLinkCalls += 1; + if (mainLinkCalls === 1) { + ordinaryLinkStarted.resolve(); + await allowOrdinaryPublication.promise; + await link(from, to); + ordinaryPublished.resolve(); + await allowOrdinaryLinkReturn.promise; + return; + } + if (mainLinkCalls === 2) { + timerMainLinkStarted.resolve(); + await ordinaryPublished.promise; + } + await link(from, to); + }); + + try { + const ordinary = lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + ordinaryEntered = true; + }, + options({ timeoutMs: 2_000 }), + ); + await ordinaryLinkStarted.promise; + + const deadline = lifecycleLock.withMcpLifecycleDeadlineFence( + "alpha", + processToken, + async () => { + timerEntered.resolve(); + await releaseTimer.promise; + }, + options({ timeoutMs: 2_000 }), + ); + await timerMainLinkStarted.promise; + expect(fs.existsSync(deadlinePath)).toBe(true); + + allowOrdinaryPublication.resolve(); + await ordinaryPublished.promise; + allowOrdinaryLinkReturn.resolve(); + await timerEntered.promise; + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(ordinaryEntered).toBe(false); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(ordinaryEntered).toBe(false); + + releaseTimer.resolve(); + await Promise.all([deadline, ordinary]); + expect(ordinaryEntered).toBe(true); + expect(mainLinkCalls).toBeGreaterThanOrEqual(3); + } finally { + allowOrdinaryPublication.resolve(); + allowOrdinaryLinkReturn.resolve(); + releaseTimer.resolve(); + linkSpy.mockRestore(); + } + }); + + it("waits for a live same-generation owner to release naturally", async () => { + const processToken = "d".repeat(32); + writeTimerMarker("alpha", processToken); + const releasePath = path.join(stateDir, "release-owner"); + const script = String.raw` +const fs = require("node:fs"); +const lock = require(process.argv[1]); +const stateDir = process.argv[2]; +const releasePath = process.argv[3]; +(async () => { + await lock.withMcpLifecycleLock("alpha", async () => { + process.stdout.write("READY\n"); + while (!fs.existsSync(releasePath)) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + }, { stateDir, pollIntervalMs: 5, timeoutMs: 2000 }); +})().then(() => process.exit(0), () => process.exit(1)); +`; + const child = spawn(process.execPath, ["-e", script, lockModulePath, stateDir, releasePath], { + stdio: ["ignore", "pipe", "pipe"], + }); + children.add(child); + await waitForLine(child, "READY"); + const childExit = new Promise((resolve) => child.once("exit", () => resolve())); + const containmentReported = deferred(); + let entered = false; + const deadline = lifecycleLock.withMcpLifecycleDeadlineFence( + "alpha", + processToken, + () => { + entered = true; + }, + { + ...options({ timeoutMs: 10 }), + onContainment: ({ ownerPid }) => { + expect(ownerPid).toBe(child.pid); + containmentReported.resolve(); + }, + }, + ); + await containmentReported.promise; + expect(entered).toBe(false); + expect(() => process.kill(child.pid!, 0)).not.toThrow(); + + fs.writeFileSync(releasePath, "release\n"); + await Promise.all([deadline, childExit]); + expect(entered).toBe(true); + children.delete(child); + }); + + it("preserves an active owner from a different timer generation", async () => { + const processToken = "e".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + writeTimerMarker("alpha", processToken); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: currentProcessIdentity, + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + shieldsTakeoverToken: "f".repeat(32), + token: "replacement-generation", + acquiredAt: new Date().toISOString(), + })}\n`, + ); + const deadlinePath = `${lockPath}.deadline`; + const onContainment = vi.fn(() => { + expect(fs.existsSync(deadlinePath)).toBe(true); + }); + setTimeout(() => writeTimerMarker("alpha", "3".repeat(32)), 40); + + await expect( + lifecycleLock.withMcpLifecycleDeadlineFence("alpha", processToken, () => undefined, { + ...options({ timeoutMs: 10 }), + onContainment, + }), + ).rejects.toThrow("Auto-restore authority changed"); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-generation"); + }); + + it("contains an already-dead local owner because surviving descendants cannot be ruled out", async () => { + const processToken = "4".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; + writeTimerMarker("alpha", processToken); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-owner", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + shieldsTakeoverToken: "5".repeat(32), + token: "dead-foreign-generation", + acquiredAt: new Date().toISOString(), + })}\n`, + ); + const onContainment = vi.fn(); + setTimeout(() => writeTimerMarker("alpha", "a".repeat(32)), 40); + + await expect( + lifecycleLock.withMcpLifecycleDeadlineFence("alpha", processToken, () => undefined, { + ...options(), + onContainment, + }), + ).rejects.toThrow("Auto-restore authority changed"); + expect(onContainment).toHaveBeenCalled(); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(containmentPath)).toBe(true); + }); }); From 8c7c0a299883f9fdc06f528efd806f809597aa02 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 00:35:33 -0400 Subject: [PATCH 03/25] fix(snapshot): honor Shields deadline gate --- ...snapshot-baseline-exclusion-output.test.ts | 11 +++++++++- src/lib/actions/sandbox/snapshot-help.test.ts | 22 +++++++++++++++++++ .../snapshot-restore-lifecycle.test.ts | 22 +++++++++++++++++++ src/lib/actions/sandbox/snapshot.test.ts | 19 +++++----------- src/lib/actions/sandbox/snapshot.ts | 14 ++++++------ 5 files changed, 67 insertions(+), 21 deletions(-) create mode 100644 src/lib/actions/sandbox/snapshot-help.test.ts diff --git a/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts b/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts index c3017fcb8bd..97d8171de01 100644 --- a/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts +++ b/src/lib/actions/sandbox/snapshot-baseline-exclusion-output.test.ts @@ -3,11 +3,16 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { runSandboxSnapshot } from "./snapshot"; + const mocks = vi.hoisted(() => ({ backupSandboxState: vi.fn(), captureOpenshell: vi.fn(() => ({ status: 0, output: "alpha Ready\n" })), findBackup: vi.fn(), getBaselineExclusions: vi.fn(), + withSandboxMutationLock: vi.fn( + async (_sandboxName: string, operation: () => unknown) => await operation(), + ), })); vi.mock("../../adapters/openshell/runtime", () => ({ @@ -30,6 +35,11 @@ vi.mock("../../shields/timer-bound-lock", () => ({ ), })); +vi.mock("../../state/mcp-lifecycle-lock", () => ({ + withMcpLifecycleLock: mocks.withSandboxMutationLock, + withSandboxMutationLock: mocks.withSandboxMutationLock, +})); + vi.mock("../../state/registry", () => ({ getBaselineExclusions: mocks.getBaselineExclusions, getSandbox: vi.fn(() => ({ name: "alpha", agent: "hermes" })), @@ -71,7 +81,6 @@ describe("snapshot baseline exclusion output", () => { it("reports active exclusions and support impact after a successful snapshot (#7178)", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); - const { runSandboxSnapshot } = await import("./snapshot"); await runSandboxSnapshot("alpha", { kind: "create" }); diff --git a/src/lib/actions/sandbox/snapshot-help.test.ts b/src/lib/actions/sandbox/snapshot-help.test.ts new file mode 100644 index 00000000000..46b9cf1f11f --- /dev/null +++ b/src/lib/actions/sandbox/snapshot-help.test.ts @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, expect, it, vi } from "vitest"; + +import { runSandboxSnapshot } from "./snapshot"; + +afterEach(() => { + vi.restoreAllMocks(); +}); + +it("prints create, list, and restore usage for the bare help branch", async () => { + const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); + + await runSandboxSnapshot("alpha", { kind: "help" }); + + const output = consoleLog.mock.calls.flat().join("\n"); + expect(output).toContain("Usage:"); + expect(output).toContain("alpha snapshot create"); + expect(output).toContain("alpha snapshot list"); + expect(output).toContain("alpha snapshot restore"); +}); diff --git a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts index d1fbc6da9b1..13a20f9723d 100644 --- a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts +++ b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts @@ -11,6 +11,9 @@ import * as f from "./snapshot-restore-test-fixture"; const tempHomes: string[] = []; beforeEach(() => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-snapshot-lifecycle-")); + tempHomes.push(tempHome); + vi.stubEnv("HOME", tempHome); f.resetSnapshotRestoreMocks(); }); afterEach(() => { @@ -20,6 +23,25 @@ afterEach(() => { } }); describe("runSandboxSnapshot restore: lifecycle and destination safety", () => { + it("holds snapshot creation under the lifecycle gate before the timer-bound transition", () => { + const source = fs.readFileSync(new URL("./snapshot.ts", import.meta.url), "utf8"); + const createCase = source.indexOf('case "create":'); + const listCase = source.indexOf('case "list":', createCase); + const createDispatch = source.slice(createCase, listCase); + const createFunction = source.indexOf("function runSnapshotCreate"); + const timerBoundLock = source.indexOf( + "withTimerBoundShieldsMutationLock(sandboxName,", + createFunction, + ); + + expect(createCase).toBeGreaterThanOrEqual(0); + expect(listCase).toBeGreaterThan(createCase); + expect(createDispatch).toContain( + "await withSandboxMutationLock(sandboxName, () => runSnapshotCreate(sandboxName, request));", + ); + expect(timerBoundLock).toBeGreaterThan(createFunction); + }); + it("restores the latest snapshot into the source sandbox", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); f.getLatestBackupMock.mockReturnValue({ diff --git a/src/lib/actions/sandbox/snapshot.test.ts b/src/lib/actions/sandbox/snapshot.test.ts index 97b37d0fae6..804ef15964d 100644 --- a/src/lib/actions/sandbox/snapshot.test.ts +++ b/src/lib/actions/sandbox/snapshot.test.ts @@ -83,6 +83,7 @@ const lifecycleMock = vi.hoisted(() => { events, cleanupShieldsDestroyArtifactsMock: vi.fn(() => events.push("cleanup-shields")), readTimerMarkerMock: vi.fn(() => null as Record | null), + withSandboxMutationLockMock: vi.fn(async (_name: string, fn: () => unknown) => await fn()), withTimerBoundMock: vi.fn( (_sandboxName: string, command: string, fn: () => unknown): unknown => { events.push(`lock:${command}`); @@ -215,6 +216,11 @@ vi.mock("../../state/gateway", () => ({ ), })); +vi.mock("../../state/mcp-lifecycle-lock", () => ({ + withMcpLifecycleLock: lifecycleMock.withSandboxMutationLockMock, + withSandboxMutationLock: lifecycleMock.withSandboxMutationLockMock, +})); + vi.mock("../../state/registry", () => ({ getBaselineExclusions: vi.fn(() => []), getCustomPolicies: getCustomPoliciesMock, @@ -778,19 +784,6 @@ describe("runSandboxSnapshot", () => { expect(output).toContain("2 snapshot(s). Restore with:"); }); - it("prints create, list, and restore usage for the bare help branch", async () => { - const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); - const { runSandboxSnapshot } = await import("./snapshot"); - - await runSandboxSnapshot("alpha", { kind: "help" }); - - const output = consoleLog.mock.calls.flat().join("\n"); - expect(output).toContain("Usage:"); - expect(output).toContain("alpha snapshot create"); - expect(output).toContain("alpha snapshot list"); - expect(output).toContain("alpha snapshot restore"); - }); - it("restores the latest snapshot into the source sandbox", async () => { const consoleLog = vi.spyOn(console, "log").mockImplementation(() => {}); getLatestBackupMock.mockReturnValue({ diff --git a/src/lib/actions/sandbox/snapshot.ts b/src/lib/actions/sandbox/snapshot.ts index 9a80ebd7cc7..5756ee57233 100644 --- a/src/lib/actions/sandbox/snapshot.ts +++ b/src/lib/actions/sandbox/snapshot.ts @@ -598,9 +598,9 @@ function runSnapshotCreate( snapshotExit(1); } return withTimerBoundShieldsMutationLock(sandboxName, "create sandbox snapshot", () => { - // Keep the shields check and backup in one timer-bound interval. Normal - // auto-restore waits; at the absolute deadline it may preempt this process - // and reclaim the token rather than changing policy/config mid-copy. + // Keep the shields check and backup in one timer-bound interval. At the + // absolute deadline, auto-restore closes the outer lifecycle gate and waits + // for this exact owner to finish before changing policy or config. if (!isSnapshotCreationAllowedByShields(sandboxName)) { console.error(" Cannot create snapshot while shields are up."); console.error(` Run \`${CLI_NAME} ${sandboxName} shields down\` first, then retry.`); @@ -1112,9 +1112,9 @@ async function runSnapshotRestoreUnlocked( } withTimerBoundShieldsMutationLock(targetSandbox, "restore sandbox snapshot", () => { // Serialize filesystem restore, mutable-permission repair, and policy - // reconciliation under the active timer generation. Normal auto-restore - // waits; the absolute deadline may preempt this process and reclaim the - // token, preventing policy/config mutation after lockdown resumes. + // reconciliation under the active timer generation. At the absolute + // deadline, auto-restore keeps the outer lifecycle gate closed and waits + // for this exact owner to finish before restoring lockdown. if (targetSandbox !== sandboxName) { console.log(` Restoring snapshot from '${sandboxName}' into '${targetSandbox}'...`); } else { @@ -1181,7 +1181,7 @@ export async function runSandboxSnapshot( ) { switch (request.kind) { case "create": { - runSnapshotCreate(sandboxName, request); + await withSandboxMutationLock(sandboxName, () => runSnapshotCreate(sandboxName, request)); break; } case "list": { From dd0a63e88945d8b594b21e9ebd9cb71fa9c51b56 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 02:21:44 -0400 Subject: [PATCH 04/25] test(state): isolate non-live fixture state --- ci/env-var-doc-allowlist.json | 8 +++++++ ci/source-shape-test-budget.json | 5 +++++ src/lib/state/paths.test.ts | 26 +++++++++++++++++++++-- src/lib/state/paths.ts | 13 +++++++++--- test/helpers/isolate-test-state.ts | 33 +++++++++++++++++++++++++++++ test/vitest-temp-root.test.ts | 34 ++++++++++++++++++++++++++++++ vitest.config.ts | 23 +++++++++++++++----- 7 files changed, 132 insertions(+), 10 deletions(-) create mode 100644 test/helpers/isolate-test-state.ts diff --git a/ci/env-var-doc-allowlist.json b/ci/env-var-doc-allowlist.json index aed046f4cbb..26b886da7ce 100644 --- a/ci/env-var-doc-allowlist.json +++ b/ci/env-var-doc-allowlist.json @@ -59,6 +59,14 @@ "name": "NEMOCLAW_TEST_NO_SLEEP", "reason": "Test sentinel that bypasses real-time sleep() calls in onboard inference probes. Set to '1' only by Vitest tests; never user-set." }, + { + "name": "NEMOCLAW_TEST_BASE_HOME", + "reason": "Internal Vitest-only baseline used with NEMOCLAW_TEST_STATE_DIR so tests that explicitly replace HOME retain their fixture paths. Never user-set in production." + }, + { + "name": "NEMOCLAW_TEST_STATE_DIR", + "reason": "Internal Vitest-only state root that keeps lifecycle locks and Shields artifacts out of the caller's real NemoClaw state. The production resolver honors it only while Vitest is active." + }, { "name": "NEMOCLAW_TELEGRAM_STARTUP_GRACE_MS", "reason": "Internal Vitest-only override that shortens the Telegram diagnostics startup-grace timer. Production uses the built-in default." diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 3dd45f3afe5..32ee72f504b 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -681,6 +681,11 @@ "test": "wires cleanup into root and standalone plugin test runs", "category": "compatibility" }, + { + "file": "test/vitest-temp-root.test.ts", + "test": "isolates stateful non-live projects without redirecting live E2E state", + "category": "security" + }, { "file": "test/vitest-watch-triggers.test.ts", "test": "registers the focused mappings at the root configuration boundary (#6692)", diff --git a/src/lib/state/paths.test.ts b/src/lib/state/paths.test.ts index a43b016c656..2124a9e9a99 100644 --- a/src/lib/state/paths.test.ts +++ b/src/lib/state/paths.test.ts @@ -3,9 +3,9 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; -import { ROOT, SCRIPTS } from "./paths"; +import { ROOT, resolveNemoclawHomeDir, resolveNemoclawStateDir, SCRIPTS } from "./paths"; describe("paths", () => { it("resolves the repo root", () => { @@ -17,4 +17,26 @@ describe("paths", () => { expect(SCRIPTS).toBe(join(ROOT, "scripts")); expect(existsSync(join(SCRIPTS, "debug.sh"))).toBe(true); }); + + it("isolates default state during Vitest without changing explicit home resolution", () => { + const isolatedState = process.env.NEMOCLAW_TEST_STATE_DIR; + + expect(isolatedState).toBeDefined(); + expect(resolveNemoclawStateDir()).toBe(isolatedState); + expect(resolveNemoclawStateDir("/explicit-home")).toBe( + join("/explicit-home", ".nemoclaw", "state"), + ); + }); + + it("honors a test fixture that explicitly changes HOME", () => { + vi.stubEnv("HOME", "/fixture-home"); + + expect(resolveNemoclawStateDir()).toBe(join("/fixture-home", ".nemoclaw", "state")); + }); + + it("does not honor the internal state override outside Vitest", () => { + vi.stubEnv("VITEST", "false"); + + expect(resolveNemoclawStateDir()).toBe(join(resolveNemoclawHomeDir(), "state")); + }); }); diff --git a/src/lib/state/paths.ts b/src/lib/state/paths.ts index ffd5d9d2856..72b9e0b2e7e 100644 --- a/src/lib/state/paths.ts +++ b/src/lib/state/paths.ts @@ -14,8 +14,15 @@ export function resolveNemoclawHomeDir(homeDir: string = process.env.HOME ?? os. return nemoclawStateRoot(homeDir, GATEWAY_PORT); } -export function resolveNemoclawStateDir( - homeDir: string = process.env.HOME ?? os.homedir(), -): string { +export function resolveNemoclawStateDir(homeDir?: string): string { + if ( + homeDir === undefined && + process.env.VITEST === "true" && + (process.env.HOME ?? "") === process.env.NEMOCLAW_TEST_BASE_HOME && + process.env.NEMOCLAW_TEST_STATE_DIR && + path.isAbsolute(process.env.NEMOCLAW_TEST_STATE_DIR) + ) { + return process.env.NEMOCLAW_TEST_STATE_DIR; + } return path.join(resolveNemoclawHomeDir(homeDir), "state"); } diff --git a/test/helpers/isolate-test-state.ts b/test/helpers/isolate-test-state.ts new file mode 100644 index 00000000000..b0e80b287ef --- /dev/null +++ b/test/helpers/isolate-test-state.ts @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; + +import { afterAll } from "vitest"; + +const tempRoot = process.env.TMPDIR; +if (!tempRoot || !path.isAbsolute(tempRoot)) { + throw new Error("Vitest state isolation requires the shared absolute temporary root"); +} + +const previousStateDir = process.env.NEMOCLAW_TEST_STATE_DIR; +const previousBaseHome = process.env.NEMOCLAW_TEST_BASE_HOME; +process.env.NEMOCLAW_TEST_BASE_HOME = process.env.HOME ?? ""; +process.env.NEMOCLAW_TEST_STATE_DIR = fs.mkdtempSync(path.join(tempRoot, `state-${process.pid}-`), { + encoding: "utf8", +}); +fs.chmodSync(process.env.NEMOCLAW_TEST_STATE_DIR, 0o700); + +afterAll(() => { + if (previousBaseHome === undefined) { + delete process.env.NEMOCLAW_TEST_BASE_HOME; + } else { + process.env.NEMOCLAW_TEST_BASE_HOME = previousBaseHome; + } + if (previousStateDir === undefined) { + delete process.env.NEMOCLAW_TEST_STATE_DIR; + } else { + process.env.NEMOCLAW_TEST_STATE_DIR = previousStateDir; + } +}); diff --git a/test/vitest-temp-root.test.ts b/test/vitest-temp-root.test.ts index bfeafce5e40..eb781c21a8c 100644 --- a/test/vitest-temp-root.test.ts +++ b/test/vitest-temp-root.test.ts @@ -13,6 +13,7 @@ import { setupVitestTempRoot } from "./helpers/vitest-temp-root"; const TEMP_ENV_KEYS = ["TMPDIR", "TMP", "TEMP"] as const; const ROOT_SETUP = "test/helpers/vitest-temp-root.ts"; +const STATE_SETUP = "test/helpers/isolate-test-state.ts"; type TempEnv = Record<(typeof TEMP_ENV_KEYS)[number], string | undefined>; @@ -51,6 +52,17 @@ describe("Vitest temp root", () => { expect(fs.statSync(root).isDirectory()).toBe(true); }); + it("isolates each test file's NemoClaw state inside the run root", () => { + const root = process.env.TMPDIR as string; + const stateDir = process.env.NEMOCLAW_TEST_STATE_DIR as string; + const relativeStateDir = path.relative(root, stateDir); + + expect(relativeStateDir).toMatch(/^state-\d+-/); + expect(relativeStateDir.startsWith(`..${path.sep}`)).toBe(false); + expect(path.isAbsolute(stateDir)).toBe(true); + expect(fs.statSync(stateDir).mode & 0o777).toBe(0o700); + }); + it("removes run artifacts and restores the caller temp environment", () => { const outerEnv = readTempEnv(); const previousKeep = process.env.NEMOCLAW_TEST_KEEP_TEMP; @@ -209,4 +221,26 @@ describe("Vitest temp root", () => { path.resolve(import.meta.dirname, "..", ROOT_SETUP), ); }); + + // source-shape-contract: security -- Non-live state isolation must never redirect credential-bearing live E2E state + it("isolates stateful non-live projects without redirecting live E2E state", () => { + const projects = (rootVitestConfig.test?.projects ?? []) as Array<{ + test?: { name?: string; setupFiles?: string[] }; + }>; + const setupFilesByProject = new Map( + projects.map((project) => [project.test?.name, project.test?.setupFiles ?? []]), + ); + + for (const name of [ + "cli", + "integration", + "installer-integration", + "package-contract", + "e2e-support", + ]) { + expect(setupFilesByProject.get(name), name).toContain(STATE_SETUP); + } + expect(setupFilesByProject.get("plugin")).not.toContain(STATE_SETUP); + expect(setupFilesByProject.get("e2e-live")).not.toContain(STATE_SETUP); + }); }); diff --git a/vitest.config.ts b/vitest.config.ts index 151ed56ac6c..06352b7153d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -67,6 +67,7 @@ const controlledNonLiveEnv = { // intentionally excluded below and keep their own stricter umask handling. See // test/helpers/normalize-fixture-umask.ts (#6448). const fixtureUmaskSetup = "test/helpers/normalize-fixture-umask.ts"; +const isolatedTestStateSetup = "test/helpers/isolate-test-state.ts"; const pluginVitestProject = defineProject(pluginVitestProjectOptions); const integrationProjectScheduling = resolveIntegrationProjectScheduling({ isCi, @@ -100,7 +101,11 @@ export default defineConfig({ alias: canonicalSourceAliases, env: controlledNonLiveEnv, testTimeout: testTimeout(), - setupFiles: [fixtureUmaskSetup, "test/helpers/onboard-script-mocks.cjs"], + setupFiles: [ + fixtureUmaskSetup, + isolatedTestStateSetup, + "test/helpers/onboard-script-mocks.cjs", + ], include: ["src/**/*.test.ts"], exclude: ["**/node_modules/**", "**/.claude/**"], }, @@ -114,7 +119,11 @@ export default defineConfig({ // Source-backed process fixtures can exceed the unit-test budget // when several coverage shards transpile and spawn them concurrently. testTimeout: testTimeout(15_000), - setupFiles: [fixtureUmaskSetup, "test/helpers/onboard-script-mocks.cjs"], + setupFiles: [ + fixtureUmaskSetup, + isolatedTestStateSetup, + "test/helpers/onboard-script-mocks.cjs", + ], // Integration fixtures often spawn short Node programs. Coverage // stays serial because concurrent source-loader forks exhaust the // 7 GiB CI runner. The canonical local full suite instead runs this @@ -164,7 +173,7 @@ export default defineConfig({ name: "installer-integration", alias: canonicalSourceAliases, env: controlledNonLiveEnv, - setupFiles: [fixtureUmaskSetup], + setupFiles: [fixtureUmaskSetup, isolatedTestStateSetup], include: [ "test/install-express-prompt.test.ts", "test/install-express-wsl-ollama.test.ts", @@ -195,7 +204,7 @@ export default defineConfig({ name: "package-contract", alias: canonicalSourceAliases, env: controlledNonLiveEnv, - setupFiles: [fixtureUmaskSetup], + setupFiles: [fixtureUmaskSetup, isolatedTestStateSetup], include: ["test/package-contract/**/*.test.ts"], }, }, @@ -210,7 +219,11 @@ export default defineConfig({ alias: canonicalSourceAliases, env: controlledNonLiveEnv, testTimeout: testTimeout(), - setupFiles: [fixtureUmaskSetup, "test/helpers/onboard-script-mocks.cjs"], + setupFiles: [ + fixtureUmaskSetup, + isolatedTestStateSetup, + "test/helpers/onboard-script-mocks.cjs", + ], include: ["test/e2e/support/**/*.test.ts"], }, }, From 7581fe06ad46e6bb0eb004abe62884492ad443be Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 02:42:00 -0400 Subject: [PATCH 05/25] test(shields): linearize regression fixtures Signed-off-by: Julie Yaunches --- src/lib/actions/maintenance.test.ts | 38 +++++++++---------- src/lib/shields/flow.test.ts | 9 +++-- src/lib/shields/index.test.ts | 45 +++++++++++++++------- src/lib/shields/timer.test.ts | 51 ++++++++++++------------- test/mcp-lifecycle-lock.test.ts | 59 +++++++++++++++-------------- 5 files changed, 111 insertions(+), 91 deletions(-) diff --git a/src/lib/actions/maintenance.test.ts b/src/lib/actions/maintenance.test.ts index f9cc0b05b2d..9b354fe31d4 100644 --- a/src/lib/actions/maintenance.test.ts +++ b/src/lib/actions/maintenance.test.ts @@ -18,12 +18,17 @@ const mocks = vi.hoisted(() => ({ isSandboxContainerDefinitivelyAbsent: vi.fn(), openBackupShieldsWindow: vi.fn(), relockBackupShieldsWindow: vi.fn(), - withSandboxMutationLock: vi.fn( - async (_sandboxName: string, action: () => unknown, _options?: { timeoutMs?: number }) => - action(), - ), + withSandboxMutationLock: vi.fn(), })); +async function runSandboxMutationAction( + _sandboxName: string, + action: () => unknown, + _options?: { timeoutMs?: number }, +): Promise { + return action(); +} + vi.mock("../state/registry", () => ({ isRouteOnlySandboxReservation: (entry: { pendingRouteReservation?: true; createdAt?: string }) => entry.pendingRouteReservation === true && entry.createdAt === undefined, @@ -89,10 +94,7 @@ describe("backupAll", () => { beforeEach(() => { vi.clearAllMocks(); mocks.backupStartedSandboxState.mockReset(); - mocks.withSandboxMutationLock.mockImplementation( - async (_sandboxName: string, action: () => unknown, _options?: { timeoutMs?: number }) => - action(), - ); + mocks.withSandboxMutationLock.mockImplementation(runSandboxMutationAction); delete process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS; mocks.captureSandboxListWithGatewayPreflightOrExit.mockResolvedValue({ status: 0, @@ -467,21 +469,19 @@ describe("backupAll", () => { throw backupError; }); const relockLockError = new Error("mutation lock timed out"); - let lockAttempt = 0; - mocks.withSandboxMutationLock.mockImplementation( - async (_sandboxName: string, action: () => unknown, options?: { timeoutMs?: number }) => { - lockAttempt += 1; - if (lockAttempt === 2) { - expect(options).toEqual({ timeoutMs: 30_000 }); - throw relockLockError; - } - return action(); - }, - ); + mocks.withSandboxMutationLock + .mockImplementationOnce(runSandboxMutationAction) + .mockRejectedValueOnce(relockLockError); vi.spyOn(console, "log").mockImplementation(() => undefined); const failure = await backupAll().catch((error: unknown) => error); + expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith( + 2, + "alpha", + expect.any(Function), + { timeoutMs: 30_000 }, + ); expect(failure).toBeInstanceOf(AggregateError); expect((failure as AggregateError).message).toContain( "Backup for 'alpha' failed and Shields lockdown could not be restored", diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 068c237ee70..2d23143e7f0 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -102,6 +102,10 @@ function throwHarnessError(error: Error): never { throw error; } +function recordPolicySetBody(policySetBodies: string[], file: unknown): void { + policySetBodies.push(fs.readFileSync(String(file), "utf-8")); +} + function createHarness(options: HarnessOptions = {}): ShieldsHarness { vi.stubEnv("NEMOCLAW_INVOKED_AS", options.invokedAs ?? "nemoclaw"); delete require.cache[requireDist.resolve(shieldsModulePath)]; @@ -136,10 +140,7 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { options.fork && vi.spyOn(childProcess, "fork").mockImplementation(options.fork); vi.spyOn(policy, "buildPolicyGetCommand").mockReturnValue(["openshell", "policy", "get"]); vi.spyOn(policy, "buildPolicySetCommand").mockImplementation((file: unknown) => { - const policyFile = String(file); - if (fs.existsSync(policyFile)) { - policySetBodies.push(fs.readFileSync(policyFile, "utf-8")); - } + recordPolicySetBody(policySetBodies, file); return ["openshell", "policy", "set"]; }); vi.spyOn(policy, "parseCurrentPolicy").mockImplementation((raw: unknown) => String(raw)); diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index 4670556b9f1..cc84b0da75e 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -103,6 +103,35 @@ function withDefaultNodeExecFileSync( return defaultNodeExecFileSync(file, argv) || fallback(); } +function throwRegistryPermissionDenied(): never { + throw Object.assign(new Error("registry permission denied"), { code: "EACCES" }); +} + +function readFileWithUnreadableRegistry( + originalReadFileSync: typeof fs.readFileSync, + file: fs.PathOrFileDescriptor, + options?: unknown, +): unknown { + const readers = new Map unknown>([ + [true, throwRegistryPermissionDenied], + [false, () => originalReadFileSync(file, options as never)], + ]); + return readers.get(String(file).endsWith(`${path.sep}sandboxes.json`))!(); +} + +function throwProcessNotRunning(): never { + throw Object.assign(new Error("not running"), { code: "ESRCH" }); +} + +function reportProcessRunning(): true { + return true; +} + +function routeProcessKill(pid: number, signal?: string | number): true { + const processActions = new Map true>([["2147483647:0", throwProcessNotRunning]]); + return (processActions.get(`${pid}:${signal}`) ?? reportProcessRunning)(); +} + beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-test-")); vi.stubEnv("HOME", tmpDir); @@ -515,20 +544,10 @@ describe("shields — unit logic", () => { processToken, }); const originalReadFileSync = fs.readFileSync.bind(fs); - vi.spyOn(fs, "readFileSync").mockImplementation( - (file: fs.PathOrFileDescriptor, options?: unknown) => { - if (String(file).endsWith(`${path.sep}sandboxes.json`)) { - throw Object.assign(new Error("registry permission denied"), { code: "EACCES" }); - } - return originalReadFileSync(file, options as never) as never; - }, - ); - vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { - if (pid === 2_147_483_647 && signal === 0) { - throw Object.assign(new Error("not running"), { code: "ESRCH" }); - } - return true; + vi.spyOn(fs, "readFileSync").mockImplementation((file, options) => { + return readFileWithUnreadableRegistry(originalReadFileSync, file, options) as never; }); + vi.spyOn(process, "kill").mockImplementation(routeProcessKill); const { applyShieldsPolicySnapshot } = await loadShieldsModule(); const result = applyShieldsPolicySnapshot(sandboxName, snapshotPath, { diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index 1eb4741ff46..b6693933f96 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -72,6 +72,16 @@ describe("shields timer authorization", () => { } } + async function waitForRetryBoundary(deadlinePath: string, auditPath: string): Promise { + await vi.waitFor( + () => { + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(fs.existsSync(auditPath)).toBe(true); + }, + { interval: 1, timeout: 200 }, + ); + } + async function invokeTimerAndExpectRetry( runRestoreTimer: (args: any, options?: { retryDelayMs?: number }) => Promise, args: unknown, @@ -79,38 +89,25 @@ describe("shields timer authorization", () => { const exitSpy = vi .spyOn(process, "exit") .mockImplementation((() => undefined) as typeof process.exit); - const markerPath = (args as { markerPath?: string }).markerPath; - const markerContents = - markerPath && fs.existsSync(markerPath) ? fs.readFileSync(markerPath) : null; + const { markerPath, sandboxName } = args as { + markerPath: string; + sandboxName: string; + }; + const markerContents = fs.readFileSync(markerPath); + const deadlinePath = `${getMcpLifecycleLockPath( + sandboxName, + path.join(tmpHome, ".nemoclaw", "state"), + )}.deadline`; + const auditPath = path.join(path.dirname(markerPath), "shields-audit.jsonl"); try { const pending = runRestoreTimer(args, { retryDelayMs: 50 }); - const sandboxName = (args as { sandboxName?: string }).sandboxName; - const deadlinePath = sandboxName - ? `${getMcpLifecycleLockPath( - sandboxName, - path.join(tmpHome, ".nemoclaw", "state"), - )}.deadline` - : null; - const auditPath = markerPath - ? path.join(path.dirname(markerPath), "shields-audit.jsonl") - : null; - for (let attempt = 0; attempt < 200; attempt += 1) { - if ( - (!deadlinePath || fs.existsSync(deadlinePath)) && - (!auditPath || fs.existsSync(auditPath)) - ) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 1)); - } + await waitForRetryBoundary(deadlinePath, auditPath); expect(exitSpy).not.toHaveBeenCalled(); - if (deadlinePath) expect(fs.existsSync(deadlinePath)).toBe(true); - if (markerPath) fs.rmSync(markerPath, { force: true }); + expect(fs.existsSync(deadlinePath)).toBe(true); + fs.rmSync(markerPath, { force: true }); await pending; } finally { - if (markerPath && markerContents) { - fs.writeFileSync(markerPath, markerContents); - } + fs.writeFileSync(markerPath, markerContents); exitSpy.mockRestore(); } } diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts index 00f93a21d02..c11a359f4e7 100644 --- a/test/mcp-lifecycle-lock.test.ts +++ b/test/mcp-lifecycle-lock.test.ts @@ -55,6 +55,15 @@ function writeTimerMarker(sandboxName: string, processToken: string): void { ); } +function routeLinkToPath( + targetPath: string, + targetLink: typeof fs.promises.link, + fallback: typeof fs.promises.link, +): typeof fs.promises.link { + const routes = new Map([[targetPath, targetLink]]); + return (from, to) => (routes.get(String(to)) ?? fallback)(from, to); +} + function waitForLine(child: ChildProcess, expected: string): Promise { return new Promise((resolve, reject) => { let output = ""; @@ -698,14 +707,14 @@ const releasePath = process.argv[3]; const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); writeTimerMarker("alpha", firstProcessToken); const link = fs.promises.link.bind(fs.promises); - let replaced = false; - const linkSpy = vi.spyOn(fs.promises, "link").mockImplementation(async (from, to) => { + const lockPathLink = vi.fn(link); + lockPathLink.mockImplementationOnce(async (from, to) => { await link(from, to); - if (!replaced && String(to) === lockPath) { - replaced = true; - writeTimerMarker("alpha", replacementProcessToken); - } + writeTimerMarker("alpha", replacementProcessToken); }); + const linkSpy = vi + .spyOn(fs.promises, "link") + .mockImplementation(routeLinkToPath(lockPath, lockPathLink, link)); try { await lifecycleLock.withMcpLifecycleLock( @@ -720,7 +729,7 @@ const releasePath = process.argv[3]; } finally { linkSpy.mockRestore(); } - expect(replaced).toBe(true); + expect(lockPathLink).toHaveBeenCalled(); }); it("keeps the deadline fence closed through restore and configuration relock", async () => { @@ -769,29 +778,23 @@ const releasePath = process.argv[3]; const timerEntered = deferred(); const releaseTimer = deferred(); const link = fs.promises.link.bind(fs.promises); - let mainLinkCalls = 0; let ordinaryEntered = false; - const linkSpy = vi.spyOn(fs.promises, "link").mockImplementation(async (from, to) => { - if (String(to) !== lockPath) { - await link(from, to); - return; - } - - mainLinkCalls += 1; - if (mainLinkCalls === 1) { - ordinaryLinkStarted.resolve(); - await allowOrdinaryPublication.promise; - await link(from, to); - ordinaryPublished.resolve(); - await allowOrdinaryLinkReturn.promise; - return; - } - if (mainLinkCalls === 2) { - timerMainLinkStarted.resolve(); - await ordinaryPublished.promise; - } + const lockPathLink = vi.fn(link); + lockPathLink.mockImplementationOnce(async (from, to) => { + ordinaryLinkStarted.resolve(); + await allowOrdinaryPublication.promise; + await link(from, to); + ordinaryPublished.resolve(); + await allowOrdinaryLinkReturn.promise; + }); + lockPathLink.mockImplementationOnce(async (from, to) => { + timerMainLinkStarted.resolve(); + await ordinaryPublished.promise; await link(from, to); }); + const linkSpy = vi + .spyOn(fs.promises, "link") + .mockImplementation(routeLinkToPath(lockPath, lockPathLink, link)); try { const ordinary = lifecycleLock.withMcpLifecycleLock( @@ -828,7 +831,7 @@ const releasePath = process.argv[3]; releaseTimer.resolve(); await Promise.all([deadline, ordinary]); expect(ordinaryEntered).toBe(true); - expect(mainLinkCalls).toBeGreaterThanOrEqual(3); + expect(lockPathLink.mock.calls.length).toBeGreaterThanOrEqual(3); } finally { allowOrdinaryPublication.resolve(); allowOrdinaryLinkReturn.resolve(); From b8206781fb477509541cca24c1e2f42220d5b4eb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 03:10:36 -0400 Subject: [PATCH 06/25] fix(shields): harden recovery authority Signed-off-by: Julie Yaunches --- docs/manage-sandboxes/runtime-controls.mdx | 17 +- docs/reference/commands.mdx | 14 +- .../snapshot-restore-lifecycle.test.ts | 49 +++-- .../sandbox/snapshot-restore-test-fixture.ts | 1 + src/lib/shields/flow.test.ts | 135 +++++++++++++- src/lib/shields/index.test.ts | 34 ++++ src/lib/shields/index.ts | 167 ++++++++++++++---- src/lib/shields/timer.test.ts | 46 +++++ src/lib/shields/timer.ts | 21 ++- .../state/mcp-lifecycle-lock-acquisition.ts | 18 +- .../shields-timer-authority.ts | 23 ++- test/e2e/support/mcp-bridge-sandbox.test.ts | 104 +++-------- test/mcp-lifecycle-lock.test.ts | 78 +++++++- 13 files changed, 545 insertions(+), 162 deletions(-) diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 82db54f47f6..8baa44b9659 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -113,11 +113,18 @@ The gate blocks new mutations and lets the recorded live owner finish without se After that owner releases its exact lock generation, auto-restore restores the restrictive policy and config posture. The ownership check includes both the process ID and process start identity so PID reuse does not grant control over an unrelated process. -During restoration, NemoClaw reconciles the current exact managed MCP entries into the saved policy. -A removed MCP server stays removed, while an unrelated surviving server keeps its recorded endpoint and address pins. - -If the recorded owner exits before releasing its lock generation, NemoClaw leaves Shields down and records permanent containment. -New mutations remain blocked so an untracked descendant cannot change the sandbox after restoration. +Before a manual Shields transition replaces a policy, NemoClaw requires exact agreement among the sandbox registry, generated-policy record, and live gateway policy. +`shields down` carries the proven managed MCP policy entries into the relaxed policy. +Restoration removes snapshot-time managed MCP entries before it overlays current exact entries. +If exact agreement is absent, a manual Shields transition refuses the replacement policy. +At an expired deadline, auto-restore omits unproven managed MCP policy entries, restores lockdown, and records the omission count in its audit entry. +An MCP server removed during the shields-down window stays removed. +A surviving server keeps its recorded endpoint and address pins while its policy ownership remains exact. + +When an interactive command takes over an expired timer, NemoClaw retries restoration for one additional completion-grace window while the deadline gate stays closed. +If the command cannot complete and commit restoration within that window, NemoClaw converts the deadline gate into durable containment. +NemoClaw commits the containment record before it returns the failure, so new mutations remain blocked until operator resolution. +NemoClaw also records durable containment if the recorded owner exits before releasing its exact lock generation, because surviving descendants cannot be ruled out. Stop all NemoClaw processes for that sandbox, then follow the exact lock-generation recovery guidance in the reported error or audit entry. Do not remove a recorded lifecycle lock while any NemoClaw process for that sandbox is running. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index beee1d048bd..2bd8fca0d91 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1152,11 +1152,19 @@ Host-side config and inference writes, snapshot mutation, sandbox destruction, a When a timed shields-down window reaches its deadline, auto-restore closes the per-sandbox lifecycle deadline gate. The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation before auto-restore restores lockdown. NemoClaw does not signal that process because portable process inspection cannot prove that all descendants are contained. -If the owner exits before releasing the generation, NemoClaw leaves Shields down, records permanent containment, and reports operator recovery guidance. +When an interactive command takes over an expired timer, NemoClaw retries restoration for one additional completion-grace window while the deadline gate stays closed. +If the command cannot complete and commit restoration within that window, NemoClaw converts the deadline gate into durable containment. +NemoClaw commits the containment record before it returns the failure, so new mutations remain blocked until operator resolution. +NemoClaw also records durable containment if the recorded owner exits before releasing its exact lock generation, because surviving descendants cannot be ruled out. Stop all NemoClaw processes for that sandbox, then follow the exact lock-generation guidance before retrying. -Policy restoration reconciles the current exact managed MCP entries into the saved restrictive policy. -An MCP server removed during the shields-down window stays removed, while an unrelated surviving server keeps its recorded endpoint and address pins. +Before a manual Shields transition replaces a policy, NemoClaw requires exact agreement among the sandbox registry, generated-policy record, and live gateway policy. +`shields down` carries the proven managed MCP policy entries into the relaxed policy. +Restoration removes snapshot-time managed MCP entries before it overlays current exact entries. +If exact agreement is absent, a manual Shields transition refuses the replacement policy. +At an expired deadline, auto-restore omits unproven managed MCP policy entries, restores lockdown, and records the omission count in its audit entry. +An MCP server removed during the shields-down window stays removed. +A surviving server keeps its recorded endpoint and address pins while its policy ownership remains exact. diff --git a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts index 13a20f9723d..6fc31261d99 100644 --- a/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts +++ b/src/lib/actions/sandbox/snapshot-restore-lifecycle.test.ts @@ -6,7 +6,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { withSandboxMutationLock } from "../../state/mcp-lifecycle-lock"; +import { isMcpLifecycleLockHeld, withSandboxMutationLock } from "../../state/mcp-lifecycle-lock"; import * as f from "./snapshot-restore-test-fixture"; const tempHomes: string[] = []; @@ -23,23 +23,42 @@ afterEach(() => { } }); describe("runSandboxSnapshot restore: lifecycle and destination safety", () => { - it("holds snapshot creation under the lifecycle gate before the timer-bound transition", () => { - const source = fs.readFileSync(new URL("./snapshot.ts", import.meta.url), "utf8"); - const createCase = source.indexOf('case "create":'); - const listCase = source.indexOf('case "list":', createCase); - const createDispatch = source.slice(createCase, listCase); - const createFunction = source.indexOf("function runSnapshotCreate"); - const timerBoundLock = source.indexOf( - "withTimerBoundShieldsMutationLock(sandboxName,", - createFunction, + it("holds snapshot creation under the lifecycle gate before the timer-bound transition (#7952)", async () => { + const manifest = { + timestamp: "2026-06-15T00:00:00.000Z", + backupPath: "/tmp/backup-alpha", + }; + f.backupSandboxStateMock.mockReturnValue({ + success: true, + backedUpDirs: ["workspace"], + backedUpFiles: [], + failedDirs: [], + failedFiles: [], + manifest, + }); + f.findBackupMock.mockReturnValue({ + match: { ...manifest, snapshotVersion: 1 }, + }); + let lifecycleHeldDuringTimerTransition = false; + f.lifecycleMock.withTimerBoundMock.mockImplementationOnce( + (sandboxName: string, _command: string, operation: () => unknown) => { + lifecycleHeldDuringTimerTransition = isMcpLifecycleLockHeld(sandboxName); + return operation(); + }, ); + vi.spyOn(console, "log").mockImplementation(() => {}); + const { runSandboxSnapshot } = await import("./snapshot"); + + await runSandboxSnapshot("alpha", { kind: "create" }); - expect(createCase).toBeGreaterThanOrEqual(0); - expect(listCase).toBeGreaterThan(createCase); - expect(createDispatch).toContain( - "await withSandboxMutationLock(sandboxName, () => runSnapshotCreate(sandboxName, request));", + expect(lifecycleHeldDuringTimerTransition).toBe(true); + expect(f.lifecycleMock.withTimerBoundMock).toHaveBeenCalledWith( + "alpha", + "create sandbox snapshot", + expect.any(Function), ); - expect(timerBoundLock).toBeGreaterThan(createFunction); + expect(f.backupSandboxStateMock).toHaveBeenCalledWith("alpha", { name: null }); + expect(isMcpLifecycleLockHeld("alpha")).toBe(false); }); it("restores the latest snapshot into the source sandbox", async () => { diff --git a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts index 07810b81f04..5eeb8f4e6c8 100644 --- a/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts +++ b/src/lib/actions/sandbox/snapshot-restore-test-fixture.ts @@ -275,6 +275,7 @@ vi.mock("../../state/gateway", () => ({ })); vi.mock("../../state/registry", () => ({ + getBaselineExclusions: vi.fn(() => []), getConfiguredMessagingChannelsFromEntry: vi.fn(() => []), getCustomPolicies: getCustomPoliciesMock, getDisabledMessagingChannelsFromEntry: vi.fn(() => []), diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 2d23143e7f0..0acc9b9b437 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -30,6 +30,9 @@ type ShieldsHarness = { }; let tmpDir: string; +const currentProcessStartIdentity = ( + requireDist("./timer-control.js") as typeof import("./timer-control.js") +).readProcessStartIdentity(process.pid); type HarnessOptions = { directSandboxUnavailable?: boolean; @@ -516,12 +519,13 @@ describe("shields command flow", () => { it("loads 257 managed keys recorded by Shields down (#7952)", () => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); const snapshotPath = path.join(stateDir, "policy-snapshot-many-managed-keys.yaml"); - const keys = Array.from({ length: 257 }, (_, index) => `mcp_bridge_server_${index}`); - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync( - snapshotPath, - YAML.stringify({ network_policies: Object.fromEntries(keys.map((key) => [key, {}])) }), + const policies = Array.from({ length: 257 }, (_, index) => managedMcpPolicy(`server${index}`)); + const keys = policies.map(({ server }) => `mcp_bridge_${server}`); + const networkPolicies = Object.fromEntries( + policies.map(({ networkPolicy, server }) => [`mcp_bridge_${server}`, networkPolicy]), ); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(snapshotPath, YAML.stringify({ network_policies: networkPolicies })); fs.writeFileSync( path.join(stateDir, "shields-openclaw.json"), JSON.stringify({ @@ -530,9 +534,17 @@ describe("shields command flow", () => { shieldsManagedMcpPolicyKeys: keys, }), ); - const harness = createHarness(); + const harness = createHarness({ + livePolicy: YAML.stringify({ version: 1, network_policies: networkPolicies }), + sandboxEntry: managedMcpSandbox(policies), + }); expect(harness.applyShieldsPolicySnapshot("openclaw", snapshotPath).status).toBe(0); + const applied = YAML.parse(harness.policySetBodies.at(-1)!); + const appliedKeys = Object.keys(applied.network_policies); + expect([...appliedKeys].sort()).toEqual([...keys].sort()); + expect(appliedKeys).toHaveLength(257); + expect(appliedKeys).toContain("mcp_bridge_server256"); }); it("binds manual shields-up to the active auto-restore timer generation", () => { @@ -865,6 +877,48 @@ describe("shields command flow", () => { expect(fs.existsSync(path.join(stateDir, "shields-timer-openclaw.json"))).toBe(true); }); + it.skipIf(process.platform === "win32")( + "atomically replaces a timer marker symlink without modifying its target", + () => { + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const markerPath = path.join(stateDir, "shields-timer-openclaw.json"); + const markerTargetPath = path.join(stateDir, "operator-owned-marker.json"); + const markerTarget = "operator-owned marker contents"; + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(markerTargetPath, markerTarget); + const originalRename = fs.renameSync.bind(fs); + const plantMarkerSymlink = () => fs.symlinkSync(markerTargetPath, markerPath); + const publicationRoutes = new Map void>([[markerPath, plantMarkerSymlink]]); + const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => { + (publicationRoutes.get(String(destination)) ?? (() => undefined))(); + originalRename(source, destination); + }); + const harness = createHarness({ + fork: () => ({ + pid: 4242, + disconnect: vi.fn(), + unref: vi.fn(), + send: vi.fn(() => true), + kill: vi.fn(() => true), + }), + }); + + harness.shieldsDown("openclaw", { + timeout: "5m", + reason: "marker publication coverage", + throwOnError: true, + }); + + expect(renameSpy).toHaveBeenCalledWith(expect.stringContaining(".tmp"), markerPath); + expect(fs.lstatSync(markerPath).isSymbolicLink()).toBe(false); + expect(JSON.parse(fs.readFileSync(markerPath, "utf-8"))).toMatchObject({ + pid: 4242, + sandboxName: "openclaw", + }); + expect(fs.readFileSync(markerTargetPath, "utf-8")).toBe(markerTarget); + }, + ); + it("shieldsUp refuses to mark lockdown active when the saved restrictive policy snapshot is missing", () => { const harness = createHarness(); const stateDir = path.join(tmpDir, ".nemoclaw", "state"); @@ -1191,4 +1245,73 @@ describe("shields command flow", () => { expect.anything(), ); }); + + it.skipIf(currentProcessStartIdentity === null)( + "bounds live transition takeover before committing durable containment", + () => { + const sandboxName = "openclaw"; + const processToken = "8".repeat(32); + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const snapshotPath = path.join(stateDir, "policy-snapshot-takeover-exhausted.yaml"); + const timerMarkerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const transitionLockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); + const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); + const containmentPath = `${lifecycleLock.getMcpLifecycleLockPath(sandboxName)}.containment`; + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}\n"); + fs.writeFileSync( + path.join(stateDir, `shields-${sandboxName}.json`), + JSON.stringify({ + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), + shieldsDownTimeout: 60, + shieldsDownReason: "takeover exhaustion coverage", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: snapshotPath, + }), + ); + fs.writeFileSync( + timerMarkerPath, + JSON.stringify({ + pid: 2_147_483_647, + sandboxName, + snapshotPath, + restoreAt: new Date(Date.now() - 60_000).toISOString(), + processToken, + }), + ); + fs.writeFileSync( + transitionLockPath, + JSON.stringify({ + version: 1, + sandboxName, + pid: process.pid, + processStartIdentity: currentProcessStartIdentity, + command: "shields down", + acquiredAtMs: Date.now(), + takeoverToken: processToken, + }), + ); + const waitSpy = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); + const harness = createHarness(); + + expect(() => harness.shieldsStatus(sandboxName)).toThrow( + "Auto-restore transition takeover exhausted 7 attempts", + ); + + expect(waitSpy.mock.calls.map((call) => call[3])).toEqual([ + 5_000, 5_000, 5_000, 5_000, 5_000, 5_000, + ]); + expect(fs.existsSync(containmentPath)).toBe(true); + expect(harness.auditSpy).toHaveBeenCalledTimes(1); + expect(harness.auditSpy).toHaveBeenCalledWith( + expect.objectContaining({ + action: "shields_up_failed", + sandbox: sandboxName, + error: + "Shields transition owner is still active; automatic recovery is waiting behind the deadline gate", + }), + ); + }, + ); }); diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index cc84b0da75e..9f49f47f97a 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -613,6 +613,40 @@ describe("shields — unit logic", () => { expect(fs.existsSync(path.join(stateDir(), `shields-timer-${sandboxName}.json`))).toBe(true); }); + it("bounds current-generation inline recovery when the snapshot is missing (#7952)", async () => { + const sandboxName = "openclaw"; + const processToken = "c".repeat(32); + const missingSnapshotPath = path.join(stateDir(), "missing-current-snapshot.yaml"); + writeState(sandboxName, { + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 60_000).toISOString(), + shieldsDownTimeout: 300, + shieldsDownReason: "testing", + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: missingSnapshotPath, + updatedAt: new Date().toISOString(), + }); + writeMarker(sandboxName, { + pid: 2_147_483_647, + sandboxName, + snapshotPath: missingSnapshotPath, + restoreAt: new Date(Date.now() - 30_000).toISOString(), + processToken, + }); + vi.spyOn(process, "kill").mockImplementation(routeProcessKill); + vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); + vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + const { shieldsStatus } = await loadShieldsModule(); + + expect(() => shieldsStatus(sandboxName)).toThrow("Inline auto-restore exhausted 7 attempts"); + const { getMcpLifecycleLockPath } = await import("../state/mcp-lifecycle-lock"); + expect(fs.existsSync(`${getMcpLifecycleLockPath(sandboxName, stateDir())}.containment`)).toBe( + true, + ); + expect(fs.existsSync(path.join(stateDir(), `shields-timer-${sandboxName}.json`))).toBe(true); + }); + it("shieldsStatus attempts inline recovery when expired marker PID is alive but cmdline does not match recorded timer", async () => { const sandboxName = "openclaw"; const snapshotPath = path.join(stateDir(), "policy-snapshot-test.yaml"); diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index dd5465896e7..cc77232a00d 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -114,6 +114,11 @@ const SHIELDS_TRANSITION_POLL_MS = 50; const SHIELDS_TRANSITION_HANDOFF_GRACE_MS = 500; const SHIELDS_TRANSITION_TERMINATE_GRACE_MS = 1000; const AUTO_RESTORE_COMPLETION_GRACE_MS = 30_000; +// Retry on the detached timer's cadence for one additional completion-grace +// window before converting the live deadline fence into durable containment. +const INTERACTIVE_AUTO_RESTORE_RETRY_MS = 5_000; +const INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS = + Math.floor(AUTO_RESTORE_COMPLETION_GRACE_MS / INTERACTIVE_AUTO_RESTORE_RETRY_MS) + 1; const HERMES_RUNTIME_CONFIG_GUARD = "/usr/local/lib/nemoclaw/hermes-runtime-config-guard.py"; const HERMES_PYTHON = "/opt/hermes/.venv/bin/python"; const HERMES_RESTART_SEAL_STATE = "/run/nemoclaw/hermes-restart-seal.json"; @@ -245,6 +250,22 @@ function writeShieldsDownTransition( } } +function writeTimerMarkerAtomic(sandboxName: string, marker: TimerMarker): void { + const markerPath = timerMarkerPath(sandboxName); + fs.mkdirSync(path.dirname(markerPath), { recursive: true, mode: 0o700 }); + const tempPath = `${markerPath}.${String(process.pid)}.${randomBytes(8).toString("hex")}.tmp`; + try { + fs.writeFileSync(tempPath, JSON.stringify(marker), { flag: "wx", mode: 0o600 }); + fs.renameSync(tempPath, markerPath); + } finally { + try { + fs.rmSync(tempPath, { force: true }); + } catch { + // Best effort. The authoritative path was either atomically replaced or unchanged. + } + } +} + function clearShieldsDownTransition(sandboxName: string, processToken: string): void { try { fs.rmSync(shieldsDownTransitionPath(sandboxName, processToken), { force: true }); @@ -280,17 +301,12 @@ function readExactProcessStatus( function persistUnresolvedShieldsContainment( sandboxName: string, processToken: string, - ownerPid: number, + reason: string, ): void { const containmentPath = `${getMcpLifecycleLockPath(sandboxName, STATE_DIR)}.containment`; if (fs.existsSync(containmentPath)) return; try { - beginCommittedMcpLifecycleContainmentSync( - sandboxName, - processToken, - `Shields recovery owner PID ${String(ownerPid)} exited without descendant-containment proof`, - STATE_DIR, - ); + beginCommittedMcpLifecycleContainmentSync(sandboxName, processToken, reason, STATE_DIR); } catch (error) { if (fs.existsSync(containmentPath)) return; throw error; @@ -338,7 +354,13 @@ function waitForShieldsDownForwardCommit( ); if (ownerStatus === "gone") { assertTakeoverAuthority?.(); - persistUnresolvedShieldsContainment(sandboxName, processToken, observed.ownerPid); + persistUnresolvedShieldsContainment( + sandboxName, + processToken, + `Shields recovery owner PID ${String( + observed.ownerPid, + )} exited without descendant-containment proof`, + ); throw new Error( "Shields-down forward owner exited before committing its final mutation; permanent containment requires operator resolution", ); @@ -854,9 +876,54 @@ function inspectExpiredAutoRestoreTakeover( }; } -function retryInlineAutoRestore(sandboxName: string, marker: TimerMarker): void { +function failInteractiveAutoRestoreClosed( + sandboxName: string, + marker: TimerMarker & { processToken: string }, + message: string, +): never { + const containmentPath = `${getMcpLifecycleLockPath(sandboxName, STATE_DIR)}.containment`; let notifiedError: string | null = null; + // The deadline fence unwinds when this function throws. Keep retrying the + // durable containment commit first so a bounded interactive recovery can + // return without reopening the sandbox mutation gate. for (;;) { + assertTimerMarkerGeneration(sandboxName, marker); + try { + persistUnresolvedShieldsContainment( + sandboxName, + marker.processToken, + `Interactive auto-restore could not complete safely: ${message}`, + ); + break; + } catch (error) { + if (fs.existsSync(containmentPath)) break; + assertTimerMarkerGeneration(sandboxName, marker); + const containmentError = error instanceof Error ? error.message : String(error); + if (containmentError !== notifiedError) { + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: `Permanent containment commit failed; retrying behind the deadline gate: ${containmentError}`, + }); + notifiedError = containmentError; + } + Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); + } + } + throw new Error( + `${message}. Permanent sandbox mutation containment requires operator resolution`, + ); +} + +function retryInlineAutoRestore( + sandboxName: string, + marker: TimerMarker & { processToken: string }, +): void { + let notifiedError: string | null = null; + for (let attempt = 0; attempt < INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS; attempt += 1) { try { const recoveredState = recoverExpiredAutoRestoreGate(sandboxName, true); if (!recoveredState._isCorrupt && recoveredState.shieldsDown !== true) { @@ -890,8 +957,17 @@ function retryInlineAutoRestore(sandboxName: string, marker: TimerMarker): void notifiedError = message; } } - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); + if (attempt + 1 < INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS) { + Atomics.wait(transitionPollBuffer, 0, 0, INTERACTIVE_AUTO_RESTORE_RETRY_MS); + } } + failInteractiveAutoRestoreClosed( + sandboxName, + marker, + `Inline auto-restore exhausted ${String( + INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS, + )} attempts: ${notifiedError ?? "recovery did not complete"}`, + ); } function withExpiredAutoRestoreDeadlineFence( @@ -905,7 +981,7 @@ function withExpiredAutoRestoreDeadlineFence( withTimerBoundShieldsMutationLock(sandboxName, command, callback); const recoverThenRun = () => withTimerBoundAutoRestoreLock(sandboxName, command, () => { - if (expiredMarker) retryInlineAutoRestore(sandboxName, expiredMarker); + if (takeover) retryInlineAutoRestore(sandboxName, takeover.marker); return operation(false); }); if (isMcpLifecycleLockHeld(sandboxName, STATE_DIR)) { @@ -933,7 +1009,8 @@ function withExpiredAutoRestoreDeadlineFence( sandboxName, marker.processToken, () => { - for (;;) { + let notifiedError: string | null = null; + for (let attempt = 0; attempt < INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS; attempt += 1) { try { prepareAutoRestoreTransitionTakeover( sandboxName, @@ -941,21 +1018,33 @@ function withExpiredAutoRestoreDeadlineFence( marker.snapshotPath, assertTakeoverAuthority, ); - break; + return recoverThenRun(); } catch (error) { assertTakeoverAuthority(); - appendAuditEntryBestEffort({ - action: "shields_up_failed", - sandbox: sandboxName, - timestamp: new Date().toISOString(), - restored_by: "auto_timer", - policy_snapshot: marker.snapshotPath, - error: error instanceof Error ? error.message : String(error), - }); - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); + const message = error instanceof Error ? error.message : String(error); + if (message !== notifiedError) { + appendAuditEntryBestEffort({ + action: "shields_up_failed", + sandbox: sandboxName, + timestamp: new Date().toISOString(), + restored_by: "auto_timer", + policy_snapshot: marker.snapshotPath, + error: message, + }); + notifiedError = message; + } + if (attempt + 1 < INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS) { + Atomics.wait(transitionPollBuffer, 0, 0, INTERACTIVE_AUTO_RESTORE_RETRY_MS); + } } } - return recoverThenRun(); + return failInteractiveAutoRestoreClosed( + sandboxName, + marker, + `Auto-restore transition takeover exhausted ${String( + INTERACTIVE_AUTO_RESTORE_MAX_ATTEMPTS, + )} attempts: ${notifiedError ?? "transition ownership did not become available"}`, + ); }, { stateDir: STATE_DIR, @@ -2405,7 +2494,11 @@ function prepareAutoRestoreTransitionTakeover( ); if (ownerStatus === "gone") { assertTakeoverAuthority?.(); - persistUnresolvedShieldsContainment(sandboxName, processToken, owner.pid); + persistUnresolvedShieldsContainment( + sandboxName, + processToken, + `Shields recovery owner PID ${String(owner.pid)} exited without descendant-containment proof`, + ); throw new Error( "Shields transition owner exited without descendant-containment proof; permanent containment requires operator resolution", ); @@ -3136,21 +3229,17 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = }, ); if (!timerChild.pid) throw new Error("auto-restore timer did not report a process id"); - fs.writeFileSync( - timerMarkerPath(sandboxName), - JSON.stringify({ - pid: timerChild.pid, - sandboxName, - snapshotPath, - restoreAt: restoreAt.toISOString(), - processToken, - allowLegacyHermesProtocol: opts.allowLegacyHermesProtocol === true, - ...(leaseOwnerPid !== null && leaseOwnerStartIdentity - ? { leaseOwnerPid, leaseOwnerStartIdentity } - : {}), - }), - { mode: 0o600 }, - ); + writeTimerMarkerAtomic(sandboxName, { + pid: timerChild.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAt.toISOString(), + processToken, + allowLegacyHermesProtocol: opts.allowLegacyHermesProtocol === true, + ...(leaseOwnerPid !== null && leaseOwnerStartIdentity + ? { leaseOwnerPid, leaseOwnerStartIdentity } + : {}), + }); if (!timerChild.send({ type: "authorize", processToken })) { throw new Error("auto-restore timer authorization channel closed early"); } diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index b6693933f96..9081a4d1b6e 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -179,6 +179,52 @@ describe("shields timer authorization", () => { expect(fs.existsSync(markerPath)).toBe(true); }); + it.skipIf(process.platform === "win32")( + "does not restore or rewrite state through a symlinked timer marker", + async () => { + const timer = await import("./timer"); + const stateDir = path.join(tmpHome, ".nemoclaw", "state"); + fs.mkdirSync(stateDir, { recursive: true }); + + const sandboxName = "alpha"; + const snapshotPath = path.join(stateDir, "snapshot.yaml"); + const restoreAtIso = new Date(Date.now() + 60_000).toISOString(); + const stateFile = path.join(stateDir, `shields-${sandboxName}.json`); + const markerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const markerTargetPath = path.join(stateDir, "operator-owned-marker.json"); + const initialState = { shieldsDown: true, updatedAt: "2026-01-01T00:00:00.000Z" }; + const markerTarget = JSON.stringify({ + pid: process.pid, + sandboxName, + snapshotPath, + restoreAt: restoreAtIso, + processToken: PROCESS_TOKEN, + }); + + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n default: {}\n"); + fs.writeFileSync(stateFile, JSON.stringify(initialState, null, 2)); + fs.writeFileSync(markerTargetPath, markerTarget); + fs.symlinkSync(markerTargetPath, markerPath); + const args = timer.parseTimerArgs([ + sandboxName, + snapshotPath, + restoreAtIso, + "", + "", + PROCESS_TOKEN, + ]); + expect(args).not.toBeNull(); + + const exitCode = await invokeTimerAndCaptureExit(timer.runRestoreTimer, args); + + expect(exitCode).toBe(0); + expect(shieldsIndexMock.applyShieldsPolicySnapshot).not.toHaveBeenCalled(); + expect(JSON.parse(fs.readFileSync(stateFile, "utf-8"))).toEqual(initialState); + expect(fs.lstatSync(markerPath).isSymbolicLink()).toBe(true); + expect(fs.readFileSync(markerTargetPath, "utf-8")).toBe(markerTarget); + }, + ); + it("binds rebuild-only legacy authorization to both argv and the root-owned marker", async () => { const timer = await import("./timer"); const stateDir = path.join(tmpHome, ".nemoclaw", "state"); diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index ddb6c908ec4..5632d8f59b0 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -13,6 +13,10 @@ import path from "node:path"; import { isObjectRecord, type UnknownRecord } from "../core/json-types"; import { resolveAgentConfig } from "../sandbox/config"; import { withMcpLifecycleDeadlineFence } from "../state/mcp-lifecycle-lock"; +import { + readShieldsTimerMarkerFile, + type ShieldsTimerMarker, +} from "../state/mcp-lifecycle-lock/shields-timer-authority"; import { resolveNemoclawStateDir } from "../state/paths"; import { appendAuditEntry, type ShieldsAuditEntry } from "./audit"; import * as shields from "./index"; @@ -153,19 +157,14 @@ function updateState(stateFile: string, patch: ShieldsStatePatch): void { } } -function readTimerMarker(markerPath: string): UnknownRecord | null { - try { - if (!fs.existsSync(markerPath)) { - return null; - } - const parsed = JSON.parse(fs.readFileSync(markerPath, "utf-8")); - return isObjectRecord(parsed) ? parsed : null; - } catch { - return null; - } +function readTimerMarker(markerPath: string): ShieldsTimerMarker | null { + return readShieldsTimerMarkerFile(markerPath); } -function markerRecordMatchesCurrentTimer(marker: UnknownRecord | null, args: TimerArgs): boolean { +function markerRecordMatchesCurrentTimer( + marker: ShieldsTimerMarker | null, + args: TimerArgs, +): boolean { if (!marker) return false; return ( marker.pid === process.pid && diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.ts index 86642eddc5c..440fb3909a0 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.ts @@ -832,7 +832,16 @@ function clearDeadlineProtectedPathSync( Boolean(owner.processIdentity) && owner.hostIdentity === readMcpLockHostIdentity() && owner.pidNamespaceIdentity === readMcpLockPidNamespaceIdentity(); - if (exactLocalOwner && owner?.pid === process.pid) { + const currentProcessIdentity = + exactLocalOwner && owner?.pid === process.pid + ? readMcpLockProcessIdentity(process.pid, true) + : null; + if ( + exactLocalOwner && + owner?.pid === process.pid && + currentProcessIdentity !== null && + owner.processIdentity === currentProcessIdentity + ) { const error = new Error( "Synchronous auto-restore cannot wait behind a sibling lifecycle operation in this process", ) as Error & { code: string }; @@ -861,6 +870,13 @@ function clearDeadlineProtectedPathSync( ), ); } + if (exactLocalOwner && owner?.pid === process.pid && currentProcessIdentity === null) { + const error = new Error( + "Synchronous auto-restore cannot verify the process identity of a same-PID lifecycle owner", + ) as Error & { code: string }; + error.code = "NEMOCLAW_SYNC_REENTRANT_OWNER"; + throw error; + } const generation = `${String(observed.dev)}:${String(observed.ino)}:${ owner?.token ?? "invalid" diff --git a/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts b/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts index e446b8c2055..3ca1633c5f0 100644 --- a/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts +++ b/src/lib/state/mcp-lifecycle-lock/shields-timer-authority.ts @@ -64,10 +64,25 @@ export function readShieldsTimerMarker( stateDir = resolveNemoclawStateDir(), ): ShieldsTimerMarker | null { try { - const markerPath = shieldsTimerMarkerPath(sandboxName, stateDir); - if (!fs.existsSync(markerPath)) return null; - const parsed = JSON.parse(fs.readFileSync(markerPath, "utf-8")); - return isShieldsTimerMarker(parsed) ? parsed : null; + return readShieldsTimerMarkerFile(shieldsTimerMarkerPath(sandboxName, stateDir)); + } catch { + return null; + } +} + +export function readShieldsTimerMarkerFile(markerPath: string): ShieldsTimerMarker | null { + try { + const markerFd = fs.openSync( + markerPath, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK, + ); + try { + if (!fs.fstatSync(markerFd).isFile()) return null; + const parsed = JSON.parse(fs.readFileSync(markerFd, "utf-8")); + return isShieldsTimerMarker(parsed) ? parsed : null; + } finally { + fs.closeSync(markerFd); + } } catch { return null; } diff --git a/test/e2e/support/mcp-bridge-sandbox.test.ts b/test/e2e/support/mcp-bridge-sandbox.test.ts index 5d504d05322..c2e6017ee9f 100644 --- a/test/e2e/support/mcp-bridge-sandbox.test.ts +++ b/test/e2e/support/mcp-bridge-sandbox.test.ts @@ -12,6 +12,7 @@ import YAML from "yaml"; import { testTimeout } from "../../helpers/timeouts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { + assertManagedMcpPolicySurvivedRemoval, buildMcpDnsRebindingProbeScript, hostAddressForSandbox, hostPrivateAddressForSandbox, @@ -302,83 +303,34 @@ network_policies: expect(contractSource).not.toContain("assertAdapterDnsRebindingDenied"); }); - it("runs the zero-upstream rebinding proof and preserves the surviving policy for all three adapters", () => { - const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); - const helper = source.indexOf("async function assertAdapterDnsRebindingDenied"); - const survivingPolicyBeforeAdd = source.indexOf("const survivingPolicyBeforeAddResult", helper); - const add = source.indexOf("const add = await host.nemoclaw", survivingPolicyBeforeAdd); - const remove = source.indexOf("const remove = await host.nemoclaw", add); - const survivingPolicyAfterRemove = source.indexOf( - "const survivingPolicyAfterRemoveResult", - remove, - ); - - expect(source.match(/await assertAdapterDnsRebindingDenied/g)).toHaveLength(3); - expect(source.match(/survivingMcpUrl: mcpUrl/g)).toHaveLength(3); - for (const adapter of [ - 'adapter: "mcporter"', - 'adapter: "hermes-config"', - 'adapter: "deepagents-config"', - ]) { - expect(source).toContain(adapter); - } - expect(source).toContain("rebound request must not reach the upstream MCP server"); - expect(source).toContain(").toHaveLength(0);"); - expect(survivingPolicyBeforeAdd).toBeGreaterThan(helper); - expect(add).toBeGreaterThan(survivingPolicyBeforeAdd); - expect(remove).toBeGreaterThan(add); - expect(survivingPolicyAfterRemove).toBeGreaterThan(remove); - expect(source).toContain( - "assertManagedMcpPolicySurvivedRemoval(\n survivingPolicyBeforeAdd,\n survivingPolicyAfterRemoveResult,\n REBIND_POLICY_KEY,", - ); - }); - - it("requires the Hermes E2E journey to call the surviving route before and after the unrelated route lifecycle", () => { - const source = fs.readFileSync("test/e2e/live/mcp-bridge.test.ts", "utf8"); - const denialProof = source.indexOf("rebound request must not reach the upstream MCP server"); - const restore = source.indexOf("await restoreDnsRebindingHostsFixture", denialProof); - const remove = source.indexOf("const remove = await host.nemoclaw", denialProof); - const hermesTest = source.indexOf('mcpBridgeShardTest("hermes")'); - const shieldsDown = source.indexOf( - "assertHermesManagedAddSurvivesLockedGatewayRestartAndStateLayout", - hermesTest, - ); - const afterShieldsDownToolCall = source.indexOf( - "hermes-real-mcp-tool-call-immediately-after-shields-down", - shieldsDown, - ); - const rebinding = source.indexOf("await assertAdapterDnsRebindingDenied", hermesTest); - const afterRemoveToolCall = source.indexOf( - "hermes-real-mcp-tool-call-after-dns-rebinding-remove", - rebinding, - ); - const offset = source.indexOf( - "const survivingDiscoveryOffset = fakeMcp.requests.length", - afterRemoveToolCall, - ); - const restart = source.indexOf("await restartBridgeWithoutHostSecret", offset); - const afterRestartToolCall = source.indexOf( - "hermes-real-mcp-tool-call-after-rediscovery-restart", - restart, - ); - const rediscovery = source.indexOf( - "await assertAuthenticatedMcpRediscovery", - afterRestartToolCall, - ); + it("accepts an unchanged surviving policy only after the unrelated policy is absent", () => { + const survivingPolicy = { + endpoints: [{ host: "surviving.example.test", allowed_ips: ["203.0.113.10"] }], + }; - expect(denialProof).toBeGreaterThanOrEqual(0); - expect(restore).toBeGreaterThan(denialProof); - expect(remove).toBeGreaterThan(restore); - expect(shieldsDown).toBeGreaterThan(hermesTest); - expect(afterShieldsDownToolCall).toBeGreaterThan(shieldsDown); - expect(rebinding).toBeGreaterThan(hermesTest); - expect(rebinding).toBeGreaterThan(afterShieldsDownToolCall); - expect(afterRemoveToolCall).toBeGreaterThan(rebinding); - expect(offset).toBeGreaterThan(afterRemoveToolCall); - expect(restart).toBeGreaterThan(offset); - expect(afterRestartToolCall).toBeGreaterThan(restart); - expect(rediscovery).toBeGreaterThan(afterRestartToolCall); - expect(source).toContain("Hermes MCP rediscovery after explicit restart"); + expect(() => + assertManagedMcpPolicySurvivedRemoval( + survivingPolicy, + { + networkPolicies: { mcp_bridge_surviving: survivingPolicy }, + policy: survivingPolicy, + }, + "mcp_bridge_rebinding", + ), + ).not.toThrow(); + expect(() => + assertManagedMcpPolicySurvivedRemoval( + survivingPolicy, + { + networkPolicies: { + mcp_bridge_rebinding: { endpoints: [] }, + mcp_bridge_surviving: survivingPolicy, + }, + policy: survivingPolicy, + }, + "mcp_bridge_rebinding", + ), + ).toThrow(); }); it("restores host DNS strictly while treating the ephemeral sandbox as best effort", async () => { diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts index c11a359f4e7..72c7bbcd2ff 100644 --- a/test/mcp-lifecycle-lock.test.ts +++ b/test/mcp-lifecycle-lock.test.ts @@ -615,6 +615,40 @@ const releasePath = process.argv[3]; }, ); + it.skipIf(currentProcessIdentity === null)( + "does not treat a recycled same PID as synchronous reentrancy", + () => { + const processToken = "6".repeat(32); + const replacementProcessToken = "7".repeat(32); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + writeTimerMarker("alpha", processToken); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: `${String(currentProcessIdentity)}-different-start`, + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + shieldsTakeoverToken: processToken, + token: "recycled-sync-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + const onContainment = vi.fn(() => writeTimerMarker("alpha", replacementProcessToken)); + + expect(() => + lifecycleLock.withMcpLifecycleDeadlineFenceSync("alpha", processToken, () => undefined, { + ...options({ timeoutMs: 10 }), + onContainment, + }), + ).toThrow("Auto-restore authority changed"); + expect(onContainment).toHaveBeenCalled(); + }, + ); + it("does not break a long-lived lock owned by the same process identity", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); fs.mkdirSync(path.dirname(lockPath), { recursive: true }); @@ -701,6 +735,41 @@ const releasePath = process.argv[3]; } }); + it.skipIf(process.platform === "win32")( + "does not derive Shields authority from a symlinked timer marker", + async () => { + const processToken = "8".repeat(32); + const targetPath = path.join(stateDir, "operator-owned-timer-marker.json"); + const markerPath = path.join(stateDir, "shields-timer-alpha.json"); + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.writeFileSync( + targetPath, + JSON.stringify({ + pid: process.pid, + sandboxName: "alpha", + snapshotPath: path.join(stateDir, "snapshot.yaml"), + restoreAt: new Date(Date.now() + 60_000).toISOString(), + processToken, + }), + ); + fs.symlinkSync(targetPath, markerPath); + let observedTakeoverToken: string | undefined; + + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + observedTakeoverToken = JSON.parse( + fs.readFileSync(lockPath, "utf8"), + ).shieldsTakeoverToken; + }, + options(), + ); + + expect(observedTakeoverToken).toBeUndefined(); + expect(JSON.parse(fs.readFileSync(targetPath, "utf8")).processToken).toBe(processToken); + }, + ); + it("retries without entering when the timer generation changes during lock publication", async () => { const firstProcessToken = "a".repeat(32); const replacementProcessToken = "b".repeat(32); @@ -866,6 +935,7 @@ const releasePath = process.argv[3]; const childExit = new Promise((resolve) => child.once("exit", () => resolve())); const containmentReported = deferred(); let entered = false; + let reportedOwnerPid: number | null = null; const deadline = lifecycleLock.withMcpLifecycleDeadlineFence( "alpha", processToken, @@ -875,12 +945,13 @@ const releasePath = process.argv[3]; { ...options({ timeoutMs: 10 }), onContainment: ({ ownerPid }) => { - expect(ownerPid).toBe(child.pid); + reportedOwnerPid = ownerPid; containmentReported.resolve(); }, }, ); await containmentReported.promise; + expect(reportedOwnerPid).toBe(child.pid); expect(entered).toBe(false); expect(() => process.kill(child.pid!, 0)).not.toThrow(); @@ -910,8 +981,9 @@ const releasePath = process.argv[3]; })}\n`, ); const deadlinePath = `${lockPath}.deadline`; + const deadlineObservations: boolean[] = []; const onContainment = vi.fn(() => { - expect(fs.existsSync(deadlinePath)).toBe(true); + deadlineObservations.push(fs.existsSync(deadlinePath)); }); setTimeout(() => writeTimerMarker("alpha", "3".repeat(32)), 40); @@ -921,6 +993,8 @@ const releasePath = process.argv[3]; onContainment, }), ).rejects.toThrow("Auto-restore authority changed"); + expect(deadlineObservations).not.toHaveLength(0); + expect(deadlineObservations.every(Boolean)).toBe(true); expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-generation"); }); From 6a182f26aeee169348b979cebd58c6d58cf5c231 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 03:26:53 -0400 Subject: [PATCH 07/25] docs(shields): clarify recovery timing Signed-off-by: Julie Yaunches --- docs/manage-sandboxes/backup-restore.mdx | 3 ++- docs/manage-sandboxes/runtime-controls.mdx | 4 ++-- docs/reference/commands.mdx | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/manage-sandboxes/backup-restore.mdx b/docs/manage-sandboxes/backup-restore.mdx index b0d1c7de1fb..241ad354f15 100644 --- a/docs/manage-sandboxes/backup-restore.mdx +++ b/docs/manage-sandboxes/backup-restore.mdx @@ -198,7 +198,8 @@ If a sandbox is not running and its container cannot be started this way, start When Shields are UP for an eligible sandbox, `backup-all` opens a 30-minute shields-down window before it creates that sandbox's snapshot. A sandbox that starts with Shields down remains down. An unlock failure marks that sandbox as failed, and `backup-all` continues with the next sandbox. -Because the timer does not defer to the backup process, it can restore lockdown when the 30-minute deadline expires. +`backup-all` does not hold the lifecycle mutation lock while it copies sandbox state. +The timer can restore lockdown if the 30-minute deadline expires during that copy. NemoClaw always attempts to restore Shields lockdown before it processes the next sandbox, including when the backup fails. If lockdown cannot be restored, `backup-all` stops and does not process the remaining sandboxes. Correct the reported issue, run the printed `$$nemoclaw shields up` command, and rerun `$$nemoclaw backup-all`. diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 8baa44b9659..d7f014a1ad8 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -113,7 +113,7 @@ The gate blocks new mutations and lets the recorded live owner finish without se After that owner releases its exact lock generation, auto-restore restores the restrictive policy and config posture. The ownership check includes both the process ID and process start identity so PID reuse does not grant control over an unrelated process. -Before a manual Shields transition replaces a policy, NemoClaw requires exact agreement among the sandbox registry, generated-policy record, and live gateway policy. +Before a manual Shields transition replaces a policy, NemoClaw requires exact Model Context Protocol (MCP) agreement among the sandbox registry, generated-policy record, and live gateway policy. `shields down` carries the proven managed MCP policy entries into the relaxed policy. Restoration removes snapshot-time managed MCP entries before it overlays current exact entries. If exact agreement is absent, a manual Shields transition refuses the replacement policy. @@ -121,7 +121,7 @@ At an expired deadline, auto-restore omits unproven managed MCP policy entries, An MCP server removed during the shields-down window stays removed. A surviving server keeps its recorded endpoint and address pins while its policy ownership remains exact. -When an interactive command takes over an expired timer, NemoClaw retries restoration for one additional completion-grace window while the deadline gate stays closed. +When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window while the deadline gate stays closed. If the command cannot complete and commit restoration within that window, NemoClaw converts the deadline gate into durable containment. NemoClaw commits the containment record before it returns the failure, so new mutations remain blocked until operator resolution. NemoClaw also records durable containment if the recorded owner exits before releasing its exact lock generation, because surviving descendants cannot be ruled out. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 2bd8fca0d91..689f5f5e626 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1152,7 +1152,7 @@ Host-side config and inference writes, snapshot mutation, sandbox destruction, a When a timed shields-down window reaches its deadline, auto-restore closes the per-sandbox lifecycle deadline gate. The gate blocks new mutations and waits for the recorded live owner to release its exact lock generation before auto-restore restores lockdown. NemoClaw does not signal that process because portable process inspection cannot prove that all descendants are contained. -When an interactive command takes over an expired timer, NemoClaw retries restoration for one additional completion-grace window while the deadline gate stays closed. +When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window while the deadline gate stays closed. If the command cannot complete and commit restoration within that window, NemoClaw converts the deadline gate into durable containment. NemoClaw commits the containment record before it returns the failure, so new mutations remain blocked until operator resolution. NemoClaw also records durable containment if the recorded owner exits before releasing its exact lock generation, because surviving descendants cannot be ruled out. From d61bcbcdf429d19caf9e593e2ec3ccf88ea6f343 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 03:34:39 -0400 Subject: [PATCH 08/25] test(shields): assert retry revocation Signed-off-by: Julie Yaunches --- src/lib/shields/timer.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index 9081a4d1b6e..e655d0816b3 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -104,8 +104,13 @@ describe("shields timer authorization", () => { await waitForRetryBoundary(deadlinePath, auditPath); expect(exitSpy).not.toHaveBeenCalled(); expect(fs.existsSync(deadlinePath)).toBe(true); + const policyApplicationsBeforeRevocation = + shieldsIndexMock.applyShieldsPolicySnapshot.mock.calls.length; fs.rmSync(markerPath, { force: true }); await pending; + expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes( + policyApplicationsBeforeRevocation, + ); } finally { fs.writeFileSync(markerPath, markerContents); exitSpy.mockRestore(); From 4fa5cbd53efc9b84c422aa1e0ab6978b4445ea48 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 04:16:41 -0400 Subject: [PATCH 09/25] fix(shields): close lifecycle recovery gaps Signed-off-by: Julie Yaunches --- docs/manage-sandboxes/runtime-controls.mdx | 5 +- docs/reference/commands.mdx | 5 +- src/lib/shields/flow.test.ts | 90 +++++++- src/lib/shields/index.test.ts | 25 ++- src/lib/shields/index.ts | 198 +++++++++--------- src/lib/shields/permissive-runtime.ts | 4 +- .../state/mcp-lifecycle-lock-acquisition.ts | 116 ++++++++++ test/config-set-nested-ssrf.test.ts | 2 + test/mcp-lifecycle-lock.test.ts | 100 ++++++++- 9 files changed, 418 insertions(+), 127 deletions(-) diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index d7f014a1ad8..3fb3f2211fb 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -124,7 +124,10 @@ A surviving server keeps its recorded endpoint and address pins while its policy When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window while the deadline gate stays closed. If the command cannot complete and commit restoration within that window, NemoClaw converts the deadline gate into durable containment. NemoClaw commits the containment record before it returns the failure, so new mutations remain blocked until operator resolution. -NemoClaw also records durable containment if the recorded owner exits before releasing its exact lock generation, because surviving descendants cannot be ruled out. +For an ordinary lifecycle lock, NemoClaw reclaims only an exact, structurally valid stale generation for the same sandbox. +It rechecks the generation under an exclusive stale-lock reaper gate before removal. +An expired deadline owner or interrupted reaper still records durable containment because surviving descendants cannot be ruled out. +NemoClaw fails closed for corrupt, non-regular, wrong-sandbox, ambiguous, foreign, and live identities instead of deleting those generations. Stop all NemoClaw processes for that sandbox, then follow the exact lock-generation recovery guidance in the reported error or audit entry. Do not remove a recorded lifecycle lock while any NemoClaw process for that sandbox is running. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 689f5f5e626..ccf5c8044b6 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1155,7 +1155,10 @@ NemoClaw does not signal that process because portable process inspection cannot When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window while the deadline gate stays closed. If the command cannot complete and commit restoration within that window, NemoClaw converts the deadline gate into durable containment. NemoClaw commits the containment record before it returns the failure, so new mutations remain blocked until operator resolution. -NemoClaw also records durable containment if the recorded owner exits before releasing its exact lock generation, because surviving descendants cannot be ruled out. +For an ordinary lifecycle lock, NemoClaw reclaims only an exact, structurally valid stale generation for the same sandbox. +It rechecks the generation under an exclusive stale-lock reaper gate before removal. +An expired deadline owner or interrupted reaper still records durable containment because surviving descendants cannot be ruled out. +NemoClaw fails closed for corrupt, non-regular, wrong-sandbox, ambiguous, foreign, and live identities instead of deleting those generations. Stop all NemoClaw processes for that sandbox, then follow the exact lock-generation guidance before retrying. Before a manual Shields transition replaces a policy, NemoClaw requires exact agreement among the sandbox registry, generated-policy record, and live gateway policy. diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 0acc9b9b437..2057242ac04 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -18,6 +18,7 @@ const shieldsModulePath = "./index.js"; type ShieldsHarness = { applyShieldsPolicySnapshot: typeof import("./index.js").applyShieldsPolicySnapshot; auditSpy: MockInstance; + cleanupTempDirSpy: MockInstance; errorSpy: MockInstance; logSpy: MockInstance; policySetBodies: string[]; @@ -38,6 +39,7 @@ type HarnessOptions = { directSandboxUnavailable?: boolean; dockerExecFileSync?: (argv: unknown) => string; failOpenClawGuardActions?: Array<"lock" | "unlock">; + failStateSave?: boolean; invokedAs?: "nemoclaw" | "nemohermes"; openClawGuardFailure?: { code: string; @@ -62,15 +64,18 @@ type HarnessOptions = { }; function managedMcpPolicy(server: string, address = "8.8.8.8") { - const key = `mcp_bridge_${server}`; const content = buildMcpBridgePolicyYaml( server, `https://${server}.example.com/mcp`, "hermes-config", [address], ); - const networkPolicy = YAML.parse(content).network_policies[key]; - return { content, networkPolicy, server }; + const entries = Object.entries(YAML.parse(content).network_policies as Record); + if (entries.length !== 1) { + throw new Error(`Expected one rendered MCP policy for ${server}, found ${entries.length}`); + } + const [key, networkPolicy] = entries[0]!; + return { content, key, networkPolicy, server }; } function managedMcpSandbox(policies: Array>): SandboxEntry { @@ -129,6 +134,7 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { const privilegedExec = requireDist("../sandbox/privileged-exec.js"); const dockerExec = requireDist("../adapters/docker/exec.js"); const audit = requireDist("./audit.js"); + const tempFiles = requireDist("../onboard/temp-files.js"); const childProcess = requireDist("node:child_process"); const policySetBodies: string[] = []; let openClawPosture: "locked" | "mutable" = "mutable"; @@ -162,6 +168,9 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { options.sandboxEntry ?? { name: "openclaw", openshellDriver: "docker" }, ); vi.spyOn(registry, "listSandboxes").mockReturnValue({ sandboxes: [{ name: "openclaw" }] }); + const permissiveRuntime = requireDist( + "./permissive-runtime.js", + ) as typeof import("./permissive-runtime.js"); const directSandboxUnavailableError = new Error( "No running direct OpenShell sandbox container found for 'openclaw' (driver: docker). Expected a running container named openshell-openclaw or openshell-openclaw-*. Is the sandbox running?", ); @@ -257,6 +266,19 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { : ""; }); const auditSpy = vi.spyOn(audit, "appendAuditEntry").mockImplementation(() => undefined); + const cleanupTempDirSpy = vi.spyOn(tempFiles, "cleanupTempDir"); + if (options.failStateSave) { + const buildRuntimePermissivePolicy = permissiveRuntime.buildRuntimePermissivePolicy; + vi.spyOn(permissiveRuntime, "buildRuntimePermissivePolicy").mockImplementation( + (basePath, deps) => { + const runtimePolicy = buildRuntimePermissivePolicy(basePath, deps); + fs.mkdirSync(path.join(tmpDir, ".nemoclaw", "state", "shields-openclaw.json"), { + recursive: true, + }); + return runtimePolicy; + }, + ); + } const shields = requireDist(shieldsModulePath); logSpy.mockClear(); @@ -265,6 +287,7 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { return { applyShieldsPolicySnapshot: shields.applyShieldsPolicySnapshot, auditSpy, + cleanupTempDirSpy, errorSpy, logSpy, policySetBodies, @@ -369,6 +392,63 @@ describe("shields command flow", () => { expect(applied.network_policies).not.toHaveProperty("restrictive_baseline"); }); + it("cleans the staged managed MCP policy when timer startup fails", () => { + const alpha = managedMcpPolicy("alpha"); + const harness = createHarness({ + fork: () => { + throw new Error("timer startup failed"); + }, + livePolicy: YAML.stringify({ + version: 1, + network_policies: { [alpha.key]: alpha.networkPolicy }, + }), + sandboxEntry: managedMcpSandbox([alpha]), + }); + + expect(() => + harness.shieldsDown("openclaw", { + timeout: "5m", + reason: "cleanup coverage", + throwOnError: true, + }), + ).toThrow("Cannot start auto-restore timer: timer startup failed"); + + expect(harness.cleanupTempDirSpy).toHaveBeenCalledWith( + expect.stringContaining("nemoclaw-permissive-runtime"), + "nemoclaw-permissive-runtime", + ); + const stagedPolicyPath = String(harness.cleanupTempDirSpy.mock.calls.at(-1)?.[0]); + expect(fs.existsSync(path.dirname(stagedPolicyPath))).toBe(false); + }); + + it("cleans the staged managed MCP policy when state persistence fails", () => { + const alpha = managedMcpPolicy("alpha"); + const harness = createHarness({ + failStateSave: true, + livePolicy: YAML.stringify({ + version: 1, + network_policies: { [alpha.key]: alpha.networkPolicy }, + }), + sandboxEntry: managedMcpSandbox([alpha]), + }); + + expect(() => + harness.shieldsDown("openclaw", { + timeout: "5m", + reason: "cleanup coverage", + skipTimer: true, + throwOnError: true, + }), + ).toThrow(/EISDIR|directory/i); + + expect(harness.cleanupTempDirSpy).toHaveBeenCalledWith( + expect.stringContaining("nemoclaw-permissive-runtime"), + "nemoclaw-permissive-runtime", + ); + const stagedPolicyPath = String(harness.cleanupTempDirSpy.mock.calls.at(-1)?.[0]); + expect(fs.existsSync(path.dirname(stagedPolicyPath))).toBe(false); + }); + it("timer restore uses persisted MCP ownership after its transition marker clears (#7952)", () => { const alpha = managedMcpPolicy("alpha", "8.8.8.8"); const beta = managedMcpPolicy("beta", "1.1.1.1"); @@ -520,9 +600,9 @@ describe("shields command flow", () => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); const snapshotPath = path.join(stateDir, "policy-snapshot-many-managed-keys.yaml"); const policies = Array.from({ length: 257 }, (_, index) => managedMcpPolicy(`server${index}`)); - const keys = policies.map(({ server }) => `mcp_bridge_${server}`); + const keys = policies.map(({ key }) => key); const networkPolicies = Object.fromEntries( - policies.map(({ networkPolicy, server }) => [`mcp_bridge_${server}`, networkPolicy]), + policies.map(({ key, networkPolicy }) => [key, networkPolicy]), ); fs.mkdirSync(stateDir, { recursive: true }); fs.writeFileSync(snapshotPath, YAML.stringify({ network_policies: networkPolicies })); diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index 9f49f47f97a..df94f265fbf 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -548,6 +548,21 @@ describe("shields — unit logic", () => { return readFileWithUnreadableRegistry(originalReadFileSync, file, options) as never; }); vi.spyOn(process, "kill").mockImplementation(routeProcessKill); + const originalRmSync = fs.rmSync.bind(fs); + let appliedPolicy = ""; + vi.spyOn(fs, "rmSync").mockImplementation((target, options) => { + const cleanupDir = String(target); + if ( + path.basename(cleanupDir).startsWith("nemoclaw-permissive-runtime-") && + fs.existsSync(cleanupDir) + ) { + const policyFile = fs.readdirSync(cleanupDir).find((name) => name.endsWith(".yaml")); + if (policyFile) { + appliedPolicy = originalReadFileSync(path.join(cleanupDir, policyFile), "utf-8"); + } + } + originalRmSync(target, options); + }); const { applyShieldsPolicySnapshot } = await loadShieldsModule(); const result = applyShieldsPolicySnapshot(sandboxName, snapshotPath, { @@ -559,14 +574,8 @@ describe("shields — unit logic", () => { expect(result.managedMcpOmissions).toEqual([ expect.objectContaining({ reason: expect.stringMatching(/Cannot read config file:/) }), ]); - const { composeDeadlineManagedMcpPolicies } = await import("./mcp-policy-transition"); - const composition = composeDeadlineManagedMcpPolicies( - fs.readFileSync(snapshotPath, "utf-8"), - [], - ["mcp_bridge_alpha"], - ); - expect(composition.yaml).toContain("restrictive_baseline"); - expect(composition.yaml).not.toContain("mcp_bridge_alpha"); + expect(appliedPolicy).toContain("restrictive_baseline"); + expect(appliedPolicy).not.toContain("mcp_bridge_alpha"); }); it("shieldsStatus warns and stays DOWN when inline recovery fails", async () => { diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index cc77232a00d..22b4b413b02 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -3166,114 +3166,114 @@ function shieldsDownWithoutHostLock(sandboxName: string, opts: ShieldsDownOpts = const now = new Date().toISOString(); let transition: ShieldsDownTransition | null = null; - // Commit the host-side recovery authority before weakening policy or file - // permissions. If this process is killed later, the detached timer and its - // marker already exist and the persisted state honestly reports shields - // down. A crash can therefore never leave an untracked mutable window. - if (!opts.skipTimer) { - const restoreAt = new Date(Date.now() + timeoutSeconds * 1000); - const processToken = opts.processToken ?? randomBytes(16).toString("hex"); - if (!/^[0-9a-f]{32}$/.test(processToken)) { - throw new Error("Invalid shields-down recovery process token"); - } - const timerScript = path.join(__dirname, "timer.ts"); - const timerScriptJs = timerScript.replace(/\.ts$/, ".js"); - const actualScript = fs.existsSync(timerScriptJs) ? timerScriptJs : timerScript; - transition = { - version: 1, - phase: "preparing", - ownerPid: process.pid, - ownerStartIdentity: - readProcessStartIdentity(process.pid) ?? - (() => { - throw new Error("Cannot identify shields-down owner process"); - })(), - ownerMcpProcessIdentity: - readMcpLockProcessIdentity(process.pid, true) ?? - (() => { - throw new Error("Cannot identify shields-down lifecycle owner process"); - })(), - processToken, - sandboxName, - snapshotPath, - managedMcpPolicyKeys: snapshotManagedMcpPolicyKeys, - }; - const leaseOwnerPid = opts.deferAutoRestoreWhileOwnerAlive ? transition.ownerPid : null; - const leaseOwnerStartIdentity = opts.deferAutoRestoreWhileOwnerAlive - ? transition.ownerStartIdentity - : null; - let timerChild: ReturnType | null = null; + try { + // Commit the host-side recovery authority before weakening policy or file + // permissions. If this process is killed later, the detached timer and its + // marker already exist and the persisted state honestly reports shields + // down. A crash can therefore never leave an untracked mutable window. + if (!opts.skipTimer) { + const restoreAt = new Date(Date.now() + timeoutSeconds * 1000); + const processToken = opts.processToken ?? randomBytes(16).toString("hex"); + if (!/^[0-9a-f]{32}$/.test(processToken)) { + throw new Error("Invalid shields-down recovery process token"); + } + const timerScript = path.join(__dirname, "timer.ts"); + const timerScriptJs = timerScript.replace(/\.ts$/, ".js"); + const actualScript = fs.existsSync(timerScriptJs) ? timerScriptJs : timerScript; + transition = { + version: 1, + phase: "preparing", + ownerPid: process.pid, + ownerStartIdentity: + readProcessStartIdentity(process.pid) ?? + (() => { + throw new Error("Cannot identify shields-down owner process"); + })(), + ownerMcpProcessIdentity: + readMcpLockProcessIdentity(process.pid, true) ?? + (() => { + throw new Error("Cannot identify shields-down lifecycle owner process"); + })(), + processToken, + sandboxName, + snapshotPath, + managedMcpPolicyKeys: snapshotManagedMcpPolicyKeys, + }; + const leaseOwnerPid = opts.deferAutoRestoreWhileOwnerAlive ? transition.ownerPid : null; + const leaseOwnerStartIdentity = opts.deferAutoRestoreWhileOwnerAlive + ? transition.ownerStartIdentity + : null; + let timerChild: ReturnType | null = null; - try { - // Publish the forward-transition ownership marker before authorizing the - // timer. If the timeout expires while this command is still weakening - // policy/config, the timer waits for phase=active or owner death instead - // of racing the forward mutations. - writeShieldsDownTransition(transition, null); - timerChild = fork( - actualScript, - [ + try { + // Publish the forward-transition ownership marker before authorizing the + // timer. If the timeout expires while this command is still weakening + // policy/config, the timer waits for phase=active or owner death instead + // of racing the forward mutations. + writeShieldsDownTransition(transition, null); + timerChild = fork( + actualScript, + [ + sandboxName, + snapshotPath, + restoreAt.toISOString(), + target.configPath, + target.configDir, + processToken, + opts.allowLegacyHermesProtocol === true ? "1" : "0", + leaseOwnerPid === null ? "" : String(leaseOwnerPid), + leaseOwnerStartIdentity ?? "", + ], + { + detached: true, + stdio: ["ignore", "ignore", "ignore", "ipc"], + }, + ); + if (!timerChild.pid) throw new Error("auto-restore timer did not report a process id"); + writeTimerMarkerAtomic(sandboxName, { + pid: timerChild.pid, sandboxName, snapshotPath, - restoreAt.toISOString(), - target.configPath, - target.configDir, + restoreAt: restoreAt.toISOString(), processToken, - opts.allowLegacyHermesProtocol === true ? "1" : "0", - leaseOwnerPid === null ? "" : String(leaseOwnerPid), - leaseOwnerStartIdentity ?? "", - ], - { - detached: true, - stdio: ["ignore", "ignore", "ignore", "ipc"], - }, - ); - if (!timerChild.pid) throw new Error("auto-restore timer did not report a process id"); - writeTimerMarkerAtomic(sandboxName, { - pid: timerChild.pid, - sandboxName, - snapshotPath, - restoreAt: restoreAt.toISOString(), - processToken, - allowLegacyHermesProtocol: opts.allowLegacyHermesProtocol === true, - ...(leaseOwnerPid !== null && leaseOwnerStartIdentity - ? { leaseOwnerPid, leaseOwnerStartIdentity } - : {}), - }); - if (!timerChild.send({ type: "authorize", processToken })) { - throw new Error("auto-restore timer authorization channel closed early"); + allowLegacyHermesProtocol: opts.allowLegacyHermesProtocol === true, + ...(leaseOwnerPid !== null && leaseOwnerStartIdentity + ? { leaseOwnerPid, leaseOwnerStartIdentity } + : {}), + }); + if (!timerChild.send({ type: "authorize", processToken })) { + throw new Error("auto-restore timer authorization channel closed early"); + } + timerChild.disconnect(); + timerChild.unref(); + } catch (err) { + clearTimerMarker(sandboxName); + clearShieldsDownTransition(sandboxName, processToken); + const message = err instanceof Error ? err.message : String(err); + console.error(` Cannot start auto-restore timer: ${message}`); + return failShieldsCommand(`Cannot start auto-restore timer: ${message}`, opts.throwOnError); } - timerChild.disconnect(); - timerChild.unref(); - } catch (err) { - clearTimerMarker(sandboxName); - clearShieldsDownTransition(sandboxName, processToken); - const message = err instanceof Error ? err.message : String(err); - console.error(` Cannot start auto-restore timer: ${message}`); - return failShieldsCommand(`Cannot start auto-restore timer: ${message}`, opts.throwOnError); } - } - try { - saveShieldsState(sandboxName, { - shieldsDown: true, - shieldsDownAt: now, - shieldsDownTimeout: timeoutSeconds, - shieldsDownReason: reason, - shieldsDownPolicy: policyName, - shieldsPolicySnapshotPath: snapshotPath, - shieldsManagedMcpPolicyKeys: snapshotManagedMcpPolicyKeys, - }); - } catch (error) { - if (transition) { - clearShieldsDownTransition(sandboxName, transition.processToken); - killTimer(sandboxName); + try { + saveShieldsState(sandboxName, { + shieldsDown: true, + shieldsDownAt: now, + shieldsDownTimeout: timeoutSeconds, + shieldsDownReason: reason, + shieldsDownPolicy: policyName, + shieldsPolicySnapshotPath: snapshotPath, + shieldsManagedMcpPolicyKeys: snapshotManagedMcpPolicyKeys, + }); + } catch (error) { + if (transition) { + clearShieldsDownTransition(sandboxName, transition.processToken); + killTimer(sandboxName); + } + throw error; } - throw error; - } - console.log(` Applying ${policyName} policy...`); - try { + console.log(` Applying ${policyName} policy...`); run(buildPolicySetCommand(policyFile, sandboxName)); } finally { if (policyFileIsTemp) { diff --git a/src/lib/shields/permissive-runtime.ts b/src/lib/shields/permissive-runtime.ts index ddc53900a76..897ed4d52a9 100644 --- a/src/lib/shields/permissive-runtime.ts +++ b/src/lib/shields/permissive-runtime.ts @@ -193,7 +193,7 @@ export interface ManagedMcpRuntimePolicyDeps { * shields-down window. */ export function buildRuntimeManagedMcpPolicy( - basePolicyPath: string, + _basePolicyPath: string, deps: ManagedMcpRuntimePolicyDeps, ): string { const snapshotManagedPolicyKeys = deps.snapshotManagedPolicyKeys ?? []; @@ -240,7 +240,7 @@ export interface DeadlineManagedMcpRuntimePolicy { } export function buildDeadlineRuntimeManagedMcpPolicy( - basePolicyPath: string, + _basePolicyPath: string, deps: ManagedMcpRuntimePolicyDeps, ): DeadlineManagedMcpRuntimePolicy { const baseYaml = deps.readBasePolicy(); diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.ts index 440fb3909a0..07035a09aad 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.ts @@ -24,6 +24,8 @@ import { mcpLifecycleLockPathExistsSync, readMcpLifecycleLockObservation, readMcpLifecycleLockObservationSync, + reclaimStaleMcpLifecycleLockGeneration, + reclaimStaleMcpLifecycleLockGenerationSync, safelyReleaseMcpLifecycleLock, safelyReleaseMcpLifecycleLockSync, writeMcpLifecycleLockCandidateAndLink, @@ -203,6 +205,100 @@ function classifyObservedMcpLifecycleLock( ); } +function isValidMainOwnerForSandbox(observation: LockObservation, sandboxName: string): boolean { + return observation.owner?.sandboxName === sandboxName; +} + +async function tryReapStaleMainLock( + lockPath: string, + sandboxName: string, + stateDir: string, + corruptLockGraceMs: number, + corruptTracker: CorruptGenerationTracker, +): Promise { + const containmentPath = committedContainmentPath(lockPath); + const deadlinePath = `${lockPath}.deadline`; + if ( + (await mcpLifecycleLockPathExists(containmentPath)) || + (await mcpLifecycleLockPathExists(deadlinePath)) + ) { + return false; + } + + const takeoverToken = readShieldsTimerTakeoverToken(sandboxName, stateDir); + const reaperPath = `${lockPath}.reaper`; + const reaperToken = crypto.randomUUID(); + const reaperOwner = createMcpLifecycleLockOwner(sandboxName, reaperToken, takeoverToken); + if (!(await writeMcpLifecycleLockCandidateAndLink(reaperPath, reaperOwner))) return false; + + try { + if ( + (await mcpLifecycleLockPathExists(containmentPath)) || + (await mcpLifecycleLockPathExists(deadlinePath)) || + readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken + ) { + return false; + } + const latest = await readMcpLifecycleLockObservation(lockPath); + if (!latest) return true; + if ( + !isValidMainOwnerForSandbox(latest, sandboxName) || + classifyObservedMcpLifecycleLock(latest, sandboxName, corruptLockGraceMs, corruptTracker) !== + "stale" + ) { + return false; + } + return reclaimStaleMcpLifecycleLockGeneration(lockPath, latest); + } finally { + await safelyReleaseMcpLifecycleLock(reaperPath, reaperToken); + } +} + +function tryReapStaleMainLockSync( + lockPath: string, + sandboxName: string, + stateDir: string, + corruptLockGraceMs: number, + corruptTracker: CorruptGenerationTracker, +): boolean { + const containmentPath = committedContainmentPath(lockPath); + const deadlinePath = `${lockPath}.deadline`; + if ( + mcpLifecycleLockPathExistsSync(containmentPath) || + mcpLifecycleLockPathExistsSync(deadlinePath) + ) { + return false; + } + + const takeoverToken = readShieldsTimerTakeoverToken(sandboxName, stateDir); + const reaperPath = `${lockPath}.reaper`; + const reaperToken = crypto.randomUUID(); + const reaperOwner = createMcpLifecycleLockOwner(sandboxName, reaperToken, takeoverToken); + if (!writeMcpLifecycleLockCandidateAndLinkSync(reaperPath, reaperOwner)) return false; + + try { + if ( + mcpLifecycleLockPathExistsSync(containmentPath) || + mcpLifecycleLockPathExistsSync(deadlinePath) || + readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken + ) { + return false; + } + const latest = readMcpLifecycleLockObservationSync(lockPath); + if (!latest) return true; + if ( + !isValidMainOwnerForSandbox(latest, sandboxName) || + classifyObservedMcpLifecycleLock(latest, sandboxName, corruptLockGraceMs, corruptTracker) !== + "stale" + ) { + return false; + } + return reclaimStaleMcpLifecycleLockGenerationSync(lockPath, latest); + } finally { + safelyReleaseMcpLifecycleLockSync(reaperPath, reaperToken); + } +} + async function acquireMcpLifecycleLock( sandboxName: string, options: McpLifecycleLockOptions, @@ -328,6 +424,16 @@ async function acquireMcpLifecycleLock( corruptMainTracker, ) === "stale" ) { + if (isValidMainOwnerForSandbox(observation, sandboxName)) { + await tryReapStaleMainLock( + lockPath, + sandboxName, + stateDir, + corruptLockGraceMs, + corruptMainTracker, + ); + continue; + } ensurePermanentContainmentForStaleGenerationSync( lockPath, sandboxName, @@ -463,6 +569,16 @@ function acquireMcpLifecycleLockSync( corruptMainTracker, ) === "stale" ) { + if (isValidMainOwnerForSandbox(observation, sandboxName)) { + tryReapStaleMainLockSync( + lockPath, + sandboxName, + options.stateDir, + corruptLockGraceMs, + corruptMainTracker, + ); + continue; + } ensurePermanentContainmentForStaleGenerationSync( lockPath, sandboxName, diff --git a/test/config-set-nested-ssrf.test.ts b/test/config-set-nested-ssrf.test.ts index 14325ce61e6..7bec805e8fe 100644 --- a/test/config-set-nested-ssrf.test.ts +++ b/test/config-set-nested-ssrf.test.ts @@ -45,6 +45,8 @@ function installMockPrivilegedExec( exports: { // Transition-lock behavior has dedicated coverage. Keep this SSRF suite // independent from host process-identity discovery while it mocks ps. + // configSet holds the lifecycle lock before entering this boundary. + isMcpLifecycleLockHeld: () => true, withTimerBoundShieldsMutationLock: ( _sandboxName: string, _command: string, diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts index 72c7bbcd2ff..743a3aea3b0 100644 --- a/test/mcp-lifecycle-lock.test.ts +++ b/test/mcp-lifecycle-lock.test.ts @@ -306,7 +306,7 @@ const releasePath = process.argv[3]; children.delete(child); }); - it("permanently contains an atomic lock left by a dead owner", async () => { + it("recovers an atomic lock left by a dead owner", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); const containmentPath = `${lockPath}.containment`; fs.mkdirSync(path.dirname(lockPath), { recursive: true }); @@ -324,11 +324,89 @@ const releasePath = process.argv[3]; })}\n`, ); - await expect( - lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), - ).rejects.toThrow("Sandbox mutation containment is active"); - expect(fs.existsSync(lockPath)).toBe(true); - expect(fs.existsSync(containmentPath)).toBe(true); + let entered = false; + await lifecycleLock.withMcpLifecycleLock( + "alpha", + () => { + entered = true; + }, + options(), + ); + expect(entered).toBe(true); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(containmentPath)).toBe(false); + }); + + it("recovers an atomic lock left by a dead owner for synchronous callers", () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + const containmentPath = `${lockPath}.containment`; + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-process", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "stale-sync-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + + expect(lifecycleLock.withMcpLifecycleLockSync("alpha", () => "acquired", options())).toBe( + "acquired", + ); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(containmentPath)).toBe(false); + }); + + it("preserves a replacement main lock published during stale recovery", async () => { + const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: "alpha", + pid: 2_147_483_647, + processIdentity: "dead-process", + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "observed-stale-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + const replacement = { + version: 1, + sandboxName: "alpha", + pid: process.pid, + processIdentity: currentProcessIdentity, + hostIdentity: currentHostIdentity, + pidNamespaceIdentity: currentPidNamespaceIdentity, + token: "replacement-main-token", + acquiredAt: new Date().toISOString(), + }; + const rename = fs.promises.rename.bind(fs.promises); + let injectedReplacement = false; + const renameSpy = vi.spyOn(fs.promises, "rename").mockImplementation(async (from, to) => { + if (!injectedReplacement && String(from) === lockPath) { + injectedReplacement = true; + fs.unlinkSync(lockPath); + fs.writeFileSync(lockPath, `${JSON.stringify(replacement)}\n`); + } + return rename(from, to); + }); + + try { + await expect( + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 50 })), + ).rejects.toThrow("Timed out waiting for the sandbox mutation lock"); + } finally { + renameSpy.mockRestore(); + } + expect(JSON.parse(fs.readFileSync(lockPath, "utf8")).token).toBe("replacement-main-token"); }); it("permanently contains a stale deadline generation before an ordinary mutation", async () => { @@ -588,7 +666,7 @@ const releasePath = process.argv[3]; }); it.skipIf(currentProcessIdentity === null)( - "permanently contains a recycled PID because prior descendants are unknown", + "recovers a recycled PID after confirming a fresh process-start mismatch", async () => { const lockPath = lifecycleLock.getMcpLifecycleLockPath("alpha", stateDir); const containmentPath = `${lockPath}.containment`; @@ -608,10 +686,10 @@ const releasePath = process.argv[3]; ); await expect( - lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options({ timeoutMs: 40 })), - ).rejects.toThrow("Sandbox mutation containment is active"); - expect(fs.existsSync(lockPath)).toBe(true); - expect(fs.existsSync(containmentPath)).toBe(true); + lifecycleLock.withMcpLifecycleLock("alpha", () => undefined, options()), + ).resolves.toBeUndefined(); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(containmentPath)).toBe(false); }, ); From dc6de570f5397b08aa538256b29ed479c4aaa81f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 04:28:31 -0400 Subject: [PATCH 10/25] test(shields): linearize recovery assertions Signed-off-by: Julie Yaunches --- src/lib/shields/flow.test.ts | 26 +++++++++++++------------- src/lib/shields/index.test.ts | 28 +++++++++++++++++++--------- test/mcp-lifecycle-lock.test.ts | 9 +++++---- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 2057242ac04..780d9e92aff 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -71,9 +71,7 @@ function managedMcpPolicy(server: string, address = "8.8.8.8") { [address], ); const entries = Object.entries(YAML.parse(content).network_policies as Record); - if (entries.length !== 1) { - throw new Error(`Expected one rendered MCP policy for ${server}, found ${entries.length}`); - } + expect(entries, `rendered MCP policies for ${server}`).toHaveLength(1); const [key, networkPolicy] = entries[0]!; return { content, key, networkPolicy, server }; } @@ -267,18 +265,20 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { }); const auditSpy = vi.spyOn(audit, "appendAuditEntry").mockImplementation(() => undefined); const cleanupTempDirSpy = vi.spyOn(tempFiles, "cleanupTempDir"); - if (options.failStateSave) { - const buildRuntimePermissivePolicy = permissiveRuntime.buildRuntimePermissivePolicy; - vi.spyOn(permissiveRuntime, "buildRuntimePermissivePolicy").mockImplementation( - (basePath, deps) => { - const runtimePolicy = buildRuntimePermissivePolicy(basePath, deps); + const prepareStateSaveFailure = options.failStateSave + ? () => fs.mkdirSync(path.join(tmpDir, ".nemoclaw", "state", "shields-openclaw.json"), { recursive: true, - }); - return runtimePolicy; - }, - ); - } + }) + : () => undefined; + const buildRuntimePermissivePolicy = permissiveRuntime.buildRuntimePermissivePolicy; + vi.spyOn(permissiveRuntime, "buildRuntimePermissivePolicy").mockImplementation( + (basePath, deps) => { + const runtimePolicy = buildRuntimePermissivePolicy(basePath, deps); + prepareStateSaveFailure(); + return runtimePolicy; + }, + ); const shields = requireDist(shieldsModulePath); logSpy.mockClear(); diff --git a/src/lib/shields/index.test.ts b/src/lib/shields/index.test.ts index df94f265fbf..f3668784086 100644 --- a/src/lib/shields/index.test.ts +++ b/src/lib/shields/index.test.ts @@ -132,6 +132,23 @@ function routeProcessKill(pid: number, signal?: string | number): true { return (processActions.get(`${pid}:${signal}`) ?? reportProcessRunning)(); } +function readRuntimePolicyBeforeCleanup( + cleanupDir: string, + readFile: typeof fs.readFileSync, +): string | null { + switch ( + path.basename(cleanupDir).startsWith("nemoclaw-permissive-runtime-") && + fs.existsSync(cleanupDir) + ) { + case false: + return null; + case true: { + const policyFile = fs.readdirSync(cleanupDir).find((name) => name.endsWith(".yaml")); + return policyFile ? readFile(path.join(cleanupDir, policyFile), "utf-8") : null; + } + } +} + beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-test-")); vi.stubEnv("HOME", tmpDir); @@ -552,15 +569,8 @@ describe("shields — unit logic", () => { let appliedPolicy = ""; vi.spyOn(fs, "rmSync").mockImplementation((target, options) => { const cleanupDir = String(target); - if ( - path.basename(cleanupDir).startsWith("nemoclaw-permissive-runtime-") && - fs.existsSync(cleanupDir) - ) { - const policyFile = fs.readdirSync(cleanupDir).find((name) => name.endsWith(".yaml")); - if (policyFile) { - appliedPolicy = originalReadFileSync(path.join(cleanupDir, policyFile), "utf-8"); - } - } + appliedPolicy = + readRuntimePolicyBeforeCleanup(cleanupDir, originalReadFileSync) ?? appliedPolicy; originalRmSync(target, options); }); const { applyShieldsPolicySnapshot } = await loadShieldsModule(); diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts index 743a3aea3b0..729115af6fe 100644 --- a/test/mcp-lifecycle-lock.test.ts +++ b/test/mcp-lifecycle-lock.test.ts @@ -391,10 +391,11 @@ const releasePath = process.argv[3]; const rename = fs.promises.rename.bind(fs.promises); let injectedReplacement = false; const renameSpy = vi.spyOn(fs.promises, "rename").mockImplementation(async (from, to) => { - if (!injectedReplacement && String(from) === lockPath) { - injectedReplacement = true; - fs.unlinkSync(lockPath); - fs.writeFileSync(lockPath, `${JSON.stringify(replacement)}\n`); + switch (!injectedReplacement && String(from) === lockPath) { + case true: + injectedReplacement = true; + fs.unlinkSync(lockPath); + fs.writeFileSync(lockPath, `${JSON.stringify(replacement)}\n`); } return rename(from, to); }); From 719023feb13f433b2ef32131dff92f41dfa4b28f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 04:44:56 -0400 Subject: [PATCH 11/25] docs(shields): clarify containment recovery Signed-off-by: Julie Yaunches --- docs/manage-sandboxes/runtime-controls.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 3fb3f2211fb..500fc67976e 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -124,10 +124,10 @@ A surviving server keeps its recorded endpoint and address pins while its policy When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window while the deadline gate stays closed. If the command cannot complete and commit restoration within that window, NemoClaw converts the deadline gate into durable containment. NemoClaw commits the containment record before it returns the failure, so new mutations remain blocked until operator resolution. -For an ordinary lifecycle lock, NemoClaw reclaims only an exact, structurally valid stale generation for the same sandbox. -It rechecks the generation under an exclusive stale-lock reaper gate before removal. -An expired deadline owner or interrupted reaper still records durable containment because surviving descendants cannot be ruled out. -NemoClaw fails closed for corrupt, non-regular, wrong-sandbox, ambiguous, foreign, and live identities instead of deleting those generations. +For an ordinary lifecycle lock, NemoClaw reclaims only an exact, structurally valid stale generation for your sandbox. +Before removal, NemoClaw rechecks the generation under an exclusive stale-lock reaper gate. +If a deadline owner expires or a reaper is interrupted, NemoClaw records durable containment because it cannot rule out surviving descendants. +NemoClaw fails closed for corrupt, non-regular, wrong-sandbox, ambiguous, foreign, and live identities, and it leaves those generations in place. Stop all NemoClaw processes for that sandbox, then follow the exact lock-generation recovery guidance in the reported error or audit entry. Do not remove a recorded lifecycle lock while any NemoClaw process for that sandbox is running. From 4f124e64eb1db8d8d4c99ef2d2e3c2df1dda030f Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 05:00:29 -0400 Subject: [PATCH 12/25] test(shields): allow coverage boundary runtime Signed-off-by: Julie Yaunches --- src/lib/shields/flow.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 780d9e92aff..b398ac80885 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -596,7 +596,7 @@ describe("shields command flow", () => { ); }); - it("loads 257 managed keys recorded by Shields down (#7952)", () => { + it("loads 257 managed keys recorded by Shields down (#7952)", { timeout: 15_000 }, () => { const stateDir = path.join(tmpDir, ".nemoclaw", "state"); const snapshotPath = path.join(stateDir, "policy-snapshot-many-managed-keys.yaml"); const policies = Array.from({ length: 257 }, (_, index) => managedMcpPolicy(`server${index}`)); From 948ed64e3f97eee4a7b705fcb868f711e17f813b Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 05:33:52 -0400 Subject: [PATCH 13/25] test(state): attribute lifecycle lock coverage Signed-off-by: Julie Yaunches --- test/mcp-lifecycle-lock.test.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts index 729115af6fe..ed92b0f7c4b 100644 --- a/test/mcp-lifecycle-lock.test.ts +++ b/test/mcp-lifecycle-lock.test.ts @@ -10,13 +10,16 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as lifecycleLock from "../src/lib/state/mcp-lifecycle-lock"; import "./helpers/mcp-lifecycle-lock-properties"; -type LifecycleLockModule = typeof import("../src/lib/state/mcp-lifecycle-lock"); - const requireDist = createRequire(import.meta.url); const lockModulePath = requireDist.resolve("../src/lib/state/mcp-lifecycle-lock.js"); -const lifecycleLock = requireDist(lockModulePath) as LifecycleLockModule; +// Keep one CommonJS instance for the macOS probe spy. Behavior tests use the +// static source import so Vitest attributes their coverage to the split modules. +const requiredLifecycleLock = requireDist( + lockModulePath, +) as typeof import("../src/lib/state/mcp-lifecycle-lock"); const currentProcessIdentity = lifecycleLock.readMcpLockProcessIdentity(process.pid); const currentHostIdentity = lifecycleLock.readMcpLockHostIdentity(); const currentPidNamespaceIdentity = lifecycleLock.readMcpLockPidNamespaceIdentity(); @@ -107,7 +110,7 @@ describe("MCP lifecycle lock", () => { process.env.OPENSHELL_GATEWAY = "nemoclaw-19080"; try { - expect(lifecycleLock.readMcpLockProcessIdentity(4242, true)).toBe( + expect(requiredLifecycleLock.readMcpLockProcessIdentity(4242, true)).toBe( "darwin:Mon Jun 30 12:00:00 2026", ); const options = spawnSync.mock.calls[0]?.[2] as { env?: NodeJS.ProcessEnv }; From c650528e152659cc7e8a5076e0dadab42378ec59 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 05:54:46 -0400 Subject: [PATCH 14/25] test(state): cover lifecycle lock acquisition Signed-off-by: Julie Yaunches --- .../mcp-lifecycle-lock-acquisition.test.ts | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 src/lib/state/mcp-lifecycle-lock-acquisition.test.ts diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts new file mode 100644 index 00000000000..d55c9595cd8 --- /dev/null +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { + beginCommittedMcpLifecycleContainmentSync, + isMcpLifecycleLockHeld, + withMcpLifecycleDeadlineFence, + withMcpLifecycleDeadlineFenceSync, + withMcpLifecycleLockSync, +} from "./mcp-lifecycle-lock-acquisition"; +import { createMcpLifecycleLockOwner } from "./mcp-lifecycle-lock-identity"; +import { getMcpLifecycleLockPath } from "./mcp-lifecycle-lock-storage"; + +const SANDBOX_NAME = "alpha"; +let stateDir: string; + +function options() { + return { + stateDir, + pollIntervalMs: 1, + timeoutMs: 20, + corruptLockGraceMs: 1, + }; +} + +function writeTimerMarker(processToken: string): void { + fs.writeFileSync( + path.join(stateDir, `shields-timer-${SANDBOX_NAME}.json`), + JSON.stringify({ + pid: process.pid, + sandboxName: SANDBOX_NAME, + snapshotPath: path.join(stateDir, "snapshot.yaml"), + restoreAt: new Date(Date.now() + 60_000).toISOString(), + processToken, + }), + ); +} + +beforeEach(() => { + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-lock-acquisition-")); +}); + +afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(stateDir, { recursive: true, force: true }); +}); + +describe("MCP lifecycle lock acquisition", () => { + it("releases a synchronous lock after nested work completes", () => { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const events: string[] = []; + + const result = withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + expect(isMcpLifecycleLockHeld(SANDBOX_NAME, stateDir)).toBe(true); + events.push("outer"); + return withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + expect(isMcpLifecycleLockHeld(SANDBOX_NAME, stateDir)).toBe(true); + events.push("nested"); + return "complete"; + }, + options(), + ); + }, + options(), + ); + + expect(result).toBe("complete"); + expect(events).toEqual(["outer", "nested"]); + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it("keeps synchronous deadline recovery reentrant until cleanup completes", () => { + const processToken = "a".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + + const result = withMcpLifecycleDeadlineFenceSync( + SANDBOX_NAME, + processToken, + () => { + expect(isMcpLifecycleLockHeld(SANDBOX_NAME, stateDir)).toBe(true); + return withMcpLifecycleLockSync(SANDBOX_NAME, () => "restored", options()); + }, + options(), + ); + + expect(result).toBe("restored"); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + }); + + it("blocks synchronous mutation while committed containment is active", () => { + const processToken = "b".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + beginCommittedMcpLifecycleContainmentSync( + SANDBOX_NAME, + processToken, + "test containment", + stateDir, + ); + + expect(() => withMcpLifecycleLockSync(SANDBOX_NAME, () => "entered", options())).toThrow( + "Sandbox mutation containment is active", + ); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("keeps an active deadline gate closed when containment reporting fails", async () => { + const processToken = "c".repeat(32); + const replacementToken = "d".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const deadlinePath = `${lockPath}.deadline`; + writeTimerMarker(processToken); + fs.mkdirSync(path.dirname(deadlinePath), { recursive: true }); + fs.writeFileSync( + deadlinePath, + `${JSON.stringify( + createMcpLifecycleLockOwner(SANDBOX_NAME, "active-deadline-owner", processToken), + )}\n`, + ); + const onContainment = vi.fn(() => { + writeTimerMarker(replacementToken); + throw new Error("audit unavailable"); + }); + + await expect( + withMcpLifecycleDeadlineFence(SANDBOX_NAME, processToken, () => "entered", { + ...options(), + onContainment, + }), + ).rejects.toThrow("Auto-restore authority changed"); + expect(onContainment).toHaveBeenCalledOnce(); + expect(fs.existsSync(deadlinePath)).toBe(true); + }); +}); From db0494e068859f98b7874355984b5c673f7b2beb Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 05:58:09 -0400 Subject: [PATCH 15/25] test(state): clarify deadline recovery assertion Signed-off-by: Julie Yaunches --- src/lib/state/mcp-lifecycle-lock-acquisition.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts index d55c9595cd8..3a696c54385 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts @@ -79,7 +79,7 @@ describe("MCP lifecycle lock acquisition", () => { expect(fs.existsSync(lockPath)).toBe(false); }); - it("keeps synchronous deadline recovery reentrant until cleanup completes", () => { + it("allows a nested synchronous lock during deadline recovery and releases the main lock and deadline gate afterward", () => { const processToken = "a".repeat(32); const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); writeTimerMarker(processToken); From f31b034f8dc5bcd60c76aa4712bcb4dc74ee6f32 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 06:15:38 -0400 Subject: [PATCH 16/25] test(shields): avoid marker check-use race Signed-off-by: Julie Yaunches --- src/lib/shields/flow.test.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index b398ac80885..7ce2dc883c0 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -990,11 +990,16 @@ describe("shields command flow", () => { }); expect(renameSpy).toHaveBeenCalledWith(expect.stringContaining(".tmp"), markerPath); - expect(fs.lstatSync(markerPath).isSymbolicLink()).toBe(false); - expect(JSON.parse(fs.readFileSync(markerPath, "utf-8"))).toMatchObject({ - pid: 4242, - sandboxName: "openclaw", - }); + const markerFd = fs.openSync(markerPath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + expect(fs.fstatSync(markerFd).isFile()).toBe(true); + expect(JSON.parse(fs.readFileSync(markerFd, "utf-8"))).toMatchObject({ + pid: 4242, + sandboxName: "openclaw", + }); + } finally { + fs.closeSync(markerFd); + } expect(fs.readFileSync(markerTargetPath, "utf-8")).toBe(markerTarget); }, ); From 21d87864ffdd2f11f96b911124e20b5d8e5da01c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 07:04:33 -0400 Subject: [PATCH 17/25] chore(ci): retry trusted e2e gate Signed-off-by: Julie Yaunches From f65c804ea70cc44add95ffea26666c8b095da0b1 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 07:17:20 -0400 Subject: [PATCH 18/25] fix(shields): clear stale restore warnings Signed-off-by: Julie Yaunches --- src/lib/shields/timer.test.ts | 29 +++++++++++++++++++++++++++-- src/lib/shields/timer.ts | 10 +++++----- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index e655d0816b3..abfa35a06e4 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -9,7 +9,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { getMcpLifecycleLockPath, withMcpLifecycleLock } from "../state/mcp-lifecycle-lock"; const shieldsIndexMock = vi.hoisted(() => ({ - applyShieldsPolicySnapshot: vi.fn(() => ({ status: 0 })), + applyShieldsPolicySnapshot: vi.fn( + (): { + status: number; + managedMcpOmissions?: Array<{ server: string; reason: string }>; + } => ({ status: 0 }), + ), completeAutoRestoreTransition: vi.fn(() => true), lockAgentConfig: vi.fn() as unknown, prepareAutoRestoreTransitionTakeover: vi.fn(), @@ -357,7 +362,14 @@ describe("shields timer authorization", () => { shieldsIndexMock.applyShieldsPolicySnapshot.mockImplementationOnce(() => { expect(fs.existsSync(sandboxMutationLockPath)).toBe(true); expect(fs.existsSync(`${sandboxMutationLockPath}.deadline`)).toBe(true); - return { status: 17 }; + return { + status: 17, + managedMcpOmissions: [{ server: "beta", reason: "incomplete add" }], + }; + }); + shieldsIndexMock.applyShieldsPolicySnapshot.mockReturnValueOnce({ + status: 0, + managedMcpOmissions: [], }); const args = timer.parseTimerArgs([ sandboxName, @@ -378,6 +390,19 @@ describe("shields timer authorization", () => { expect(shieldsIndexMock.applyShieldsPolicySnapshot).toHaveBeenCalledTimes(2); expect(fs.existsSync(markerPath)).toBe(false); expect(fs.existsSync(sandboxMutationLockPath)).toBe(false); + const audits = fs + .readFileSync(path.join(stateDir, "shields-audit.jsonl"), "utf-8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + const successAudits = audits.filter((audit) => audit.action === "shields_auto_restore"); + expect(successAudits).toEqual([ + expect.objectContaining({ + action: "shields_auto_restore", + sandbox: sandboxName, + }), + ]); + expect(successAudits[0]).not.toHaveProperty("warning"); }); it("does not restore or rewrite state when marker pid mismatches", async () => { diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index 5632d8f59b0..d81e844bdb5 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -304,11 +304,11 @@ async function runRestoreTimer( deadlineAuthoritative: true, }); const status = typeof result.status === "number" ? result.status : 1; - if (result.managedMcpOmissions?.length) { - managedMcpWarning = `Auto-restore omitted ${String( - result.managedMcpOmissions.length, - )} unproven managed MCP policy entries`; - } + managedMcpWarning = result.managedMcpOmissions?.length + ? `Auto-restore omitted ${String( + result.managedMcpOmissions.length, + )} unproven managed MCP policy entries` + : undefined; if (status !== 0) { appendAudit({ From f731be0ce921c01ad011c985a9f93c3fd7eba4f2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 07:21:59 -0400 Subject: [PATCH 19/25] fix(shields): timestamp restore retries accurately Signed-off-by: Julie Yaunches --- src/lib/shields/timer.test.ts | 11 ++++++++++- src/lib/shields/timer.ts | 17 ++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/lib/shields/timer.test.ts b/src/lib/shields/timer.test.ts index abfa35a06e4..b5711037fd4 100644 --- a/src/lib/shields/timer.test.ts +++ b/src/lib/shields/timer.test.ts @@ -337,7 +337,7 @@ describe("shields timer authorization", () => { } }); - it("retains a dead rebuild owner's timer and retries a transient restore failure", async () => { + it("audits a successful restore retry without stale MCP warnings or timestamps", async () => { const timer = await import("./timer"); const stateDir = path.join(tmpHome, ".nemoclaw", "state"); fs.mkdirSync(stateDir, { recursive: true }); @@ -403,6 +403,15 @@ describe("shields timer authorization", () => { }), ]); expect(successAudits[0]).not.toHaveProperty("warning"); + const failedAudit = audits.find( + (audit) => + audit.action === "shields_up_failed" && + audit.error === "Policy restore exited with status 17", + ); + expect(failedAudit).toEqual(expect.objectContaining({ timestamp: expect.any(String) })); + expect(Date.parse(successAudits[0].timestamp)).toBeGreaterThan( + Date.parse(failedAudit.timestamp), + ); }); it("does not restore or rewrite state when marker pid mismatches", async () => { diff --git a/src/lib/shields/timer.ts b/src/lib/shields/timer.ts index d81e844bdb5..bcb31e00901 100644 --- a/src/lib/shields/timer.ts +++ b/src/lib/shields/timer.ts @@ -233,7 +233,6 @@ async function runRestoreTimer( args: TimerArgs, runtimeOptions: TimerRuntimeOptions = {}, ): Promise { - const now = new Date().toISOString(); const retryDelayMs = Number.isFinite(runtimeOptions.retryDelayMs) && (runtimeOptions.retryDelayMs ?? 0) >= 0 ? Math.floor(runtimeOptions.retryDelayMs!) @@ -290,7 +289,7 @@ async function runRestoreTimer( appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", error: "Policy snapshot file missing", }); @@ -314,7 +313,7 @@ async function runRestoreTimer( appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", error: `Policy restore exited with status ${String(status)}`, }); @@ -367,7 +366,7 @@ async function runRestoreTimer( appendAudit({ action: "shields_auto_restore_lock_warning", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", warning: "Missing config directory for auto-restore re-lock verification", lock_verified: false, @@ -400,7 +399,7 @@ async function runRestoreTimer( appendAudit({ action: "shields_auto_restore_lock_warning", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", warning: relock.error ?? "Config re-lock did not re-confirm after settle window", @@ -412,7 +411,7 @@ async function runRestoreTimer( appendAudit({ action: "shields_auto_restore_lock_warning", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", warning: error instanceof Error ? error.message : String(error), lock_verified: false, @@ -450,7 +449,7 @@ async function runRestoreTimer( appendAudit({ action: "shields_auto_restore", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", policy_snapshot: args.snapshotPath, scheduled_restore_at: args.restoreAtIso, @@ -468,7 +467,7 @@ async function runRestoreTimer( appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", error: "Config re-lock verification failed — shields remain DOWN", }); @@ -534,7 +533,7 @@ async function runRestoreTimer( appendAudit({ action: "shields_up_failed", sandbox: args.sandboxName, - timestamp: now, + timestamp: new Date().toISOString(), restored_by: "auto_timer", error: error instanceof Error ? error.message : String(error), }); From 27d1056a0d26606b62afc1084143369093ed7c35 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 08:43:54 -0400 Subject: [PATCH 20/25] fix(shields): keep failed containment closed --- docs/manage-sandboxes/runtime-controls.mdx | 7 +- docs/reference/commands.mdx | 7 +- src/lib/shields/flow.test.ts | 233 ++++++--- src/lib/shields/index.ts | 66 ++- src/lib/shields/timer-bound-lock.ts | 1 + .../mcp-lifecycle-lock-acquisition.test.ts | 484 +++++++++++++++++- .../state/mcp-lifecycle-lock-acquisition.ts | 233 +++++++-- src/lib/state/mcp-lifecycle-lock.ts | 1 + test/mcp-lifecycle-lock.test.ts | 23 +- 9 files changed, 897 insertions(+), 158 deletions(-) diff --git a/docs/manage-sandboxes/runtime-controls.mdx b/docs/manage-sandboxes/runtime-controls.mdx index 500fc67976e..80ac2950acb 100644 --- a/docs/manage-sandboxes/runtime-controls.mdx +++ b/docs/manage-sandboxes/runtime-controls.mdx @@ -123,8 +123,13 @@ A surviving server keeps its recorded endpoint and address pins while its policy When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window while the deadline gate stays closed. If the command cannot complete and commit restoration within that window, NemoClaw converts the deadline gate into durable containment. -NemoClaw commits the containment record before it returns the failure, so new mutations remain blocked until operator resolution. +When the containment write succeeds, NemoClaw commits the record before it returns the failure, so new mutations remain blocked until operator resolution. +If NemoClaw cannot commit that record after its bounded write retry budget, the command returns an operator-resolution error with the last state-directory write failure and keeps every exact deadline or main lifecycle lock generation that it owns. +Correct the reported state-directory write failure, then retry the command once. +After the retained owner exits, the retry records durable containment for the stale timer-bound generation and returns exact-generation operator-resolution instructions. +Complete those instructions before running another sandbox mutation. For an ordinary lifecycle lock, NemoClaw reclaims only an exact, structurally valid stale generation for your sandbox. +NemoClaw records durable containment for a stale main owner bound to a Shields timer instead of reclaiming it as an ordinary stale generation. Before removal, NemoClaw rechecks the generation under an exclusive stale-lock reaper gate. If a deadline owner expires or a reaper is interrupted, NemoClaw records durable containment because it cannot rule out surviving descendants. NemoClaw fails closed for corrupt, non-regular, wrong-sandbox, ambiguous, foreign, and live identities, and it leaves those generations in place. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index ccf5c8044b6..abd5cb5f7f8 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1154,8 +1154,13 @@ The gate blocks new mutations and waits for the recorded live owner to release i NemoClaw does not signal that process because portable process inspection cannot prove that all descendants are contained. When an interactive command takes over an expired timer, NemoClaw makes up to 7 restoration attempts over an additional 30-second completion-grace window while the deadline gate stays closed. If the command cannot complete and commit restoration within that window, NemoClaw converts the deadline gate into durable containment. -NemoClaw commits the containment record before it returns the failure, so new mutations remain blocked until operator resolution. +When the containment write succeeds, NemoClaw commits the record before it returns the failure, so new mutations remain blocked until operator resolution. +If NemoClaw cannot commit that record after its bounded write retry budget, the command returns an operator-resolution error with the last state-directory write failure and keeps every exact deadline or main lifecycle lock generation that it owns. +Correct the reported state-directory write failure, then retry the command once. +After the retained owner exits, the retry records durable containment for the stale timer-bound generation and returns exact-generation operator-resolution instructions. +Complete those instructions before running another sandbox mutation. For an ordinary lifecycle lock, NemoClaw reclaims only an exact, structurally valid stale generation for the same sandbox. +NemoClaw records durable containment for a stale main owner bound to a Shields timer instead of reclaiming it as an ordinary stale generation. It rechecks the generation under an exclusive stale-lock reaper gate before removal. An expired deadline owner or interrupted reaper still records durable containment because surviving descendants cannot be ruled out. NemoClaw fails closed for corrupt, non-regular, wrong-sandbox, ambiguous, foreign, and live identities instead of deleting those generations. diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 7ce2dc883c0..3472ab59325 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -36,6 +36,7 @@ const currentProcessStartIdentity = ( ).readProcessStartIdentity(process.pid); type HarnessOptions = { + beginContainment?: typeof import("../state/mcp-lifecycle-lock.js").beginCommittedMcpLifecycleContainmentSync; directSandboxUnavailable?: boolean; dockerExecFileSync?: (argv: unknown) => string; failOpenClawGuardActions?: Array<"lock" | "unlock">; @@ -121,6 +122,14 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { delete require.cache[requireDist.resolve("../actions/sandbox/mcp-bridge-policy.js")]; delete require.cache[requireDist.resolve("../sandbox/privileged-exec.js")]; delete require.cache[requireDist.resolve("../cli/branding.js")]; + const lifecycleLock = requireDist( + "../state/mcp-lifecycle-lock.js", + ) as typeof import("../state/mcp-lifecycle-lock.js"); + if (options.beginContainment) { + vi.spyOn(lifecycleLock, "beginCommittedMcpLifecycleContainmentSync").mockImplementation( + options.beginContainment, + ); + } const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(console, "warn").mockImplementation(() => undefined); @@ -316,6 +325,55 @@ function expectStagedDriverNeutralRecovery( return output; } +function writeExpiredShieldsFixture( + processToken: string, + reason: string, + ownerState: "dead" | "live", +) { + const liveOwner = ownerState === "live"; + const sandboxName = "openclaw"; + const stateDir = path.join(tmpDir, ".nemoclaw", "state"); + const snapshotPath = path.join(stateDir, `snapshot-${processToken.slice(0, 8)}.yaml`); + const timerMarkerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); + const transitionLockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); + fs.mkdirSync(stateDir, { recursive: true }); + fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}\n"); + fs.writeFileSync( + path.join(stateDir, `shields-${sandboxName}.json`), + JSON.stringify({ + shieldsDown: true, + shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), + shieldsDownTimeout: 60, + shieldsDownReason: reason, + shieldsDownPolicy: "permissive", + shieldsPolicySnapshotPath: snapshotPath, + }), + ); + fs.writeFileSync( + timerMarkerPath, + JSON.stringify({ + pid: liveOwner ? 2_147_483_647 : 4242, + sandboxName, + snapshotPath, + restoreAt: new Date(Date.now() - 60_000).toISOString(), + processToken, + }), + ); + fs.writeFileSync( + transitionLockPath, + JSON.stringify({ + version: 1, + sandboxName, + pid: liveOwner ? process.pid : 4242, + processStartIdentity: liveOwner ? currentProcessStartIdentity : "dead-timer", + command: liveOwner ? "shields down" : "shields auto-restore", + acquiredAtMs: Date.now() - 60_000, + takeoverToken: processToken, + }), + ); + return { stateDir, timerMarkerPath, transitionLockPath }; +} + describe("shields command flow", () => { beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "shields-flow-")); @@ -417,6 +475,7 @@ describe("shields command flow", () => { expect.stringContaining("nemoclaw-permissive-runtime"), "nemoclaw-permissive-runtime", ); + expect(harness.cleanupTempDirSpy).toHaveBeenCalledTimes(1); const stagedPolicyPath = String(harness.cleanupTempDirSpy.mock.calls.at(-1)?.[0]); expect(fs.existsSync(path.dirname(stagedPolicyPath))).toBe(false); }); @@ -445,6 +504,7 @@ describe("shields command flow", () => { expect.stringContaining("nemoclaw-permissive-runtime"), "nemoclaw-permissive-runtime", ); + expect(harness.cleanupTempDirSpy).toHaveBeenCalledTimes(1); const stagedPolicyPath = String(harness.cleanupTempDirSpy.mock.calls.at(-1)?.[0]); expect(fs.existsSync(path.dirname(stagedPolicyPath))).toBe(false); }); @@ -1263,45 +1323,11 @@ describe("shields command flow", () => { const sandboxMutationLockPath = lifecycleLock.getMcpLifecycleLockPath("openclaw"); const containmentPath = `${sandboxMutationLockPath}.containment`; const harness = createHarness(); - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - const lockPath = path.join(stateDir, "shields-transition-lock-openclaw.json"); - const timerMarkerPath = path.join(stateDir, "shields-timer-openclaw.json"); - fs.mkdirSync(stateDir, { recursive: true }); - const snapshotPath = path.join(stateDir, "policy-snapshot-expired.yaml"); - fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}\n"); - fs.writeFileSync( - path.join(stateDir, "shields-openclaw.json"), - JSON.stringify({ - shieldsDown: true, - shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), - shieldsDownTimeout: 60, - shieldsDownReason: "coverage", - shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, - }), - ); - fs.writeFileSync( + const { + stateDir, timerMarkerPath, - JSON.stringify({ - pid: 4242, - sandboxName: "openclaw", - snapshotPath, - restoreAt: new Date(Date.now() - 30_000).toISOString(), - processToken, - }), - ); - fs.writeFileSync( - lockPath, - JSON.stringify({ - version: 1, - sandboxName: "openclaw", - pid: 4242, - processStartIdentity: "dead-timer", - command: "shields auto-restore", - acquiredAtMs: Date.now() - 60_000, - takeoverToken: processToken, - }), - ); + transitionLockPath: lockPath, + } = writeExpiredShieldsFixture(processToken, "coverage", "dead"); vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { const failDeadTimerProbe = () => { const error = new Error("timer is gone") as NodeJS.ErrnoException; @@ -1331,52 +1357,66 @@ describe("shields command flow", () => { ); }); + it("retains the timer-bound lifecycle generation when a caller handles a failed containment write", async () => { + const processToken = "a".repeat(32); + const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); + const mainLockPath = lifecycleLock.getMcpLifecycleLockPath("openclaw"); + const containmentPath = `${mainLockPath}.containment`; + const { timerMarkerPath, transitionLockPath } = writeExpiredShieldsFixture( + processToken, + "containment write failure coverage", + "dead", + ); + vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { + if (`${pid}:${signal}` === "4242:0") { + const error = new Error("timer is gone") as NodeJS.ErrnoException; + error.code = "ESRCH"; + throw error; + } + return true; + }); + const harness = createHarness({ + beginContainment: () => { + throw new Error("state directory is read-only"); + }, + }); + let containmentFailure: unknown; + + const result = await lifecycleLock.withSandboxMutationLock("openclaw", () => { + try { + return harness.shieldsStatus("openclaw"); + } catch (error) { + containmentFailure = error; + return "handled"; + } + }); + + expect(result).toBe("handled"); + expect(containmentFailure).toMatchObject({ + code: "NEMOCLAW_PERMANENT_CONTAINMENT", + }); + expect(String(containmentFailure)).toContain("state directory is read-only"); + expect(fs.existsSync(containmentPath)).toBe(false); + expect(JSON.parse(fs.readFileSync(mainLockPath, "utf8"))).toMatchObject({ + sandboxName: "openclaw", + shieldsTakeoverToken: processToken, + }); + expect(fs.existsSync(timerMarkerPath)).toBe(true); + expect(fs.existsSync(transitionLockPath)).toBe(true); + expect(harness.runSpy).not.toHaveBeenCalledWith( + ["openshell", "policy", "set"], + expect.anything(), + ); + }); + it.skipIf(currentProcessStartIdentity === null)( "bounds live transition takeover before committing durable containment", () => { const sandboxName = "openclaw"; const processToken = "8".repeat(32); - const stateDir = path.join(tmpDir, ".nemoclaw", "state"); - const snapshotPath = path.join(stateDir, "policy-snapshot-takeover-exhausted.yaml"); - const timerMarkerPath = path.join(stateDir, `shields-timer-${sandboxName}.json`); - const transitionLockPath = path.join(stateDir, `shields-transition-lock-${sandboxName}.json`); const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); const containmentPath = `${lifecycleLock.getMcpLifecycleLockPath(sandboxName)}.containment`; - fs.mkdirSync(stateDir, { recursive: true }); - fs.writeFileSync(snapshotPath, "version: 1\nnetwork_policies:\n test: {}\n"); - fs.writeFileSync( - path.join(stateDir, `shields-${sandboxName}.json`), - JSON.stringify({ - shieldsDown: true, - shieldsDownAt: new Date(Date.now() - 120_000).toISOString(), - shieldsDownTimeout: 60, - shieldsDownReason: "takeover exhaustion coverage", - shieldsDownPolicy: "permissive", - shieldsPolicySnapshotPath: snapshotPath, - }), - ); - fs.writeFileSync( - timerMarkerPath, - JSON.stringify({ - pid: 2_147_483_647, - sandboxName, - snapshotPath, - restoreAt: new Date(Date.now() - 60_000).toISOString(), - processToken, - }), - ); - fs.writeFileSync( - transitionLockPath, - JSON.stringify({ - version: 1, - sandboxName, - pid: process.pid, - processStartIdentity: currentProcessStartIdentity, - command: "shields down", - acquiredAtMs: Date.now(), - takeoverToken: processToken, - }), - ); + writeExpiredShieldsFixture(processToken, "takeover exhaustion coverage", "live"); const waitSpy = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); const harness = createHarness(); @@ -1399,4 +1439,43 @@ describe("shields command flow", () => { ); }, ); + + it.skipIf(currentProcessStartIdentity === null)( + "returns after bounded containment commit failures without reopening the deadline gate", + () => { + const sandboxName = "openclaw"; + const processToken = "9".repeat(32); + const lifecycleLock = requireDist("../state/mcp-lifecycle-lock.js"); + const mainLockPath = lifecycleLock.getMcpLifecycleLockPath(sandboxName); + const containmentPath = `${mainLockPath}.containment`; + writeExpiredShieldsFixture(processToken, "containment write failure coverage", "live"); + const waitSpy = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); + let containmentAttempts = 0; + const harness = createHarness({ + beginContainment: () => { + containmentAttempts += 1; + throw new Error("state directory is read-only"); + }, + }); + + expect(() => harness.shieldsStatus(sandboxName)).toThrow( + /Permanent containment could not be committed after 11 attempts: state directory is read-only.*Correct the state-directory write failure/, + ); + + expect(containmentAttempts).toBe(11); + expect(waitSpy.mock.calls.filter((call) => call[3] === 5_000)).toHaveLength(6); + expect(waitSpy.mock.calls.filter((call) => call[3] === 50)).toHaveLength(10); + expect(fs.existsSync(containmentPath)).toBe(false); + expect(fs.existsSync(mainLockPath)).toBe(true); + expect(fs.existsSync(`${mainLockPath}.deadline`)).toBe(true); + expect(harness.auditSpy).toHaveBeenCalledWith( + expect.objectContaining({ + action: "shields_up_failed", + sandbox: sandboxName, + error: + "Permanent containment commit failed; retrying behind the deadline gate: state directory is read-only", + }), + ); + }, + ); }); diff --git a/src/lib/shields/index.ts b/src/lib/shields/index.ts index 22b4b413b02..07783b29244 100644 --- a/src/lib/shields/index.ts +++ b/src/lib/shields/index.ts @@ -76,6 +76,7 @@ const { beginCommittedMcpLifecycleContainmentSync, getMcpLifecycleLockPath, isMcpLifecycleLockHeld, + permanentMcpLifecycleContainmentFailure, readMcpLockProcessIdentity, withMcpLifecycleDeadlineFenceSync, withMcpLifecycleLockSync, @@ -113,6 +114,8 @@ const STATE_DIR = resolveNemoclawStateDir(); const SHIELDS_TRANSITION_POLL_MS = 50; const SHIELDS_TRANSITION_HANDOFF_GRACE_MS = 500; const SHIELDS_TRANSITION_TERMINATE_GRACE_MS = 1000; +const INTERACTIVE_CONTAINMENT_COMMIT_MAX_ATTEMPTS = + Math.floor(SHIELDS_TRANSITION_HANDOFF_GRACE_MS / SHIELDS_TRANSITION_POLL_MS) + 1; const AUTO_RESTORE_COMPLETION_GRACE_MS = 30_000; // Retry on the detached timer's cadence for one additional completion-grace // window before converting the live deadline fence into durable containment. @@ -354,13 +357,20 @@ function waitForShieldsDownForwardCommit( ); if (ownerStatus === "gone") { assertTakeoverAuthority?.(); - persistUnresolvedShieldsContainment( - sandboxName, - processToken, - `Shields recovery owner PID ${String( - observed.ownerPid, - )} exited without descendant-containment proof`, - ); + try { + persistUnresolvedShieldsContainment( + sandboxName, + processToken, + `Shields recovery owner PID ${String( + observed.ownerPid, + )} exited without descendant-containment proof`, + ); + } catch (error) { + throw permanentMcpLifecycleContainmentFailure( + error, + getMcpLifecycleLockPath(sandboxName, STATE_DIR), + ); + } throw new Error( "Shields-down forward owner exited before committing its final mutation; permanent containment requires operator resolution", ); @@ -883,10 +893,11 @@ function failInteractiveAutoRestoreClosed( ): never { const containmentPath = `${getMcpLifecycleLockPath(sandboxName, STATE_DIR)}.containment`; let notifiedError: string | null = null; - // The deadline fence unwinds when this function throws. Keep retrying the - // durable containment commit first so a bounded interactive recovery can - // return without reopening the sandbox mutation gate. - for (;;) { + let lastContainmentError: string | null = null; + // Retry a durable containment commit for one normal transition-handoff + // window. A persistent state-directory failure returns to the operator only + // through the coded failure that keeps the owned lifecycle gates. + for (let attempt = 0; attempt < INTERACTIVE_CONTAINMENT_COMMIT_MAX_ATTEMPTS; attempt += 1) { assertTimerMarkerGeneration(sandboxName, marker); try { persistUnresolvedShieldsContainment( @@ -899,6 +910,7 @@ function failInteractiveAutoRestoreClosed( if (fs.existsSync(containmentPath)) break; assertTimerMarkerGeneration(sandboxName, marker); const containmentError = error instanceof Error ? error.message : String(error); + lastContainmentError = containmentError; if (containmentError !== notifiedError) { appendAuditEntryBestEffort({ action: "shields_up_failed", @@ -910,9 +922,21 @@ function failInteractiveAutoRestoreClosed( }); notifiedError = containmentError; } - Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); + if (attempt + 1 < INTERACTIVE_CONTAINMENT_COMMIT_MAX_ATTEMPTS) { + Atomics.wait(transitionPollBuffer, 0, 0, SHIELDS_TRANSITION_POLL_MS); + } } } + if (!fs.existsSync(containmentPath)) { + throw permanentMcpLifecycleContainmentFailure( + new Error( + `${message}. Permanent containment could not be committed after ${String( + INTERACTIVE_CONTAINMENT_COMMIT_MAX_ATTEMPTS, + )} attempts: ${lastContainmentError ?? "unknown state-directory failure"}. Correct the state-directory write failure and retry the command before running another sandbox mutation`, + ), + getMcpLifecycleLockPath(sandboxName, STATE_DIR), + ); + } throw new Error( `${message}. Permanent sandbox mutation containment requires operator resolution`, ); @@ -1048,6 +1072,7 @@ function withExpiredAutoRestoreDeadlineFence( }, { stateDir: STATE_DIR, + throwOnCommittedContainment: true, onContainment: ({ ownerPid, reason }) => { appendAuditEntryBestEffort({ action: "shields_up_failed", @@ -2494,11 +2519,18 @@ function prepareAutoRestoreTransitionTakeover( ); if (ownerStatus === "gone") { assertTakeoverAuthority?.(); - persistUnresolvedShieldsContainment( - sandboxName, - processToken, - `Shields recovery owner PID ${String(owner.pid)} exited without descendant-containment proof`, - ); + try { + persistUnresolvedShieldsContainment( + sandboxName, + processToken, + `Shields recovery owner PID ${String(owner.pid)} exited without descendant-containment proof`, + ); + } catch (error) { + throw permanentMcpLifecycleContainmentFailure( + error, + getMcpLifecycleLockPath(sandboxName, STATE_DIR), + ); + } throw new Error( "Shields transition owner exited without descendant-containment proof; permanent containment requires operator resolution", ); diff --git a/src/lib/shields/timer-bound-lock.ts b/src/lib/shields/timer-bound-lock.ts index 7a94aa0fe4c..cbce8aefb9a 100644 --- a/src/lib/shields/timer-bound-lock.ts +++ b/src/lib/shields/timer-bound-lock.ts @@ -12,6 +12,7 @@ export { beginCommittedMcpLifecycleContainmentSync, getMcpLifecycleLockPath, isMcpLifecycleLockHeld, + permanentMcpLifecycleContainmentFailure, readMcpLockProcessIdentity, withMcpLifecycleDeadlineFenceSync, withMcpLifecycleLockSync, diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts index 3a696c54385..6264a314376 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts @@ -10,11 +10,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { beginCommittedMcpLifecycleContainmentSync, isMcpLifecycleLockHeld, + permanentMcpLifecycleContainmentFailure, withMcpLifecycleDeadlineFence, withMcpLifecycleDeadlineFenceSync, + withMcpLifecycleLock, withMcpLifecycleLockSync, } from "./mcp-lifecycle-lock-acquisition"; -import { createMcpLifecycleLockOwner } from "./mcp-lifecycle-lock-identity"; +import { + createMcpLifecycleLockOwner, + readMcpLockHostIdentity, + readMcpLockPidNamespaceIdentity, +} from "./mcp-lifecycle-lock-identity"; import { getMcpLifecycleLockPath } from "./mcp-lifecycle-lock-storage"; const SANDBOX_NAME = "alpha"; @@ -42,6 +48,26 @@ function writeTimerMarker(processToken: string): void { ); } +function writeStaleMainOwner(shieldsTakeoverToken?: string): string { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + version: 1, + sandboxName: SANDBOX_NAME, + pid: 2_147_483_647, + processIdentity: "dead-process", + hostIdentity: readMcpLockHostIdentity(), + pidNamespaceIdentity: readMcpLockPidNamespaceIdentity(), + ...(shieldsTakeoverToken ? { shieldsTakeoverToken } : {}), + token: "stale-main-token", + acquiredAt: "2026-01-01T00:00:00.000Z", + })}\n`, + ); + return lockPath; +} + beforeEach(() => { stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-mcp-lock-acquisition-")); }); @@ -99,6 +125,462 @@ describe("MCP lifecycle lock acquisition", () => { expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); }); + it("releases the synchronous deadline and main generations after an ordinary error", () => { + const processToken = "e".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + + expect(() => + withMcpLifecycleDeadlineFenceSync( + SANDBOX_NAME, + processToken, + () => { + throw new Error("ordinary recovery failure"); + }, + options(), + ), + ).toThrow("ordinary recovery failure"); + + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + }); + + it("retains exact synchronous deadline and main generations after an uncommitted permanent-containment failure", () => { + const processToken = "f".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const deadlinePath = `${lockPath}.deadline`; + writeTimerMarker(processToken); + + expect(() => + withMcpLifecycleDeadlineFenceSync( + SANDBOX_NAME, + processToken, + () => { + throw permanentMcpLifecycleContainmentFailure( + new Error("containment state is read-only"), + lockPath, + ); + }, + options(), + ), + ).toThrow("containment state is read-only"); + + const mainOwner = JSON.parse(fs.readFileSync(lockPath, "utf8")); + const deadlineOwner = JSON.parse(fs.readFileSync(deadlinePath, "utf8")); + expect(mainOwner).toMatchObject({ + sandboxName: SANDBOX_NAME, + pid: process.pid, + shieldsTakeoverToken: processToken, + }); + expect(deadlineOwner).toMatchObject({ + sandboxName: SANDBOX_NAME, + pid: process.pid, + shieldsTakeoverToken: processToken, + }); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(false); + }); + + it("releases owned generations when committed containment is proven present", () => { + const processToken = "1".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + + expect(() => + withMcpLifecycleDeadlineFenceSync( + SANDBOX_NAME, + processToken, + () => { + beginCommittedMcpLifecycleContainmentSync( + SANDBOX_NAME, + processToken, + "test containment", + stateDir, + ); + throw permanentMcpLifecycleContainmentFailure( + new Error("containment reporting stopped"), + lockPath, + ); + }, + options(), + ), + ).toThrow("containment reporting stopped"); + + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("retains owned generations when committed containment cannot be inspected", () => { + const processToken = "2".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const containmentPath = `${lockPath}.containment`; + const realLstatSync = fs.lstatSync.bind(fs); + let denyContainmentInspection = false; + vi.spyOn(fs, "lstatSync").mockImplementation((target, options) => { + if (denyContainmentInspection && String(target) === containmentPath) { + const error = new Error("containment inspection denied") as NodeJS.ErrnoException; + error.code = "EACCES"; + throw error; + } + return realLstatSync(target, options as never); + }); + writeTimerMarker(processToken); + + expect(() => + withMcpLifecycleDeadlineFenceSync( + SANDBOX_NAME, + processToken, + () => { + denyContainmentInspection = true; + throw permanentMcpLifecycleContainmentFailure( + new Error("containment commit could not be verified"), + lockPath, + ); + }, + options(), + ), + ).toThrow("containment commit could not be verified"); + + denyContainmentInspection = false; + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(true); + }); + + it("retains an async lifecycle generation only for an uncommitted coded failure", async () => { + const retainedLockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker("8".repeat(32)); + + await expect( + withMcpLifecycleLock( + SANDBOX_NAME, + () => { + throw permanentMcpLifecycleContainmentFailure( + new Error("nested containment commit failed"), + retainedLockPath, + ); + }, + options(), + ), + ).rejects.toThrow("nested containment commit failed"); + expect(fs.existsSync(retainedLockPath)).toBe(true); + + fs.rmSync(retainedLockPath, { force: true }); + await expect( + withMcpLifecycleLock( + SANDBOX_NAME, + () => { + throw new Error("ordinary nested failure"); + }, + options(), + ), + ).rejects.toThrow("ordinary nested failure"); + expect(fs.existsSync(retainedLockPath)).toBe(false); + }); + + it("retains a synchronous lifecycle generation only for an uncommitted coded failure", () => { + const retainedLockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker("9".repeat(32)); + + expect(() => + withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + throw permanentMcpLifecycleContainmentFailure( + new Error("synchronous nested containment commit failed"), + retainedLockPath, + ); + }, + options(), + ), + ).toThrow("synchronous nested containment commit failed"); + expect(fs.existsSync(retainedLockPath)).toBe(true); + + fs.rmSync(retainedLockPath, { force: true }); + expect(() => + withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + throw new Error("ordinary synchronous nested failure"); + }, + options(), + ), + ).toThrow("ordinary synchronous nested failure"); + expect(fs.existsSync(retainedLockPath)).toBe(false); + }); + + it("retains a timer-bound lifecycle generation when nested code handles the containment failure", () => { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const processToken = "4".repeat(32); + writeTimerMarker(processToken); + + const result = withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + try { + throw permanentMcpLifecycleContainmentFailure( + new Error("nested containment commit failed"), + lockPath, + ); + } catch { + return "handled"; + } + }, + options(), + ); + + expect(result).toBe("handled"); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8"))).toMatchObject({ + sandboxName: SANDBOX_NAME, + shieldsTakeoverToken: processToken, + }); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(false); + }); + + it("retains an async timer-bound lifecycle generation when nested code handles the containment failure", async () => { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const processToken = "5".repeat(32); + writeTimerMarker(processToken); + + const result = await withMcpLifecycleLock( + SANDBOX_NAME, + async () => { + try { + throw permanentMcpLifecycleContainmentFailure( + new Error("nested async containment commit failed"), + lockPath, + ); + } catch { + await Promise.resolve(); + return "handled"; + } + }, + options(), + ); + + expect(result).toBe("handled"); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8"))).toMatchObject({ + sandboxName: SANDBOX_NAME, + shieldsTakeoverToken: processToken, + }); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(false); + }); + + it("releases a non-timer-bound lifecycle generation after a coded containment failure", () => { + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + + expect(() => + withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + throw permanentMcpLifecycleContainmentFailure( + new Error("non-timer containment failure"), + lockPath, + ); + }, + options(), + ), + ).toThrow("non-timer containment failure"); + + expect(fs.existsSync(lockPath)).toBe(false); + }); + + it("contains a retained timer-bound main generation after its owner exits", () => { + const processToken = "a".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + writeTimerMarker(processToken); + expect(() => + withMcpLifecycleLockSync( + SANDBOX_NAME, + () => { + throw permanentMcpLifecycleContainmentFailure( + new Error("retain this timer-bound generation"), + lockPath, + ); + }, + options(), + ), + ).toThrow("retain this timer-bound generation"); + expect(JSON.parse(fs.readFileSync(lockPath, "utf8"))).toMatchObject({ + pid: process.pid, + shieldsTakeoverToken: processToken, + }); + + const realProcessKill = process.kill.bind(process); + vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { + if (pid === process.pid && signal === 0) { + const error = new Error("retained owner exited") as NodeJS.ErrnoException; + error.code = "ESRCH"; + throw error; + } + return realProcessKill(pid, signal as never); + }); + const waitSpy = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); + + expect(() => withMcpLifecycleLockSync(SANDBOX_NAME, () => "must not enter", options())).toThrow( + "Sandbox mutation containment is active", + ); + expect(waitSpy).not.toHaveBeenCalled(); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("returns operator guidance after recording durable containment for a stale deadline generation", () => { + const processToken = "6".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const deadlinePath = `${lockPath}.deadline`; + const operation = vi.fn(); + const containmentReasons: string[] = []; + writeTimerMarker(processToken); + fs.mkdirSync(path.dirname(deadlinePath), { recursive: true }); + fs.writeFileSync( + deadlinePath, + JSON.stringify({ + ...createMcpLifecycleLockOwner(SANDBOX_NAME, "stale-deadline-token", processToken), + pid: 2_147_483_647, + processIdentity: "dead-process", + }), + ); + + let failure: unknown; + try { + withMcpLifecycleDeadlineFenceSync(SANDBOX_NAME, processToken, operation, { + ...options(), + throwOnCommittedContainment: true, + onContainment: ({ reason }) => containmentReasons.push(reason), + }); + } catch (error) { + failure = error; + } + + expect(failure).toMatchObject({ code: "NEMOCLAW_PERMANENT_CONTAINMENT" }); + expect(String(failure)).toContain( + "A committed process-tree containment requires operator resolution", + ); + expect(containmentReasons).toEqual([ + expect.stringContaining("remove only the exact stale owner generations"), + ]); + expect(operation).not.toHaveBeenCalled(); + expect(fs.existsSync(deadlinePath)).toBe(true); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("returns operator guidance after recording durable containment for a stale timer-bound main generation", () => { + const processToken = "b".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const operation = vi.fn(); + const containmentReasons: string[] = []; + writeTimerMarker(processToken); + writeStaleMainOwner(processToken); + + let failure: unknown; + try { + withMcpLifecycleDeadlineFenceSync(SANDBOX_NAME, processToken, operation, { + ...options(), + throwOnCommittedContainment: true, + onContainment: ({ reason }) => containmentReasons.push(reason), + }); + } catch (error) { + failure = error; + } + + expect(failure).toMatchObject({ code: "NEMOCLAW_PERMANENT_CONTAINMENT" }); + expect(String(failure)).toContain("remove only the exact stale owner generations"); + expect(containmentReasons).toEqual([ + expect.stringContaining("remove only the exact stale owner generations"), + ]); + expect(operation).not.toHaveBeenCalled(); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(`${lockPath}.deadline`)).toBe(false); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("retains async deadline generations only for an uncommitted coded failure", async () => { + const processToken = "7".repeat(32); + const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); + const deadlinePath = `${lockPath}.deadline`; + writeTimerMarker(processToken); + + await expect( + withMcpLifecycleDeadlineFence( + SANDBOX_NAME, + processToken, + () => { + throw permanentMcpLifecycleContainmentFailure( + new Error("async deadline containment commit failed"), + lockPath, + ); + }, + options(), + ), + ).rejects.toThrow("async deadline containment commit failed"); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(deadlinePath)).toBe(true); + + fs.rmSync(lockPath, { force: true }); + fs.rmSync(deadlinePath, { force: true }); + await expect( + withMcpLifecycleDeadlineFence( + SANDBOX_NAME, + processToken, + () => { + throw new Error("ordinary async deadline failure"); + }, + options(), + ), + ).rejects.toThrow("ordinary async deadline failure"); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(deadlinePath)).toBe(false); + }); + + it("contains a stale async main generation that records a rotated Shields timer token", async () => { + const ownerToken = "3".repeat(32); + const currentToken = "4".repeat(32); + const lockPath = writeStaleMainOwner(ownerToken); + writeTimerMarker(currentToken); + let entered = false; + + await expect( + withMcpLifecycleLock( + SANDBOX_NAME, + () => { + entered = true; + }, + options(), + ), + ).rejects.toThrow("Sandbox mutation containment is active"); + + expect(entered).toBe(false); + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("contains a stale synchronous main generation that records the current Shields timer token", () => { + const processToken = "5".repeat(32); + const lockPath = writeStaleMainOwner(processToken); + writeTimerMarker(processToken); + + expect(() => withMcpLifecycleLockSync(SANDBOX_NAME, () => "entered", options())).toThrow( + "Sandbox mutation containment is active", + ); + + expect(fs.existsSync(lockPath)).toBe(true); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(true); + }); + + it("reclaims a stale main generation whose owner has no Shields timer token", () => { + const lockPath = writeStaleMainOwner(); + writeTimerMarker("6".repeat(32)); + + expect( + withMcpLifecycleLockSync(SANDBOX_NAME, () => "entered", { + ...options(), + timeoutMs: 2_000, + }), + ).toBe("entered"); + expect(fs.existsSync(lockPath)).toBe(false); + expect(fs.existsSync(`${lockPath}.containment`)).toBe(false); + }); + it("blocks synchronous mutation while committed containment is active", () => { const processToken = "b".repeat(32); const lockPath = getMcpLifecycleLockPath(SANDBOX_NAME, stateDir); diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.ts index 07035a09aad..4176592a5c1 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.ts @@ -45,6 +45,7 @@ interface CorruptGenerationTracker { interface AcquiredMcpLifecycleLock { lockPath: string; token: string; + shieldsTakeoverToken?: string; } export interface McpLifecycleDeadlineFenceOptions extends McpLifecycleLockOptions { @@ -55,6 +56,8 @@ export interface McpLifecycleDeadlineFenceOptions extends McpLifecycleLockOption export interface McpLifecycleDeadlineFenceSyncOptions extends McpLifecycleLockOptions { /** Audit an owner that keeps the deadline gate closed while it exits naturally. */ onContainment?: (details: McpLifecycleDeadlineContainment) => void; + /** Return operator guidance instead of waiting when durable containment already exists. */ + throwOnCommittedContainment?: boolean; } export interface McpLifecycleDeadlineContainment { @@ -72,6 +75,7 @@ export interface McpLifecycleLockOptions { interface HeldLockLease { active: boolean; + retainForPermanentContainment: boolean; } type HeldLockContext = ReadonlyMap; @@ -209,6 +213,17 @@ function isValidMainOwnerForSandbox(observation: LockObservation, sandboxName: s return observation.owner?.sandboxName === sandboxName; } +function committedContainmentActiveError( + sandboxName: string, + lockPath: string, + containment: LockObservation, +): Error { + const containmentPath = committedContainmentPath(lockPath); + return new Error( + `Sandbox mutation containment is active for '${sandboxName}' at '${containmentPath}' (generation token '${containment.owner?.token ?? "invalid"}'). A previous owner or stale-lock reaper exited without proof that every descendant stopped. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', and '${lockPath}.deadline'; record each target's file kind, device/inode, and owner token when present; verify those identities and this containment token are unchanged; remove only those exact stale owner generations first and this exact containment generation last before retrying.`, + ); +} + async function tryReapStaleMainLock( lockPath: string, sandboxName: string, @@ -248,6 +263,16 @@ async function tryReapStaleMainLock( ) { return false; } + if (latest.owner?.shieldsTakeoverToken) { + ensurePermanentContainmentForStaleGenerationSync( + lockPath, + sandboxName, + stateDir, + latest, + "A timer-bound sandbox mutation owner exited before permanent containment was committed", + ); + return false; + } return reclaimStaleMcpLifecycleLockGeneration(lockPath, latest); } finally { await safelyReleaseMcpLifecycleLock(reaperPath, reaperToken); @@ -293,6 +318,16 @@ function tryReapStaleMainLockSync( ) { return false; } + if (latest.owner?.shieldsTakeoverToken) { + ensurePermanentContainmentForStaleGenerationSync( + lockPath, + sandboxName, + stateDir, + latest, + "A timer-bound sandbox mutation owner exited before permanent containment was committed", + ); + return false; + } return reclaimStaleMcpLifecycleLockGenerationSync(lockPath, latest); } finally { safelyReleaseMcpLifecycleLockSync(reaperPath, reaperToken); @@ -322,26 +357,18 @@ async function acquireMcpLifecycleLock( const corruptDeadlineTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; let lastOwnerPid: number | null = null; for (;;) { + const containmentPath = committedContainmentPath(lockPath); + const containment = await readMcpLifecycleLockObservation(containmentPath); + if (containment) { + throw committedContainmentActiveError(sandboxName, lockPath, containment); + } if (performance.now() - startedAt >= timeoutMs) { - const containmentPath = committedContainmentPath(lockPath); - const containment = await readMcpLifecycleLockObservation(containmentPath); - if (containment) { - throw new Error( - `Sandbox mutation containment is active for '${sandboxName}' at '${containmentPath}' (generation token '${containment.owner?.token ?? "invalid"}'). A previous owner or stale-lock reaper exited without proof that every descendant stopped. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', and '${lockPath}.deadline'; record each target's file kind, device/inode, and owner token when present; verify those identities and this containment token are unchanged; remove only those exact stale owner generations first and this exact containment generation last before retrying.`, - ); - } const ownerSuffix = lastOwnerPid ? ` (owner pid ${lastOwnerPid})` : ""; throw new Error( `Timed out waiting for the sandbox mutation lock for '${sandboxName}'${ownerSuffix}. Another lifecycle, policy, channel, shields, or snapshot operation is still running.`, ); } - const containmentPath = committedContainmentPath(lockPath); - if (await mcpLifecycleLockPathExists(containmentPath)) { - await sleep(pollIntervalMs); - continue; - } - const deadlinePath = `${lockPath}.deadline`; const deadlineObservation = await readMcpLifecycleLockObservation(deadlinePath); if (deadlineObservation) { @@ -407,7 +434,11 @@ async function acquireMcpLifecycleLock( !(await mcpLifecycleLockPathExists(reaperPath)) && readShieldsTimerTakeoverToken(sandboxName, stateDir) === shieldsTakeoverToken ) { - return { lockPath, token }; + return { + lockPath, + token, + ...(shieldsTakeoverToken ? { shieldsTakeoverToken } : {}), + }; } await safelyReleaseMcpLifecycleLock(lockPath, token); } @@ -469,14 +500,12 @@ function acquireMcpLifecycleLockSync( const corruptDeadlineTracker: CorruptGenerationTracker = { generation: null, firstSeenAt: 0 }; let lastOwnerPid: number | null = null; for (;;) { + const containmentPath = committedContainmentPath(lockPath); + const containment = readMcpLifecycleLockObservationSync(containmentPath); + if (containment) { + throw committedContainmentActiveError(sandboxName, lockPath, containment); + } if (performance.now() - startedAt >= timeoutMs) { - const containmentPath = committedContainmentPath(lockPath); - const containment = readMcpLifecycleLockObservationSync(containmentPath); - if (containment) { - throw new Error( - `Sandbox mutation containment is active for '${sandboxName}' at '${containmentPath}' (generation token '${containment.owner?.token ?? "invalid"}'). A previous owner or stale-lock reaper exited without proof that every descendant stopped. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', and '${lockPath}.deadline'; record each target's file kind, device/inode, and owner token when present; verify those identities and this containment token are unchanged; remove only those exact stale owner generations first and this exact containment generation last before retrying.`, - ); - } throw new Error( `Timed out waiting for sandbox mutation lock for '${sandboxName}'${ lastOwnerPid ? ` (owner PID ${lastOwnerPid})` : "" @@ -484,12 +513,6 @@ function acquireMcpLifecycleLockSync( ); } - const containmentPath = committedContainmentPath(lockPath); - if (mcpLifecycleLockPathExistsSync(containmentPath)) { - sleepSync(pollIntervalMs); - continue; - } - const deadlinePath = `${lockPath}.deadline`; const deadlineObservation = readMcpLifecycleLockObservationSync(deadlinePath); if (deadlineObservation) { @@ -552,7 +575,11 @@ function acquireMcpLifecycleLockSync( !mcpLifecycleLockPathExistsSync(reaperPath) && readShieldsTimerTakeoverToken(sandboxName, options.stateDir) === shieldsTakeoverToken ) { - return { lockPath, token }; + return { + lockPath, + token, + ...(shieldsTakeoverToken ? { shieldsTakeoverToken } : {}), + }; } safelyReleaseMcpLifecycleLockSync(lockPath, token); } @@ -630,7 +657,15 @@ function isPermanentContainmentError(error: unknown): boolean { ); } -function permanentContainmentFailure(error: unknown): Error & { code: string } { +export function permanentMcpLifecycleContainmentFailure( + error: unknown, + lockPath: string, +): Error & { code: string } { + const lease = heldLocks.getStore()?.get(lockPath); + if (lease?.active) lease.retainForPermanentContainment = true; + if (isPermanentContainmentError(error)) { + return error as Error & { code: string }; + } const failure = new Error( `Permanent sandbox mutation containment requires operator resolution: ${ error instanceof Error ? error.message : String(error) @@ -640,6 +675,39 @@ function permanentContainmentFailure(error: unknown): Error & { code: string } { return failure; } +async function ownedLifecycleGateMustRemainClosed(lockPath: string): Promise { + try { + return !(await mcpLifecycleLockPathExists(committedContainmentPath(lockPath))); + } catch { + // If committed containment cannot be inspected, retaining the exact owned + // generation is the only fail-closed outcome. + return true; + } +} + +function ownedLifecycleGateMustRemainClosedSync(lockPath: string): boolean { + try { + return !mcpLifecycleLockPathExistsSync(committedContainmentPath(lockPath)); + } catch { + // If committed containment cannot be inspected, retaining the exact + // owned generation is the only fail-closed outcome. + return true; + } +} + +async function retainOwnedLifecycleGateAfterFailure( + error: unknown, + lockPath: string, +): Promise { + if (!isPermanentContainmentError(error)) return false; + return await ownedLifecycleGateMustRemainClosed(lockPath); +} + +function retainOwnedLifecycleGateAfterFailureSync(error: unknown, lockPath: string): boolean { + if (!isPermanentContainmentError(error)) return false; + return ownedLifecycleGateMustRemainClosedSync(lockPath); +} + async function deadlineMainStillPresent(lockPath: string): Promise { try { return (await readMcpLifecycleLockObservation(lockPath)) !== null; @@ -787,7 +855,16 @@ function acquireDeadlineFenceSync( if (readShieldsTimerTakeoverToken(sandboxName, options.stateDir) !== takeoverToken) { throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); } - if (mcpLifecycleLockPathExistsSync(committedContainmentPath(lockPath))) { + const containmentPath = committedContainmentPath(lockPath); + if (mcpLifecycleLockPathExistsSync(containmentPath)) { + if (options.throwOnCommittedContainment) { + const reason = `A committed process-tree containment requires operator resolution before auto-restore can continue. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${deadlinePath}', and '${containmentPath}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`; + reportDeadlineContainmentSync(options, { + ownerPid: null, + reason, + }); + throw permanentMcpLifecycleContainmentFailure(new Error(reason), lockPath); + } if ( notifiedGeneration !== "committed-containment" && performance.now() - blockedAt >= timeoutMs @@ -896,10 +973,11 @@ async function clearDeadlineProtectedPath( observed, `The ${targetLabel} owner PID ${String(owner?.pid)} was already gone before takeover`, ); - throw permanentContainmentFailure( + throw permanentMcpLifecycleContainmentFailure( new Error( `The exact ${targetLabel} owner was already gone, so surviving descendants cannot be ruled out`, ), + lifecyclePath, ); } @@ -980,10 +1058,11 @@ function clearDeadlineProtectedPathSync( observed, `The ${targetLabel} owner PID ${String(owner?.pid)} was already gone before takeover`, ); - throw permanentContainmentFailure( + throw permanentMcpLifecycleContainmentFailure( new Error( `The exact ${targetLabel} owner was already gone, so surviving descendants cannot be ruled out`, ), + lifecyclePath, ); } if (exactLocalOwner && owner?.pid === process.pid && currentProcessIdentity === null) { @@ -1158,13 +1237,19 @@ function publishDeadlineMainOwnerSync( } const message = error instanceof Error ? error.message : String(error); if (isPermanentContainmentError(error)) { + const resolutionReason = `${message} The deadline gate remains closed. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${lockPath}.deadline', and '${committedContainmentPath(lockPath)}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`; if (message !== notifiedError) { reportDeadlineContainmentSync(options, { ownerPid: null, - reason: `${message} The deadline gate remains closed. Stop all NemoClaw processes for this sandbox; inspect '${lockPath}', '${lockPath}.reaper', '${lockPath}.deadline', and '${committedContainmentPath(lockPath)}'; record each target's file kind, device/inode, and owner token when present; verify every recorded identity is unchanged; remove only the exact stale owner generations first and the exact containment generation last before retrying.`, + reason: resolutionReason, }); notifiedError = message; } + if (options.throwOnCommittedContainment) { + const failure = permanentMcpLifecycleContainmentFailure(error, lockPath); + failure.message = resolutionReason; + throw failure; + } while ( readShieldsTimerTakeoverToken(sandboxName, stateDir) === takeoverToken && (mcpLifecycleLockPathExistsSync(committedContainmentPath(lockPath)) || @@ -1216,6 +1301,8 @@ export async function withMcpLifecycleDeadlineFence( stateDir, }); let mainToken: string | null = null; + let retainOwnedGate = false; + let lease: HeldLockLease | null = null; try { // An ordinary acquirer can pass its pre-publication deadline check before // this fence exists, then link the main path after an earlier clear. Keep @@ -1228,9 +1315,13 @@ export async function withMcpLifecycleDeadlineFence( options, ); - const lease: HeldLockLease = { active: true }; + const activeLease: HeldLockLease = { + active: true, + retainForPermanentContainment: false, + }; + lease = activeLease; const context = new Map(inherited ?? []); - context.set(lockPath, lease); + context.set(lockPath, activeLease); return await heldLocks.run(context, async () => { try { if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { @@ -1238,12 +1329,20 @@ export async function withMcpLifecycleDeadlineFence( } return await operation(); } finally { - lease.active = false; + activeLease.active = false; } }); + } catch (error) { + retainOwnedGate = await retainOwnedLifecycleGateAfterFailure(error, lockPath); + throw error; } finally { - if (mainToken) await safelyReleaseMcpLifecycleLock(lockPath, mainToken); - await safelyReleaseMcpLifecycleLock(fence.lockPath, fence.token); + if (!retainOwnedGate && lease?.retainForPermanentContainment) { + retainOwnedGate = await ownedLifecycleGateMustRemainClosed(lockPath); + } + if (!retainOwnedGate) { + if (mainToken) await safelyReleaseMcpLifecycleLock(lockPath, mainToken); + await safelyReleaseMcpLifecycleLock(fence.lockPath, fence.token); + } } } @@ -1269,6 +1368,8 @@ export function withMcpLifecycleDeadlineFenceSync( stateDir, }); let mainToken: string | null = null; + let retainOwnedGate = false; + let lease: HeldLockLease | null = null; try { mainToken = publishDeadlineMainOwnerSync( lockPath, @@ -1278,20 +1379,32 @@ export function withMcpLifecycleDeadlineFenceSync( options, ); - const lease: HeldLockLease = { active: true }; + const activeLease: HeldLockLease = { + active: true, + retainForPermanentContainment: false, + }; + lease = activeLease; const context = new Map(inherited ?? []); - context.set(lockPath, lease); + context.set(lockPath, activeLease); try { if (readShieldsTimerTakeoverToken(sandboxName, stateDir) !== takeoverToken) { throw new Error(`Auto-restore authority changed for sandbox '${sandboxName}'`); } return heldLocks.run(context, operation); } finally { - lease.active = false; + activeLease.active = false; } + } catch (error) { + retainOwnedGate = retainOwnedLifecycleGateAfterFailureSync(error, lockPath); + throw error; } finally { - if (mainToken) safelyReleaseMcpLifecycleLockSync(lockPath, mainToken); - safelyReleaseMcpLifecycleLockSync(fence.lockPath, fence.token); + if (!retainOwnedGate && lease?.retainForPermanentContainment) { + retainOwnedGate = ownedLifecycleGateMustRemainClosedSync(lockPath); + } + if (!retainOwnedGate) { + if (mainToken) safelyReleaseMcpLifecycleLockSync(lockPath, mainToken); + safelyReleaseMcpLifecycleLockSync(fence.lockPath, fence.token); + } } } @@ -1313,14 +1426,25 @@ export function withMcpLifecycleLockSync( if (inherited?.get(lockPath)?.active) return operation(); const acquired = acquireMcpLifecycleLockSync(sandboxName, { ...options, stateDir }); - const lease: HeldLockLease = { active: true }; + const lease: HeldLockLease = { active: true, retainForPermanentContainment: false }; const context = new Map(inherited ?? []); context.set(lockPath, lease); + let retainOwnedGate = false; try { return heldLocks.run(context, operation); + } catch (error) { + retainOwnedGate = + Boolean(acquired.shieldsTakeoverToken) && + retainOwnedLifecycleGateAfterFailureSync(error, lockPath); + throw error; } finally { lease.active = false; - safelyReleaseMcpLifecycleLockSync(acquired.lockPath, acquired.token); + if (!retainOwnedGate && acquired.shieldsTakeoverToken && lease.retainForPermanentContainment) { + retainOwnedGate = ownedLifecycleGateMustRemainClosedSync(lockPath); + } + if (!retainOwnedGate) { + safelyReleaseMcpLifecycleLockSync(acquired.lockPath, acquired.token); + } } } @@ -1351,18 +1475,33 @@ export async function withMcpLifecycleLock( ...options, stateDir, }); - const lease: HeldLockLease = { active: true }; + const lease: HeldLockLease = { active: true, retainForPermanentContainment: false }; const context = new Map(inherited ?? []); context.set(lockKey, lease); return heldLocks.run(context, async () => { + let retainOwnedGate = false; try { return await operation(); + } catch (error) { + retainOwnedGate = + Boolean(acquired.shieldsTakeoverToken) && + (await retainOwnedLifecycleGateAfterFailure(error, lockKey)); + throw error; } finally { // Async resources created by the callback retain their ALS store. Mark // the lease inactive before releasing so a detached/later promise cannot // mistake an ended parent operation for a still-held reentrant lock. lease.active = false; - await safelyReleaseMcpLifecycleLock(acquired.lockPath, acquired.token); + if ( + !retainOwnedGate && + acquired.shieldsTakeoverToken && + lease.retainForPermanentContainment + ) { + retainOwnedGate = await ownedLifecycleGateMustRemainClosed(lockKey); + } + if (!retainOwnedGate) { + await safelyReleaseMcpLifecycleLock(acquired.lockPath, acquired.token); + } } }); } diff --git a/src/lib/state/mcp-lifecycle-lock.ts b/src/lib/state/mcp-lifecycle-lock.ts index da7cf069840..715cfb03b90 100644 --- a/src/lib/state/mcp-lifecycle-lock.ts +++ b/src/lib/state/mcp-lifecycle-lock.ts @@ -8,6 +8,7 @@ export { type McpLifecycleDeadlineFenceOptions, type McpLifecycleDeadlineFenceSyncOptions, type McpLifecycleLockOptions, + permanentMcpLifecycleContainmentFailure, withMcpLifecycleDeadlineFence, withMcpLifecycleDeadlineFenceSync, withMcpLifecycleLock, diff --git a/test/mcp-lifecycle-lock.test.ts b/test/mcp-lifecycle-lock.test.ts index ed92b0f7c4b..9ee5fe5f4a1 100644 --- a/test/mcp-lifecycle-lock.test.ts +++ b/test/mcp-lifecycle-lock.test.ts @@ -460,16 +460,14 @@ const releasePath = process.argv[3]; acquiredAt: "2026-01-01T00:00:00.000Z", })}\n`, ); - setTimeout(() => writeTimerMarker("alpha", "8".repeat(32)), 40); - + const onContainment = vi.fn(() => writeTimerMarker("alpha", "8".repeat(32))); await expect( - lifecycleLock.withMcpLifecycleDeadlineFence( - "alpha", - processToken, - () => undefined, - options({ timeoutMs: 40 }), - ), + lifecycleLock.withMcpLifecycleDeadlineFence("alpha", processToken, () => undefined, { + ...options({ timeoutMs: 40 }), + onContainment, + }), ).rejects.toThrow("Auto-restore authority changed"); + expect(onContainment).toHaveBeenCalledOnce(); expect(fs.existsSync(deadlinePath)).toBe(true); expect(fs.existsSync(containmentPath)).toBe(true); }); @@ -1066,9 +1064,8 @@ const releasePath = process.argv[3]; const deadlineObservations: boolean[] = []; const onContainment = vi.fn(() => { deadlineObservations.push(fs.existsSync(deadlinePath)); + writeTimerMarker("alpha", "3".repeat(32)); }); - setTimeout(() => writeTimerMarker("alpha", "3".repeat(32)), 40); - await expect( lifecycleLock.withMcpLifecycleDeadlineFence("alpha", processToken, () => undefined, { ...options({ timeoutMs: 10 }), @@ -1100,16 +1097,14 @@ const releasePath = process.argv[3]; acquiredAt: new Date().toISOString(), })}\n`, ); - const onContainment = vi.fn(); - setTimeout(() => writeTimerMarker("alpha", "a".repeat(32)), 40); - + const onContainment = vi.fn(() => writeTimerMarker("alpha", "a".repeat(32))); await expect( lifecycleLock.withMcpLifecycleDeadlineFence("alpha", processToken, () => undefined, { ...options(), onContainment, }), ).rejects.toThrow("Auto-restore authority changed"); - expect(onContainment).toHaveBeenCalled(); + expect(onContainment).toHaveBeenCalledOnce(); expect(fs.existsSync(lockPath)).toBe(true); expect(fs.existsSync(containmentPath)).toBe(true); }); From eb8e4c00677292177cda0f0624b81ecf5ba42e69 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 31 Jul 2026 08:50:02 -0400 Subject: [PATCH 21/25] test(shields): linearize containment fixtures --- src/lib/shields/flow.test.ts | 16 +++++++------- .../mcp-lifecycle-lock-acquisition.test.ts | 21 ++++++++++++------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/lib/shields/flow.test.ts b/src/lib/shields/flow.test.ts index 3472ab59325..49056036840 100644 --- a/src/lib/shields/flow.test.ts +++ b/src/lib/shields/flow.test.ts @@ -125,11 +125,11 @@ function createHarness(options: HarnessOptions = {}): ShieldsHarness { const lifecycleLock = requireDist( "../state/mcp-lifecycle-lock.js", ) as typeof import("../state/mcp-lifecycle-lock.js"); - if (options.beginContainment) { - vi.spyOn(lifecycleLock, "beginCommittedMcpLifecycleContainmentSync").mockImplementation( - options.beginContainment, - ); - } + const beginContainment = + options.beginContainment ?? lifecycleLock.beginCommittedMcpLifecycleContainmentSync; + vi.spyOn(lifecycleLock, "beginCommittedMcpLifecycleContainmentSync").mockImplementation( + beginContainment, + ); const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); vi.spyOn(console, "warn").mockImplementation(() => undefined); @@ -1368,11 +1368,13 @@ describe("shields command flow", () => { "dead", ); vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { - if (`${pid}:${signal}` === "4242:0") { + const failDeadTimerProbe = () => { const error = new Error("timer is gone") as NodeJS.ErrnoException; error.code = "ESRCH"; throw error; - } + }; + const deadTimerProbe = `${pid}:${signal}` === "4242:0" ? failDeadTimerProbe : undefined; + deadTimerProbe?.(); return true; }); const harness = createHarness({ diff --git a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts index 6264a314376..13612002ca6 100644 --- a/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts +++ b/src/lib/state/mcp-lifecycle-lock-acquisition.test.ts @@ -216,13 +216,15 @@ describe("MCP lifecycle lock acquisition", () => { const containmentPath = `${lockPath}.containment`; const realLstatSync = fs.lstatSync.bind(fs); let denyContainmentInspection = false; + const rejectContainmentInspection = (): never => { + const error = new Error("containment inspection denied") as NodeJS.ErrnoException; + error.code = "EACCES"; + throw error; + }; vi.spyOn(fs, "lstatSync").mockImplementation((target, options) => { - if (denyContainmentInspection && String(target) === containmentPath) { - const error = new Error("containment inspection denied") as NodeJS.ErrnoException; - error.code = "EACCES"; - throw error; - } - return realLstatSync(target, options as never); + return denyContainmentInspection && String(target) === containmentPath + ? rejectContainmentInspection() + : realLstatSync(target, options as never); }); writeTimerMarker(processToken); @@ -407,11 +409,14 @@ describe("MCP lifecycle lock acquisition", () => { const realProcessKill = process.kill.bind(process); vi.spyOn(process, "kill").mockImplementation((pid: number, signal?: string | number) => { - if (pid === process.pid && signal === 0) { + const failRetainedOwnerProbe = () => { const error = new Error("retained owner exited") as NodeJS.ErrnoException; error.code = "ESRCH"; throw error; - } + }; + const retainedOwnerProbe = + pid === process.pid && signal === 0 ? failRetainedOwnerProbe : undefined; + retainedOwnerProbe?.(); return realProcessKill(pid, signal as never); }); const waitSpy = vi.spyOn(Atomics, "wait").mockReturnValue("timed-out"); From 46e43e51435643748ee351f7d2c1b70a2ee83103 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Sat, 1 Aug 2026 04:26:15 -0400 Subject: [PATCH 22/25] test(e2e): stay within MCP bridge size budget --- test/e2e/live/mcp-bridge.test.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/e2e/live/mcp-bridge.test.ts b/test/e2e/live/mcp-bridge.test.ts index 65648f9a1a0..d4150983fa0 100644 --- a/test/e2e/live/mcp-bridge.test.ts +++ b/test/e2e/live/mcp-bridge.test.ts @@ -76,12 +76,10 @@ const COMPATIBLE_MODEL = "mock/mcp-bridge"; const TOOL_CHALLENGE = "nemoclaw-authenticated-mcp-proof"; const REGISTRY_FILE = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json"); const selectedMcpBridgeShard = resolveMcpBridgeShard(); - function mcpBridgeShardTest(shard: McpBridgeShard) { return selectedMcpBridgeShard === shard ? e2eTest : e2eTest.skip; } const test = mcpBridgeShardTest("openclaw"); - type McpAgent = "openclaw" | "hermes" | "langchain-deepagents-code"; type McpAdapter = "mcporter" | "hermes-config" | "deepagents-config"; const MCP_MUTATION_TIMEOUT_MS: Record = { @@ -91,7 +89,6 @@ const MCP_MUTATION_TIMEOUT_MS: Record = { }; const MCP_BRIDGE_ALREADY_ABSENT = /No MCP servers are registered|No MCP server '.+' is registered|MCP server '.+' not found/iu; - async function cleanupMcpBridge( host: HostCliClient, sandboxName: string, @@ -109,7 +106,6 @@ async function cleanupMcpBridge( `cleanup MCP bridge ${server} on sandbox ${sandboxName}`, ); } - async function onboardAgent( host: HostCliClient, cleanup: CleanupRegistry, @@ -147,7 +143,6 @@ async function onboardAgent( ); expectExitZero(result, `onboard ${options.agent} sandbox for MCP bridge`); } - async function assertSecretAbsentFromSandbox( sandbox: SandboxClient, sandboxName: string, @@ -359,7 +354,6 @@ async function addBridgeAndReadStatus( }, ); expectExitZero(add, `${options.artifactPrefix} mcp add fake server`); - const status = await host.nemoclaw( [options.sandboxName, "mcp", "status", SERVER_NAME, "--json"], { From 7d3e33e7c6a5cf077c2f22caf8dec8a3b79a2fe4 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Sun, 2 Aug 2026 04:37:20 -0400 Subject: [PATCH 23/25] docs: refresh documentation review receipt From 93c32f5ecf65688343b89732ceb45569ee4dc85c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 3 Aug 2026 04:29:15 -0400 Subject: [PATCH 24/25] chore(ci): ratchet runner fan-in budget --- ci/source-architecture-budget.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 8d1def477b4..825a7032186 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -24,7 +24,7 @@ "src/lib/inference/web-search.ts": 21, "src/lib/messaging/channels/index.ts": 26, "src/lib/onboard/gateway-binding.ts": 48, - "src/lib/runner.ts": 89, + "src/lib/runner.ts": 88, "src/lib/security/redact.ts": 51, "src/lib/state/onboard-session.ts": 36, "src/lib/state/registry.ts": 101, From 0f0987864b97f404b8be94844d2c178bdaeaf68d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 3 Aug 2026 04:50:58 -0400 Subject: [PATCH 25/25] fix(backup): preserve current mutation lock scope --- src/lib/actions/maintenance.test.ts | 48 ++--------------------------- src/lib/actions/maintenance.ts | 18 ++--------- 2 files changed, 4 insertions(+), 62 deletions(-) diff --git a/src/lib/actions/maintenance.test.ts b/src/lib/actions/maintenance.test.ts index 79bdb8a2a09..515ef28c6d1 100644 --- a/src/lib/actions/maintenance.test.ts +++ b/src/lib/actions/maintenance.test.ts @@ -22,14 +22,6 @@ const mocks = vi.hoisted(() => ({ withSandboxMutationLock: vi.fn(), })); -async function runSandboxMutationAction( - _sandboxName: string, - action: () => unknown, - _options?: { timeoutMs?: number }, -): Promise { - return action(); -} - vi.mock("../state/registry", () => ({ isRouteOnlySandboxReservation: (entry: { pendingRouteReservation?: true; createdAt?: string }) => entry.pendingRouteReservation === true && entry.createdAt === undefined, @@ -99,7 +91,6 @@ describe("backupAll", () => { beforeEach(() => { vi.clearAllMocks(); mocks.backupStartedSandboxState.mockReset(); - mocks.withSandboxMutationLock.mockImplementation(runSandboxMutationAction); delete process.env.NEMOCLAW_REQUIRE_ALL_SANDBOX_BACKUPS; mocks.captureSandboxListWithGatewayPreflightOrExit.mockResolvedValue({ status: 0, @@ -331,23 +322,13 @@ describe("backupAll", () => { logSpy.mockRestore(); }); - it("serializes each Shields transition without holding the lock during backup (#7952)", async () => { + it("closes each shields window before backing up the next sandbox (#6455)", async () => { mocks.listSandboxes.mockReturnValue({ sandboxes: [{ name: "alpha" }, { name: "beta" }], defaultSandbox: "alpha", }); mocks.parseReadySandboxNames.mockReturnValue(new Set(["alpha", "beta"])); const events: string[] = []; - mocks.withSandboxMutationLock.mockImplementation( - async (name: string, action: () => unknown) => { - events.push(`lock:start:${name}`); - try { - return await action(); - } finally { - events.push(`lock:end:${name}`); - } - }, - ); mocks.openBackupShieldsWindow.mockImplementation( ( name: string, @@ -387,27 +368,13 @@ describe("backupAll", () => { await backupAll(); expect(events).toEqual([ - "lock:start:alpha", "open:alpha", - "lock:end:alpha", "backup:alpha", - "lock:start:alpha", "relock:alpha", - "lock:end:alpha", - "lock:start:beta", "open:beta", - "lock:end:beta", "backup:beta", - "lock:start:beta", "relock:beta", - "lock:end:beta", ]); - expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith( - 2, - "alpha", - expect.any(Function), - { timeoutMs: 30_000 }, - ); }); it("relocks shields after a credential permission failure and keeps the failure hard (#6455)", async () => { @@ -511,20 +478,11 @@ describe("backupAll", () => { mocks.backupSandboxState.mockImplementation(() => { throw backupError; }); - const relockLockError = new Error("mutation lock timed out"); - mocks.withSandboxMutationLock - .mockImplementationOnce(runSandboxMutationAction) - .mockRejectedValueOnce(relockLockError); + mocks.relockBackupShieldsWindow.mockReturnValue(false); vi.spyOn(console, "log").mockImplementation(() => undefined); const failure = await backupAll().catch((error: unknown) => error); - expect(mocks.withSandboxMutationLock).toHaveBeenNthCalledWith( - 2, - "alpha", - expect.any(Function), - { timeoutMs: 30_000 }, - ); expect(failure).toBeInstanceOf(AggregateError); expect((failure as AggregateError).message).toContain( "Backup for 'alpha' failed and Shields lockdown could not be restored", @@ -532,13 +490,11 @@ describe("backupAll", () => { expect((failure as AggregateError).errors).toEqual([ backupError, expect.objectContaining({ - cause: relockLockError, message: expect.stringContaining( "Shields lockdown could not be restored for 'alpha' after backup-all", ), }), ]); - expect(mocks.relockBackupShieldsWindow).not.toHaveBeenCalled(); }); it("preserves an orphan-manifest error when shields restoration also fails (#6455)", async () => { diff --git a/src/lib/actions/maintenance.ts b/src/lib/actions/maintenance.ts index 17b5782a5ee..1025b327fd7 100644 --- a/src/lib/actions/maintenance.ts +++ b/src/lib/actions/maintenance.ts @@ -81,9 +81,7 @@ async function backupSandboxWithinShieldsWindow( backup: () => sandboxState.BackupResult | Promise, ): Promise { const shieldsWindowOptions = backupAllShieldsWindowOptions(sandboxName); - const window = await withSandboxMutationLock(sandboxName, () => - openBackupShieldsWindow(sandboxName, shieldsWindowOptions), - ); + const window = openBackupShieldsWindow(sandboxName, shieldsWindowOptions); if (!window) { return { result: null, @@ -111,21 +109,9 @@ async function backupSandboxWithinShieldsWindow( hasBackupError = true; } } finally { - try { - const relocked = await withSandboxMutationLock( - sandboxName, - () => relockBackupShieldsWindow(sandboxName, window, true, shieldsWindowOptions), - { timeoutMs: 30_000 }, - ); - if (!relocked) { - relockError = new Error( - `Shields lockdown could not be restored for '${sandboxName}' after backup-all; aborting remaining backups.`, - ); - } - } catch (error) { + if (!relockBackupShieldsWindow(sandboxName, window, true, shieldsWindowOptions)) { relockError = new Error( `Shields lockdown could not be restored for '${sandboxName}' after backup-all; aborting remaining backups.`, - { cause: error }, ); } }