From 083fa1df61fae32bf04ede26ec97518df4b8310d Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 29 Jul 2026 06:11:01 +0000 Subject: [PATCH 01/25] fix(rebuild): journal same-name sandbox replacement `rebuild` deleted the sandbox and only then handed off to resumed onboarding, so nothing durable proved the source, target, or completed phase. A termination in that window left a later command with no replacement intent to reconcile, and the ordinary path had already dropped the registry row. Open the canonical recreate transaction before the destroy phase, bound to the recorded gateway, the source registry fingerprint and live identity, and a target fingerprint built from validated non-secret rebuild inputs. Record `deleting` before the delete command and `deleted` only after the journal re-proves absence on that exact gateway; a probe that shows neither a live sandbox nor explicit absence stops the command with the backup, MCP state, and registry recovery data preserved. Keep the journaled source row across the delete for every rebuild, not just the MCP-bearing and baseline-exclusion cases, so a restart between deletion and replacement registration still has a source contract to reconcile. Prepared recovery therefore no longer reverses a default-sandbox transition, because none happens. Carry the journal across the session reset the recreate phase performs, rebinding only the sandbox identity, gateway authority, and transaction onto a checkpoint derived from the new session, and hand the journaled target fingerprint to the inner run so it adopts the open transaction instead of starting its own. Signed-off-by: Tinson Lai --- ci/source-architecture-budget.json | 20 +- .../sandbox/rebuild-destroy-phase.test.ts | 103 +++++- .../actions/sandbox/rebuild-destroy-phase.ts | 48 +-- .../actions/sandbox/rebuild-gpu-opt-out.ts | 2 + src/lib/actions/sandbox/rebuild-pipeline.ts | 17 + .../sandbox/rebuild-prepared-recovery.test.ts | 4 +- .../sandbox/rebuild-recreate-journal.test.ts | 331 ++++++++++++++++++ .../sandbox/rebuild-recreate-journal.ts | 194 ++++++++++ .../rebuild-recreate-observability.test.ts | 60 ++++ .../actions/sandbox/rebuild-recreate-phase.ts | 20 +- .../sandbox/rebuild-shields-finally.test.ts | 25 +- src/lib/onboard.ts | 1 + src/lib/onboard/machine/core-flow-phases.ts | 2 + src/lib/onboard/machine/handlers/sandbox.ts | 21 +- .../onboard/sandbox-recreate-transaction.ts | 7 + src/lib/onboard/types.ts | 2 + test/helpers/rebuild-flow-harness.ts | 13 + test/helpers/rebuild-flow-lifecycle-cases.ts | 24 +- test/helpers/rebuild-flow-recovery-cases.ts | 56 +-- test/helpers/rebuild-flow-test-support.ts | 13 + test/mcp-destroy-lifecycle.test.ts | 13 + 21 files changed, 872 insertions(+), 104 deletions(-) create mode 100644 src/lib/actions/sandbox/rebuild-recreate-journal.test.ts create mode 100644 src/lib/actions/sandbox/rebuild-recreate-journal.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index b1cb940c153..429f215d151 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -8,8 +8,8 @@ "src/lib/adapters/docker/index.ts": 45, "src/lib/adapters/openshell/client.ts": 22, "src/lib/adapters/openshell/resolve.ts": 28, - "src/lib/adapters/openshell/runtime.ts": 50, - "src/lib/adapters/openshell/timeouts.ts": 36, + "src/lib/adapters/openshell/runtime.ts": 51, + "src/lib/adapters/openshell/timeouts.ts": 37, "src/lib/agent/defs.ts": 32, "src/lib/cli/branding.ts": 85, "src/lib/cli/nemoclaw-oclif-command.ts": 103, @@ -26,8 +26,8 @@ "src/lib/onboard/gateway-binding.ts": 47, "src/lib/runner.ts": 89, "src/lib/security/redact.ts": 51, - "src/lib/state/onboard-session.ts": 34, - "src/lib/state/registry.ts": 99, + "src/lib/state/onboard-session.ts": 35, + "src/lib/state/registry.ts": 100, "src/lib/state/state-root.ts": 23, "src/lib/subprocess-env.ts": 23, "src/lib/validation.ts": 25 @@ -42,7 +42,7 @@ "src/lib/actions/sandbox/doctor.ts": 29, "src/lib/actions/sandbox/policy-channel.ts": 28, "src/lib/actions/sandbox/process-recovery.ts": 22, - "src/lib/actions/sandbox/rebuild-pipeline.ts": 27, + "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, "src/lib/actions/sandbox/snapshot.ts": 38, "src/lib/actions/uninstall/run-plan.ts": 25, "src/lib/inference/onboard-probes.ts": 21, @@ -62,9 +62,6 @@ "src/lib/actions/sandbox/connect-boundary-refusal.ts", "src/lib/actions/sandbox/connect-hermes-light-skin.ts", "src/lib/actions/sandbox/connect.ts", - "src/lib/actions/sandbox/destroy-execution.ts", - "src/lib/actions/sandbox/destroy-preflight.ts", - "src/lib/actions/sandbox/destroy.ts", "src/lib/actions/sandbox/gateway-restart.ts", "src/lib/actions/sandbox/gateway-state.ts", "src/lib/actions/sandbox/mcp-bridge-adapter-deepagents-capability.ts", @@ -116,6 +113,7 @@ "src/lib/actions/sandbox/rebuild-post-restore-phase.ts", "src/lib/actions/sandbox/rebuild-preflight-phase.ts", "src/lib/actions/sandbox/rebuild-preflight-target-phase.ts", + "src/lib/actions/sandbox/rebuild-recreate-journal.ts", "src/lib/actions/sandbox/rebuild-recreate-phase.ts", "src/lib/actions/sandbox/rebuild-shields-phase.ts", "src/lib/actions/sandbox/rebuild-shields.ts", @@ -137,15 +135,13 @@ "src/lib/onboard/setup-inference.ts", "src/lib/openshell-sandbox-list.ts", "src/lib/sandbox/config.ts", - "src/lib/shields/index.ts", - "src/lib/tunnel/allowed-origins.ts", - "src/lib/tunnel/services.ts" + "src/lib/shields/index.ts" ] ], "maxRootFiles": { "src/lib/onboard": 307, "src/lib/actions": 19, - "src/lib/actions/sandbox": 182, + "src/lib/actions/sandbox": 183, "src/lib/state": 38, "src/lib/inference": 62, "scripts": 44 diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index 993d3a6a92a..bd233f1c8dd 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -71,6 +71,17 @@ vi.mock("./rebuild-mcp-phase", () => ({ })); import { runRebuildDestroyPhase, waitForRebuildDeleteAbsence } from "./rebuild-destroy-phase"; +import type { RebuildRecreateJournal } from "./rebuild-recreate-journal"; + +function stubRecreateJournal(): RebuildRecreateJournal { + return { + id: "journal-1", + targetGeneration: "generation-1", + targetIntentFingerprint: "intent-1", + markDeleting: vi.fn(), + confirmDeleted: vi.fn(), + }; +} describe("rebuild destroy phase", () => { beforeEach(() => { @@ -134,6 +145,7 @@ describe("rebuild destroy phase", () => { }, }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, log: vi.fn(), bail, @@ -162,6 +174,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "langchain-deepagents-code" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, log, bail, @@ -192,6 +205,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log, @@ -227,6 +241,7 @@ describe("rebuild destroy phase", () => { gatewayPort: 19080, }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -281,6 +296,7 @@ describe("rebuild destroy phase", () => { gatewayPort: 8080, }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -326,6 +342,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -369,6 +386,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -420,6 +438,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -538,6 +557,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -582,6 +602,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -630,6 +651,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -676,6 +698,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -811,6 +834,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, log: vi.fn(), bail: vi.fn((message: string): never => { @@ -828,7 +852,7 @@ describe("rebuild destroy phase", () => { expect(mocks.listSandboxes).not.toHaveBeenCalled(); }); - it("removes registry state only after the gateway reports the deleted sandbox missing", async () => { + it("keeps the journaled source row after the gateway reports the deleted sandbox missing (#7734)", async () => { const events: string[] = []; let getAttempts = 0; mocks.runOpenshell.mockImplementation(() => { @@ -843,15 +867,13 @@ describe("rebuild destroy phase", () => { ? { status: 0, stdout: "Name: alpha\nPhase: Terminating", stderr: "" } : { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" }; }); - mocks.removeSandboxRegistryEntryWithReceipt.mockImplementation(() => { - events.push("remove-registry"); - return null; - }); + const recreateJournal = stubRecreateJournal(); const result = await runRebuildDestroyPhase({ sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal, backupManifest: null, log: vi.fn(), bail: vi.fn((message: string): never => { @@ -862,14 +884,80 @@ describe("rebuild destroy phase", () => { }); expect(result).not.toBeNull(); - expect(events).toEqual(["delete", "get-live", "get-missing", "on-deleted", "remove-registry"]); + expect(result?.removalReceipt).toBeNull(); + expect(events).toEqual(["delete", "get-live", "get-missing", "on-deleted"]); expect(mocks.waitUntil).toHaveBeenCalledOnce(); expect(mocks.captureOpenshell).toHaveBeenNthCalledWith( 1, ["sandbox", "get", "-g", "nemoclaw", "alpha"], expect.objectContaining({ timeout: expect.any(Number) }), ); - expect(mocks.removeSandboxRegistryEntryWithReceipt).toHaveBeenCalledWith("alpha"); + expect(recreateJournal.markDeleting).toHaveBeenCalledOnce(); + expect(recreateJournal.confirmDeleted).toHaveBeenCalledOnce(); + }); + + it("journals the delete boundary before the destructive command (#7734)", async () => { + const order: string[] = []; + const recreateJournal = stubRecreateJournal(); + vi.mocked(recreateJournal.markDeleting).mockImplementation(() => { + order.push("journal:deleting"); + }); + mocks.runOpenshell.mockImplementation((args: string[]) => { + if (args[1] === "delete") { + order.push("openshell:delete"); + return { status: 0, stdout: "deleted", stderr: "" }; + } + return { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" }; + }); + + await runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, + staleRecovery: false, + recreateJournal, + backupManifest: null, + log: vi.fn(), + bail: vi.fn((message: string): never => { + throw new Error(message); + }), + relockShieldsIfNeeded: vi.fn(() => true), + onDeleted: vi.fn(), + }); + + expect(order).toEqual(["journal:deleting", "openshell:delete"]); + }); + + it("stops before inference and registry mutation when absence cannot be journaled (#7734)", async () => { + const recreateJournal = stubRecreateJournal(); + vi.mocked(recreateJournal.confirmDeleted).mockImplementation(() => { + throw new Error("OpenShell still reports the journaled source after delete"); + }); + mocks.runOpenshell.mockReturnValue({ status: 0, stdout: "deleted", stderr: "" }); + const onDeleted = vi.fn(); + const onDeleteStateAmbiguous = vi.fn(); + + await expect( + runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, + staleRecovery: false, + recreateJournal, + backupManifest: null, + log: vi.fn(), + bail: vi.fn((message: string): never => { + throw new Error(message); + }), + relockShieldsIfNeeded: vi.fn(() => true), + onDeleted, + onDeleteStateAmbiguous, + }), + ).rejects.toThrow("Sandbox deletion could not be journaled"); + + expect(onDeleteStateAmbiguous).toHaveBeenCalledOnce(); + expect(onDeleted).not.toHaveBeenCalled(); + expect(mocks.stopNimContainer).not.toHaveBeenCalled(); + expect(mocks.stopNimContainerByName).not.toHaveBeenCalled(); + expect(mocks.removeSandboxRegistryEntryWithReceipt).not.toHaveBeenCalled(); }); it("marks accepted deletion as ambiguous when transport failures prevent confirmation", async () => { @@ -888,6 +976,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: { backupPath: "/tmp/rebuild-backups/alpha/backup" } as never, log: vi.fn(), bail: vi.fn((message: string): never => { diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index 9226de56d55..d3e805b96bd 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -12,7 +12,6 @@ import { redactFull } from "../../security/redact"; import { parseSandboxPhase } from "../../state/gateway"; import { registryEntryGatewayPort } from "../../state/gateway-registry"; import * as registry from "../../state/registry"; -import { removeSandboxRegistryEntryWithReceipt } from "./destroy"; import { isExplicitMissingSandboxGatewayOutput } from "./gateway-state"; import type { RebuildBackupManifest } from "./rebuild-backup-phase"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; @@ -24,6 +23,7 @@ import { reattachMcpAfterDeleteFailure, } from "./rebuild-mcp-phase"; import { blockRebuildOnPendingBaselineTransition } from "./rebuild-preflight-guards"; +import type { RebuildRecreateJournal } from "./rebuild-recreate-journal"; export type RebuildDeleteValidationResult = | { ok: true } @@ -33,6 +33,7 @@ export interface RebuildDestroyPhaseInput { sandboxName: string; sandboxEntry: RebuildSandboxEntry; staleRecovery: boolean; + recreateJournal: RebuildRecreateJournal; backupManifest: RebuildBackupManifest; log: RebuildLog; bail: RebuildBail; @@ -224,6 +225,7 @@ export async function runRebuildDestroyPhase( const { sandboxName, staleRecovery, + recreateJournal, backupManifest, log, bail, @@ -312,7 +314,6 @@ export async function runRebuildDestroyPhase( log, }); if (!mcpPreparation) return null; - const rebuildMcpEntries = mcpPreparation.entries; const rebuildDetachedMcpProviderEntries = mcpPreparation.detachedProviderEntries; const rebuildScrubbedMcpAdapterEntries = mcpPreparation.scrubbedAdapterEntries; @@ -355,6 +356,7 @@ export async function runRebuildDestroyPhase( return null; } + recreateJournal.markDeleting(); log(`Running: openshell sandbox delete -g ${gatewayName} ${sandboxName}`); const deleteResult = runOpenshell(["sandbox", "delete", "-g", gatewayName, sandboxName], { ignoreError: true, @@ -425,29 +427,31 @@ export async function runRebuildDestroyPhase( bail("Sandbox deletion could not be confirmed."); return null; } + try { + recreateJournal.confirmDeleted(); + } catch (error) { + console.error( + " Sandbox delete was accepted, but the replacement journal could not confirm absence.", + ); + if (backupManifest) { + console.error(" State backup is preserved at: " + backupManifest.backupPath); + } + input.onDeleteStateAmbiguous?.(); + const detail = error instanceof Error ? error.message : String(error); + bail(`Sandbox deletion could not be journaled: ${redactFull(detail)}`); + return null; + } stopNimBestEffort(); onDeleted(); - let removalReceipt: registry.SandboxRemovalReceipt | null = null; - const hasBaselineExclusions = (input.sandboxEntry.baselineExclusions?.length ?? 0) > 0; - if (rebuildMcpEntries.length === 0 && !hasBaselineExclusions) { - removalReceipt = removeSandboxRegistryEntryWithReceipt(sandboxName); - } - if (rebuildMcpEntries.length > 0) { - // The registry entry is the durable MCP rebuild transaction. The inner - // onboard run observes that the sandbox is absent, carries the MCP state - // into the replacement registration, and never enters generic live - // recreation. Keeping it here closes every process-death window between - // successful delete and fresh registry registration. - log("Preserving MCP-bearing registry entry across sandbox recreation"); - } - if (hasBaselineExclusions) { - // Baseline exclusions are also registry-only rebuild intent. Keep the row - // until inner onboard snapshots it and replacement registration atomically - // publishes the fresh row. - log("Preserving baseline-exclusion registry entry across sandbox recreation"); - } + const removalReceipt: registry.SandboxRemovalReceipt | null = null; + // The journaled source row is the durable replacement transaction. The inner + // onboard run observes that the sandbox is absent, carries the recorded state + // into the replacement registration, and never enters generic live + // recreation. Keeping it here closes every process-death window between + // successful delete and fresh registry registration. + log("Preserving journaled source registry entry across sandbox recreation"); log( - `Registry after remove: ${JSON.stringify(registry.listSandboxes().sandboxes.map((s: { name: string }) => s.name))}`, + `Registry after delete: ${JSON.stringify(registry.listSandboxes().sandboxes.map((s: { name: string }) => s.name))}`, ); console.log(` ${G}\u2713${R} Old sandbox deleted`); diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts index eb951816065..3beb8444a58 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts @@ -110,6 +110,8 @@ export type RebuildRecreateOnboardOpts = { targetGatewayName: string; targetGatewayPort: number; onboardLockAlreadyHeld: true; + /** Target fingerprint of the replacement journal opened before deletion. */ + recreateJournalTargetIntentFingerprint?: string; preparedDcodeRebuild?: PreparedDcodeRebuildHandoff; rebuildRegistryInferenceRoute?: RebuildRouteHandoff; rebuildProviderReconfigure?: RebuildProviderReconfigureHandoff; diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index 1f973340629..4d500ffe54d 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -31,6 +31,10 @@ import { revalidatePreparedRecoveryBeforeDelete, } from "./rebuild-prepared-recovery"; import { inspectRebuildGatewayProviderRegistration } from "./rebuild-provider-preflight"; +import { + fingerprintRebuildRecreateTargetIntent, + openRebuildRecreateJournal, +} from "./rebuild-recreate-journal"; import { runRebuildRecreatePhase } from "./rebuild-recreate-phase"; import { createRebuildRegistryRollback } from "./rebuild-registry-rollback"; import { runRebuildRestorePhase } from "./rebuild-restore-phase"; @@ -210,10 +214,22 @@ async function rebuildSandboxUnlocked( return; } + const recreateJournal = openRebuildRecreateJournal({ + target: { + sandboxName, + gatewayName: recreateOptions.targetGatewayName, + gatewayPort: recreateOptions.targetGatewayPort, + }, + agentName: rebuildAgent || "openclaw", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent(recreateOptions), + log, + }); + const mcpPreparation = await runRebuildDestroyPhase({ sandboxName, sandboxEntry, staleRecovery, + recreateJournal, backupManifest: backup.backupManifest, force: normalized.force, log, @@ -272,6 +288,7 @@ async function rebuildSandboxUnlocked( durableConfig, resumeConfig, recreateOptions, + recreateJournal, fromDockerfile, rebuildAgent, messagingPlan, diff --git a/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts b/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts index ff683736b44..705c43e791a 100644 --- a/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts +++ b/src/lib/actions/sandbox/rebuild-prepared-recovery.test.ts @@ -260,9 +260,11 @@ describe("prepared rebuild recovery", () => { ).rejects.toThrow("Recreate failed"); expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); + // The journaled source row survives the delete, so no default-sandbox + // transition happened and none has to be reversed (#7734). expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), - { defaultTransition: { from: null, to: "alpha", expectedRevision: 1 } }, + {}, ); expect(harness.restoreSandboxStateSpy).not.toHaveBeenCalled(); }); diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts new file mode 100644 index 00000000000..5d28a0e1b8e --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts @@ -0,0 +1,331 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureOpenshell: vi.fn(), + resolveGatewayTeardownAuthority: vi.fn(), +})); + +vi.mock("../../adapters/openshell/runtime", () => ({ + captureOpenshell: mocks.captureOpenshell, + runOpenshell: vi.fn(), +})); + +vi.mock("../../onboard/gateway-teardown-authority", () => ({ + resolveGatewayTeardownAuthority: mocks.resolveGatewayTeardownAuthority, +})); + +import type { Session } from "../../state/onboard-session"; +import * as onboardSession from "../../state/onboard-session"; +import * as registry from "../../state/registry"; +import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; +import { + fingerprintRebuildRecreateTargetIntent, + observeRebuildSandbox, + openRebuildRecreateJournal, +} from "./rebuild-recreate-journal"; + +const SANDBOX_ID = "sbx-0d6f4c2a91"; + +const NON_DEFAULT_TARGET = { + sandboxName: "alpha", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, +}; + +const recreateOptions: RebuildRecreateOnboardOpts = { + resume: true, + nonInteractive: true, + recreateSandbox: true, + authoritativeResumeConfig: true, + acceptThirdPartySoftware: true, + agent: "langchain-deepagents-code", + fromDockerfile: null, + sandboxGpu: null, + sandboxGpuDevice: null, + controlUiPort: null, + targetGatewayName: "nemoclaw-9090", + targetGatewayPort: 9090, + onboardLockAlreadyHeld: true, + autoYes: true, + toolDisclosure: "progressive", + dcodeAutoApprovalMode: "disabled", + dcodeAutoApprovalRequestedExplicitly: false, + observabilityEnabled: true, + observabilityRequestedExplicitly: true, + policyTier: "restricted", + baseImageResolutionHint: null, +}; + +function livePresentProbe(phase = "Ready") { + return { + status: 0, + output: `Name: alpha\nId: ${SANDBOX_ID}\nPhase: ${phase}\n`, + stdout: `Name: alpha\nId: ${SANDBOX_ID}\nPhase: ${phase}\n`, + stderr: "", + }; +} + +function absentProbe() { + return { + status: 1, + output: "", + stdout: "", + stderr: "Error: sandbox alpha not found", + }; +} + +describe("rebuild replacement target fingerprint", () => { + it("carries only validated non-secret rebuild inputs", () => { + const withTransientHandoffs: RebuildRecreateOnboardOpts = { + ...recreateOptions, + autoYes: false, + dcodeAutoApprovalRequestedExplicitly: true, + observabilityRequestedExplicitly: false, + preparedDcodeRebuild: { + stagingDir: "/tmp/rebuild-xyz", + } as unknown as RebuildRecreateOnboardOpts["preparedDcodeRebuild"], + }; + + expect(fingerprintRebuildRecreateTargetIntent(withTransientHandoffs)).toBe( + fingerprintRebuildRecreateTargetIntent(recreateOptions), + ); + }); + + it("changes when a recorded replacement input changes", () => { + expect( + fingerprintRebuildRecreateTargetIntent({ + ...recreateOptions, + dcodeAutoApprovalMode: "thread-opt-in", + }), + ).not.toBe(fingerprintRebuildRecreateTargetIntent(recreateOptions)); + }); + + it("changes when the replacement targets another gateway", () => { + expect( + fingerprintRebuildRecreateTargetIntent({ + ...recreateOptions, + targetGatewayName: "nemoclaw", + targetGatewayPort: 8080, + }), + ).not.toBe(fingerprintRebuildRecreateTargetIntent(recreateOptions)); + }); +}); + +describe("rebuild replacement observation", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("queries only the journaled gateway", () => { + mocks.captureOpenshell.mockReturnValue(absentProbe()); + + observeRebuildSandbox(NON_DEFAULT_TARGET); + + expect(mocks.captureOpenshell).toHaveBeenCalledTimes(1); + expect(mocks.captureOpenshell.mock.calls[0]?.[0]).toEqual([ + "sandbox", + "get", + "-g", + "nemoclaw-9090", + "alpha", + ]); + }); + + it("reports explicit absence without an identity", () => { + mocks.captureOpenshell.mockReturnValue(absentProbe()); + + expect(observeRebuildSandbox(NON_DEFAULT_TARGET)).toEqual({ + state: "missing", + liveIdentityFingerprint: null, + }); + }); + + it("hashes a live identity instead of recording it", () => { + mocks.captureOpenshell.mockReturnValue(livePresentProbe()); + + const observation = observeRebuildSandbox(NON_DEFAULT_TARGET); + + expect(observation.state).toBe("ready"); + expect(observation.liveIdentityFingerprint).toMatch(/^[0-9a-f]{64}$/); + expect(observation.liveIdentityFingerprint).not.toContain(SANDBOX_ID); + }); + + it("separates a live sandbox that is not ready", () => { + mocks.captureOpenshell.mockReturnValue(livePresentProbe("Pending")); + + expect(observeRebuildSandbox(NON_DEFAULT_TARGET).state).toBe("not_ready"); + }); + + it("fails closed when a live sandbox has no stable identity", () => { + mocks.captureOpenshell.mockReturnValue({ + status: 0, + output: "Name: alpha\nPhase: Ready\n", + stdout: "Name: alpha\nPhase: Ready\n", + stderr: "", + }); + + expect(() => observeRebuildSandbox(NON_DEFAULT_TARGET)).toThrow( + /did not report a stable sandbox Id/, + ); + }); + + it("fails closed when the gateway proves neither presence nor absence", () => { + mocks.captureOpenshell.mockReturnValue({ + status: 1, + output: "", + stdout: "", + stderr: "Error: connection refused", + }); + + expect(() => observeRebuildSandbox(NON_DEFAULT_TARGET)).toThrow( + /neither a live sandbox nor explicit absence/, + ); + }); +}); + +describe("rebuild replacement journal", () => { + let session: Session; + + beforeEach(() => { + vi.clearAllMocks(); + session = onboardSession.createSession({ sandboxName: "alpha" }); + vi.spyOn(onboardSession, "loadSession").mockImplementation(() => session); + vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator) => { + session = mutator(session) ?? session; + return session; + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + } as registry.SandboxEntry); + mocks.resolveGatewayTeardownAuthority.mockReturnValue({ + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }); + mocks.captureOpenshell.mockReturnValue(livePresentProbe()); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + function open() { + return openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent(recreateOptions), + log: vi.fn(), + }); + } + + it("binds a secret-free replacement to the non-default gateway before deletion", () => { + const journal = open(); + + const recorded = session.checkpoint?.sandboxRecreate; + expect(recorded?.phase).toBe("planned"); + expect(recorded?.gatewayName).toBe("nemoclaw-9090"); + expect(recorded?.gatewayPort).toBe(9090); + expect(recorded?.sourceLiveIdentityFingerprint).toMatch(/^[0-9a-f]{64}$/); + expect(recorded?.targetIntentFingerprint).toBe(journal.targetIntentFingerprint); + expect(JSON.stringify(session.checkpoint)).not.toContain(SANDBOX_ID); + }); + + it("selects the exact gateway authority the journal records", () => { + open(); + + expect(mocks.resolveGatewayTeardownAuthority).toHaveBeenCalledWith({ + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + }); + const authority = session.checkpoint?.gatewayAuthority; + expect(authority?.kind).toBe("selected"); + expect(authority?.kind === "selected" && authority.value.gatewayPort).toBe(9090); + }); + + it("starts at deleted when the source sandbox is already absent", () => { + mocks.captureOpenshell.mockReturnValue(absentProbe()); + + open(); + + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleted"); + expect(session.checkpoint?.sandboxRecreate?.sourceLiveIdentityFingerprint).toBeNull(); + }); + + it("records the delete boundary before and after the destructive command", () => { + const journal = open(); + + journal.markDeleting(); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); + + mocks.captureOpenshell.mockReturnValue(absentProbe()); + journal.confirmDeleted(); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleted"); + }); + + it("keeps a proven deletion when a recovery rebuild deletes an already absent source", () => { + mocks.captureOpenshell.mockReturnValue(absentProbe()); + const journal = open(); + + journal.markDeleting(); + + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleted"); + }); + + it("stops before the next mutation when the source outlives its delete", () => { + const journal = open(); + journal.markDeleting(); + + expect(() => journal.confirmDeleted()).toThrow( + /OpenShell still reports the journaled source after delete/, + ); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); + }); + + it("refuses to resume when a different same-name sandbox holds the name", () => { + open(); + mocks.captureOpenshell.mockReturnValue({ + status: 0, + output: "Name: alpha\nId: sbx-replaced-by-another\nPhase: Ready\n", + stdout: "Name: alpha\nId: sbx-replaced-by-another\nPhase: Ready\n", + stderr: "", + }); + + expect(() => open()).toThrow(/no longer has the journaled source identity/); + }); + + it("refuses to resume a journal that targets another replacement", () => { + open(); + + expect(() => + openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent({ + ...recreateOptions, + dcodeAutoApprovalMode: "thread-opt-in", + }), + log: vi.fn(), + }), + ).toThrow(/different recreate transaction in progress/); + }); + + it("resumes the same replacement without restarting its generation", () => { + const first = open(); + + const second = open(); + + expect(second.id).toBe(first.id); + expect(second.targetGeneration).toBe(first.targetGeneration); + }); +}); diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.ts new file mode 100644 index 00000000000..f4272cb778d --- /dev/null +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.ts @@ -0,0 +1,194 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { captureOpenshell } from "../../adapters/openshell/runtime"; +import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; +import { checkpointGatewayAuthority } from "../../onboard/gateway-authority-checkpoint"; +import { resolveGatewayTeardownAuthority } from "../../onboard/gateway-teardown-authority"; +import { + advanceSandboxRecreateTransaction, + beginSandboxRecreateTransaction, + fingerprintSandboxLiveIdentity, + fingerprintSandboxRecreateValue, + planSandboxRecreateRecovery, + type SandboxRecreateObservation, + sandboxRecreatePhaseReached, +} from "../../onboard/sandbox-recreate-transaction"; +import { parseSandboxPhase } from "../../state/gateway"; +import { decisionSelected } from "../../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; +import type { CheckpointSandboxRecreatePhase } from "../../state/onboard-checkpoint-types"; +import * as onboardSession from "../../state/onboard-session"; +import * as registry from "../../state/registry"; +import { isExplicitMissingSandboxGatewayOutput } from "./gateway-state"; +import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; + +export interface RebuildRecreateJournalTarget { + readonly sandboxName: string; + readonly gatewayName: string; + readonly gatewayPort: number; +} + +export type RebuildSandboxObserver = ( + target: RebuildRecreateJournalTarget, +) => SandboxRecreateObservation; + +export interface RebuildRecreateJournal { + readonly id: string; + readonly targetGeneration: string; + readonly targetIntentFingerprint: string; + markDeleting(): void; + confirmDeleted(): void; +} + +export function fingerprintRebuildRecreateTargetIntent( + options: Pick< + RebuildRecreateOnboardOpts, + | "agent" + | "fromDockerfile" + | "sandboxGpu" + | "sandboxGpuDevice" + | "controlUiPort" + | "targetGatewayName" + | "targetGatewayPort" + | "toolDisclosure" + | "dcodeAutoApprovalMode" + | "observabilityEnabled" + | "policyTier" + >, +): string { + return fingerprintSandboxRecreateValue({ + version: 1, + agent: options.agent ?? null, + fromDockerfile: options.fromDockerfile, + sandboxGpu: options.sandboxGpu, + sandboxGpuDevice: options.sandboxGpuDevice, + controlUiPort: options.controlUiPort, + gatewayName: options.targetGatewayName, + gatewayPort: options.targetGatewayPort, + toolDisclosure: options.toolDisclosure, + dcodeAutoApprovalMode: options.dcodeAutoApprovalMode, + observabilityEnabled: options.observabilityEnabled, + policyTier: options.policyTier, + }); +} + +export function observeRebuildSandbox( + target: RebuildRecreateJournalTarget, +): SandboxRecreateObservation { + const probe = captureOpenshell(["sandbox", "get", "-g", target.gatewayName, target.sandboxName], { + ignoreError: true, + includeStderr: true, + includeStreams: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); + const stdout = String(probe.stdout ?? (probe.status === 0 ? probe.output : "")).trim(); + const combined = `${stdout}\n${String(probe.stderr ?? probe.output ?? "")}`.trim(); + const failedCleanly = + !probe.error && !probe.signal && probe.status !== null && probe.status !== 0; + if (failedCleanly && isExplicitMissingSandboxGatewayOutput(combined, target.sandboxName)) { + return { state: "missing", liveIdentityFingerprint: null }; + } + if (probe.status === 0 && stdout.length > 0) { + const liveIdentityFingerprint = fingerprintSandboxLiveIdentity(stdout); + if (!liveIdentityFingerprint) { + throw new Error( + `Cannot journal sandbox '${target.sandboxName}' replacement: OpenShell did not report a stable sandbox Id on gateway '${target.gatewayName}'.`, + ); + } + const phase = parseSandboxPhase(combined); + return { + state: phase === "Ready" || phase === "Running" ? "ready" : "not_ready", + liveIdentityFingerprint, + }; + } + throw new Error( + `Cannot journal sandbox '${target.sandboxName}' replacement: gateway '${target.gatewayName}' reported neither a live sandbox nor explicit absence.`, + ); +} + +export interface OpenRebuildRecreateJournalInput { + readonly target: RebuildRecreateJournalTarget; + readonly agentName: string; + readonly targetIntentFingerprint: string; + readonly log: (message: string) => void; + readonly observe?: RebuildSandboxObserver; +} + +export function openRebuildRecreateJournal( + input: OpenRebuildRecreateJournalInput, +): RebuildRecreateJournal { + const { target, agentName, targetIntentFingerprint, log } = input; + const observe = input.observe ?? observeRebuildSandbox; + const authority = resolveGatewayTeardownAuthority({ + gatewayName: target.gatewayName, + gatewayPort: target.gatewayPort, + }); + const sourceEntry = registry.getSandbox(target.sandboxName); + const observation = observe(target); + const active = onboardSession.loadSession()?.checkpoint?.sandboxRecreate ?? null; + if (active) { + const recovery = planSandboxRecreateRecovery(active, observation, sourceEntry); + if (recovery.action === "reject") { + throw new Error( + `Cannot resume sandbox '${target.sandboxName}' replacement: ${recovery.reason}.`, + ); + } + } + + const session = onboardSession.updateSession((current) => { + const checkpoint = current.checkpoint ?? deriveCheckpointFromSession(current); + current.checkpoint = { + ...checkpoint, + machineState: current.machine.state, + updatedAt: new Date().toISOString(), + sandboxIdentity: decisionSelected({ name: target.sandboxName, agent: agentName }), + gatewayAuthority: decisionSelected(checkpointGatewayAuthority(authority)), + }; + beginSandboxRecreateTransaction(current, { + sandboxName: target.sandboxName, + gatewayName: target.gatewayName, + gatewayPort: target.gatewayPort, + sourceEntry, + observation, + targetIntentFingerprint, + }); + return current; + }); + + const transaction = session.checkpoint?.sandboxRecreate; + if (!transaction) { + throw new Error( + `Sandbox '${target.sandboxName}' replacement journal could not be recorded before deletion.`, + ); + } + log( + `Journaled replacement ${transaction.id} for '${target.sandboxName}' on ${target.gatewayName}:${String(target.gatewayPort)} at phase '${transaction.phase}'`, + ); + + let phase: CheckpointSandboxRecreatePhase = transaction.phase; + const advance = (next: CheckpointSandboxRecreatePhase): void => { + onboardSession.updateSession((current) => { + phase = advanceSandboxRecreateTransaction(current, transaction.id, next).phase; + return current; + }); + }; + + return { + id: transaction.id, + targetGeneration: transaction.targetGeneration, + targetIntentFingerprint: transaction.targetIntentFingerprint, + markDeleting: () => { + if (sandboxRecreatePhaseReached(phase, "deleted")) return; + advance("deleting"); + }, + confirmDeleted: () => { + if (observe(target).state !== "missing") { + throw new Error( + `Cannot continue sandbox '${target.sandboxName}' replacement: OpenShell still reports the journaled source after delete.`, + ); + } + advance("deleted"); + }, + }; +} diff --git a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts index dde431881ab..97bc3ae01a1 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts @@ -4,6 +4,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { restoreEnv } from "../../../../test/helpers/env-test-helpers"; +import { decisionSelected } from "../../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import type { RebuildDurableConfig } from "./rebuild-durable-config"; @@ -83,6 +85,13 @@ function makeInput(overrides: Partial = {}): RebuildR durableConfig, resumeConfig, recreateOptions, + recreateJournal: { + id: "journal-1", + targetGeneration: "generation-1", + targetIntentFingerprint: "intent-1", + markDeleting: vi.fn(), + confirmDeleted: vi.fn(), + }, fromDockerfile: null, rebuildAgent: DCODE_AGENT, messagingPlan: null, @@ -174,6 +183,57 @@ describe("runRebuildRecreatePhase handoff", () => { expect(onboardSession.loadSession()?.observabilityRequestedExplicitly).toBe(false); }); + it("carries the replacement journal and its target fingerprint into inner onboard (#7734)", async () => { + session.checkpoint = { + ...deriveCheckpointFromSession(session), + sandboxIdentity: decisionSelected({ name: "alpha", agent: DCODE_AGENT }), + gatewayAuthority: decisionSelected({ + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + sandboxRecreate: { + version: 1, + id: "journal-1", + revision: 3, + sandboxName: "alpha", + gatewayName: "nemoclaw", + gatewayPort: 8080, + sourceRegistryFingerprint: "source-registry", + sourceLiveIdentityFingerprint: "source-identity", + targetIntentFingerprint: "intent-1", + targetGeneration: "generation-1", + targetLiveIdentityFingerprint: null, + phase: "deleted", + startedAt: "2026-07-28T00:00:00.000Z", + updatedAt: "2026-07-28T00:00:01.000Z", + }, + }; + const retiredSessionId = session.sessionId; + let observedFingerprint: string | null | undefined; + let observedJournalPhase: string | undefined; + let observedCheckpointSessionId: string | undefined; + vi.spyOn(rebuildOnboardDependencies, "onboard").mockImplementation(async (options) => { + observedFingerprint = options.recreateJournalTargetIntentFingerprint; + const carried = onboardSession.loadSession()?.checkpoint; + observedJournalPhase = carried?.sandboxRecreate?.phase; + observedCheckpointSessionId = carried?.sessionId; + }); + + await expect(runRebuildRecreatePhase(makeInput())).resolves.toBe(true); + + expect(observedFingerprint).toBe("intent-1"); + expect(observedJournalPhase).toBe("deleted"); + expect(observedCheckpointSessionId).toBe(onboardSession.loadSession()?.sessionId); + expect(observedCheckpointSessionId).not.toBe(retiredSessionId); + expect(onboardSession.loadSession()?.checkpoint?.effectGroups).toEqual({}); + }); + it("pins the authoritative restricted tier during recreate and restores ambient policy input", async () => { const previousPolicyTier = process.env.NEMOCLAW_POLICY_TIER; process.env.NEMOCLAW_POLICY_TIER = "open"; diff --git a/src/lib/actions/sandbox/rebuild-recreate-phase.ts b/src/lib/actions/sandbox/rebuild-recreate-phase.ts index 2bf430b00a2..b3f2508ff0a 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-phase.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-phase.ts @@ -6,6 +6,7 @@ import { RD as _RD, R } from "../../cli/terminal-style"; import type { SandboxMessagingPlan } from "../../messaging"; import { markLastStartedStepFailed } from "../../onboard/exit-step-failure"; import * as shields from "../../shields"; +import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; @@ -28,6 +29,7 @@ import { restoreMcpRegistryForRebuildRetry, } from "./rebuild-mcp-phase"; import { rebuildOnboardDependencies } from "./rebuild-onboard-dependencies"; +import type { RebuildRecreateJournal } from "./rebuild-recreate-journal"; import type { RebuildRegistryRollback } from "./rebuild-registry-rollback"; import type { RebuildResumeConfig } from "./rebuild-resume-config"; import { printRebuildShieldsRecovery, type RebuildShieldsWindow } from "./rebuild-shields"; @@ -40,6 +42,7 @@ export interface RebuildRecreatePhaseInput { durableConfig: RebuildDurableConfig; resumeConfig: RebuildResumeConfig; recreateOptions: RebuildRecreateOnboardOpts; + recreateJournal: RebuildRecreateJournal; fromDockerfile: string | null; rebuildAgent: string | null; messagingPlan: SandboxMessagingPlan | null; @@ -74,6 +77,7 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): durableConfig: rebuildDurableConfig, resumeConfig, recreateOptions, + recreateJournal, fromDockerfile: storedFromDockerfile, rebuildAgent, messagingPlan: rebuildMessagingPlan, @@ -103,6 +107,7 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): ); onboardSession.updateSession((s: Session) => { + const journaledCheckpoint = s.checkpoint; Object.assign( s, onboardSession.createSession({ @@ -154,6 +159,16 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): s.toolDisclosure = rebuildDurableConfig.toolDisclosure; s.observabilityEnabled = recreateOptions.observabilityEnabled; s.observabilityRequestedExplicitly = recreateOptions.observabilityRequestedExplicitly; + // The journal outlives this reset, but the retired session owns its effect + // receipts and bindings. Rebind only the values the journal invariant needs. + s.checkpoint = journaledCheckpoint + ? { + ...deriveCheckpointFromSession(s), + sandboxIdentity: journaledCheckpoint.sandboxIdentity, + gatewayAuthority: journaledCheckpoint.gatewayAuthority, + sandboxRecreate: journaledCheckpoint.sandboxRecreate, + } + : null; return s; }); const sessionAfter = onboardSession.loadSession(); @@ -196,7 +211,10 @@ export async function runRebuildRecreatePhase(input: RebuildRecreatePhaseInput): const restoreRebuildBaseImageOverride = pinRebuildAgentBaseImageForRecreate(rebuildBaseImagePreflight); try { - await rebuildOnboardDependencies.onboard(recreateOptions); + await rebuildOnboardDependencies.onboard({ + ...recreateOptions, + recreateJournalTargetIntentFingerprint: recreateJournal.targetIntentFingerprint, + }); log("onboard() returned successfully"); } catch (error) { onboardFailed = true; diff --git a/src/lib/actions/sandbox/rebuild-shields-finally.test.ts b/src/lib/actions/sandbox/rebuild-shields-finally.test.ts index 000da2e2ce2..b1f643b51c8 100644 --- a/src/lib/actions/sandbox/rebuild-shields-finally.test.ts +++ b/src/lib/actions/sandbox/rebuild-shields-finally.test.ts @@ -8,6 +8,12 @@ const phaseMocks = vi.hoisted(() => ({ runDestroy: vi.fn(), runPreflight: vi.fn(), runShields: vi.fn(), + openRecreateJournal: vi.fn(), +})); + +vi.mock("./rebuild-recreate-journal", () => ({ + fingerprintRebuildRecreateTargetIntent: () => "intent-1", + openRebuildRecreateJournal: phaseMocks.openRecreateJournal, })); vi.mock("./rebuild-backup-phase", () => ({ @@ -44,7 +50,11 @@ describe("rebuild shields relock guard", () => { phaseMocks.runPreflight.mockResolvedValue({ sandboxEntry: { name: "alpha", customPolicies: [] }, targetConfig: { durableConfig: { webSearchConfig: null } }, - recreateOptions: { observabilityEnabled: false }, + recreateOptions: { + observabilityEnabled: false, + targetGatewayName: "nemoclaw", + targetGatewayPort: 8080, + }, liveState: { staleRecovery: false, staleRegistrySnapshot: null }, recoveryManifest: null, dcodePreflight: { @@ -64,6 +74,13 @@ describe("rebuild shields relock guard", () => { phaseMocks.runBackup.mockImplementation(() => { throw new Error("unexpected backup exception"); }); + phaseMocks.openRecreateJournal.mockReturnValue({ + id: "journal-1", + targetGeneration: "generation-1", + targetIntentFingerprint: "intent-1", + markDeleting: vi.fn(), + confirmDeleted: vi.fn(), + }); }); it("relocks shields when an unexpected rebuild phase exception escapes after auto-unlock (#6245)", async () => { @@ -114,7 +131,11 @@ describe("rebuild shields relock guard", () => { }, }, targetConfig: { durableConfig: { webSearchConfig: null } }, - recreateOptions: { observabilityEnabled: false }, + recreateOptions: { + observabilityEnabled: false, + targetGatewayName: "nemoclaw", + targetGatewayPort: 8080, + }, liveState: { staleRecovery: false, staleRegistrySnapshot: null }, recoveryManifest: null, dcodePreflight: { cleanup: cleanupDcodePreflight }, diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 1b70da187fc..dbe58bcd158 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4380,6 +4380,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { authoritativeResumeConfig: opts.authoritativeResumeConfig === true, authoritativePolicyTier: opts.authoritativeResumeConfig === true ? (opts.policyTier ?? null) : undefined, + recreateJournalTargetIntentFingerprint: opts.recreateJournalTargetIntentFingerprint ?? null, resumeAgentChanged, requestedObservabilityEnabled: runtimeControlRequests.requestedObservabilityEnabled, requestedDcodeAutoApprovalMode: runtimeControlRequests.requestedDcodeAutoApprovalMode, diff --git a/src/lib/onboard/machine/core-flow-phases.ts b/src/lib/onboard/machine/core-flow-phases.ts index 962e1871dba..110069f12ad 100644 --- a/src/lib/onboard/machine/core-flow-phases.ts +++ b/src/lib/onboard/machine/core-flow-phases.ts @@ -66,6 +66,7 @@ export interface SandboxOnboardFlowPhaseOptions< gatewayName: string; authoritativeResumeConfig?: boolean; authoritativePolicyTier?: string | null; + recreateJournalTargetIntentFingerprint?: string | null; resumeAgentChanged: boolean; requestedObservabilityEnabled?: boolean | null; requestedDcodeAutoApprovalMode?: DcodeAutoApprovalMode | null; @@ -212,6 +213,7 @@ export function createSandboxOnboardFlowPhase< gatewayName: options.gatewayName, authoritativeResumeConfig: options.authoritativeResumeConfig, authoritativePolicyTier: options.authoritativePolicyTier, + recreateJournalTargetIntentFingerprint: options.recreateJournalTargetIntentFingerprint, endpointSource: endpointProvenance.endpointSource, resumeAgentChanged: options.resumeAgentChanged, requestedObservabilityEnabled: options.requestedObservabilityEnabled, diff --git a/src/lib/onboard/machine/handlers/sandbox.ts b/src/lib/onboard/machine/handlers/sandbox.ts index 16e428e3e7d..5fb54256ba5 100644 --- a/src/lib/onboard/machine/handlers/sandbox.ts +++ b/src/lib/onboard/machine/handlers/sandbox.ts @@ -132,6 +132,8 @@ export interface SandboxStateOptions< authoritativePolicyTier?: string | null; /** Endpoint source to preserve during an authoritative rebuild. */ endpointSource?: InferenceEndpointSource | null; + /** Internal rebuild target fingerprint recorded by the journal opened before deletion. */ + recreateJournalTargetIntentFingerprint?: string | null; resumeAgentChanged: boolean; requestedObservabilityEnabled?: boolean | null; requestedDcodeAutoApprovalMode?: DcodeAutoApprovalMode | null; @@ -1223,9 +1225,6 @@ class SandboxStateFlow< const sourceEntry = this.deps.getSandboxRegistryEntry(sandboxName); if (!existing && !sourceEntry) return null; const observation = this.deps.getSandboxRecreateObservation(sandboxName); - const targetIntentFingerprint = fingerprintSandboxRecreateValue( - this.currentSandboxCreateFingerprint(sandboxName, createIntent.resolved), - ); const updated = this.deps.updateSession((current) => { beginSandboxRecreateTransaction(current, { sandboxName, @@ -1233,13 +1232,27 @@ class SandboxStateFlow< gatewayPort: gateway.gatewayPort, sourceEntry, observation, - targetIntentFingerprint, + targetIntentFingerprint: this.sandboxRecreateTargetIntentFingerprint( + sandboxName, + createIntent, + ), }); return current; }); return updated.checkpoint?.sandboxRecreate ?? null; } + private sandboxRecreateTargetIntentFingerprint( + sandboxName: string, + createIntent: CompleteSandboxCreateIntent, + ): string { + const journaled = this.options.recreateJournalTargetIntentFingerprint; + if (journaled) return journaled; + return fingerprintSandboxRecreateValue( + this.currentSandboxCreateFingerprint(sandboxName, createIntent.resolved), + ); + } + private recordSandboxRecreatePhase( transaction: CheckpointSandboxRecreateTransaction, phase: Parameters[2], diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index 03cb17c8c48..b429fd86edd 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -142,6 +142,13 @@ function phaseIndex(phase: CheckpointSandboxRecreatePhase): number { return ORDERED_PHASES.indexOf(phase); } +export function sandboxRecreatePhaseReached( + phase: CheckpointSandboxRecreatePhase, + target: CheckpointSandboxRecreatePhase, +): boolean { + return phaseIndex(phase) >= phaseIndex(target); +} + export function advanceSandboxRecreateTransaction( session: Session, id: string, diff --git a/src/lib/onboard/types.ts b/src/lib/onboard/types.ts index b60f238fff3..5227b6d44c6 100644 --- a/src/lib/onboard/types.ts +++ b/src/lib/onboard/types.ts @@ -92,6 +92,8 @@ export type OnboardOptions = { targetGatewayPort?: number | null; /** Internal rebuild handoff: the outer destructive lifecycle owns the onboard lock. */ onboardLockAlreadyHeld?: boolean; + /** Internal rebuild handoff: target fingerprint of the journal opened before deletion. */ + recreateJournalTargetIntentFingerprint?: string | null; /** Internal one-shot handoff for a prevalidated managed DCode replacement. */ preparedDcodeRebuild?: import("./prepared-dcode-rebuild").PreparedDcodeRebuildHandoff; /** Internal authoritative registry route captured before rebuild deletion. */ diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index ef2846b26c6..93e73636f9c 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -225,10 +225,23 @@ function createStep(status: string): RebuildFlowStep { function createRebuildFlowSession(machineSnapshotVersion: number): RebuildFlowSession { return { + sessionId: "rebuild-flow-session", + updatedAt: "2026-06-01T00:00:00.000Z", sandboxName: "alpha", + agent: null, provider: "ollama-local", model: "nvidia/nemotron", credentialEnv: null, + checkpoint: null, + webSearchConfig: null, + resourceProfile: null, + messagingPlan: null, + sandboxPromptProgress: { + sandboxName: true, + webSearch: false, + messaging: false, + resourceProfile: false, + }, metadata: {}, hermesToolGateways: [], lastStepStarted: null, diff --git a/test/helpers/rebuild-flow-lifecycle-cases.ts b/test/helpers/rebuild-flow-lifecycle-cases.ts index af32d4ade64..470e71f1e12 100644 --- a/test/helpers/rebuild-flow-lifecycle-cases.ts +++ b/test/helpers/rebuild-flow-lifecycle-cases.ts @@ -113,7 +113,7 @@ export function registerRebuildFlowLifecycleTests(): void { expect(harness.restoreMcpBridgesAfterRebuildSpy).toHaveBeenCalledWith("alpha", [mcpEntry]); expect(harness.removeSandboxRegistryEntryWithReceiptSpy).not.toHaveBeenCalled(); expect(harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "Preserving MCP-bearing registry entry across sandbox recreation", + "Preserving journaled source registry entry across sandbox recreation", ); expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "npm"); expect(harness.applyPresetSpy).toHaveBeenCalledWith("alpha", "bad"); @@ -159,7 +159,7 @@ export function registerRebuildFlowLifecycleTests(): void { expect(harness.removeSandboxRegistryEntryWithReceiptSpy).not.toHaveBeenCalled(); expect(harness.onboardSpy).toHaveBeenCalledOnce(); expect(harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "Preserving baseline-exclusion registry entry across sandbox recreation", + "Preserving journaled source registry entry across sandbox recreation", ); expect(harness.restoreSandboxEntrySpy).not.toHaveBeenCalled(); expect(harness.restoreSandboxEntryIfMissingSpy).not.toHaveBeenCalled(); @@ -238,7 +238,7 @@ network_policies: expect(harness.restoreSandboxEntrySpy).not.toHaveBeenCalled(); expect(harness.restoreSandboxEntryIfMissingSpy).not.toHaveBeenCalled(); expect(harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "Preserving baseline-exclusion registry entry across sandbox recreation", + "Preserving journaled source registry entry across sandbox recreation", ); }); @@ -248,7 +248,7 @@ network_policies: const probeSequence = [ { event: "stale-live", - result: { status: 0, output: "Sandbox: alpha\nPhase: Ready" }, + result: { status: 0, output: "Sandbox: alpha\nId: sbx-0d6f4c2a91\nPhase: Ready" }, }, { event: "absent", @@ -271,12 +271,14 @@ network_policies: harness.rebuildSandbox("alpha", ["--yes", "--verbose"], { throwOnError: true }), ).resolves.toBeUndefined(); - expect(events).toEqual(["stale-live", "absent", "onboard"]); + // Open the journal against the live source, wait for absence, then prove + // absence once more before the journal records the deleted phase (#7734). + expect(events).toEqual(["stale-live", "absent", "absent", "onboard"]); expect( harness.captureOpenshellSpy.mock.calls.filter( ([args]) => Array.isArray(args) && args.join(" ") === "sandbox get -g nemoclaw alpha", ), - ).toHaveLength(2); + ).toHaveLength(3); }); it("accepts the agent version cached by the confirmation probe before lock acquisition", async () => { @@ -383,18 +385,18 @@ network_policies: } }); - it("relocks as absent when registry cleanup throws after confirmed delete", async () => { + it("relocks as absent and keeps the journaled row when replacement creation fails (#7734)", async () => { const harness = createRebuildFlowHarness({ - removeSandboxRegistryEntryWithReceipt: () => { - throw new Error("registry cleanup after delete failed"); + onboard: () => { + throw new Error("recreate failed"); }, }); await expect( harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), - ).rejects.toThrow("registry cleanup after delete failed"); + ).rejects.toThrow("Recreate failed"); - expect(harness.onboardSpy).not.toHaveBeenCalled(); + expect(harness.removeSandboxRegistryEntryWithReceiptSpy).not.toHaveBeenCalled(); expect(harness.relockSpy).toHaveBeenLastCalledWith( "alpha", expect.any(Object), diff --git a/test/helpers/rebuild-flow-recovery-cases.ts b/test/helpers/rebuild-flow-recovery-cases.ts index 1aba1431732..633a7415c5d 100644 --- a/test/helpers/rebuild-flow-recovery-cases.ts +++ b/test/helpers/rebuild-flow-recovery-cases.ts @@ -201,28 +201,16 @@ export function registerRebuildFlowRecoveryTests(): void { expect(harness.backupSandboxStateSpy).not.toHaveBeenCalled(); expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( expect.objectContaining({ name: "alpha", agentVersion: "0.1.0" }), - { - defaultTransition: { - from: null, - to: "alpha", - expectedRevision: 11, - }, - }, + {}, ); expect(harness.restoreSandboxStateSpy).not.toHaveBeenCalled(); }); - it("preserves an explicit same-fallback default choice during prepared rollback", async () => { + it("keeps an explicit default choice made while the replacement was in flight (#7734)", async () => { let harness!: ReturnType; harness = createRebuildFlowHarness({ defaultSandbox: "alpha", defaultSelectionRevision: 10, - removalReceipt: { - entry: { name: "alpha", agentVersion: "0.1.0" }, - wasDefault: true, - fallbackDefault: "beta", - postRemovalDefaultSelectionRevision: 11, - }, onboard: () => { expect(harness.setDefault("beta")).toBe(true); throw new Error("recreate failed after explicit default choice"); @@ -238,35 +226,23 @@ export function registerRebuildFlowRecoveryTests(): void { expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( expect.objectContaining({ name: "alpha" }), - { - defaultTransition: { - from: "beta", - to: "alpha", - expectedRevision: 11, - }, - }, + {}, ); expect(harness.getDefaultSelectionState()).toEqual({ defaultSandbox: "beta", - defaultSelectionRevision: 12, + defaultSelectionRevision: 11, }); }); - it("preserves replacement registry metadata after a custom removal receipt", async () => { + it("keeps the default selection untouched across a journaled replacement (#7734)", async () => { let harness!: ReturnType; harness = createRebuildFlowHarness({ defaultSandbox: "alpha", defaultSelectionRevision: 10, - removeSandboxRegistryEntryWithReceipt: () => ({ - entry: { name: "alpha", model: "old-model" }, - wasDefault: true, - fallbackDefault: "beta", - postRemovalDefaultSelectionRevision: 11, - }), onboard: () => { expect(harness.getDefaultSelectionState()).toEqual({ - defaultSandbox: "beta", - defaultSelectionRevision: 11, + defaultSandbox: "alpha", + defaultSelectionRevision: 10, }); harness.registerSandboxEntry("alpha"); throw new Error("recreate failed after replacement registration"); @@ -277,14 +253,12 @@ export function registerRebuildFlowRecoveryTests(): void { harness.rebuildSandbox("alpha", ["--yes", "--verbose"], { throwOnError: true }), ).rejects.toThrow("Recreate failed"); - expect(harness.restoreSandboxEntryIfMissingSpy).toHaveReturnedWith(false); + expect(harness.removeSandboxRegistryEntryWithReceiptSpy).not.toHaveBeenCalled(); + expect(harness.restoreSandboxEntryIfMissingSpy).not.toHaveBeenCalled(); expect(harness.getDefaultSelectionState()).toEqual({ - defaultSandbox: "beta", - defaultSelectionRevision: 11, + defaultSandbox: "alpha", + defaultSelectionRevision: 10, }); - expect(harness.errorSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain( - "Recreate failed: kept the replacement registry metadata already present", - ); }); it("performs exactly one prepared-recovery rollback when MCP state is present", async () => { @@ -343,13 +317,7 @@ export function registerRebuildFlowRecoveryTests(): void { expect(harness.restoreSandboxEntrySpy).toHaveBeenCalledWith( expect.objectContaining({ name: "alpha", toolDisclosure: "progressive" }), - { - defaultTransition: { - from: null, - to: "alpha", - expectedRevision: 11, - }, - }, + {}, ); expect(harness.errorSpy).toHaveBeenCalledWith( expect.stringContaining("onboard --resume --name alpha --tool-disclosure direct"), diff --git a/test/helpers/rebuild-flow-test-support.ts b/test/helpers/rebuild-flow-test-support.ts index 195f3a5e8f3..69459905b7d 100644 --- a/test/helpers/rebuild-flow-test-support.ts +++ b/test/helpers/rebuild-flow-test-support.ts @@ -177,10 +177,23 @@ function createStep(status: string): RebuildFlowStep { } export function createRebuildFlowSession(machineSnapshotVersion: number): RebuildFlowSession { return { + sessionId: "rebuild-flow-session", + updatedAt: "2026-06-01T00:00:00.000Z", sandboxName: "alpha", + agent: null, provider: "ollama-local", model: "nvidia/nemotron", credentialEnv: null, + checkpoint: null, + webSearchConfig: null, + resourceProfile: null, + messagingPlan: null, + sandboxPromptProgress: { + sandboxName: true, + webSearch: false, + messaging: false, + resourceProfile: false, + }, metadata: {}, hermesToolGateways: [], lastStepStarted: null, diff --git a/test/mcp-destroy-lifecycle.test.ts b/test/mcp-destroy-lifecycle.test.ts index b47fae403ac..a7d4c72447d 100644 --- a/test/mcp-destroy-lifecycle.test.ts +++ b/test/mcp-destroy-lifecycle.test.ts @@ -91,8 +91,19 @@ vi.mock("../src/lib/inference/nim", () => ({ import * as bridge from "../src/lib/actions/sandbox/mcp-bridge"; import { isAgentMcpAdapter } from "../src/lib/actions/sandbox/mcp-bridge-contracts"; import { runRebuildDestroyPhase } from "../src/lib/actions/sandbox/rebuild-destroy-phase"; +import type { RebuildRecreateJournal } from "../src/lib/actions/sandbox/rebuild-recreate-journal"; import * as registry from "../src/lib/state/registry"; +function stubRecreateJournal(): RebuildRecreateJournal { + return { + id: "journal-1", + targetGeneration: "generation-1", + targetIntentFingerprint: "intent-1", + markDeleting: vi.fn(), + confirmDeleted: vi.fn(), + }; +} + const MATCHING_OPENSHELL = path.resolve("test/fixtures/openshell-v0.0.85"); const bridgeEntries: Record<"github" | "slack", McpBridgeEntry> = { @@ -872,6 +883,7 @@ describe("authenticated MCP sandbox destroy lifecycle", () => { sandboxName: "alpha", sandboxEntry: before ?? { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), @@ -948,6 +960,7 @@ describe("authenticated MCP sandbox destroy lifecycle", () => { sandboxName: "alpha", sandboxEntry: beforeRegistry ?? { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, force: true, log: vi.fn(), From dc778c1e63d0d181bcf274a12d3ac67c22626812 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 29 Jul 2026 08:59:37 +0000 Subject: [PATCH 02/25] fix(onboard): journal non-resumed same-name recreation `createSandbox` re-derives its own recreation triggers, so a plain onboard run could delete and replace a live same-name sandbox with no durable replacement intent. The sandbox handler only opened the canonical journal when resuming, and without one the lifecycle runtime was an inert stub: the delete boundary, the post-delete absence proof, and the registry-row reservation all became no-ops. Open the canonical recreate transaction inside `createSandbox`, after every required confirmation and non-mutating check and before provider cleanup, the OpenShell delete, or registry removal. Bind it to the sandbox name, the resolved gateway name and port, the source registry fingerprint and generation, the hashed source OpenShell identity, and a target fingerprint built from validated non-secret create inputs. An active journal is reconciled through `planSandboxRecreateRecovery` before any mutation, so a later invocation resumes the replacement without `--resume`. Fail closed instead of guessing: a missing source registry row, a live sandbox with no stable OpenShell Id, a changed source identity, a changed target fingerprint, and a probe that reports neither a live sandbox nor explicit absence all stop the command before the next mutation. Name the gateway explicitly on the sandbox get, list, and delete commands that drive replacement, so a host running several gateways cannot answer or mutate for a sibling. Extract the shared strict-absence classifier and gateway-scoped observation so the rebuild and onboard journals prove absence the same way. Signed-off-by: Tinson Lai --- ci/source-architecture-budget.json | 14 +- src/lib/actions/sandbox/gateway-state.ts | 31 -- .../actions/sandbox/rebuild-destroy-phase.ts | 2 +- .../sandbox/rebuild-recreate-journal.ts | 55 +--- src/lib/onboard.ts | 12 +- .../onboard/onboard-recreate-journal.test.ts | 277 ++++++++++++++++++ src/lib/onboard/onboard-recreate-journal.ts | 121 ++++++++ src/lib/onboard/sandbox-recreate-probe.ts | 82 ++++++ src/lib/onboard/sandbox-reuse.ts | 15 +- test/onboard-sandbox-recreation.test.ts | 68 +++-- 10 files changed, 564 insertions(+), 113 deletions(-) create mode 100644 src/lib/onboard/onboard-recreate-journal.test.ts create mode 100644 src/lib/onboard/onboard-recreate-journal.ts create mode 100644 src/lib/onboard/sandbox-recreate-probe.ts diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 429f215d151..2590c4f6a9e 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -6,7 +6,7 @@ "src/lib/actions/sandbox/mcp-bridge-contracts.ts": 26, "src/lib/actions/sandbox/process-recovery.ts": 26, "src/lib/adapters/docker/index.ts": 45, - "src/lib/adapters/openshell/client.ts": 22, + "src/lib/adapters/openshell/client.ts": 23, "src/lib/adapters/openshell/resolve.ts": 28, "src/lib/adapters/openshell/runtime.ts": 51, "src/lib/adapters/openshell/timeouts.ts": 37, @@ -26,8 +26,8 @@ "src/lib/onboard/gateway-binding.ts": 47, "src/lib/runner.ts": 89, "src/lib/security/redact.ts": 51, - "src/lib/state/onboard-session.ts": 35, - "src/lib/state/registry.ts": 100, + "src/lib/state/onboard-session.ts": 36, + "src/lib/state/registry.ts": 101, "src/lib/state/state-root.ts": 23, "src/lib/subprocess-env.ts": 23, "src/lib/validation.ts": 25 @@ -47,7 +47,7 @@ "src/lib/actions/uninstall/run-plan.ts": 25, "src/lib/inference/onboard-probes.ts": 21, "src/lib/inference/vllm.ts": 23, - "src/lib/onboard.ts": 224, + "src/lib/onboard.ts": 225, "src/lib/onboard/machine/handlers/sandbox.ts": 21, "src/lib/sandbox/config.ts": 23, "src/lib/shields/index.ts": 23 @@ -112,9 +112,7 @@ "src/lib/actions/sandbox/rebuild-pipeline.ts", "src/lib/actions/sandbox/rebuild-post-restore-phase.ts", "src/lib/actions/sandbox/rebuild-preflight-phase.ts", - "src/lib/actions/sandbox/rebuild-preflight-target-phase.ts", - "src/lib/actions/sandbox/rebuild-recreate-journal.ts", - "src/lib/actions/sandbox/rebuild-recreate-phase.ts", + "src/lib/actions/sandbox/rebuild-preflight-target-phase.ts", "src/lib/actions/sandbox/rebuild-recreate-phase.ts", "src/lib/actions/sandbox/rebuild-shields-phase.ts", "src/lib/actions/sandbox/rebuild-shields.ts", "src/lib/actions/sandbox/rebuild-target-config.ts", @@ -139,7 +137,7 @@ ] ], "maxRootFiles": { - "src/lib/onboard": 307, + "src/lib/onboard": 309, "src/lib/actions": 19, "src/lib/actions/sandbox": 183, "src/lib/state": 38, diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index 37e35a75d41..256227bae1f 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -101,37 +101,6 @@ export function isMissingSandboxGatewayOutput(output = ""): boolean { ); } -/** - * Strict absence classifier for destructive owner-gateway reconciliation. - * Bare NotFound is not sufficient because OpenShell uses it for missing - * gateways and providers as well as sandboxes. - */ -export function isExplicitMissingSandboxGatewayOutput( - output: string, - sandboxName: string, -): boolean { - const clean = stripAnsi(String(output)).replace(/\r/g, "").trim(); - const exactNoSpec = - /^(?:error:\s*)?status:\s*Internal,\s*message:\s*["']sandbox has no spec["'](?:,\s*details:\s*\[\])?(?:,\s*metadata:\s*MetadataMap\s*\{\s*\})?$/i; - if (exactNoSpec.test(clean)) return true; - // OpenShell can omit the requested name from an owner-scoped lookup. - // Require both exact structured fields so gateway/provider absence and - // transport diagnostics remain ambiguous. - const exactStructuredNotFound = - /^(?:error:\s*)?(?:×\s*)?code:\s*["']Some requested entity was not found["']\s*,\s*message:\s*["']sandbox not found["']$/i; - if (exactStructuredNotFound.test(clean)) return true; - - const escapedName = sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - const namedSandbox = `(?:['\"]${escapedName}['\"]|${escapedName})`; - return ( - new RegExp( - `^(?:error:\\s*)?sandbox\\s+${namedSandbox}\\s+(?:(?:is\\s+)?not\\s+(?:found|present)|does\\s+not\\s+exist)[.!]?$`, - "i", - ).test(clean) || - new RegExp(`^(?:error:\\s*)?no\\s+such\\s+sandbox\\s+${namedSandbox}[.!]?$`, "i").test(clean) - ); -} - function formatGatewaySchemaMismatchOutput( issue: OpenShellStateRpcIssue, action: string, diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index d3e805b96bd..49181bedeab 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -8,11 +8,11 @@ import { waitUntil } from "../../core/wait"; import { getSandboxDeleteOutcome } from "../../domain/sandbox/destroy"; import * as nim from "../../inference/nim"; import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding"; +import { isExplicitMissingSandboxGatewayOutput } from "../../onboard/sandbox-recreate-probe"; import { redactFull } from "../../security/redact"; import { parseSandboxPhase } from "../../state/gateway"; import { registryEntryGatewayPort } from "../../state/gateway-registry"; import * as registry from "../../state/registry"; -import { isExplicitMissingSandboxGatewayOutput } from "./gateway-state"; import type { RebuildBackupManifest } from "./rebuild-backup-phase"; import type { RebuildBail, RebuildLog } from "./rebuild-credential-preflight"; import { type RebuildSandboxEntry, warnUnpreservedUserManagedFiles } from "./rebuild-flow-helpers"; diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.ts index f4272cb778d..4ca7848e768 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.ts @@ -1,37 +1,30 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { captureOpenshell } from "../../adapters/openshell/runtime"; -import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import { checkpointGatewayAuthority } from "../../onboard/gateway-authority-checkpoint"; import { resolveGatewayTeardownAuthority } from "../../onboard/gateway-teardown-authority"; +import { + observeSandboxOnGateway, + type SandboxRecreateObserver, + type SandboxRecreateTarget, +} from "../../onboard/sandbox-recreate-probe"; import { advanceSandboxRecreateTransaction, beginSandboxRecreateTransaction, - fingerprintSandboxLiveIdentity, fingerprintSandboxRecreateValue, planSandboxRecreateRecovery, - type SandboxRecreateObservation, sandboxRecreatePhaseReached, } from "../../onboard/sandbox-recreate-transaction"; -import { parseSandboxPhase } from "../../state/gateway"; import { decisionSelected } from "../../state/onboard-checkpoint-decision"; import { deriveCheckpointFromSession } from "../../state/onboard-checkpoint-migrate"; import type { CheckpointSandboxRecreatePhase } from "../../state/onboard-checkpoint-types"; import * as onboardSession from "../../state/onboard-session"; import * as registry from "../../state/registry"; -import { isExplicitMissingSandboxGatewayOutput } from "./gateway-state"; import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; -export interface RebuildRecreateJournalTarget { - readonly sandboxName: string; - readonly gatewayName: string; - readonly gatewayPort: number; -} +export type RebuildRecreateJournalTarget = SandboxRecreateTarget; -export type RebuildSandboxObserver = ( - target: RebuildRecreateJournalTarget, -) => SandboxRecreateObservation; +export type RebuildSandboxObserver = SandboxRecreateObserver; export interface RebuildRecreateJournal { readonly id: string; @@ -73,39 +66,7 @@ export function fingerprintRebuildRecreateTargetIntent( }); } -export function observeRebuildSandbox( - target: RebuildRecreateJournalTarget, -): SandboxRecreateObservation { - const probe = captureOpenshell(["sandbox", "get", "-g", target.gatewayName, target.sandboxName], { - ignoreError: true, - includeStderr: true, - includeStreams: true, - timeout: OPENSHELL_PROBE_TIMEOUT_MS, - }); - const stdout = String(probe.stdout ?? (probe.status === 0 ? probe.output : "")).trim(); - const combined = `${stdout}\n${String(probe.stderr ?? probe.output ?? "")}`.trim(); - const failedCleanly = - !probe.error && !probe.signal && probe.status !== null && probe.status !== 0; - if (failedCleanly && isExplicitMissingSandboxGatewayOutput(combined, target.sandboxName)) { - return { state: "missing", liveIdentityFingerprint: null }; - } - if (probe.status === 0 && stdout.length > 0) { - const liveIdentityFingerprint = fingerprintSandboxLiveIdentity(stdout); - if (!liveIdentityFingerprint) { - throw new Error( - `Cannot journal sandbox '${target.sandboxName}' replacement: OpenShell did not report a stable sandbox Id on gateway '${target.gatewayName}'.`, - ); - } - const phase = parseSandboxPhase(combined); - return { - state: phase === "Ready" || phase === "Running" ? "ready" : "not_ready", - liveIdentityFingerprint, - }; - } - throw new Error( - `Cannot journal sandbox '${target.sandboxName}' replacement: gateway '${target.gatewayName}' reported neither a live sandbox nor explicit absence.`, - ); -} +export const observeRebuildSandbox = observeSandboxOnGateway; export interface OpenRebuildRecreateJournalInput { readonly target: RebuildRecreateJournalTarget; diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index dbe58bcd158..7c98a529986 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -597,6 +597,10 @@ import { filterEnabledChannelsByAgent } from "./onboard/messaging-state"; import { getValidatedMessagingTokenByEnvKey } from "./onboard/messaging-token"; import * as ollamaFlow from "./onboard/ollama-probe-failure"; import { runOllamaStartupOrGate } from "./onboard/ollama-startup"; +import { + fingerprintOnboardRecreateTargetIntent, + openOnboardRecreateJournal, +} from "./onboard/onboard-recreate-journal"; import type { DockerDriverBinaryOverrides, OpenShellInstallDeps, @@ -786,7 +790,7 @@ const { getGatewayReuseSnapshot, selectNamedGatewayForReuseIfNeeded } = }); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. -const { getSandboxReuseState, getSandboxRecreateObservation, repairRecordedSandbox } = sandboxReuse.createSandboxReuseHelpers({ runCaptureOpenshell, runOpenshell, getSandboxStateFromOutputs, note }); +const { getSandboxReuseState, getSandboxRecreateObservation, repairRecordedSandbox } = sandboxReuse.createSandboxReuseHelpers({ runCaptureOpenshell, runOpenshell, getSandboxStateFromOutputs, note, getGatewayName: () => GATEWAY_NAME }); const { executeSandboxCommandForVerification, @@ -2287,7 +2291,7 @@ async function createSandboxWithBaseImageResolution( // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const { existingEntry, preservedMcpState, liveExists, effectiveToolDisclosure, toolDisclosureMigrationNeeded, toolDisclosureMigrationNote } = toolDisclosureFlow.prepareSandboxToolDisclosure(sandboxName, preparedBuildContext?.rebuildTarget?.fromDockerfile ? preparedBuildContext.stagedDockerfile : fromDockerfile, isRecreateSandbox(createIntent?.recreate), inspectSandboxForCreate, createIntent?.toolDisclosure ?? null); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const recreateRuntime = sandboxRecreateTransaction.createSandboxRecreateRuntime(onboardSession, createIntent?.recreateTransaction, sandboxName, GATEWAY_NAME, existingEntry, getSandboxRecreateObservation, note); + let recreateRuntime = sandboxRecreateTransaction.createSandboxRecreateRuntime(onboardSession, createIntent?.recreateTransaction, sandboxName, GATEWAY_NAME, existingEntry, getSandboxRecreateObservation, note); const restoreReusedSandboxDashboard = (selectionVerified: boolean): void => { ({ chatUiUrl } = sandboxReuse.applyReusedSandboxDashboardState({ sandboxName, @@ -2559,6 +2563,8 @@ async function createSandboxWithBaseImageResolution( process.exit(1); } + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + if (!createIntent?.recreateTransaction) recreateRuntime = openOnboardRecreateJournal({ target: { sandboxName, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT }, agentName: getRequestedSandboxAgentName(agent) || "openclaw", note, observe: (probeTarget) => getSandboxRecreateObservation(probeTarget.sandboxName), targetIntentFingerprint: fingerprintOnboardRecreateTargetIntent({ agent: getRequestedSandboxAgentName(agent) || null, fromDockerfile: fromDockerfile ?? null, provider: provider ?? null, model: model ?? null, preferredInferenceApi: preferredInferenceApi ?? null, sandboxGpuConfig: effectiveSandboxGpuConfig ?? null, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, toolDisclosure: effectiveToolDisclosure, dcodeAutoApprovalMode: createIntent?.dcodeAutoApprovalMode ?? null, observabilityEnabled: createIntent?.observabilityEnabled === true, policyTier: createIntent?.policyTier ?? null }) }); const previousEntry: SandboxEntry | null = registry.getSandbox(sandboxName); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. baseImageResolutionFlow.captureBaseResolution(baseImageResolutionContext, previousEntry?.imageTag); @@ -2582,7 +2588,7 @@ async function createSandboxWithBaseImageResolution( recreateRuntime.advance("deleting"); runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); - runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); + runOpenshell(["sandbox", "delete", "-g", GATEWAY_NAME, sandboxName], { ignoreError: true }); recreateRuntime.confirmDeleted(); if (previousEntry?.imageTag) { // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. diff --git a/src/lib/onboard/onboard-recreate-journal.test.ts b/src/lib/onboard/onboard-recreate-journal.test.ts new file mode 100644 index 00000000000..b0ab921d5ee --- /dev/null +++ b/src/lib/onboard/onboard-recreate-journal.test.ts @@ -0,0 +1,277 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + captureOpenshell: vi.fn(), + resolveGatewayTeardownAuthority: vi.fn(), +})); + +vi.mock("../adapters/openshell/runtime", () => ({ + captureOpenshell: mocks.captureOpenshell, + runOpenshell: vi.fn(), +})); + +vi.mock("./gateway-teardown-authority", () => ({ + resolveGatewayTeardownAuthority: mocks.resolveGatewayTeardownAuthority, +})); + +import type { Session } from "../state/onboard-session"; +import * as onboardSession from "../state/onboard-session"; +import * as registry from "../state/registry"; +import { + fingerprintOnboardRecreateTargetIntent, + type OnboardRecreateTargetIntent, + openOnboardRecreateJournal, +} from "./onboard-recreate-journal"; + +const BASE_INTENT: OnboardRecreateTargetIntent = { + agent: "openclaw", + fromDockerfile: null, + provider: "nvidia-prod", + model: "gpt-5.4", + preferredInferenceApi: "openai-completions", + sandboxGpuConfig: { sandboxGpuEnabled: false, mode: "0" }, + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + toolDisclosure: "progressive", + dcodeAutoApprovalMode: null, + observabilityEnabled: false, + policyTier: "restricted", +}; + +describe("non-resumed replacement target fingerprint (#7735)", () => { + it("is stable for the same replacement intent", () => { + expect(fingerprintOnboardRecreateTargetIntent({ ...BASE_INTENT })).toBe( + fingerprintOnboardRecreateTargetIntent(BASE_INTENT), + ); + }); + + it("changes when a recorded replacement input changes", () => { + for (const drift of [ + { observabilityEnabled: true }, + { toolDisclosure: "direct" }, + { sandboxGpuConfig: { sandboxGpuEnabled: true, mode: "all" } }, + { dcodeAutoApprovalMode: "thread-opt-in" }, + { policyTier: "balanced" }, + ]) { + expect(fingerprintOnboardRecreateTargetIntent({ ...BASE_INTENT, ...drift })).not.toBe( + fingerprintOnboardRecreateTargetIntent(BASE_INTENT), + ); + } + }); + + it("changes when the replacement targets another gateway", () => { + expect( + fingerprintOnboardRecreateTargetIntent({ + ...BASE_INTENT, + gatewayName: "nemoclaw", + gatewayPort: 8080, + }), + ).not.toBe(fingerprintOnboardRecreateTargetIntent(BASE_INTENT)); + }); +}); + +const SANDBOX_ID = "sbx-71c9a4e08b"; +const TARGET_FINGERPRINT = "a".repeat(64); + +const NON_DEFAULT_TARGET = { + sandboxName: "alpha", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, +}; + +function livePresentProbe(phase = "Ready") { + const rendered = `Name: alpha\nId: ${SANDBOX_ID}\nPhase: ${phase}\n`; + return { status: 0, output: rendered, stdout: rendered, stderr: "" }; +} + +function absentProbe() { + return { + status: 1, + output: "", + stdout: "", + stderr: "Error: sandbox alpha not found", + }; +} + +describe("non-resumed onboard replacement journal (#7735)", () => { + let session: Session; + + beforeEach(() => { + vi.clearAllMocks(); + session = onboardSession.createSession({ sandboxName: "alpha" }); + vi.spyOn(onboardSession, "loadSession").mockImplementation(() => session); + vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator) => { + session = mutator(session) ?? session; + return session; + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + } as registry.SandboxEntry); + mocks.resolveGatewayTeardownAuthority.mockReturnValue({ + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }); + mocks.captureOpenshell.mockReturnValue(livePresentProbe()); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + function open(targetIntentFingerprint: string = TARGET_FINGERPRINT) { + return openOnboardRecreateJournal({ + target: NON_DEFAULT_TARGET, + agentName: "openclaw", + targetIntentFingerprint, + note: vi.fn(), + }); + } + + it("journals an explicit recreation before the delete command runs", () => { + open(); + + const recorded = session.checkpoint?.sandboxRecreate; + expect(recorded?.phase).toBe("planned"); + expect(recorded?.sandboxName).toBe("alpha"); + expect(recorded?.targetIntentFingerprint).toBe(TARGET_FINGERPRINT); + expect(recorded?.sourceLiveIdentityFingerprint).toMatch(/^[0-9a-f]{64}$/); + expect(JSON.stringify(session.checkpoint)).not.toContain(SANDBOX_ID); + }); + + it("binds the journal to the selected sandbox identity and gateway authority", () => { + open(); + + expect(mocks.resolveGatewayTeardownAuthority).toHaveBeenCalledWith({ + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + }); + const identity = session.checkpoint?.sandboxIdentity; + expect(identity?.kind === "selected" && identity.value).toEqual({ + name: "alpha", + agent: "openclaw", + }); + const authority = session.checkpoint?.gatewayAuthority; + expect(authority?.kind === "selected" && authority.value.gatewayPort).toBe(9090); + expect(session.checkpoint?.sandboxRecreate?.gatewayName).toBe("nemoclaw-9090"); + expect(session.checkpoint?.sandboxRecreate?.gatewayPort).toBe(9090); + }); + + it("queries only the journaled gateway so a sibling gateway is never reached", () => { + open(); + + for (const call of mocks.captureOpenshell.mock.calls) { + expect(call[0]).toEqual(["sandbox", "get", "-g", "nemoclaw-9090", "alpha"]); + } + expect(mocks.captureOpenshell).toHaveBeenCalled(); + }); + + it("journals a not-ready repair before the delete boundary", () => { + mocks.captureOpenshell.mockReturnValue(livePresentProbe("Pending")); + + open(); + + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("planned"); + }); + + it("records the delete boundary through the returned runtime", () => { + const runtime = open(); + + runtime.advance("deleting"); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); + + mocks.captureOpenshell.mockReturnValue(absentProbe()); + runtime.confirmDeleted(); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleted"); + }); + + it("stops before the next mutation when the source outlives its delete", () => { + const runtime = open(); + runtime.advance("deleting"); + + expect(() => runtime.confirmDeleted()).toThrow( + /OpenShell still reports the journaled source after delete/, + ); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); + }); + + it("resumes the same replacement after a restart without a resume flag", () => { + const first = open(); + first.advance("deleting"); + const firstId = session.checkpoint?.sandboxRecreate?.id; + + open(); + + expect(session.checkpoint?.sandboxRecreate?.id).toBe(firstId); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); + }); + + it("refuses a replacement whose target intent changed mid-transaction", () => { + open(); + + expect(() => open("b".repeat(64))).toThrow(/different recreate transaction in progress/); + }); + + it("refuses to continue when the live source identity no longer matches", () => { + open(); + mocks.captureOpenshell.mockReturnValue({ + status: 0, + output: "Name: alpha\nId: sbx-000000000\nPhase: Ready\n", + stdout: "Name: alpha\nId: sbx-000000000\nPhase: Ready\n", + stderr: "", + }); + + expect(() => open()).toThrow(/no longer has the journaled source identity/); + }); + + it("refuses to journal a replacement without its source registry row", () => { + vi.spyOn(registry, "getSandbox").mockReturnValue(null); + + expect(() => open()).toThrow(/without its source registry row/); + expect(session.checkpoint?.sandboxRecreate ?? null).toBeNull(); + }); + + it("fails closed when the gateway reports neither a live sandbox nor explicit absence", () => { + mocks.captureOpenshell.mockReturnValue({ + status: 1, + output: "", + stdout: "", + stderr: "Error: connection refused", + }); + + expect(() => open()).toThrow(/neither a live sandbox nor explicit absence/); + expect(session.checkpoint?.sandboxRecreate ?? null).toBeNull(); + }); + + it("fails closed when the live sandbox has no stable OpenShell Id", () => { + mocks.captureOpenshell.mockReturnValue({ + status: 0, + output: "Name: alpha\nPhase: Ready\n", + stdout: "Name: alpha\nPhase: Ready\n", + stderr: "", + }); + + expect(() => open()).toThrow(/did not report a stable sandbox Id/); + expect(session.checkpoint?.sandboxRecreate ?? null).toBeNull(); + }); + + it("opens at the deleted phase when the source sandbox is already gone", () => { + mocks.captureOpenshell.mockReturnValue(absentProbe()); + + open(); + + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleted"); + expect(session.checkpoint?.sandboxRecreate?.sourceLiveIdentityFingerprint).toBeNull(); + }); +}); diff --git a/src/lib/onboard/onboard-recreate-journal.ts b/src/lib/onboard/onboard-recreate-journal.ts new file mode 100644 index 00000000000..b68a3ef62dc --- /dev/null +++ b/src/lib/onboard/onboard-recreate-journal.ts @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { decisionSelected } from "../state/onboard-checkpoint-decision"; +import { deriveCheckpointFromSession } from "../state/onboard-checkpoint-migrate"; +import * as onboardSession from "../state/onboard-session"; +import * as registry from "../state/registry"; +import { checkpointGatewayAuthority } from "./gateway-authority-checkpoint"; +import { resolveGatewayTeardownAuthority } from "./gateway-teardown-authority"; +import { + observeSandboxOnGateway, + type SandboxRecreateObserver, + type SandboxRecreateTarget, +} from "./sandbox-recreate-probe"; +import { + beginSandboxRecreateTransaction, + createSandboxRecreateRuntime, + fingerprintSandboxRecreateValue, + planSandboxRecreateRecovery, + type SandboxRecreateRuntime, +} from "./sandbox-recreate-transaction"; + +export interface OnboardRecreateTargetIntent { + readonly agent: string | null; + readonly fromDockerfile: string | null; + readonly provider: string | null; + readonly model: string | null; + readonly preferredInferenceApi: string | null; + readonly sandboxGpuConfig: unknown; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly toolDisclosure: string; + readonly dcodeAutoApprovalMode: string | null; + readonly observabilityEnabled: boolean; + readonly policyTier: string | null; +} + +export function fingerprintOnboardRecreateTargetIntent( + intent: OnboardRecreateTargetIntent, +): string { + return fingerprintSandboxRecreateValue({ version: 1, ...intent }); +} + +export interface OpenOnboardRecreateJournalInput { + readonly target: SandboxRecreateTarget; + readonly agentName: string; + readonly targetIntentFingerprint: string; + readonly note: (message: string) => void; + readonly observe?: SandboxRecreateObserver; +} + +export function openOnboardRecreateJournal( + input: OpenOnboardRecreateJournalInput, +): SandboxRecreateRuntime { + const { target, agentName, targetIntentFingerprint, note } = input; + const observe = input.observe ?? observeSandboxOnGateway; + const authority = resolveGatewayTeardownAuthority({ + gatewayName: target.gatewayName, + gatewayPort: target.gatewayPort, + }); + const sourceEntry = registry.getSandbox(target.sandboxName); + if (!sourceEntry) { + throw new Error( + `Cannot start sandbox '${target.sandboxName}' recreate transaction without its source registry row.`, + ); + } + const observation = observe(target); + const active = onboardSession.loadSession()?.checkpoint?.sandboxRecreate ?? null; + if (active) { + const recovery = planSandboxRecreateRecovery(active, observation, sourceEntry); + if (recovery.action === "reject") { + throw new Error( + `Cannot resume sandbox '${target.sandboxName}' replacement: ${recovery.reason}.`, + ); + } + } + + const session = onboardSession.updateSession((current) => { + const checkpoint = current.checkpoint ?? deriveCheckpointFromSession(current); + current.checkpoint = { + ...checkpoint, + machineState: current.machine.state, + updatedAt: new Date().toISOString(), + sandboxIdentity: decisionSelected({ name: target.sandboxName, agent: agentName }), + gatewayAuthority: decisionSelected(checkpointGatewayAuthority(authority)), + }; + beginSandboxRecreateTransaction(current, { + sandboxName: target.sandboxName, + gatewayName: target.gatewayName, + gatewayPort: target.gatewayPort, + sourceEntry, + observation, + targetIntentFingerprint, + }); + return current; + }); + + const transaction = session.checkpoint?.sandboxRecreate; + if (!transaction) { + throw new Error( + `Sandbox '${target.sandboxName}' replacement journal could not be recorded before deletion.`, + ); + } + note( + ` Journaled replacement ${transaction.id} for '${target.sandboxName}' on ${target.gatewayName}:${String(target.gatewayPort)} at phase '${transaction.phase}'.`, + ); + + return createSandboxRecreateRuntime( + onboardSession, + { + id: transaction.id, + targetGeneration: transaction.targetGeneration, + targetIntentFingerprint: transaction.targetIntentFingerprint, + }, + target.sandboxName, + target.gatewayName, + sourceEntry, + (sandboxName) => observe({ ...target, sandboxName }), + note, + ); +} diff --git a/src/lib/onboard/sandbox-recreate-probe.ts b/src/lib/onboard/sandbox-recreate-probe.ts new file mode 100644 index 00000000000..eee61c9d3d8 --- /dev/null +++ b/src/lib/onboard/sandbox-recreate-probe.ts @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { stripAnsi } from "../adapters/openshell/client"; +import { captureOpenshell } from "../adapters/openshell/runtime"; +import { OPENSHELL_PROBE_TIMEOUT_MS } from "../adapters/openshell/timeouts"; +import { parseSandboxPhase } from "../state/gateway"; +import { + fingerprintSandboxLiveIdentity, + type SandboxRecreateObservation, +} from "./sandbox-recreate-transaction"; + +export interface SandboxRecreateTarget { + readonly sandboxName: string; + readonly gatewayName: string; + readonly gatewayPort: number; +} + +export type SandboxRecreateObserver = (target: SandboxRecreateTarget) => SandboxRecreateObservation; + +/** + * Strict absence classifier for destructive owner-gateway reconciliation. + * Bare NotFound is not sufficient because OpenShell uses it for missing + * gateways and providers as well as sandboxes. + */ +export function isExplicitMissingSandboxGatewayOutput( + output: string, + sandboxName: string, +): boolean { + const clean = stripAnsi(String(output)).replace(/\r/g, "").trim(); + const exactNoSpec = + /^(?:error:\s*)?status:\s*Internal,\s*message:\s*["']sandbox has no spec["'](?:,\s*details:\s*\[\])?(?:,\s*metadata:\s*MetadataMap\s*\{\s*\})?$/i; + if (exactNoSpec.test(clean)) return true; + // OpenShell can omit the requested name from an owner-scoped lookup. + // Require both exact structured fields so gateway/provider absence and + // transport diagnostics remain ambiguous. + const exactStructuredNotFound = + /^(?:error:\s*)?(?:×\s*)?code:\s*["']Some requested entity was not found["']\s*,\s*message:\s*["']sandbox not found["']$/i; + if (exactStructuredNotFound.test(clean)) return true; + + const escapedName = sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const namedSandbox = `(?:['\"]${escapedName}['\"]|${escapedName})`; + return ( + new RegExp( + `^(?:error:\\s*)?sandbox\\s+${namedSandbox}\\s+(?:(?:is\\s+)?not\\s+(?:found|present)|does\\s+not\\s+exist)[.!]?$`, + "i", + ).test(clean) || + new RegExp(`^(?:error:\\s*)?no\\s+such\\s+sandbox\\s+${namedSandbox}[.!]?$`, "i").test(clean) + ); +} + +export function observeSandboxOnGateway(target: SandboxRecreateTarget): SandboxRecreateObservation { + const probe = captureOpenshell(["sandbox", "get", "-g", target.gatewayName, target.sandboxName], { + ignoreError: true, + includeStderr: true, + includeStreams: true, + timeout: OPENSHELL_PROBE_TIMEOUT_MS, + }); + const stdout = String(probe.stdout ?? (probe.status === 0 ? probe.output : "")).trim(); + const combined = `${stdout}\n${String(probe.stderr ?? probe.output ?? "")}`.trim(); + const failedCleanly = + !probe.error && !probe.signal && probe.status !== null && probe.status !== 0; + if (failedCleanly && isExplicitMissingSandboxGatewayOutput(combined, target.sandboxName)) { + return { state: "missing", liveIdentityFingerprint: null }; + } + if (probe.status === 0 && stdout.length > 0) { + const liveIdentityFingerprint = fingerprintSandboxLiveIdentity(stdout); + if (!liveIdentityFingerprint) { + throw new Error( + `Cannot journal sandbox '${target.sandboxName}' replacement: OpenShell did not report a stable sandbox Id on gateway '${target.gatewayName}'.`, + ); + } + const phase = parseSandboxPhase(combined); + return { + state: phase === "Ready" || phase === "Running" ? "ready" : "not_ready", + liveIdentityFingerprint, + }; + } + throw new Error( + `Cannot journal sandbox '${target.sandboxName}' replacement: gateway '${target.gatewayName}' reported neither a live sandbox nor explicit absence.`, + ); +} diff --git a/src/lib/onboard/sandbox-reuse.ts b/src/lib/onboard/sandbox-reuse.ts index f75a799c75b..e81519822c6 100644 --- a/src/lib/onboard/sandbox-reuse.ts +++ b/src/lib/onboard/sandbox-reuse.ts @@ -21,6 +21,8 @@ export interface SandboxReuseDeps { runOpenshell(args: string[], opts?: Record): unknown; getSandboxStateFromOutputs(sandboxName: string, getOutput: string, listOutput: string): string; note(message: string): void; + // Read at call time: onboarding rebinds the gateway after preflight resolves it. + getGatewayName?(): string; } export interface SandboxReuseHelpers { @@ -112,12 +114,19 @@ export function applyReusedSandboxDashboardState( } export function createSandboxReuseHelpers(deps: SandboxReuseDeps): SandboxReuseHelpers { + function gatewayArgs(): string[] { + const gatewayName = deps.getGatewayName?.(); + return gatewayName ? ["-g", gatewayName] : []; + } + function readSandboxState(sandboxName: string | null): { state: string; getOutput: string } { if (!sandboxName) return { state: "missing", getOutput: "" }; - const getOutput = deps.runCaptureOpenshell(["sandbox", "get", sandboxName], { + const getOutput = deps.runCaptureOpenshell(["sandbox", "get", ...gatewayArgs(), sandboxName], { + ignoreError: true, + }); + const listOutput = deps.runCaptureOpenshell(["sandbox", "list", ...gatewayArgs()], { ignoreError: true, }); - const listOutput = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true }); const state = deps.getSandboxStateFromOutputs(sandboxName, getOutput, listOutput); return { state, getOutput }; } @@ -144,7 +153,7 @@ export function createSandboxReuseHelpers(deps: SandboxReuseDeps): SandboxReuseH if (!sandboxName) return; deps.note(` [resume] Cleaning up recorded sandbox '${sandboxName}' before recreating it.`); bestEffortForwardStop(deps.runOpenshell, DASHBOARD_PORT); - deps.runOpenshell(["sandbox", "delete", sandboxName], { ignoreError: true }); + deps.runOpenshell(["sandbox", "delete", ...gatewayArgs(), sandboxName], { ignoreError: true }); registry.removeSandbox(sandboxName); } diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index 06a6837a093..82243a3672e 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -31,10 +31,12 @@ describe("onboard helpers", () => { const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const childProcess = require("node:child_process"); runner.run = (command) => { + if (_n(command).includes("sandbox delete")) _deleted = true; if (_n(command).includes("sandbox delete")) { throw new Error("unexpected sandbox delete"); } @@ -42,8 +44,8 @@ runner.run = (command) => { }; runner.runCapture = (command) => { // Existing sandbox that is NOT ready - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; - if (_n(command).includes("sandbox list")) return "my-assistant NotReady"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant NotReady"; return ""; }; registry.getSandbox = () => ({ name: "my-assistant", toolDisclosure: "progressive" }); @@ -109,18 +111,20 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const childProcess = require("node:child_process"); const { EventEmitter } = require("node:events"); const commands = []; let registeredSandbox = null; runner.run = (command, opts = {}) => { + if (_n(command).includes("sandbox delete")) _deleted = true; commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { @@ -140,6 +144,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -219,6 +224,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -226,13 +232,14 @@ const { EventEmitter } = require("node:events"); const events = []; runner.run = (command) => { + if (_n(command).includes("sandbox delete")) _deleted = true; events.push({ kind: "run", cmd: _n(command) }); return { status: 0 }; }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get my-assistant")) return "my-assistant"; - if (cmd.includes("sandbox list")) return "my-assistant Ready"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { @@ -274,6 +281,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -366,6 +374,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -373,13 +382,14 @@ const { EventEmitter } = require("node:events"); const events = []; runner.run = (command) => { + if (_n(command).includes("sandbox delete")) _deleted = true; events.push({ kind: "run", cmd: _n(command) }); return { status: 0 }; }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get my-assistant")) return "my-assistant"; - if (cmd.includes("sandbox list")) return "my-assistant Ready"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (cmd.includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { @@ -408,6 +418,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -487,6 +498,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); const childProcess = require("node:child_process"); @@ -495,6 +507,7 @@ const { EventEmitter } = require("node:events"); const events = []; let sandboxDeleted = false; runner.run = (command) => { + if (_n(command).includes("sandbox delete")) _deleted = true; const cmd = _n(command); events.push({ kind: "run", cmd }); if (cmd.includes("sandbox delete")) sandboxDeleted = true; @@ -502,9 +515,9 @@ runner.run = (command) => { }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get my-assistant")) return "my-assistant"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { - return sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; + return _deleted ? "" : sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; } if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { @@ -547,6 +560,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -632,6 +646,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const onboardSession = require(${sessionModulePath}); const childProcess = require("node:child_process"); @@ -639,12 +654,13 @@ const { EventEmitter } = require("node:events"); const commands = []; runner.run = (command, opts = {}) => { + if (_n(command).includes("sandbox delete")) _deleted = true; commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { @@ -673,6 +689,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -750,6 +767,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -759,6 +777,7 @@ const path = require("node:path"); const commands = []; runner.run = (command, opts = {}) => { + if (_n(command).includes("sandbox delete")) _deleted = true; const commandString = Array.isArray(command) ? command.join(" ") : String(command); if (_n(command).includes("sandbox download")) { const parts = commandString.match(/'([^']*)'/g) || []; @@ -783,8 +802,8 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; @@ -794,6 +813,7 @@ registry.getSandbox = () => ({ name: "my-assistant", toolDisclosure: "progressiv credentials.prompt = async () => "y"; childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -879,6 +899,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -888,6 +909,7 @@ const path = require("node:path"); const commands = []; runner.run = (command, opts = {}) => { + if (_n(command).includes("sandbox delete")) _deleted = true; const commandString = Array.isArray(command) ? command.join(" ") : String(command); if (_n(command).includes("sandbox download")) { const parts = commandString.match(/'([^']*)'/g) || []; @@ -912,8 +934,8 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; - if (_n(command).includes("sandbox list")) return "my-assistant Ready"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); + if (_n(command).includes("sandbox list")) return _deleted ? "" : "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command, { @@ -936,6 +958,7 @@ preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => "y"; childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -1021,6 +1044,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const credentials = require(${credentialsPath}); const childProcess = require("node:child_process"); @@ -1029,15 +1053,16 @@ const { EventEmitter } = require("node:events"); const commands = []; let sandboxDeleted = false; runner.run = (command, opts = {}) => { + if (_n(command).includes("sandbox delete")) _deleted = true; commands.push({ command: _n(command), env: opts.env || null }); if (_n(command).includes("sandbox delete")) sandboxDeleted = true; return { status: 0 }; }; runner.runCapture = (command) => { // Existing sandbox that is NOT ready initially, becomes Ready after recreation - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return _deleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) { - return sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; + return _deleted ? "" : sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; } if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { @@ -1073,7 +1098,7 @@ const fakeSpawn = (...args) => { }); return child; }; -childProcess.spawn = fakeSpawn; +childProcess.spawn = (...args) => { _deleted = false; return fakeSpawn(...args); }; // Also patch spawn inside the compiled sandbox-create-stream module. // It imports spawn at load time from "node:child_process", so patching the @@ -1158,6 +1183,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); +let _deleted = false; const registry = require(${registryPath}); const preflight = require(${preflightPath}); const credentials = require(${credentialsPath}); @@ -1169,11 +1195,12 @@ const commands = []; let sandboxListCalls = 0; const keepAlive = setInterval(() => {}, 1000); runner.run = (command, opts = {}) => { + if (_n(command).includes("sandbox delete")) _deleted = true; commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) { sandboxListCalls += 1; return sandboxListCalls >= 2 ? "my-assistant Ready" : "my-assistant Pending"; @@ -1193,6 +1220,7 @@ preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; childProcess.spawn = (...args) => { + _deleted = false; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); From a2254bd38e08770801e5b3465007a81317e81bfb Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 29 Jul 2026 09:21:56 +0000 Subject: [PATCH 03/25] test(onboard): match gateway-scoped sandbox probes in onboard stubs Signed-off-by: Tinson Lai --- test/onboard-custom-dockerfile.test.ts | 4 ++-- ...board-extra-provider-reconciliation.test.ts | 2 +- test/onboard-installer-restore-intent.test.ts | 4 ++-- test/onboard-messaging.test.ts | 18 +++++++++--------- test/onboard-reservation-recreate.test.ts | 2 +- test/onboard-sandbox-build.test.ts | 10 +++++----- test/onboard.test.ts | 2 +- test/shellquote-sandbox.test.ts | 2 +- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/test/onboard-custom-dockerfile.test.ts b/test/onboard-custom-dockerfile.test.ts index bcd6b641d59..1a3c464cc87 100644 --- a/test/onboard-custom-dockerfile.test.ts +++ b/test/onboard-custom-dockerfile.test.ts @@ -232,7 +232,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -374,7 +374,7 @@ const destructive = []; const sandboxLive = process.env.SANDBOX_LIVE === "1"; const capture = (command) => { const text = Array.isArray(command) ? command.join(" ") : String(command); - if (/sandbox get my-assistant/.test(text)) return sandboxLive ? "my-assistant Ready" : ""; + if (/sandbox get(?: -g \S+)? my-assistant/.test(text)) return sandboxLive ? "my-assistant Ready" : ""; if (/sandbox list/.test(text)) return sandboxLive ? "my-assistant Ready" : ""; if (/forward list/.test(text)) return ""; return ""; diff --git a/test/onboard-extra-provider-reconciliation.test.ts b/test/onboard-extra-provider-reconciliation.test.ts index f0c6ce4d98c..266b781b127 100644 --- a/test/onboard-extra-provider-reconciliation.test.ts +++ b/test/onboard-extra-provider-reconciliation.test.ts @@ -77,7 +77,7 @@ runner.run = (command, opts = {}) => { }; runner.runCapture = (command) => { const normalized = _n(command); - if (normalized.includes("sandbox get my-assistant")) return ""; + if (normalized.includes("sandbox get") && normalized.includes("my-assistant")) return ""; if (normalized.includes("sandbox list")) return "my-assistant Ready"; const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); if (mockedCapture !== null) return mockedCapture; diff --git a/test/onboard-installer-restore-intent.test.ts b/test/onboard-installer-restore-intent.test.ts index fe64a99582a..ca5fa0aeab9 100644 --- a/test/onboard-installer-restore-intent.test.ts +++ b/test/onboard-installer-restore-intent.test.ts @@ -54,7 +54,7 @@ runner.run = (command) => { }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get my-assistant")) return "my-assistant"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return "my-assistant"; if (cmd.includes("sandbox list")) { return sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; } @@ -257,7 +257,7 @@ runner.run = (command) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return "my-assistant"; if (_n(command).includes("sandbox list")) return "my-assistant NotReady"; // Keep dashboard allocation inside this restore-intent fixture; host port // occupancy is unrelated to the not-ready decision under test. diff --git a/test/onboard-messaging.test.ts b/test/onboard-messaging.test.ts index cad264577f5..27623f4574a 100644 --- a/test/onboard-messaging.test.ts +++ b/test/onboard-messaging.test.ts @@ -88,7 +88,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("provider get")) return "Provider: discord-bridge"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; @@ -358,7 +358,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; { @@ -539,7 +539,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -699,7 +699,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -848,7 +848,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -1004,7 +1004,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -1217,7 +1217,7 @@ runner.run = (command, opts = {}) => { }; runner.runCapture = (command) => { // Existing sandbox that is ready - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return "my-assistant"; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; // All messaging providers already exist in gateway if (_n(command).includes("provider get")) return "Provider: exists"; @@ -1319,7 +1319,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -1448,7 +1448,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); diff --git a/test/onboard-reservation-recreate.test.ts b/test/onboard-reservation-recreate.test.ts index 72b8d78885b..9cde378c8bc 100644 --- a/test/onboard-reservation-recreate.test.ts +++ b/test/onboard-reservation-recreate.test.ts @@ -66,7 +66,7 @@ runner.run = (command) => { }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get my-assistant")) return "my-assistant"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return "my-assistant"; if (cmd.includes("sandbox list")) { return sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; } diff --git a/test/onboard-sandbox-build.test.ts b/test/onboard-sandbox-build.test.ts index f5cb4deb67a..7d4df10ead2 100644 --- a/test/onboard-sandbox-build.test.ts +++ b/test/onboard-sandbox-build.test.ts @@ -33,7 +33,7 @@ describe("onboard helpers", () => { const registry = require(${registryPath}); const runner = require(${runnerPath}); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -runner.runCapture = (command) => (_n(command).includes("sandbox get my-assistant") ? "" : ""); +runner.runCapture = (command) => (_n(command).includes("sandbox get") && _n(command).includes("my-assistant") ? "" : ""); registry.registerSandbox({ name: "my-assistant" }); @@ -105,7 +105,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -517,7 +517,7 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -619,7 +619,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -721,7 +721,7 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; // Custom port: dashboard readiness curl uses 19000 (DASHBOARD_PORT from env) { diff --git a/test/onboard.test.ts b/test/onboard.test.ts index b929f1ef725..34e90c86fa9 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -767,7 +767,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return "my-assistant"; if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; diff --git a/test/shellquote-sandbox.test.ts b/test/shellquote-sandbox.test.ts index 3dee1080caa..277bebe7882 100644 --- a/test/shellquote-sandbox.test.ts +++ b/test/shellquote-sandbox.test.ts @@ -95,7 +95,7 @@ runner.runFile = (file, args = [], opts = {}) => { }; runner.runCapture = (command) => { const text = asText(command); - if (text.includes("sandbox get my-assistant")) return ""; + if (text.includes("sandbox get") && text.includes("my-assistant")) return ""; if (text.includes("sandbox list")) return "my-assistant Ready"; if (text.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; if (text.includes("sandbox exec") && text.includes("http://localhost:") && text.includes("/health")) return "200"; From 158139da651b9e3fa374dccf13cad9041436dc8b Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 29 Jul 2026 10:40:54 +0000 Subject: [PATCH 04/25] fix(onboard): retire the self-opened same-name replacement journal The journal opened inside createSandbox has no outer owner, so it now advances to completed and clears itself once the replacement registry row commits. Without that a later replacement met a stale active transaction and refused to start. Read the committed journal back from the session the write returned rather than through a store that may not observe it yet, keep src/lib/onboard.ts net-neutral by moving the managed-MCP refusal text into the journal module, and model post-delete absence in the onboard stubs the guard now requires. Signed-off-by: Tinson Lai --- src/lib/onboard.ts | 19 ++----- .../onboard/onboard-recreate-journal.test.ts | 27 ++++++++-- src/lib/onboard/onboard-recreate-journal.ts | 50 +++++++++++++++++-- test/onboard-installer-restore-intent.test.ts | 8 +-- test/onboard-reservation-recreate.test.ts | 6 ++- test/onboard-terminal-dashboard.test.ts | 6 ++- test/onboard.test.ts | 2 +- test/rebuild-stale-recovery.test.ts | 12 +++-- test/repro-2201.test.ts | 2 +- 9 files changed, 95 insertions(+), 37 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 7c98a529986..0017cb3e736 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -597,10 +597,7 @@ import { filterEnabledChannelsByAgent } from "./onboard/messaging-state"; import { getValidatedMessagingTokenByEnvKey } from "./onboard/messaging-token"; import * as ollamaFlow from "./onboard/ollama-probe-failure"; import { runOllamaStartupOrGate } from "./onboard/ollama-startup"; -import { - fingerprintOnboardRecreateTargetIntent, - openOnboardRecreateJournal, -} from "./onboard/onboard-recreate-journal"; +import * as recreateJournal from "./onboard/onboard-recreate-journal"; import type { DockerDriverBinaryOverrides, OpenShellInstallDeps, @@ -2291,7 +2288,7 @@ async function createSandboxWithBaseImageResolution( // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. const { existingEntry, preservedMcpState, liveExists, effectiveToolDisclosure, toolDisclosureMigrationNeeded, toolDisclosureMigrationNote } = toolDisclosureFlow.prepareSandboxToolDisclosure(sandboxName, preparedBuildContext?.rebuildTarget?.fromDockerfile ? preparedBuildContext.stagedDockerfile : fromDockerfile, isRecreateSandbox(createIntent?.recreate), inspectSandboxForCreate, createIntent?.toolDisclosure ?? null); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - let recreateRuntime = sandboxRecreateTransaction.createSandboxRecreateRuntime(onboardSession, createIntent?.recreateTransaction, sandboxName, GATEWAY_NAME, existingEntry, getSandboxRecreateObservation, note); + let recreateRuntime: import("./onboard/sandbox-recreate-transaction").SandboxRecreateRuntime | recreateJournal.OwnedSandboxRecreateRuntime = sandboxRecreateTransaction.createSandboxRecreateRuntime(onboardSession, createIntent?.recreateTransaction, sandboxName, GATEWAY_NAME, existingEntry, getSandboxRecreateObservation, note); const restoreReusedSandboxDashboard = (selectionVerified: boolean): void => { ({ chatUiUrl } = sandboxReuse.applyReusedSandboxDashboardState({ sandboxName, @@ -2553,18 +2550,11 @@ async function createSandboxWithBaseImageResolution( if (preservedMcpState) { // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const explicitObservability = observabilityCommandFlag.explicitObservabilityFlag(createIntent?.observabilityEnabled === true, createIntent?.observabilityRequestedExplicitly === true); - console.error( - ` Sandbox '${sandboxName}' has managed MCP servers. Refusing the generic onboard recreation path.`, - ); - console.error( - ` Run \`${cliName()} ${sandboxName} rebuild --yes --tool-disclosure ${effectiveToolDisclosure}${explicitObservability ? ` ${explicitObservability}` : ""}${dcodeAutoApprovalPlan.rebuildFlag}\` so MCP providers and adapter state are preserved transactionally.`, - ); + for (const hint of recreateJournal.managedMcpRecreateRefusalHints({ sandboxName, cliName: cliName(), toolDisclosure: effectiveToolDisclosure, rebuildFlag: dcodeAutoApprovalPlan.rebuildFlag, observabilityFlag: observabilityCommandFlag.explicitObservabilityFlag(createIntent?.observabilityEnabled === true, createIntent?.observabilityRequestedExplicitly === true) })) console.error(hint); process.exit(1); } - // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - if (!createIntent?.recreateTransaction) recreateRuntime = openOnboardRecreateJournal({ target: { sandboxName, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT }, agentName: getRequestedSandboxAgentName(agent) || "openclaw", note, observe: (probeTarget) => getSandboxRecreateObservation(probeTarget.sandboxName), targetIntentFingerprint: fingerprintOnboardRecreateTargetIntent({ agent: getRequestedSandboxAgentName(agent) || null, fromDockerfile: fromDockerfile ?? null, provider: provider ?? null, model: model ?? null, preferredInferenceApi: preferredInferenceApi ?? null, sandboxGpuConfig: effectiveSandboxGpuConfig ?? null, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, toolDisclosure: effectiveToolDisclosure, dcodeAutoApprovalMode: createIntent?.dcodeAutoApprovalMode ?? null, observabilityEnabled: createIntent?.observabilityEnabled === true, policyTier: createIntent?.policyTier ?? null }) }); + if (!createIntent?.recreateTransaction) recreateRuntime = recreateJournal.openOnboardRecreateJournal({ target: { sandboxName, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT }, agentName: getRequestedSandboxAgentName(agent) || "openclaw", note, observe: (probeTarget) => getSandboxRecreateObservation(probeTarget.sandboxName), intent: { agent: getRequestedSandboxAgentName(agent) || null, fromDockerfile: fromDockerfile ?? null, provider: provider ?? null, model: model ?? null, preferredInferenceApi: preferredInferenceApi ?? null, sandboxGpuConfig: effectiveSandboxGpuConfig ?? null, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, toolDisclosure: effectiveToolDisclosure, dcodeAutoApprovalMode: createIntent?.dcodeAutoApprovalMode ?? null, observabilityEnabled: createIntent?.observabilityEnabled === true, policyTier: createIntent?.policyTier ?? null } }); const previousEntry: SandboxEntry | null = registry.getSandbox(sandboxName); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. baseImageResolutionFlow.captureBaseResolution(baseImageResolutionContext, previousEntry?.imageTag); @@ -2867,6 +2857,7 @@ async function createSandboxWithBaseImageResolution( }), }, ); + if ("complete" in recreateRuntime) recreateRuntime.complete(); restoreDefaultAfterRecreate(registry.setDefault, sandboxName, sandboxWasLiveDefault); // #4614: default deferred to finalization // DNS proxy — run a forwarder in the sandbox pod so the isolated diff --git a/src/lib/onboard/onboard-recreate-journal.test.ts b/src/lib/onboard/onboard-recreate-journal.test.ts index b0ab921d5ee..f9020ebdece 100644 --- a/src/lib/onboard/onboard-recreate-journal.test.ts +++ b/src/lib/onboard/onboard-recreate-journal.test.ts @@ -74,7 +74,6 @@ describe("non-resumed replacement target fingerprint (#7735)", () => { }); const SANDBOX_ID = "sbx-71c9a4e08b"; -const TARGET_FINGERPRINT = "a".repeat(64); const NON_DEFAULT_TARGET = { sandboxName: "alpha", @@ -130,11 +129,11 @@ describe("non-resumed onboard replacement journal (#7735)", () => { vi.restoreAllMocks(); }); - function open(targetIntentFingerprint: string = TARGET_FINGERPRINT) { + function open(intent: OnboardRecreateTargetIntent = BASE_INTENT) { return openOnboardRecreateJournal({ target: NON_DEFAULT_TARGET, agentName: "openclaw", - targetIntentFingerprint, + intent, note: vi.fn(), }); } @@ -145,7 +144,9 @@ describe("non-resumed onboard replacement journal (#7735)", () => { const recorded = session.checkpoint?.sandboxRecreate; expect(recorded?.phase).toBe("planned"); expect(recorded?.sandboxName).toBe("alpha"); - expect(recorded?.targetIntentFingerprint).toBe(TARGET_FINGERPRINT); + expect(recorded?.targetIntentFingerprint).toBe( + fingerprintOnboardRecreateTargetIntent(BASE_INTENT), + ); expect(recorded?.sourceLiveIdentityFingerprint).toMatch(/^[0-9a-f]{64}$/); expect(JSON.stringify(session.checkpoint)).not.toContain(SANDBOX_ID); }); @@ -206,6 +207,20 @@ describe("non-resumed onboard replacement journal (#7735)", () => { expect(session.checkpoint?.sandboxRecreate?.phase).toBe("deleting"); }); + it("retires its own transaction once the replacement registry row commits", () => { + const runtime = open(); + runtime.advance("deleting"); + mocks.captureOpenshell.mockReturnValue(absentProbe()); + runtime.confirmDeleted(); + runtime.advance("creating"); + mocks.captureOpenshell.mockReturnValue(livePresentProbe()); + runtime.recordCreated(); + + runtime.complete(); + + expect(session.checkpoint?.sandboxRecreate).toBeNull(); + }); + it("resumes the same replacement after a restart without a resume flag", () => { const first = open(); first.advance("deleting"); @@ -220,7 +235,9 @@ describe("non-resumed onboard replacement journal (#7735)", () => { it("refuses a replacement whose target intent changed mid-transaction", () => { open(); - expect(() => open("b".repeat(64))).toThrow(/different recreate transaction in progress/); + expect(() => open({ ...BASE_INTENT, observabilityEnabled: true })).toThrow( + /different recreate transaction in progress/, + ); }); it("refuses to continue when the live source identity no longer matches", () => { diff --git a/src/lib/onboard/onboard-recreate-journal.ts b/src/lib/onboard/onboard-recreate-journal.ts index b68a3ef62dc..9198206be72 100644 --- a/src/lib/onboard/onboard-recreate-journal.ts +++ b/src/lib/onboard/onboard-recreate-journal.ts @@ -13,7 +13,9 @@ import { type SandboxRecreateTarget, } from "./sandbox-recreate-probe"; import { + advanceSandboxRecreateTransaction, beginSandboxRecreateTransaction, + clearCompletedSandboxRecreateTransaction, createSandboxRecreateRuntime, fingerprintSandboxRecreateValue, planSandboxRecreateRecovery, @@ -41,18 +43,37 @@ export function fingerprintOnboardRecreateTargetIntent( return fingerprintSandboxRecreateValue({ version: 1, ...intent }); } +export interface ManagedMcpRecreateRefusal { + readonly sandboxName: string; + readonly cliName: string; + readonly toolDisclosure: string; + readonly rebuildFlag: string; + readonly observabilityFlag: string | null; +} + +export function managedMcpRecreateRefusalHints(input: ManagedMcpRecreateRefusal): string[] { + const observability = input.observabilityFlag ? ` ${input.observabilityFlag}` : ""; + return [ + ` Sandbox '${input.sandboxName}' has managed MCP servers. Refusing the generic onboard recreation path.`, + ` Run \`${input.cliName} ${input.sandboxName} rebuild --yes --tool-disclosure ${input.toolDisclosure}${observability}${input.rebuildFlag}\` so MCP providers and adapter state are preserved transactionally.`, + ]; +} + export interface OpenOnboardRecreateJournalInput { readonly target: SandboxRecreateTarget; readonly agentName: string; - readonly targetIntentFingerprint: string; + readonly intent: OnboardRecreateTargetIntent; readonly note: (message: string) => void; readonly observe?: SandboxRecreateObserver; } +export type OwnedSandboxRecreateRuntime = SandboxRecreateRuntime & { complete(): void }; + export function openOnboardRecreateJournal( input: OpenOnboardRecreateJournalInput, -): SandboxRecreateRuntime { - const { target, agentName, targetIntentFingerprint, note } = input; +): OwnedSandboxRecreateRuntime { + const { target, agentName, note } = input; + const targetIntentFingerprint = fingerprintOnboardRecreateTargetIntent(input.intent); const observe = input.observe ?? observeSandboxOnGateway; const authority = resolveGatewayTeardownAuthority({ gatewayName: target.gatewayName, @@ -105,8 +126,10 @@ export function openOnboardRecreateJournal( ` Journaled replacement ${transaction.id} for '${target.sandboxName}' on ${target.gatewayName}:${String(target.gatewayPort)} at phase '${transaction.phase}'.`, ); - return createSandboxRecreateRuntime( - onboardSession, + const runtime = createSandboxRecreateRuntime( + // The journal just committed is authoritative for this run; do not re-read + // it through a store that may not yet observe the write. + { loadSession: () => session, updateSession: onboardSession.updateSession }, { id: transaction.id, targetGeneration: transaction.targetGeneration, @@ -118,4 +141,21 @@ export function openOnboardRecreateJournal( (sandboxName) => observe({ ...target, sandboxName }), note, ); + + return { + ...runtime, + get registrationFields() { + return runtime.registrationFields; + }, + // This journal has no outer owner, so it retires its own transaction once + // the replacement registry row commits. + complete: () => { + onboardSession.updateSession((current) => { + advanceSandboxRecreateTransaction(current, transaction.id, "registry_committing"); + advanceSandboxRecreateTransaction(current, transaction.id, "completed"); + clearCompletedSandboxRecreateTransaction(current, transaction.id); + return current; + }); + }, + }; } diff --git a/test/onboard-installer-restore-intent.test.ts b/test/onboard-installer-restore-intent.test.ts index ca5fa0aeab9..4e36a437dd0 100644 --- a/test/onboard-installer-restore-intent.test.ts +++ b/test/onboard-installer-restore-intent.test.ts @@ -46,6 +46,7 @@ const { EventEmitter } = require("node:events"); const PRE_UPGRADE_BACKUP = "/tmp/fake-pre-upgrade-backup"; const events = []; let sandboxDeleted = false; +let sandboxRecreated = false; runner.run = (command) => { const cmd = _n(command); events.push({ kind: "run", cmd }); @@ -54,9 +55,9 @@ runner.run = (command) => { }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return "my-assistant"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxDeleted && !sandboxRecreated ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { - return sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; + return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { @@ -107,6 +108,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + sandboxRecreated = true; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); @@ -257,7 +259,7 @@ runner.run = (command) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return "my-assistant NotReady"; // Keep dashboard allocation inside this restore-intent fixture; host port // occupancy is unrelated to the not-ready decision under test. diff --git a/test/onboard-reservation-recreate.test.ts b/test/onboard-reservation-recreate.test.ts index 9cde378c8bc..b2ae4931658 100644 --- a/test/onboard-reservation-recreate.test.ts +++ b/test/onboard-reservation-recreate.test.ts @@ -58,6 +58,7 @@ const { EventEmitter } = require("node:events"); const events = []; let sandboxDeleted = false; +let sandboxRecreated = false; runner.run = (command) => { const cmd = _n(command); events.push({ kind: "run", cmd }); @@ -66,9 +67,9 @@ runner.run = (command) => { }; runner.runCapture = (command) => { const cmd = _n(command); - if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return "my-assistant"; + if (cmd.includes("sandbox get") && cmd.includes("my-assistant")) return sandboxRecreated ? ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)) : sandboxDeleted ? "" : ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (cmd.includes("sandbox list")) { - return sandboxDeleted ? "my-assistant Ready" : "my-assistant NotReady"; + return sandboxRecreated ? "my-assistant Ready" : sandboxDeleted ? "" : "my-assistant NotReady"; } if (cmd.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; { @@ -101,6 +102,7 @@ const preflight = require(${JSON.stringify(path.join(repoRoot, "src", "lib", "on preflight.checkPortAvailable = async () => ({ ok: true }); childProcess.spawn = (...args) => { + sandboxRecreated = true; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); diff --git a/test/onboard-terminal-dashboard.test.ts b/test/onboard-terminal-dashboard.test.ts index e45a9b5b17e..61baa83310a 100644 --- a/test/onboard-terminal-dashboard.test.ts +++ b/test/onboard-terminal-dashboard.test.ts @@ -104,8 +104,10 @@ runner.runCapture = (command) => { "Endpoint: https://inference.local/v1", ].join("\n"); } - if (normalized.includes("sandbox get " + sandboxName)) { - return scenario === "reuse" ? sandboxName : ""; + if (normalized.includes("sandbox get") && normalized.includes(sandboxName)) { + return scenario === "reuse" + ? [sandboxName, "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)) + : ""; } if (normalized.includes("sandbox list")) return sandboxName + " Ready"; if (normalized.includes("forward list")) return sandboxName + " 127.0.0.1 18789 12345 running"; diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 34e90c86fa9..0fc4b36c01c 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -767,7 +767,7 @@ runner.run = (command, opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return "my-assistant"; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ["my-assistant", "Id: sbx-4f2a91c0d7"].join(String.fromCharCode(10)); if (_n(command).includes("sandbox list")) return "my-assistant Ready"; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; diff --git a/test/rebuild-stale-recovery.test.ts b/test/rebuild-stale-recovery.test.ts index a1683a22f3c..138bee2db30 100644 --- a/test/rebuild-stale-recovery.test.ts +++ b/test/rebuild-stale-recovery.test.ts @@ -275,7 +275,9 @@ describe("stale sandbox rebuild recovery (#4497)", () => { targetGatewayPort: 8080, }), ); - expect(harness.removeSandboxRegistryEntryWithReceiptSpy).toHaveBeenCalledOnce(); + // The journaled source row is the durable replacement contract, so it is + // preserved until replacement registration commits (#7734). + expect(harness.removeSandboxRegistryEntryWithReceiptSpy).not.toHaveBeenCalled(); expect(harness.restoreSandboxEntrySpy).not.toHaveBeenCalled(); expect(harness.restoreSandboxEntryIfMissingSpy).not.toHaveBeenCalled(); }); @@ -365,10 +367,12 @@ describe("stale sandbox rebuild recovery (#4497)", () => { expect(output).not.toContain("Backing up sandbox state"); expect(output).toContain("Creating new sandbox with current image"); expect(output).toContain("Recovery recreate failed"); - // The preserved entry must survive the failed recreate. Its obsolete image - // tag is intentionally cleared so a leftover image remains eligible for GC. + // The preserved entry must survive the failed recreate carrying no obsolete + // image tag, so a leftover image remains eligible for GC. The journaled row + // is now preserved in place rather than removed and restored, so the field + // is absent instead of explicitly null (#7734). const registry = readRegistry(fixture); expect(registry.defaultSandbox).toBe(fixture.sandboxName); - expect(registry.sandboxes[fixture.sandboxName].imageTag).toBe(null); + expect(registry.sandboxes[fixture.sandboxName].imageTag ?? null).toBe(null); }); }); diff --git a/test/repro-2201.test.ts b/test/repro-2201.test.ts index 1766f1d5973..3bbafd10aaa 100644 --- a/test/repro-2201.test.ts +++ b/test/repro-2201.test.ts @@ -244,7 +244,7 @@ if (a[0]==="sandbox" && a[1]==="ssh-config") { process.stdout.write("${sshConfig if (a[0]==="sandbox" && a[1]==="delete") { fs.writeFileSync(deleteMarker, "deleted\\n"); process.exit(0); } if (a[0]==="sandbox" && a[1]==="get") { if (fs.existsSync(deleteMarker)) { process.stderr.write("Error: sandbox ${sandboxName} not found\\n"); process.exit(1); } - process.stdout.write("Name: ${sandboxName}\\nPhase: Ready\\n"); + process.stdout.write("Name: ${sandboxName}\\nId: sbx-2201a7c3f5\\nPhase: Ready\\n"); process.exit(0); } process.exit(0); From 071316774c6b2f567ef2202525db42f334b1242f Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 29 Jul 2026 10:52:13 +0000 Subject: [PATCH 05/25] test(onboard): track sandbox deletion without new test branches Signed-off-by: Tinson Lai --- test/onboard-sandbox-recreation.test.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index 82243a3672e..c22d5087e5b 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -36,7 +36,7 @@ const registry = require(${registryPath}); const childProcess = require("node:child_process"); runner.run = (command) => { - if (_n(command).includes("sandbox delete")) _deleted = true; + _deleted = _deleted || _n(command).includes("sandbox delete"); if (_n(command).includes("sandbox delete")) { throw new Error("unexpected sandbox delete"); } @@ -118,7 +118,7 @@ const { EventEmitter } = require("node:events"); const commands = []; let registeredSandbox = null; runner.run = (command, opts = {}) => { - if (_n(command).includes("sandbox delete")) _deleted = true; + _deleted = _deleted || _n(command).includes("sandbox delete"); commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; }; @@ -232,7 +232,7 @@ const { EventEmitter } = require("node:events"); const events = []; runner.run = (command) => { - if (_n(command).includes("sandbox delete")) _deleted = true; + _deleted = _deleted || _n(command).includes("sandbox delete"); events.push({ kind: "run", cmd: _n(command) }); return { status: 0 }; }; @@ -382,7 +382,7 @@ const { EventEmitter } = require("node:events"); const events = []; runner.run = (command) => { - if (_n(command).includes("sandbox delete")) _deleted = true; + _deleted = _deleted || _n(command).includes("sandbox delete"); events.push({ kind: "run", cmd: _n(command) }); return { status: 0 }; }; @@ -507,7 +507,7 @@ const { EventEmitter } = require("node:events"); const events = []; let sandboxDeleted = false; runner.run = (command) => { - if (_n(command).includes("sandbox delete")) _deleted = true; + _deleted = _deleted || _n(command).includes("sandbox delete"); const cmd = _n(command); events.push({ kind: "run", cmd }); if (cmd.includes("sandbox delete")) sandboxDeleted = true; @@ -654,7 +654,7 @@ const { EventEmitter } = require("node:events"); const commands = []; runner.run = (command, opts = {}) => { - if (_n(command).includes("sandbox delete")) _deleted = true; + _deleted = _deleted || _n(command).includes("sandbox delete"); commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; }; @@ -777,7 +777,7 @@ const path = require("node:path"); const commands = []; runner.run = (command, opts = {}) => { - if (_n(command).includes("sandbox delete")) _deleted = true; + _deleted = _deleted || _n(command).includes("sandbox delete"); const commandString = Array.isArray(command) ? command.join(" ") : String(command); if (_n(command).includes("sandbox download")) { const parts = commandString.match(/'([^']*)'/g) || []; @@ -909,7 +909,7 @@ const path = require("node:path"); const commands = []; runner.run = (command, opts = {}) => { - if (_n(command).includes("sandbox delete")) _deleted = true; + _deleted = _deleted || _n(command).includes("sandbox delete"); const commandString = Array.isArray(command) ? command.join(" ") : String(command); if (_n(command).includes("sandbox download")) { const parts = commandString.match(/'([^']*)'/g) || []; @@ -1053,7 +1053,7 @@ const { EventEmitter } = require("node:events"); const commands = []; let sandboxDeleted = false; runner.run = (command, opts = {}) => { - if (_n(command).includes("sandbox delete")) _deleted = true; + _deleted = _deleted || _n(command).includes("sandbox delete"); commands.push({ command: _n(command), env: opts.env || null }); if (_n(command).includes("sandbox delete")) sandboxDeleted = true; return { status: 0 }; @@ -1195,7 +1195,7 @@ const commands = []; let sandboxListCalls = 0; const keepAlive = setInterval(() => {}, 1000); runner.run = (command, opts = {}) => { - if (_n(command).includes("sandbox delete")) _deleted = true; + _deleted = _deleted || _n(command).includes("sandbox delete"); commands.push({ command: _n(command), env: opts.env || null }); return { status: 0 }; }; From 3109fe7b781a755b5e30a923e35652b8be2b0021 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 29 Jul 2026 11:05:53 +0000 Subject: [PATCH 06/25] test(rebuild): order the delete boundary without a test branch Signed-off-by: Tinson Lai --- src/lib/actions/sandbox/rebuild-destroy-phase.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index bd233f1c8dd..b51c6c4e268 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -903,11 +903,11 @@ describe("rebuild destroy phase", () => { order.push("journal:deleting"); }); mocks.runOpenshell.mockImplementation((args: string[]) => { - if (args[1] === "delete") { - order.push("openshell:delete"); - return { status: 0, stdout: "deleted", stderr: "" }; - } - return { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" }; + const deleting = args[1] === "delete"; + order.push(...(deleting ? ["openshell:delete"] : [])); + return deleting + ? { status: 0, stdout: "deleted", stderr: "" } + : { status: 1, stdout: "", stderr: "Error: sandbox alpha not found" }; }); await runRebuildDestroyPhase({ From 4c1c55057753c8f88b6ea5ec2ec40a8a41071773 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 29 Jul 2026 15:24:25 +0000 Subject: [PATCH 07/25] fix(rebuild): converge a restart onto its proven replacement A restart that observed a registered, ready sandbox carrying the journaled target generation and identity still entered the destroy phase, so the rebuild could delete the replacement it had already proved. Surface that accepted target from the journal, retire the transaction, and end the rebuild before deletion. Reattach the prepared MCP entries when the delete boundary cannot be journaled, observe a resumed replacement on the gateway its journal records rather than the ambient one, and stop the Docker GPU rollback tests from probing DNS through a live container. Signed-off-by: Tinson Lai --- .../sandbox/rebuild-destroy-phase.test.ts | 42 +++++++++++++ .../actions/sandbox/rebuild-destroy-phase.ts | 21 ++++++- src/lib/actions/sandbox/rebuild-pipeline.ts | 15 +++++ .../sandbox/rebuild-recreate-journal.test.ts | 59 +++++++++++++++++++ .../sandbox/rebuild-recreate-journal.ts | 36 ++++++++--- .../rebuild-recreate-observability.test.ts | 32 ++++++---- src/lib/onboard.ts | 2 +- .../onboard/docker-gpu-patch-rollback.test.ts | 15 +++++ src/lib/onboard/sandbox-reuse.test.ts | 26 ++++++++ src/lib/onboard/sandbox-reuse.ts | 28 ++++++--- test/helpers/rebuild-flow-recovery-cases.ts | 53 +++++++++++++++++ test/mcp-destroy-lifecycle.test.ts | 2 + 12 files changed, 302 insertions(+), 29 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index b51c6c4e268..c716fe02a06 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -76,10 +76,12 @@ import type { RebuildRecreateJournal } from "./rebuild-recreate-journal"; function stubRecreateJournal(): RebuildRecreateJournal { return { id: "journal-1", + acceptedTarget: false, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", markDeleting: vi.fn(), confirmDeleted: vi.fn(), + completeAcceptedTarget: vi.fn(), }; } @@ -927,6 +929,46 @@ describe("rebuild destroy phase", () => { expect(order).toEqual(["journal:deleting", "openshell:delete"]); }); + it("reattaches MCP providers when the delete boundary cannot be journaled (#7734)", async () => { + const recreateJournal = stubRecreateJournal(); + vi.mocked(recreateJournal.markDeleting).mockImplementation(() => { + throw new Error("session store is unwritable"); + }); + mocks.prepareMcpForRebuild.mockResolvedValue({ + entries: [{ server: "github" }], + detachedProviderEntries: [{ providerName: "nemoclaw-mcp-alpha-github" }], + scrubbedAdapterEntries: [{ server: "github" }], + }); + const relockShieldsIfNeeded = vi.fn(() => true); + + await expect( + runRebuildDestroyPhase({ + sandboxName: "alpha", + sandboxEntry: { name: "alpha", agent: "openclaw", gatewayName: "nemoclaw" }, + staleRecovery: false, + recreateJournal, + backupManifest: null, + log: vi.fn(), + bail: vi.fn((message: string): never => { + throw new Error(message); + }), + relockShieldsIfNeeded, + onDeleted: vi.fn(), + }), + ).rejects.toThrow("Sandbox deletion could not be journaled"); + + expect(mocks.reattachMcpAfterDeleteFailure).toHaveBeenCalledWith( + "alpha", + [{ providerName: "nemoclaw-mcp-alpha-github" }], + [{ server: "github" }], + ); + expect(relockShieldsIfNeeded).toHaveBeenCalledWith(true); + expect(mocks.runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "delete", "-g", "nemoclaw", "alpha"], + expect.anything(), + ); + }); + it("stops before inference and registry mutation when absence cannot be journaled (#7734)", async () => { const recreateJournal = stubRecreateJournal(); vi.mocked(recreateJournal.confirmDeleted).mockImplementation(() => { diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index 49181bedeab..56d09be3b59 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -356,7 +356,26 @@ export async function runRebuildDestroyPhase( return null; } - recreateJournal.markDeleting(); + // MCP adapter entries are already detached and scrubbed here. A journal write + // that fails must reattach them before the rebuild gives up, or the still + // running sandbox is left without its MCP wiring. + try { + recreateJournal.markDeleting(); + } catch (error) { + const mcpRecoveryFailure = await reattachMcpAfterDeleteFailure( + sandboxName, + rebuildDetachedMcpProviderEntries, + rebuildScrubbedMcpAdapterEntries, + ); + relockShieldsIfNeeded(true); + const detail = error instanceof Error ? error.message : String(error); + bail( + mcpRecoveryFailure + ? `Sandbox deletion could not be journaled: ${redactFull(detail)} MCP provider recovery also failed: ${mcpRecoveryFailure}` + : `Sandbox deletion could not be journaled: ${redactFull(detail)}`, + ); + return null; + } log(`Running: openshell sandbox delete -g ${gatewayName} ${sandboxName}`); const deleteResult = runOpenshell(["sandbox", "delete", "-g", gatewayName, sandboxName], { ignoreError: true, diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index 4d500ffe54d..8cbe10ec461 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -225,6 +225,21 @@ async function rebuildSandboxUnlocked( log, }); + // An earlier run of this rebuild already registered and proved the + // replacement. Retire its journal and stop before the destroy phase so a + // restart converges to that sandbox instead of deleting it. + if (recreateJournal.acceptedTarget) { + recreateJournal.completeAcceptedTarget(); + log(`Recovered journaled replacement ${recreateJournal.id} for '${sandboxName}'`); + console.log( + ` Sandbox '${sandboxName}' already holds the replacement from the interrupted rebuild.`, + ); + if (backup.backupManifest) { + console.log(` State backup is preserved at: ${backup.backupManifest.backupPath}`); + } + return; + } + const mcpPreparation = await runRebuildDestroyPhase({ sandboxName, sandboxEntry, diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts index 5d28a0e1b8e..94f3ad36cd5 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts @@ -328,4 +328,63 @@ describe("rebuild replacement journal", () => { expect(second.id).toBe(first.id); expect(second.targetGeneration).toBe(first.targetGeneration); }); + + function proveReplacement(journalId: string, targetGeneration: string): string { + const identity = session.checkpoint?.sandboxRecreate?.sourceLiveIdentityFingerprint ?? ""; + onboardSession.updateSession((current) => { + const checkpoint = current.checkpoint; + const transaction = checkpoint?.sandboxRecreate; + if (!checkpoint || !transaction || transaction.id !== journalId) { + throw new Error("journal is missing"); + } + current.checkpoint = { + ...checkpoint, + sandboxRecreate: { + ...transaction, + phase: "registry_committing", + targetLiveIdentityFingerprint: identity, + }, + }; + return current; + }); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + agent: "langchain-deepagents-code", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + lifecycleGeneration: targetGeneration, + lifecycleLiveIdentityFingerprint: identity, + } as registry.SandboxEntry); + return identity; + } + + it("reports a registered ready replacement as the proven target (#7734)", () => { + const first = open(); + proveReplacement(first.id, first.targetGeneration); + + const resumed = open(); + + expect(first.acceptedTarget).toBe(false); + expect(resumed.acceptedTarget).toBe(true); + expect(resumed.id).toBe(first.id); + }); + + it("retires the journal of a proven replacement instead of deleting it again (#7734)", () => { + const first = open(); + proveReplacement(first.id, first.targetGeneration); + const resumed = open(); + + resumed.completeAcceptedTarget(); + + expect(session.checkpoint?.sandboxRecreate).toBeNull(); + }); + + it("refuses to retire a journal whose replacement is not proven (#7734)", () => { + const journal = open(); + + expect(() => journal.completeAcceptedTarget()).toThrow( + /cannot be retired before its replacement is proven/, + ); + expect(session.checkpoint?.sandboxRecreate?.phase).toBe("planned"); + }); }); diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.ts index 4ca7848e768..2efc9be8fc2 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.ts @@ -11,6 +11,7 @@ import { import { advanceSandboxRecreateTransaction, beginSandboxRecreateTransaction, + clearCompletedSandboxRecreateTransaction, fingerprintSandboxRecreateValue, planSandboxRecreateRecovery, sandboxRecreatePhaseReached, @@ -28,10 +29,12 @@ export type RebuildSandboxObserver = SandboxRecreateObserver; export interface RebuildRecreateJournal { readonly id: string; + readonly acceptedTarget: boolean; readonly targetGeneration: string; readonly targetIntentFingerprint: string; markDeleting(): void; confirmDeleted(): void; + completeAcceptedTarget(): void; } export function fingerprintRebuildRecreateTargetIntent( @@ -88,14 +91,18 @@ export function openRebuildRecreateJournal( const sourceEntry = registry.getSandbox(target.sandboxName); const observation = observe(target); const active = onboardSession.loadSession()?.checkpoint?.sandboxRecreate ?? null; - if (active) { - const recovery = planSandboxRecreateRecovery(active, observation, sourceEntry); - if (recovery.action === "reject") { - throw new Error( - `Cannot resume sandbox '${target.sandboxName}' replacement: ${recovery.reason}.`, - ); - } + const recovery = active + ? planSandboxRecreateRecovery(active, observation, sourceEntry) + : { action: "continue_delete" as const }; + if (recovery.action === "reject") { + throw new Error( + `Cannot resume sandbox '${target.sandboxName}' replacement: ${recovery.reason}.`, + ); } + // A registered, ready same-name sandbox carrying the journaled target + // generation and identity is the replacement this rebuild already proved. + // The caller must retire the transaction instead of deleting it again. + const acceptedTarget = recovery.action === "accept_target"; const session = onboardSession.updateSession((current) => { const checkpoint = current.checkpoint ?? deriveCheckpointFromSession(current); @@ -137,6 +144,7 @@ export function openRebuildRecreateJournal( return { id: transaction.id, + acceptedTarget, targetGeneration: transaction.targetGeneration, targetIntentFingerprint: transaction.targetIntentFingerprint, markDeleting: () => { @@ -151,5 +159,19 @@ export function openRebuildRecreateJournal( } advance("deleted"); }, + completeAcceptedTarget: () => { + if (!acceptedTarget) { + throw new Error( + `Sandbox '${target.sandboxName}' replacement journal cannot be retired before its replacement is proven.`, + ); + } + for (const next of ["registry_committing", "completed"] as const) { + if (!sandboxRecreatePhaseReached(phase, next)) advance(next); + } + onboardSession.updateSession((current) => { + clearCompletedSandboxRecreateTransaction(current, transaction.id); + return current; + }); + }, }; } diff --git a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts index 97bc3ae01a1..be984ece338 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts @@ -87,10 +87,12 @@ function makeInput(overrides: Partial = {}): RebuildR recreateOptions, recreateJournal: { id: "journal-1", + acceptedTarget: false, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", markDeleting: vi.fn(), confirmDeleted: vi.fn(), + completeAcceptedTarget: vi.fn(), }, fromDockerfile: null, rebuildAgent: DCODE_AGENT, @@ -218,20 +220,26 @@ describe("runRebuildRecreatePhase handoff", () => { let observedFingerprint: string | null | undefined; let observedJournalPhase: string | undefined; let observedCheckpointSessionId: string | undefined; - vi.spyOn(rebuildOnboardDependencies, "onboard").mockImplementation(async (options) => { - observedFingerprint = options.recreateJournalTargetIntentFingerprint; - const carried = onboardSession.loadSession()?.checkpoint; - observedJournalPhase = carried?.sandboxRecreate?.phase; - observedCheckpointSessionId = carried?.sessionId; - }); + const onboardSpy = vi + .spyOn(rebuildOnboardDependencies, "onboard") + .mockImplementation(async (options) => { + observedFingerprint = options.recreateJournalTargetIntentFingerprint; + const carried = onboardSession.loadSession()?.checkpoint; + observedJournalPhase = carried?.sandboxRecreate?.phase; + observedCheckpointSessionId = carried?.sessionId; + }); - await expect(runRebuildRecreatePhase(makeInput())).resolves.toBe(true); + try { + await expect(runRebuildRecreatePhase(makeInput())).resolves.toBe(true); - expect(observedFingerprint).toBe("intent-1"); - expect(observedJournalPhase).toBe("deleted"); - expect(observedCheckpointSessionId).toBe(onboardSession.loadSession()?.sessionId); - expect(observedCheckpointSessionId).not.toBe(retiredSessionId); - expect(onboardSession.loadSession()?.checkpoint?.effectGroups).toEqual({}); + expect(observedFingerprint).toBe("intent-1"); + expect(observedJournalPhase).toBe("deleted"); + expect(observedCheckpointSessionId).toBe(onboardSession.loadSession()?.sessionId); + expect(observedCheckpointSessionId).not.toBe(retiredSessionId); + expect(onboardSession.loadSession()?.checkpoint?.effectGroups).toEqual({}); + } finally { + onboardSpy.mockRestore(); + } }); it("pins the authoritative restricted tier during recreate and restores ambient policy input", async () => { diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 0017cb3e736..522683cb0b9 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2554,7 +2554,7 @@ async function createSandboxWithBaseImageResolution( process.exit(1); } // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - if (!createIntent?.recreateTransaction) recreateRuntime = recreateJournal.openOnboardRecreateJournal({ target: { sandboxName, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT }, agentName: getRequestedSandboxAgentName(agent) || "openclaw", note, observe: (probeTarget) => getSandboxRecreateObservation(probeTarget.sandboxName), intent: { agent: getRequestedSandboxAgentName(agent) || null, fromDockerfile: fromDockerfile ?? null, provider: provider ?? null, model: model ?? null, preferredInferenceApi: preferredInferenceApi ?? null, sandboxGpuConfig: effectiveSandboxGpuConfig ?? null, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, toolDisclosure: effectiveToolDisclosure, dcodeAutoApprovalMode: createIntent?.dcodeAutoApprovalMode ?? null, observabilityEnabled: createIntent?.observabilityEnabled === true, policyTier: createIntent?.policyTier ?? null } }); + if (!createIntent?.recreateTransaction) recreateRuntime = recreateJournal.openOnboardRecreateJournal({ target: { sandboxName, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT }, agentName: getRequestedSandboxAgentName(agent) || "openclaw", note, observe: (probeTarget) => getSandboxRecreateObservation(probeTarget.sandboxName, probeTarget.gatewayName), intent: { agent: getRequestedSandboxAgentName(agent) || null, fromDockerfile: fromDockerfile ?? null, provider: provider ?? null, model: model ?? null, preferredInferenceApi: preferredInferenceApi ?? null, sandboxGpuConfig: effectiveSandboxGpuConfig ?? null, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, toolDisclosure: effectiveToolDisclosure, dcodeAutoApprovalMode: createIntent?.dcodeAutoApprovalMode ?? null, observabilityEnabled: createIntent?.observabilityEnabled === true, policyTier: createIntent?.policyTier ?? null } }); const previousEntry: SandboxEntry | null = registry.getSandbox(sandboxName); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. baseImageResolutionFlow.captureBaseResolution(baseImageResolutionContext, previousEntry?.imageTag); diff --git a/src/lib/onboard/docker-gpu-patch-rollback.test.ts b/src/lib/onboard/docker-gpu-patch-rollback.test.ts index 77f509d9c83..1c3a6b3c554 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.test.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.test.ts @@ -9,6 +9,14 @@ import { recreateOpenShellDockerSandboxWithGpu, } from "./docker-gpu-patch"; +// The recreate path probes sandbox DNS through a real `docker run` when these +// stay unstubbed, which makes the rollback assertions depend on a live Docker +// daemon and registry reachability. +const offlineDnsDeps = { + detectSandboxFallbackDns: () => null, + probeContainerDns: () => ({ ok: true }), +}; + function inspectFixture(): DockerContainerInspect { return { Id: "old-container-id", @@ -88,6 +96,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, errorPhaseDebouncePolls: 1, }, ), @@ -141,6 +150,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, }, ), ).toThrow(/Could not start GPU-enabled sandbox container/); @@ -187,6 +197,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { dockerRm: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, }, ), ).toThrow(/Could not move original sandbox container aside/); @@ -229,6 +240,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, }, ); } catch (error) { @@ -274,6 +286,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, }, ); } catch (error) { @@ -326,6 +339,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, errorPhaseDebouncePolls: 1, }, ), @@ -365,6 +379,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runCaptureOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), + ...offlineDnsDeps, errorPhaseDebouncePolls: 1, }, ), diff --git a/src/lib/onboard/sandbox-reuse.test.ts b/src/lib/onboard/sandbox-reuse.test.ts index dca7f55cf48..b2529e1f799 100644 --- a/src/lib/onboard/sandbox-reuse.test.ts +++ b/src/lib/onboard/sandbox-reuse.test.ts @@ -225,6 +225,32 @@ describe("createSandboxReuseHelpers", () => { ); }); + it("observes a resumed replacement on the gateway its journal records (#7734)", () => { + const runCaptureOpenshell = vi.fn((args: string[]) => + args[1] === "get" ? "Name: alpha\nId: openshell-source-id\nState: Ready\n" : "alpha Ready\n", + ); + const helpers = createSandboxReuseHelpers({ + runCaptureOpenshell, + runOpenshell: vi.fn(), + getSandboxStateFromOutputs: vi.fn(() => "ready"), + note: vi.fn(), + getGatewayName: () => "nemoclaw", + }); + + helpers.getSandboxRecreateObservation("alpha", "nemoclaw-9090"); + + expect(runCaptureOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "get", "-g", "nemoclaw-9090", "alpha"], + { ignoreError: true }, + ); + expect(runCaptureOpenshell).toHaveBeenNthCalledWith( + 2, + ["sandbox", "list", "-g", "nemoclaw-9090"], + { ignoreError: true }, + ); + }); + it("preserves an unknown reuse state but rejects it for recreate recovery", () => { const helpers = createSandboxReuseHelpers({ runCaptureOpenshell: vi.fn(() => ""), diff --git a/src/lib/onboard/sandbox-reuse.ts b/src/lib/onboard/sandbox-reuse.ts index e81519822c6..be6d510c42a 100644 --- a/src/lib/onboard/sandbox-reuse.ts +++ b/src/lib/onboard/sandbox-reuse.ts @@ -27,7 +27,10 @@ export interface SandboxReuseDeps { export interface SandboxReuseHelpers { getSandboxReuseState(sandboxName: string | null): string; - getSandboxRecreateObservation(sandboxName: string | null): SandboxRecreateObservation; + getSandboxRecreateObservation( + sandboxName: string | null, + gatewayName?: string, + ): SandboxRecreateObservation; repairRecordedSandbox(sandboxName: string | null): void; } @@ -114,25 +117,34 @@ export function applyReusedSandboxDashboardState( } export function createSandboxReuseHelpers(deps: SandboxReuseDeps): SandboxReuseHelpers { - function gatewayArgs(): string[] { - const gatewayName = deps.getGatewayName?.(); + // A recorded gateway wins over the ambient one: a resumed replacement must + // prove presence or absence on the gateway its journal names. + function gatewayArgs(recordedGatewayName?: string): string[] { + const gatewayName = recordedGatewayName ?? deps.getGatewayName?.(); return gatewayName ? ["-g", gatewayName] : []; } - function readSandboxState(sandboxName: string | null): { state: string; getOutput: string } { + function readSandboxState( + sandboxName: string | null, + recordedGatewayName?: string, + ): { state: string; getOutput: string } { if (!sandboxName) return { state: "missing", getOutput: "" }; - const getOutput = deps.runCaptureOpenshell(["sandbox", "get", ...gatewayArgs(), sandboxName], { + const args = gatewayArgs(recordedGatewayName); + const getOutput = deps.runCaptureOpenshell(["sandbox", "get", ...args, sandboxName], { ignoreError: true, }); - const listOutput = deps.runCaptureOpenshell(["sandbox", "list", ...gatewayArgs()], { + const listOutput = deps.runCaptureOpenshell(["sandbox", "list", ...args], { ignoreError: true, }); const state = deps.getSandboxStateFromOutputs(sandboxName, getOutput, listOutput); return { state, getOutput }; } - function getSandboxRecreateObservation(sandboxName: string | null): SandboxRecreateObservation { - const { state, getOutput } = readSandboxState(sandboxName); + function getSandboxRecreateObservation( + sandboxName: string | null, + recordedGatewayName?: string, + ): SandboxRecreateObservation { + const { state, getOutput } = readSandboxState(sandboxName, recordedGatewayName); if (state !== "missing" && state !== "not_ready" && state !== "ready") { throw new Error( `Cannot observe sandbox '${sandboxName}' for recreate recovery: OpenShell returned state '${state}'.`, diff --git a/test/helpers/rebuild-flow-recovery-cases.ts b/test/helpers/rebuild-flow-recovery-cases.ts index 633a7415c5d..ff5decdec91 100644 --- a/test/helpers/rebuild-flow-recovery-cases.ts +++ b/test/helpers/rebuild-flow-recovery-cases.ts @@ -8,6 +8,7 @@ import { makeActiveTeamsMessagingPlan, makePreparedRecoveryManifest, } from "../../src/lib/actions/sandbox/rebuild-flow-test-fixtures"; +import { fingerprintSandboxLiveIdentity } from "../../src/lib/onboard/sandbox-recreate-transaction"; import { expectNoSandboxDelete } from "./rebuild-delete-assertions"; import { createRebuildFlowHarness, installRebuildFlowTestHooks } from "./rebuild-flow-test-harness"; @@ -261,6 +262,58 @@ export function registerRebuildFlowRecoveryTests(): void { }); }); + it("keeps a registered replacement when a rebuild restarts after its commit (#7734)", async () => { + const replacementProbe = "Name: alpha\nId: sbx-replacement\nPhase: Ready\n"; + const replacementIdentity = fingerprintSandboxLiveIdentity(replacementProbe); + const provenReplacement = { + sandboxEntry: { + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: replacementIdentity, + }, + }; + + // The interrupted run journals the replacement and registers it, then + // dies before the journal is cleared. + const interrupted = createRebuildFlowHarness({ + ...provenReplacement, + onboard: (session) => { + Object.assign( + (session.checkpoint as { sandboxRecreate: Record }).sandboxRecreate, + { + phase: "registry_committing", + targetGeneration: "generation-1", + targetLiveIdentityFingerprint: replacementIdentity, + }, + ); + throw new Error("interrupted after replacement registration"); + }, + }); + await expect( + interrupted.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recreate failed"); + + const restarted = createRebuildFlowHarness({ + ...provenReplacement, + captureOpenshell: (argv) => + argv[0] === "sandbox" && argv[1] === "get" + ? { status: 0, output: replacementProbe, stdout: replacementProbe, stderr: "" } + : { status: 1, output: "", stdout: "", stderr: "Error: sandbox alpha not found" }, + }); + restarted.session.checkpoint = interrupted.session.checkpoint; + // Both harnesses share one spy per mocked module function, so the + // interrupted run's calls have to be dropped before the restart. + restarted.runOpenshellSpy.mockClear(); + restarted.onboardSpy.mockClear(); + + await restarted.rebuildSandbox("alpha", ["--yes"]); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).not.toHaveBeenCalled(); + expect( + (restarted.session.checkpoint as { sandboxRecreate: unknown }).sandboxRecreate, + ).toBeNull(); + }); + it("performs exactly one prepared-recovery rollback when MCP state is present", async () => { const mcpEntry = { server: "github", providerName: "nemoclaw-mcp-alpha-github" }; const harness = createRebuildFlowHarness({ diff --git a/test/mcp-destroy-lifecycle.test.ts b/test/mcp-destroy-lifecycle.test.ts index a7d4c72447d..0334007bcdf 100644 --- a/test/mcp-destroy-lifecycle.test.ts +++ b/test/mcp-destroy-lifecycle.test.ts @@ -97,10 +97,12 @@ import * as registry from "../src/lib/state/registry"; function stubRecreateJournal(): RebuildRecreateJournal { return { id: "journal-1", + acceptedTarget: false, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", markDeleting: vi.fn(), confirmDeleted: vi.fn(), + completeAcceptedTarget: vi.fn(), }; } From a1af774907773d40ec8a7c3f8ce26dc3d2277578 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 29 Jul 2026 16:04:46 +0000 Subject: [PATCH 08/25] test(rebuild): keep the replacement journal cases branch-free Rebuild the proven-replacement helper without a guard branch so changed test files add no conditionals, and tighten the shell-quote fan-in budget to the count the current tree reports. Signed-off-by: Tinson Lai --- ci/source-architecture-budget.json | 2 +- .../sandbox/rebuild-recreate-journal.test.ts | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 2590c4f6a9e..8253c31ba6c 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -16,7 +16,7 @@ "src/lib/cli/terminal-style.ts": 45, "src/lib/core/json-types.ts": 37, "src/lib/core/ports.ts": 86, - "src/lib/core/shell-quote.ts": 27, + "src/lib/core/shell-quote.ts": 26, "src/lib/core/url-utils.ts": 27, "src/lib/core/wait.ts": 35, "src/lib/credentials/store.ts": 44, diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts index 94f3ad36cd5..906152c8763 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts @@ -329,18 +329,14 @@ describe("rebuild replacement journal", () => { expect(second.targetGeneration).toBe(first.targetGeneration); }); - function proveReplacement(journalId: string, targetGeneration: string): string { + function proveReplacement(targetGeneration: string): string { const identity = session.checkpoint?.sandboxRecreate?.sourceLiveIdentityFingerprint ?? ""; onboardSession.updateSession((current) => { - const checkpoint = current.checkpoint; - const transaction = checkpoint?.sandboxRecreate; - if (!checkpoint || !transaction || transaction.id !== journalId) { - throw new Error("journal is missing"); - } + const checkpoint = current.checkpoint as NonNullable; current.checkpoint = { ...checkpoint, sandboxRecreate: { - ...transaction, + ...(checkpoint.sandboxRecreate as NonNullable), phase: "registry_committing", targetLiveIdentityFingerprint: identity, }, @@ -360,7 +356,7 @@ describe("rebuild replacement journal", () => { it("reports a registered ready replacement as the proven target (#7734)", () => { const first = open(); - proveReplacement(first.id, first.targetGeneration); + proveReplacement(first.targetGeneration); const resumed = open(); @@ -371,7 +367,7 @@ describe("rebuild replacement journal", () => { it("retires the journal of a proven replacement instead of deleting it again (#7734)", () => { const first = open(); - proveReplacement(first.id, first.targetGeneration); + proveReplacement(first.targetGeneration); const resumed = open(); resumed.completeAcceptedTarget(); From e8e8a28a4f71b0f8a9d0e0d395b6b0e7a5563740 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Wed, 29 Jul 2026 16:26:26 +0000 Subject: [PATCH 09/25] test(rebuild): cover restart from every post-delete journal phase Drive the rebuild pipeline from persisted creating, created, registry_committing, and completed journals. Assert a restart accepts only a ready replacement whose identity and generation match the journal, and that a foreign same-name sandbox stops the rebuild before any destructive mutation. Signed-off-by: Tinson Lai --- test/helpers/rebuild-flow-recovery-cases.ts | 60 +++++++++++++++------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/test/helpers/rebuild-flow-recovery-cases.ts b/test/helpers/rebuild-flow-recovery-cases.ts index ff5decdec91..7a08cf49bbc 100644 --- a/test/helpers/rebuild-flow-recovery-cases.ts +++ b/test/helpers/rebuild-flow-recovery-cases.ts @@ -262,48 +262,61 @@ export function registerRebuildFlowRecoveryTests(): void { }); }); - it("keeps a registered replacement when a rebuild restarts after its commit (#7734)", async () => { - const replacementProbe = "Name: alpha\nId: sbx-replacement\nPhase: Ready\n"; - const replacementIdentity = fingerprintSandboxLiveIdentity(replacementProbe); - const provenReplacement = { - sandboxEntry: { - lifecycleGeneration: "generation-1", - lifecycleLiveIdentityFingerprint: replacementIdentity, - }, - }; - - // The interrupted run journals the replacement and registers it, then - // dies before the journal is cleared. + const REPLACEMENT_PROBE = "Name: alpha\nId: sbx-replacement\nPhase: Ready\n"; + const FOREIGN_PROBE = "Name: alpha\nId: sbx-foreign\nPhase: Ready\n"; + const REPLACEMENT_IDENTITY = fingerprintSandboxLiveIdentity(REPLACEMENT_PROBE); + const POST_DELETE_PHASES = ["creating", "created", "registry_committing", "completed"] as const; + const provenReplacement = { + sandboxEntry: { + lifecycleGeneration: "generation-1", + lifecycleLiveIdentityFingerprint: REPLACEMENT_IDENTITY, + }, + }; + + // Journal a replacement through the real pipeline, then die at the given + // post-delete phase so the restart reads persisted state, not a hand-built + // checkpoint. + async function interruptAfterCreate(phase: string): Promise { const interrupted = createRebuildFlowHarness({ ...provenReplacement, onboard: (session) => { Object.assign( (session.checkpoint as { sandboxRecreate: Record }).sandboxRecreate, { - phase: "registry_committing", + phase, targetGeneration: "generation-1", - targetLiveIdentityFingerprint: replacementIdentity, + targetLiveIdentityFingerprint: REPLACEMENT_IDENTITY, }, ); - throw new Error("interrupted after replacement registration"); + throw new Error("interrupted after replacement creation"); }, }); await expect( interrupted.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), ).rejects.toThrow("Recreate failed"); + return interrupted.session.checkpoint; + } + function restartRebuild(probe: string, checkpoint: unknown) { const restarted = createRebuildFlowHarness({ ...provenReplacement, captureOpenshell: (argv) => argv[0] === "sandbox" && argv[1] === "get" - ? { status: 0, output: replacementProbe, stdout: replacementProbe, stderr: "" } + ? { status: 0, output: probe, stdout: probe, stderr: "" } : { status: 1, output: "", stdout: "", stderr: "Error: sandbox alpha not found" }, }); - restarted.session.checkpoint = interrupted.session.checkpoint; + restarted.session.checkpoint = checkpoint; // Both harnesses share one spy per mocked module function, so the // interrupted run's calls have to be dropped before the restart. restarted.runOpenshellSpy.mockClear(); restarted.onboardSpy.mockClear(); + return restarted; + } + + it.each( + POST_DELETE_PHASES, + )("keeps a registered replacement when a rebuild restarts from '%s' (#7734)", async (phase) => { + const restarted = restartRebuild(REPLACEMENT_PROBE, await interruptAfterCreate(phase)); await restarted.rebuildSandbox("alpha", ["--yes"]); @@ -314,6 +327,19 @@ export function registerRebuildFlowRecoveryTests(): void { ).toBeNull(); }); + it.each( + POST_DELETE_PHASES, + )("refuses a foreign same-name sandbox when a rebuild restarts from '%s' (#7734)", async (phase) => { + const restarted = restartRebuild(FOREIGN_PROBE, await interruptAfterCreate(phase)); + + await expect(restarted.rebuildSandbox("alpha", ["--yes"])).rejects.toThrow( + /not the journaled replacement/, + ); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).not.toHaveBeenCalled(); + }); + it("performs exactly one prepared-recovery rollback when MCP state is present", async () => { const mcpEntry = { server: "github", providerName: "nemoclaw-mcp-alpha-github" }; const harness = createRebuildFlowHarness({ From b5adecda8dcd5397e49879454065793f90f14d0e Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 05:01:00 +0000 Subject: [PATCH 10/25] fix(onboard): keep the journaled source row across its route reservation Signed-off-by: Tinson Lai --- .../sandbox-recreate-transaction.test.ts | 96 ++++++++++++++++++- .../onboard/sandbox-recreate-transaction.ts | 17 +++- 2 files changed, 110 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index 2498c0f50ba..65c134aeefc 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -1,7 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { decisionSelected } from "../state/onboard-checkpoint-decision"; import { deriveCheckpointFromSession } from "../state/onboard-checkpoint-migrate"; @@ -391,7 +394,7 @@ describe("sandbox recreate recovery", () => { planSandboxRecreateRecovery( transactionAt("planned"), { state: "ready", liveIdentityFingerprint: SOURCE_ID }, - { ...SOURCE_ENTRY, model: "changed-out-of-band" }, + { ...SOURCE_ENTRY, imageTag: "changed-out-of-band" }, ), ).toMatchObject({ action: "reject", @@ -399,6 +402,23 @@ describe("sandbox recreate recovery", () => { }); }); + it("keeps the preserved source row after the replacement reserves its inference route", () => { + expect( + planSandboxRecreateRecovery( + transactionAt("deleted"), + { state: "missing", liveIdentityFingerprint: null }, + { + ...SOURCE_ENTRY, + pendingRouteReservation: true, + reservationSessionId: "session-9", + model: "model-b", + endpointUrl: "https://api.example.test/v1", + gatewayPort: undefined, + }, + ), + ).toEqual({ action: "continue_create" }); + }); + it("rejects a same-name live sandbox with a different source identity", () => { expect( planSandboxRecreateRecovery( @@ -461,6 +481,78 @@ describe("sandbox recreate recovery", () => { }); }); +describe("source registry fingerprint", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.resetModules(); + }); + + it("survives the route reservation the replacement onboard writes (#1904)", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-recreate-journal-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("../state/registry"); + registry.registerSandbox({ + name: "alpha", + agent: "openclaw", + agentVersion: "2026.3.11", + createdAt: ISO, + imageTag: "nemoclaw/openclaw:2026.3.11", + provider: "compatible-endpoint", + model: "model-a", + endpointUrl: "https://api.example.test/v1", + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-responses", + gatewayName: "nemoclaw", + gatewayPort: 8080, + }); + const journaled = fingerprintSandboxRegistryEntry( + registry.getSandbox("alpha") as SandboxEntry, + ); + + expect( + registry.reserveSandboxInferenceRoute("alpha", { + provider: "compatible-endpoint", + model: "model-a", + endpointUrl: "https://api.example.test/v1", + endpointSource: "onboard", + credentialEnv: "COMPATIBLE_API_KEY", + preferredInferenceApi: "openai-responses", + gatewayName: "nemoclaw", + reservationSessionId: "session-9", + }), + ).toBe(true); + expect(fingerprintSandboxRegistryEntry(registry.getSandbox("alpha") as SandboxEntry)).toBe( + journaled, + ); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + + it("changes when the row records another sandbox", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-recreate-journal-")); + vi.stubEnv("HOME", home); + vi.resetModules(); + try { + const registry = await import("../state/registry"); + registry.registerSandbox({ name: "alpha", agent: "openclaw", createdAt: ISO }); + const journaled = fingerprintSandboxRegistryEntry( + registry.getSandbox("alpha") as SandboxEntry, + ); + + registry.updateSandbox("alpha", { createdAt: "2026-07-28T20:00:00.000Z" }); + + expect( + fingerprintSandboxRegistryEntry(registry.getSandbox("alpha") as SandboxEntry), + ).not.toBe(journaled); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); +}); + describe("OpenShell live identity", () => { it("hashes an ANSI-decorated Id without persisting the raw identifier", () => { const output = "Name: alpha\n\u001b[32mId: openshell-source-id\u001b[0m\nState: Ready\n"; diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index b429fd86edd..464e9025713 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -40,8 +40,23 @@ export function fingerprintSandboxRecreateValue(value: unknown): string { return createHash("sha256").update(serialized).digest("hex"); } +const ROUTE_RESERVATION_FIELDS: readonly (keyof SandboxEntry)[] = [ + "pendingRouteReservation", + "reservationSessionId", + "provider", + "model", + "endpointUrl", + "endpointSource", + "credentialEnv", + "preferredInferenceApi", + "gatewayName", + "gatewayPort", +]; + export function fingerprintSandboxRegistryEntry(entry: SandboxEntry): string { - return fingerprintSandboxRecreateValue(entry); + const durable: Record = { ...entry }; + for (const field of ROUTE_RESERVATION_FIELDS) delete durable[field]; + return fingerprintSandboxRecreateValue(durable); } export function fingerprintSandboxLiveIdentity(getOutput: string): string | null { From 9b01609414511a24e8e32ca19877996abdb2c92f Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 05:02:33 +0000 Subject: [PATCH 11/25] fix(onboard): stop deleting a same-name replacement the journal already proved Signed-off-by: Tinson Lai --- src/lib/onboard.ts | 2 + .../onboard/onboard-recreate-journal.test.ts | 53 +++++++++++++++++++ src/lib/onboard/onboard-recreate-journal.ts | 8 ++- 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 522683cb0b9..c9ac4816c72 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2555,6 +2555,8 @@ async function createSandboxWithBaseImageResolution( } // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. if (!createIntent?.recreateTransaction) recreateRuntime = recreateJournal.openOnboardRecreateJournal({ target: { sandboxName, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT }, agentName: getRequestedSandboxAgentName(agent) || "openclaw", note, observe: (probeTarget) => getSandboxRecreateObservation(probeTarget.sandboxName, probeTarget.gatewayName), intent: { agent: getRequestedSandboxAgentName(agent) || null, fromDockerfile: fromDockerfile ?? null, provider: provider ?? null, model: model ?? null, preferredInferenceApi: preferredInferenceApi ?? null, sandboxGpuConfig: effectiveSandboxGpuConfig ?? null, gatewayName: GATEWAY_NAME, gatewayPort: GATEWAY_PORT, toolDisclosure: effectiveToolDisclosure, dcodeAutoApprovalMode: createIntent?.dcodeAutoApprovalMode ?? null, observabilityEnabled: createIntent?.observabilityEnabled === true, policyTier: createIntent?.policyTier ?? null } }); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + if (recreateRuntime.acceptedTarget) { if ("complete" in recreateRuntime) recreateRuntime.complete(); restoreReusedSandboxDashboard(true); return sandboxName; } const previousEntry: SandboxEntry | null = registry.getSandbox(sandboxName); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. baseImageResolutionFlow.captureBaseResolution(baseImageResolutionContext, previousEntry?.imageTag); diff --git a/src/lib/onboard/onboard-recreate-journal.test.ts b/src/lib/onboard/onboard-recreate-journal.test.ts index f9020ebdece..04a218be278 100644 --- a/src/lib/onboard/onboard-recreate-journal.test.ts +++ b/src/lib/onboard/onboard-recreate-journal.test.ts @@ -86,6 +86,11 @@ function livePresentProbe(phase = "Ready") { return { status: 0, output: rendered, stdout: rendered, stderr: "" }; } +function replacementProbe(phase = "Ready") { + const rendered = `Name: alpha\nId: sbx-2f80d5a613\nPhase: ${phase}\n`; + return { status: 0, output: rendered, stdout: rendered, stderr: "" }; +} + function absentProbe() { return { status: 1, @@ -221,6 +226,54 @@ describe("non-resumed onboard replacement journal (#7735)", () => { expect(session.checkpoint?.sandboxRecreate).toBeNull(); }); + it("accepts the proven replacement instead of deleting it again (#7734)", () => { + const first = open(); + first.advance("deleting"); + mocks.captureOpenshell.mockReturnValue(absentProbe()); + first.confirmDeleted(); + first.advance("creating"); + mocks.captureOpenshell.mockReturnValue(replacementProbe()); + first.recordCreated(); + first.advance("registry_committing"); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + ...first.registrationFields, + } as registry.SandboxEntry); + + const resumed = open(); + + expect(resumed.acceptedTarget).toBe(true); + resumed.complete(); + expect(session.checkpoint?.sandboxRecreate).toBeNull(); + }); + + it("retires a replacement whose registration already reached completion", () => { + const first = open(); + first.advance("deleting"); + mocks.captureOpenshell.mockReturnValue(absentProbe()); + first.confirmDeleted(); + first.advance("creating"); + mocks.captureOpenshell.mockReturnValue(replacementProbe()); + first.recordCreated(); + first.advance("completed"); + vi.spyOn(registry, "getSandbox").mockReturnValue({ + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw-9090", + gatewayPort: 9090, + ...first.registrationFields, + } as registry.SandboxEntry); + + const resumed = open(); + + expect(resumed.acceptedTarget).toBe(true); + expect(() => resumed.complete()).not.toThrow(); + expect(session.checkpoint?.sandboxRecreate).toBeNull(); + }); + it("resumes the same replacement after a restart without a resume flag", () => { const first = open(); first.advance("deleting"); diff --git a/src/lib/onboard/onboard-recreate-journal.ts b/src/lib/onboard/onboard-recreate-journal.ts index 9198206be72..161226cf9a6 100644 --- a/src/lib/onboard/onboard-recreate-journal.ts +++ b/src/lib/onboard/onboard-recreate-journal.ts @@ -20,6 +20,7 @@ import { fingerprintSandboxRecreateValue, planSandboxRecreateRecovery, type SandboxRecreateRuntime, + sandboxRecreatePhaseReached, } from "./sandbox-recreate-transaction"; export interface OnboardRecreateTargetIntent { @@ -151,8 +152,11 @@ export function openOnboardRecreateJournal( // the replacement registry row commits. complete: () => { onboardSession.updateSession((current) => { - advanceSandboxRecreateTransaction(current, transaction.id, "registry_committing"); - advanceSandboxRecreateTransaction(current, transaction.id, "completed"); + for (const next of ["registry_committing", "completed"] as const) { + const phase = current.checkpoint?.sandboxRecreate?.phase; + if (phase && sandboxRecreatePhaseReached(phase, next)) continue; + advanceSandboxRecreateTransaction(current, transaction.id, next); + } clearCompletedSandboxRecreateTransaction(current, transaction.id); return current; }); From ba01650248df9a72081b5393a11cb0e2a664b7b5 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 05:10:09 +0000 Subject: [PATCH 12/25] ci: split the rebuild phase entries in the allowed-cycle list Signed-off-by: Tinson Lai --- ci/source-architecture-budget.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 8253c31ba6c..b7affdffeed 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -112,7 +112,8 @@ "src/lib/actions/sandbox/rebuild-pipeline.ts", "src/lib/actions/sandbox/rebuild-post-restore-phase.ts", "src/lib/actions/sandbox/rebuild-preflight-phase.ts", - "src/lib/actions/sandbox/rebuild-preflight-target-phase.ts", "src/lib/actions/sandbox/rebuild-recreate-phase.ts", + "src/lib/actions/sandbox/rebuild-preflight-target-phase.ts", + "src/lib/actions/sandbox/rebuild-recreate-phase.ts", "src/lib/actions/sandbox/rebuild-shields-phase.ts", "src/lib/actions/sandbox/rebuild-shields.ts", "src/lib/actions/sandbox/rebuild-target-config.ts", From 015a86f67bda760636d306e8429faf60963e0bcc Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 06:23:38 +0000 Subject: [PATCH 13/25] fix(rebuild): journal endpoint provenance in the replacement target intent Signed-off-by: Tinson Lai --- .../sandbox/rebuild-recreate-journal.test.ts | 31 +++++++++++++++---- .../sandbox/rebuild-recreate-journal.ts | 2 ++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts index 906152c8763..a598865f340 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts @@ -95,12 +95,15 @@ describe("rebuild replacement target fingerprint", () => { }); it("changes when a recorded replacement input changes", () => { - expect( - fingerprintRebuildRecreateTargetIntent({ - ...recreateOptions, - dcodeAutoApprovalMode: "thread-opt-in", - }), - ).not.toBe(fingerprintRebuildRecreateTargetIntent(recreateOptions)); + for (const drift of [ + { dcodeAutoApprovalMode: "thread-opt-in" }, + { endpointSource: "onboard" }, + { policyTier: "balanced" }, + ] as const) { + expect(fingerprintRebuildRecreateTargetIntent({ ...recreateOptions, ...drift })).not.toBe( + fingerprintRebuildRecreateTargetIntent(recreateOptions), + ); + } }); it("changes when the replacement targets another gateway", () => { @@ -320,6 +323,22 @@ describe("rebuild replacement journal", () => { ).toThrow(/different recreate transaction in progress/); }); + it("refuses to resume a journal whose endpoint provenance changed (#7734)", () => { + open(); + + expect(() => + openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent({ + ...recreateOptions, + endpointSource: "onboard", + }), + log: vi.fn(), + }), + ).toThrow(/different recreate transaction in progress/); + }); + it("resumes the same replacement without restarting its generation", () => { const first = open(); diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.ts index 2efc9be8fc2..0d3a856c70a 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.ts @@ -41,6 +41,7 @@ export function fingerprintRebuildRecreateTargetIntent( options: Pick< RebuildRecreateOnboardOpts, | "agent" + | "endpointSource" | "fromDockerfile" | "sandboxGpu" | "sandboxGpuDevice" @@ -56,6 +57,7 @@ export function fingerprintRebuildRecreateTargetIntent( return fingerprintSandboxRecreateValue({ version: 1, agent: options.agent ?? null, + endpointSource: options.endpointSource ?? null, fromDockerfile: options.fromDockerfile, sandboxGpu: options.sandboxGpu, sandboxGpuDevice: options.sandboxGpuDevice, From 0555bf4f9202c751fec746f4f653f275f1958a17 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 07:20:37 +0000 Subject: [PATCH 14/25] test(rebuild): cover restarts from pre-creation recreate journal phases Signed-off-by: Tinson Lai --- test/helpers/rebuild-flow-recovery-cases.ts | 109 ++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/test/helpers/rebuild-flow-recovery-cases.ts b/test/helpers/rebuild-flow-recovery-cases.ts index 7a08cf49bbc..9e6466b96e1 100644 --- a/test/helpers/rebuild-flow-recovery-cases.ts +++ b/test/helpers/rebuild-flow-recovery-cases.ts @@ -340,6 +340,115 @@ export function registerRebuildFlowRecoveryTests(): void { expect(restarted.onboardSpy).not.toHaveBeenCalled(); }); + const SOURCE_PROBE = "Name: alpha\nId: sbx-source\nPhase: Ready\n"; + const MISSING_SOURCE = { + status: 1, + output: "", + stdout: "", + stderr: "Error: sandbox alpha not found", + }; + const PRE_CREATE_PHASES = ["planned", "deleting", "deleted"] as const; + const LIVE_SOURCE_PHASES = ["planned", "deleting"] as const; + + function sandboxGetProbes(probes: readonly (string | null)[]) { + let gets = 0; + return (argv: string[]) => { + if (argv[0] !== "sandbox" || argv[1] !== "get") return MISSING_SOURCE; + const probe = probes[Math.min(gets++, probes.length - 1)]; + return probe ? { status: 0, output: probe, stdout: probe, stderr: "" } : MISSING_SOURCE; + }; + } + + async function interruptBeforeCreate(phase: string): Promise { + const interrupted = createRebuildFlowHarness({ + captureOpenshell: sandboxGetProbes([SOURCE_PROBE, null]), + onboard: (session) => { + Object.assign( + (session.checkpoint as { sandboxRecreate: Record }).sandboxRecreate, + { phase }, + ); + throw new Error("interrupted before replacement creation"); + }, + }); + await expect( + interrupted.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow("Recreate failed"); + return interrupted.session.checkpoint; + } + + function restartFromJournaledSource(probes: readonly (string | null)[], checkpoint: unknown) { + const restarted = createRebuildFlowHarness({ + captureOpenshell: sandboxGetProbes(probes), + }); + restarted.session.checkpoint = checkpoint; + restarted.runOpenshellSpy.mockClear(); + restarted.onboardSpy.mockClear(); + return restarted; + } + + it.each( + LIVE_SOURCE_PHASES, + )("deletes the journaled source when a rebuild restarts from '%s' (#7734)", async (phase) => { + const restarted = restartFromJournaledSource( + [SOURCE_PROBE, null], + await interruptBeforeCreate(phase), + ); + + await restarted.rebuildSandbox("alpha", ["--yes"]); + + expect(restarted.runOpenshellSpy).toHaveBeenCalledWith( + ["sandbox", "delete", "-g", "nemoclaw", "alpha"], + expect.objectContaining({ ignoreError: true }), + ); + expect(restarted.onboardSpy).toHaveBeenCalled(); + }); + + it.each( + PRE_CREATE_PHASES, + )("creates the replacement without widening the delete target when a rebuild restarts from '%s' with the source already absent (#7734)", async (phase) => { + const restarted = restartFromJournaledSource([null], await interruptBeforeCreate(phase)); + + await restarted.rebuildSandbox("alpha", ["--yes"]); + + const deleteCalls = restarted.runOpenshellSpy.mock.calls.filter( + ([args]) => Array.isArray(args) && args[0] === "sandbox" && args[1] === "delete", + ); + expect(deleteCalls.map(([args]) => args)).toEqual( + deleteCalls.map(() => ["sandbox", "delete", "-g", "nemoclaw", "alpha"]), + ); + expect(restarted.onboardSpy).toHaveBeenCalled(); + }); + + it.each( + LIVE_SOURCE_PHASES, + )("refuses a changed same-name sandbox when a rebuild restarts from '%s' (#7734)", async (phase) => { + const restarted = restartFromJournaledSource( + [FOREIGN_PROBE, null], + await interruptBeforeCreate(phase), + ); + + await expect(restarted.rebuildSandbox("alpha", ["--yes"])).rejects.toThrow( + /no longer has the journaled source identity/, + ); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).not.toHaveBeenCalled(); + }); + + it("refuses a live same-name sandbox when a rebuild restarts from 'deleted' (#7734)", async () => { + const restarted = restartFromJournaledSource( + [SOURCE_PROBE, null], + await interruptBeforeCreate("deleted"), + ); + + await expect(restarted.rebuildSandbox("alpha", ["--yes"])).rejects.toThrow( + /appeared before replacement registration committed/, + ); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).not.toHaveBeenCalled(); + }); + it("performs exactly one prepared-recovery rollback when MCP state is present", async () => { const mcpEntry = { server: "github", providerName: "nemoclaw-mcp-alpha-github" }; const harness = createRebuildFlowHarness({ From 4f1ae334f11a8c42c5e64d50c46dcb5ffbadf899 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 08:44:14 +0000 Subject: [PATCH 15/25] fix(rebuild): prove the journaled source before a restart deletes it Signed-off-by: Tinson Lai --- .../sandbox/rebuild-destroy-phase.test.ts | 2 ++ .../actions/sandbox/rebuild-destroy-phase.ts | 30 +++++++++++++------ .../sandbox/rebuild-recreate-journal.ts | 18 +++++++++++ .../rebuild-recreate-observability.test.ts | 2 ++ test/helpers/rebuild-flow-recovery-cases.ts | 27 ++++++++++++----- test/helpers/rebuild-flow-test-harness.ts | 10 +++++-- test/mcp-destroy-lifecycle.test.ts | 2 ++ 7 files changed, 72 insertions(+), 19 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index c716fe02a06..76417f20915 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -77,9 +77,11 @@ function stubRecreateJournal(): RebuildRecreateJournal { return { id: "journal-1", acceptedTarget: false, + sourceConfirmedAbsent: false, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", markDeleting: vi.fn(), + observeSourceForDelete: vi.fn(() => "source" as const), confirmDeleted: vi.fn(), completeAcceptedTarget: vi.fn(), }; diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.ts index 56d09be3b59..716502a15e5 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.ts @@ -23,7 +23,10 @@ import { reattachMcpAfterDeleteFailure, } from "./rebuild-mcp-phase"; import { blockRebuildOnPendingBaselineTransition } from "./rebuild-preflight-guards"; -import type { RebuildRecreateJournal } from "./rebuild-recreate-journal"; +import type { + RebuildRecreateJournal, + RebuildRecreateSourcePresence, +} from "./rebuild-recreate-journal"; export type RebuildDeleteValidationResult = | { ok: true } @@ -359,7 +362,9 @@ export async function runRebuildDestroyPhase( // MCP adapter entries are already detached and scrubbed here. A journal write // that fails must reattach them before the rebuild gives up, or the still // running sandbox is left without its MCP wiring. + let sourcePresence: RebuildRecreateSourcePresence; try { + sourcePresence = recreateJournal.observeSourceForDelete(); recreateJournal.markDeleting(); } catch (error) { const mcpRecoveryFailure = await reattachMcpAfterDeleteFailure( @@ -376,15 +381,22 @@ export async function runRebuildDestroyPhase( ); return null; } - log(`Running: openshell sandbox delete -g ${gatewayName} ${sandboxName}`); - const deleteResult = runOpenshell(["sandbox", "delete", "-g", gatewayName, sandboxName], { - ignoreError: true, - stdio: ["ignore", "pipe", "pipe"], - }); - const { alreadyGone } = getSandboxDeleteOutcome(deleteResult); - log(`Delete result: exit=${deleteResult.status}, alreadyGone=${alreadyGone}`); + if (sourcePresence === "missing") { + log(`Skipping delete: gateway ${gatewayName} reports '${sandboxName}' already absent`); + } else { + log(`Running: openshell sandbox delete -g ${gatewayName} ${sandboxName}`); + } + const deleteResult = + sourcePresence === "missing" + ? null + : runOpenshell(["sandbox", "delete", "-g", gatewayName, sandboxName], { + ignoreError: true, + stdio: ["ignore", "pipe", "pipe"], + }); + const alreadyGone = deleteResult === null || getSandboxDeleteOutcome(deleteResult).alreadyGone; + if (deleteResult) log(`Delete result: exit=${deleteResult.status}, alreadyGone=${alreadyGone}`); let deletionConfirmed = alreadyGone; - if (deleteResult.status !== 0) { + if (deleteResult && deleteResult.status !== 0) { const reconciledDelete = reconcileFailedSandboxDelete(sandboxName, input.sandboxEntry, log); if (reconciledDelete.state === "deleted") { log("Delete returned nonzero, but exact post-delete state confirms sandbox removal."); diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.ts index 0d3a856c70a..e4ae8343a6d 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.ts @@ -27,12 +27,16 @@ export type RebuildRecreateJournalTarget = SandboxRecreateTarget; export type RebuildSandboxObserver = SandboxRecreateObserver; +export type RebuildRecreateSourcePresence = "missing" | "source"; + export interface RebuildRecreateJournal { readonly id: string; readonly acceptedTarget: boolean; + readonly sourceConfirmedAbsent: boolean; readonly targetGeneration: string; readonly targetIntentFingerprint: string; markDeleting(): void; + observeSourceForDelete(): RebuildRecreateSourcePresence; confirmDeleted(): void; completeAcceptedTarget(): void; } @@ -147,12 +151,26 @@ export function openRebuildRecreateJournal( return { id: transaction.id, acceptedTarget, + sourceConfirmedAbsent: recovery.action === "continue_create", targetGeneration: transaction.targetGeneration, targetIntentFingerprint: transaction.targetIntentFingerprint, markDeleting: () => { if (sandboxRecreatePhaseReached(phase, "deleted")) return; advance("deleting"); }, + observeSourceForDelete: () => { + const current = observe(target); + if (current.state === "missing") return "missing"; + if ( + !transaction.sourceLiveIdentityFingerprint || + current.liveIdentityFingerprint !== transaction.sourceLiveIdentityFingerprint + ) { + throw new Error( + `Cannot delete sandbox '${target.sandboxName}': the live same-name sandbox is not the journaled source.`, + ); + } + return "source"; + }, confirmDeleted: () => { if (observe(target).state !== "missing") { throw new Error( diff --git a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts index be984ece338..0cc8cb60b20 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts @@ -88,9 +88,11 @@ function makeInput(overrides: Partial = {}): RebuildR recreateJournal: { id: "journal-1", acceptedTarget: false, + sourceConfirmedAbsent: false, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", markDeleting: vi.fn(), + observeSourceForDelete: vi.fn(() => "source" as const), confirmDeleted: vi.fn(), completeAcceptedTarget: vi.fn(), }, diff --git a/test/helpers/rebuild-flow-recovery-cases.ts b/test/helpers/rebuild-flow-recovery-cases.ts index 9e6466b96e1..f0b96afff9f 100644 --- a/test/helpers/rebuild-flow-recovery-cases.ts +++ b/test/helpers/rebuild-flow-recovery-cases.ts @@ -390,7 +390,7 @@ export function registerRebuildFlowRecoveryTests(): void { LIVE_SOURCE_PHASES, )("deletes the journaled source when a rebuild restarts from '%s' (#7734)", async (phase) => { const restarted = restartFromJournaledSource( - [SOURCE_PROBE, null], + [SOURCE_PROBE, SOURCE_PROBE, null], await interruptBeforeCreate(phase), ); @@ -405,20 +405,31 @@ export function registerRebuildFlowRecoveryTests(): void { it.each( PRE_CREATE_PHASES, - )("creates the replacement without widening the delete target when a rebuild restarts from '%s' with the source already absent (#7734)", async (phase) => { + )("creates the replacement without a second delete when a rebuild restarts from '%s' with the source already absent (#7734)", async (phase) => { const restarted = restartFromJournaledSource([null], await interruptBeforeCreate(phase)); await restarted.rebuildSandbox("alpha", ["--yes"]); - const deleteCalls = restarted.runOpenshellSpy.mock.calls.filter( - ([args]) => Array.isArray(args) && args[0] === "sandbox" && args[1] === "delete", - ); - expect(deleteCalls.map(([args]) => args)).toEqual( - deleteCalls.map(() => ["sandbox", "delete", "-g", "nemoclaw", "alpha"]), - ); + expectNoSandboxDelete(restarted.runOpenshellSpy); expect(restarted.onboardSpy).toHaveBeenCalled(); }); + it.each( + LIVE_SOURCE_PHASES, + )("stops before deletion when a same-name sandbox appears after a '%s' restart probe (#7734)", async (phase) => { + const restarted = restartFromJournaledSource( + [null, FOREIGN_PROBE], + await interruptBeforeCreate(phase), + ); + + await expect( + restarted.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), + ).rejects.toThrow(/the live same-name sandbox is not the journaled source/); + + expectNoSandboxDelete(restarted.runOpenshellSpy); + expect(restarted.onboardSpy).not.toHaveBeenCalled(); + }); + it.each( LIVE_SOURCE_PHASES, )("refuses a changed same-name sandbox when a rebuild restarts from '%s' (#7734)", async (phase) => { diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index 20aab708d39..960784fa064 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -387,12 +387,14 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): failedFiles: [], })), ); + let defaultSourceDeleted = false; const runOpenshellSpy = vi .spyOn(openshellRuntime, "runOpenshell") .mockImplementation((args: unknown) => { const argv = Array.isArray(args) ? args.map(String) : []; const overrideResult = overrides.runOpenshell?.(argv); if (overrideResult) return overrideResult; + if (argv[0] === "sandbox" && argv[1] === "delete") defaultSourceDeleted = true; if ( argv.join(" ") === "sandbox get alpha" || argv.join(" ") === "sandbox get -g nemoclaw alpha" @@ -410,8 +412,12 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .spyOn(openshellRuntime, "captureOpenshell") .mockImplementation((args: unknown, options?: unknown) => { const argv = Array.isArray(args) ? args.map(String) : []; - return overrides.captureOpenshell - ? overrides.captureOpenshell(argv, options as Record | undefined) + if (overrides.captureOpenshell) { + return overrides.captureOpenshell(argv, options as Record | undefined); + } + const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; + return argv[0] === "sandbox" && argv[1] === "get" && !defaultSourceDeleted + ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } : { status: 1, output: "", stderr: "Error: sandbox alpha not found" }; }); const defaultRemovalReceipt = { diff --git a/test/mcp-destroy-lifecycle.test.ts b/test/mcp-destroy-lifecycle.test.ts index 0334007bcdf..a4ce5170023 100644 --- a/test/mcp-destroy-lifecycle.test.ts +++ b/test/mcp-destroy-lifecycle.test.ts @@ -98,9 +98,11 @@ function stubRecreateJournal(): RebuildRecreateJournal { return { id: "journal-1", acceptedTarget: false, + sourceConfirmedAbsent: false, targetGeneration: "generation-1", targetIntentFingerprint: "intent-1", markDeleting: vi.fn(), + observeSourceForDelete: vi.fn(() => "source" as const), confirmDeleted: vi.fn(), completeAcceptedTarget: vi.fn(), }; From e2192d6ea22b35faa1070076d092f43c50f5cae0 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 09:42:43 +0000 Subject: [PATCH 16/25] test(rebuild): model a live source in the rebuild flow harness defaults Signed-off-by: Tinson Lai --- .../rebuild-local-provider-recreate.test.ts | 23 +++++++++++-------- test/helpers/rebuild-flow-harness.ts | 10 ++++++-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index 6edc921d4f3..53f7c30b55c 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -149,12 +149,7 @@ describe("rebuild local-provider recreation", () => { credentialEnv, setup, }) => { - vi.spyOn(openshellRuntime, "captureOpenshell").mockReturnValue({ - status: 1, - output: "", - stdout: "", - stderr: "Error: sandbox alpha not found", - }); + let sourceDeleted = false; let harness!: RebuildFlowHarness; let setupResult: SetupResult | undefined; harness = createRebuildFlowHarness({ @@ -174,8 +169,9 @@ describe("rebuild local-provider recreation", () => { }); harness.session.provider = provider; harness.session.model = model; - harness.runOpenshellSpy.mockImplementation((args: string[]) => - args[0] === "sandbox" && args[1] === "get" + harness.runOpenshellSpy.mockImplementation((args: string[]) => { + if (args[0] === "sandbox" && args[1] === "delete") sourceDeleted = true; + return args[0] === "sandbox" && args[1] === "get" ? { status: 1, stdout: "", @@ -185,8 +181,15 @@ describe("rebuild local-provider recreation", () => { status: args[0] === "provider" && args[1] === "get" ? 1 : 0, stdout: "", stderr: "", - }, - ); + }; + }); + const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; + harness.captureOpenshellSpy.mockImplementation((args: unknown) => { + const argv = Array.isArray(args) ? args.map(String) : []; + return argv[0] === "sandbox" && argv[1] === "get" && !sourceDeleted + ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } + : { status: 1, output: "", stdout: "", stderr: "Error: sandbox alpha not found" }; + }); await expect( harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }), diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 93e73636f9c..89d0c25b5df 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -554,8 +554,12 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): .spyOn(openshellRuntime, "captureOpenshell") .mockImplementation((args: unknown, options?: unknown) => { const argv = Array.isArray(args) ? args.map(String) : []; - return overrides.captureOpenshell - ? overrides.captureOpenshell(argv, options as Record | undefined) + if (overrides.captureOpenshell) { + return overrides.captureOpenshell(argv, options as Record | undefined); + } + const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; + return argv[0] === "sandbox" && argv[1] === "get" && !defaultSourceDeleted + ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } : { status: 1, output: "", @@ -563,8 +567,10 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): stderr: "Error: sandbox alpha not found", }; }); + let defaultSourceDeleted = false; const runOpenshellSpy = vi.spyOn(openshellRuntime, "runOpenshell").mockImplementation((args) => { const argv = args as string[]; + if (argv[0] === "sandbox" && argv[1] === "delete") defaultSourceDeleted = true; if ( argv.join(" ") === "sandbox get alpha" || argv.join(" ") === "sandbox get -g nemoclaw alpha" From d2732d938ea9cfb14a67087df9af84de583edc54 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 10:25:37 +0000 Subject: [PATCH 17/25] fix(onboard): prove the journaled source before recreation deletes it Signed-off-by: Tinson Lai --- .../sandbox/rebuild-recreate-journal.ts | 3 +- src/lib/onboard.ts | 5 +-- .../sandbox-recreate-transaction.test.ts | 44 +++++++++++++++++++ .../onboard/sandbox-recreate-transaction.ts | 24 +++++++++- 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.ts index e4ae8343a6d..4a0cf12541b 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.ts @@ -14,6 +14,7 @@ import { clearCompletedSandboxRecreateTransaction, fingerprintSandboxRecreateValue, planSandboxRecreateRecovery, + type SandboxRecreateSourcePresence, sandboxRecreatePhaseReached, } from "../../onboard/sandbox-recreate-transaction"; import { decisionSelected } from "../../state/onboard-checkpoint-decision"; @@ -27,7 +28,7 @@ export type RebuildRecreateJournalTarget = SandboxRecreateTarget; export type RebuildSandboxObserver = SandboxRecreateObserver; -export type RebuildRecreateSourcePresence = "missing" | "source"; +export type RebuildRecreateSourcePresence = SandboxRecreateSourcePresence; export interface RebuildRecreateJournal { readonly id: string; diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index c9ac4816c72..41c48bff786 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2578,9 +2578,8 @@ async function createSandboxWithBaseImageResolution( note(` Deleting and recreating sandbox '${sandboxName}'...`); - recreateRuntime.advance("deleting"); - runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); - runOpenshell(["sandbox", "delete", "-g", GATEWAY_NAME, sandboxName], { ignoreError: true }); + // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. + if (recreateRuntime.beginDelete() === "source") { runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); runOpenshell(["sandbox", "delete", "-g", GATEWAY_NAME, sandboxName], { ignoreError: true }); } recreateRuntime.confirmDeleted(); if (previousEntry?.imageTag) { // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index 65c134aeefc..8eebb2d0306 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -33,6 +33,7 @@ const TX_ID = "11111111-1111-4111-8111-111111111111"; const TARGET_GENERATION = "22222222-2222-4222-8222-222222222222"; const SOURCE_ID = fingerprintSandboxRecreateValue("openshell-source-id"); const TARGET_ID = fingerprintSandboxRecreateValue("target-id"); +const FOREIGN_ID = fingerprintSandboxRecreateValue("foreign-openshell-source-id"); const TARGET_INTENT = fingerprintSandboxRecreateValue({ agent: "openclaw", provider: "nvidia", @@ -246,6 +247,49 @@ describe("sandbox recreate journal", () => { }); }); + it("proves the journaled source at the delete edge before onboarding removes it", () => { + const session = createSession({ sandboxName: "alpha" }); + beginSandboxRecreateTransaction( + session, + beginInput({ state: "ready", liveIdentityFingerprint: SOURCE_ID }), + ); + let observation: SandboxRecreateObservation = { + state: "ready", + liveIdentityFingerprint: SOURCE_ID, + }; + const runtime = createSandboxRecreateRuntime( + { + loadSession: () => session, + updateSession: (mutator) => { + mutator(session); + return session; + }, + }, + { + id: TX_ID, + targetGeneration: TARGET_GENERATION, + targetIntentFingerprint: TARGET_INTENT, + }, + "alpha", + "nemoclaw-31818", + SOURCE_ENTRY, + () => observation, + () => undefined, + ); + + observation = { state: "ready", liveIdentityFingerprint: FOREIGN_ID }; + expect(() => runtime.beginDelete()).toThrow(/not the journaled source/i); + expect(session.checkpoint?.sandboxRecreate).toMatchObject({ phase: "planned", revision: 0 }); + + observation = { state: "ready", liveIdentityFingerprint: SOURCE_ID }; + expect(runtime.beginDelete()).toBe("source"); + expect(session.checkpoint?.sandboxRecreate).toMatchObject({ phase: "deleting" }); + + observation = { state: "missing", liveIdentityFingerprint: null }; + expect(runtime.beginDelete()).toBe("missing"); + expect(session.checkpoint?.sandboxRecreate).toMatchObject({ phase: "deleting" }); + }); + it("recovers or rejects at every resumed-onboard mutation boundary (#6492)", () => { const session = createSession({ sandboxName: "alpha" }); beginSandboxRecreateTransaction( diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index 464e9025713..c74cc3bcca0 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -363,6 +363,8 @@ interface SandboxRecreateSessionStore { updateSession(mutator: (session: Session) => Session | void): Session; } +export type SandboxRecreateSourcePresence = "missing" | "source"; + export interface SandboxRecreateRuntime { readonly acceptedTarget: boolean; readonly targetGeneration: string | undefined; @@ -371,6 +373,7 @@ export interface SandboxRecreateRuntime { "lifecycleGeneration" | "lifecycleLiveIdentityFingerprint" >; advance(phase: CheckpointSandboxRecreatePhase): void; + beginDelete(): SandboxRecreateSourcePresence; confirmDeleted(): void; recordCreated(): void; } @@ -380,6 +383,7 @@ const NO_SANDBOX_RECREATE: SandboxRecreateRuntime = { targetGeneration: undefined, registrationFields: {}, advance: () => undefined, + beginDelete: () => "source", confirmDeleted: () => undefined, recordCreated: () => undefined, }; @@ -401,9 +405,10 @@ export function createSandboxRecreateRuntime( transactionId: request.id, targetGeneration: request.targetGeneration, }); - const advance = (phase: CheckpointSandboxRecreatePhase): void => { + let phase: CheckpointSandboxRecreatePhase = transaction.phase; + const advance = (next: CheckpointSandboxRecreatePhase): void => { sessionStore.updateSession((current) => { - advanceSandboxRecreateTransaction(current, transaction.id, phase); + phase = advanceSandboxRecreateTransaction(current, transaction.id, next).phase; return current; }); }; @@ -432,6 +437,21 @@ export function createSandboxRecreateRuntime( }; }, advance, + beginDelete: () => { + const live = observe(sandboxName); + if (live.state !== "missing") { + if ( + !transaction.sourceLiveIdentityFingerprint || + live.liveIdentityFingerprint !== transaction.sourceLiveIdentityFingerprint + ) { + throw new Error( + `Cannot delete sandbox '${sandboxName}': the live same-name sandbox is not the journaled source.`, + ); + } + } + if (!sandboxRecreatePhaseReached(phase, "deleted")) advance("deleting"); + return live.state === "missing" ? "missing" : "source"; + }, confirmDeleted: () => { if (observe(sandboxName).state !== "missing") { throw new Error( From d9b86483c51320476eb1b72da213c931051a4b73 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 10:30:51 +0000 Subject: [PATCH 18/25] test(rebuild): scope the harness source mock to its sandbox and gateway Signed-off-by: Tinson Lai --- .../rebuild-local-provider-recreate.test.ts | 4 ++-- test/helpers/rebuild-flow-harness.ts | 18 +++++++++++++++--- test/helpers/rebuild-flow-test-harness.ts | 18 +++++++++++++++--- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index 53f7c30b55c..19eef519066 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -170,7 +170,7 @@ describe("rebuild local-provider recreation", () => { harness.session.provider = provider; harness.session.model = model; harness.runOpenshellSpy.mockImplementation((args: string[]) => { - if (args[0] === "sandbox" && args[1] === "delete") sourceDeleted = true; + sourceDeleted ||= args.join(" ") === "sandbox delete -g nemoclaw alpha"; return args[0] === "sandbox" && args[1] === "get" ? { status: 1, @@ -186,7 +186,7 @@ describe("rebuild local-provider recreation", () => { const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; harness.captureOpenshellSpy.mockImplementation((args: unknown) => { const argv = Array.isArray(args) ? args.map(String) : []; - return argv[0] === "sandbox" && argv[1] === "get" && !sourceDeleted + return argv.join(" ") === "sandbox get -g nemoclaw alpha" && !sourceDeleted ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } : { status: 1, output: "", stdout: "", stderr: "Error: sandbox alpha not found" }; }); diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 89d0c25b5df..2d545eb28df 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -223,6 +223,13 @@ function createStep(status: string): RebuildFlowStep { return { status, startedAt: null, completedAt: null, error: null }; } +function sourceSandboxGateway(argv: string[], verb: string): string | null { + const gatewayFlag = argv.indexOf("-g"); + return argv[0] === "sandbox" && argv[1] === verb && argv.at(-1) === "alpha" && gatewayFlag > 0 + ? (argv[gatewayFlag + 1] ?? null) + : null; +} + function createRebuildFlowSession(machineSnapshotVersion: number): RebuildFlowSession { return { sessionId: "rebuild-flow-session", @@ -557,8 +564,9 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): if (overrides.captureOpenshell) { return overrides.captureOpenshell(argv, options as Record | undefined); } + const probedGateway = sourceSandboxGateway(argv, "get"); const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; - return argv[0] === "sandbox" && argv[1] === "get" && !defaultSourceDeleted + return probedGateway && probedGateway !== deletedSourceGateway ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } : { status: 1, @@ -567,10 +575,14 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): stderr: "Error: sandbox alpha not found", }; }); - let defaultSourceDeleted = false; + let deletedSourceGateway: string | null = null; const runOpenshellSpy = vi.spyOn(openshellRuntime, "runOpenshell").mockImplementation((args) => { const argv = args as string[]; - if (argv[0] === "sandbox" && argv[1] === "delete") defaultSourceDeleted = true; + const deleteGateway = sourceSandboxGateway(argv, "delete"); + if (deleteGateway) { + deletedSourceGateway = deleteGateway; + return { status: 0, output: "" }; + } if ( argv.join(" ") === "sandbox get alpha" || argv.join(" ") === "sandbox get -g nemoclaw alpha" diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index 960784fa064..a133c78e518 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -60,6 +60,13 @@ const mcpBridge = requireDist("./mcp-bridge.js"); const messaging = requireDist("../../messaging/index.js"); const shields = requireDist("../../shields/index.js"); +function sourceSandboxGateway(argv: string[], verb: string): string | null { + const gatewayFlag = argv.indexOf("-g"); + return argv[0] === "sandbox" && argv[1] === verb && argv.at(-1) === "alpha" && gatewayFlag > 0 + ? (argv[gatewayFlag + 1] ?? null) + : null; +} + export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): RebuildFlowHarness { delete require.cache[requireDist.resolve(rebuildModulePath)]; @@ -387,14 +394,18 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): failedFiles: [], })), ); - let defaultSourceDeleted = false; + let deletedSourceGateway: string | null = null; const runOpenshellSpy = vi .spyOn(openshellRuntime, "runOpenshell") .mockImplementation((args: unknown) => { const argv = Array.isArray(args) ? args.map(String) : []; const overrideResult = overrides.runOpenshell?.(argv); if (overrideResult) return overrideResult; - if (argv[0] === "sandbox" && argv[1] === "delete") defaultSourceDeleted = true; + const deleteGateway = sourceSandboxGateway(argv, "delete"); + if (deleteGateway) { + deletedSourceGateway = deleteGateway; + return { status: 0, output: "" }; + } if ( argv.join(" ") === "sandbox get alpha" || argv.join(" ") === "sandbox get -g nemoclaw alpha" @@ -415,8 +426,9 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): if (overrides.captureOpenshell) { return overrides.captureOpenshell(argv, options as Record | undefined); } + const probedGateway = sourceSandboxGateway(argv, "get"); const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; - return argv[0] === "sandbox" && argv[1] === "get" && !defaultSourceDeleted + return probedGateway && probedGateway !== deletedSourceGateway ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } : { status: 1, output: "", stderr: "Error: sandbox alpha not found" }; }); From a40043c295499423cbde4f56e92e3a32d6943e81 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 11:05:14 +0000 Subject: [PATCH 19/25] fix(rebuild): bind the inference route to the replacement target fingerprint Signed-off-by: Tinson Lai --- .../actions/sandbox/rebuild-gpu-opt-out.ts | 9 +++++++ .../sandbox/rebuild-recreate-journal.test.ts | 26 +++++++++++++++++++ .../sandbox/rebuild-recreate-journal.ts | 6 +++++ .../rebuild-recreate-observability.test.ts | 3 +++ 4 files changed, 44 insertions(+) diff --git a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts index 3beb8444a58..1a1bc41eac5 100644 --- a/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts +++ b/src/lib/actions/sandbox/rebuild-gpu-opt-out.ts @@ -43,6 +43,9 @@ export type RebuildGpuOptOutEntry = { observabilityEnabled?: boolean; policyTier?: string | null; endpointSource?: InferenceEndpointSource | null; + provider?: string | null; + model?: string | null; + preferredInferenceApi?: string | null; }; // Modern source of truth is the persisted `sandboxGpuMode` string ("0" / "1" / @@ -103,6 +106,9 @@ export type RebuildRecreateOnboardOpts = { endpointSource?: InferenceEndpointSource | null; acceptThirdPartySoftware: true; agent: string | null | undefined; + recreateProvider: string | null; + recreateModel: string | null; + recreatePreferredInferenceApi: string | null; fromDockerfile: string | null; sandboxGpu: "enable" | "disable" | null; sandboxGpuDevice: string | null; @@ -179,6 +185,9 @@ export function buildRebuildRecreateOnboardOpts(args: { endpointSource: normalizeInferenceEndpointSource(args.sb?.endpointSource), acceptThirdPartySoftware: args.usageNoticeAccepted, agent: args.rebuildAgent, + recreateProvider: args.sb?.provider ?? null, + recreateModel: args.sb?.model ?? null, + recreatePreferredInferenceApi: args.sb?.preferredInferenceApi ?? null, fromDockerfile: args.storedFromDockerfile, sandboxGpu: gpuOverrides.sandboxGpu, sandboxGpuDevice: gpuOverrides.sandboxGpuDevice, diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts index a598865f340..ca4cb6fe832 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.test.ts @@ -42,6 +42,9 @@ const recreateOptions: RebuildRecreateOnboardOpts = { authoritativeResumeConfig: true, acceptThirdPartySoftware: true, agent: "langchain-deepagents-code", + recreateProvider: "nvidia", + recreateModel: "model-a", + recreatePreferredInferenceApi: "openai", fromDockerfile: null, sandboxGpu: null, sandboxGpuDevice: null, @@ -99,6 +102,9 @@ describe("rebuild replacement target fingerprint", () => { { dcodeAutoApprovalMode: "thread-opt-in" }, { endpointSource: "onboard" }, { policyTier: "balanced" }, + { recreateProvider: "compatible-endpoint" }, + { recreateModel: "model-b" }, + { recreatePreferredInferenceApi: "anthropic" }, ] as const) { expect(fingerprintRebuildRecreateTargetIntent({ ...recreateOptions, ...drift })).not.toBe( fingerprintRebuildRecreateTargetIntent(recreateOptions), @@ -339,6 +345,26 @@ describe("rebuild replacement journal", () => { ).toThrow(/different recreate transaction in progress/); }); + it.each([ + { recreateProvider: "compatible-endpoint" }, + { recreateModel: "model-b" }, + { recreatePreferredInferenceApi: "anthropic" }, + ] as const)("refuses to resume a journal whose inference route changed (#7734)", (drift) => { + open(); + + expect(() => + openRebuildRecreateJournal({ + target: NON_DEFAULT_TARGET, + agentName: "langchain-deepagents-code", + targetIntentFingerprint: fingerprintRebuildRecreateTargetIntent({ + ...recreateOptions, + ...drift, + }), + log: vi.fn(), + }), + ).toThrow(/different recreate transaction in progress/); + }); + it("resumes the same replacement without restarting its generation", () => { const first = open(); diff --git a/src/lib/actions/sandbox/rebuild-recreate-journal.ts b/src/lib/actions/sandbox/rebuild-recreate-journal.ts index 4a0cf12541b..bcfda70d0db 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-journal.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-journal.ts @@ -47,6 +47,9 @@ export function fingerprintRebuildRecreateTargetIntent( RebuildRecreateOnboardOpts, | "agent" | "endpointSource" + | "recreateProvider" + | "recreateModel" + | "recreatePreferredInferenceApi" | "fromDockerfile" | "sandboxGpu" | "sandboxGpuDevice" @@ -63,6 +66,9 @@ export function fingerprintRebuildRecreateTargetIntent( version: 1, agent: options.agent ?? null, endpointSource: options.endpointSource ?? null, + provider: options.recreateProvider, + model: options.recreateModel, + preferredInferenceApi: options.recreatePreferredInferenceApi, fromDockerfile: options.fromDockerfile, sandboxGpu: options.sandboxGpu, sandboxGpuDevice: options.sandboxGpuDevice, diff --git a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts index 0cc8cb60b20..190d308898d 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-observability.test.ts @@ -51,6 +51,9 @@ const recreateOptions: RebuildRecreateOnboardOpts = { authoritativeResumeConfig: true, acceptThirdPartySoftware: true, agent: DCODE_AGENT, + recreateProvider: "nvidia", + recreateModel: "model-a", + recreatePreferredInferenceApi: "openai", fromDockerfile: null, sandboxGpu: null, sandboxGpuDevice: null, From 603857d52ba85da192fd5b5cbf73ee7435b00d85 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 30 Jul 2026 12:16:11 +0000 Subject: [PATCH 20/25] fix(onboard): delete the recreate source through its journaled gateway Signed-off-by: Tinson Lai --- src/lib/onboard.ts | 2 +- src/lib/onboard/onboard-recreate-journal.ts | 2 +- .../sandbox-recreate-transaction.test.ts | 50 +++++++++++++++++++ .../onboard/sandbox-recreate-transaction.ts | 17 +++++-- test/helpers/rebuild-flow-harness.ts | 6 +-- test/helpers/rebuild-flow-test-harness.ts | 6 +-- 6 files changed, 70 insertions(+), 13 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 41c48bff786..54f31a7b523 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2579,7 +2579,7 @@ async function createSandboxWithBaseImageResolution( note(` Deleting and recreating sandbox '${sandboxName}'...`); // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - if (recreateRuntime.beginDelete() === "source") { runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); runOpenshell(["sandbox", "delete", "-g", GATEWAY_NAME, sandboxName], { ignoreError: true }); } + if (recreateRuntime.beginDelete() === "source") { runSandboxProviderPreDeleteCleanup(sandboxName, { runOpenshell, redact }); runOpenshell(["sandbox", "delete", "-g", recreateRuntime.journaledGatewayName ?? GATEWAY_NAME, sandboxName], { ignoreError: true }); } recreateRuntime.confirmDeleted(); if (previousEntry?.imageTag) { // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. diff --git a/src/lib/onboard/onboard-recreate-journal.ts b/src/lib/onboard/onboard-recreate-journal.ts index 161226cf9a6..5c1a2cbc6d2 100644 --- a/src/lib/onboard/onboard-recreate-journal.ts +++ b/src/lib/onboard/onboard-recreate-journal.ts @@ -139,7 +139,7 @@ export function openOnboardRecreateJournal( target.sandboxName, target.gatewayName, sourceEntry, - (sandboxName) => observe({ ...target, sandboxName }), + (sandboxName, gatewayName) => observe({ ...target, sandboxName, gatewayName }), note, ); diff --git a/src/lib/onboard/sandbox-recreate-transaction.test.ts b/src/lib/onboard/sandbox-recreate-transaction.test.ts index 8eebb2d0306..604d55655dc 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.test.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.test.ts @@ -290,6 +290,56 @@ describe("sandbox recreate journal", () => { expect(session.checkpoint?.sandboxRecreate).toMatchObject({ phase: "deleting" }); }); + it("scopes the delete edge to the journaled gateway, not the ambient one", () => { + const session = createSession({ sandboxName: "alpha" }); + beginSandboxRecreateTransaction( + session, + beginInput({ state: "ready", liveIdentityFingerprint: SOURCE_ID }), + ); + const probedGateways: string[] = []; + const sessionStore = { + loadSession: () => session, + updateSession: (mutator: (current: typeof session) => void) => { + mutator(session); + return session; + }, + }; + const request = { + id: TX_ID, + targetGeneration: TARGET_GENERATION, + targetIntentFingerprint: TARGET_INTENT, + }; + const observe = (_sandboxName: string, gatewayName: string): SandboxRecreateObservation => { + probedGateways.push(gatewayName); + return { state: "ready", liveIdentityFingerprint: SOURCE_ID }; + }; + + const runtime = createSandboxRecreateRuntime( + sessionStore, + request, + "alpha", + "nemoclaw-31818", + SOURCE_ENTRY, + observe, + () => undefined, + ); + runtime.beginDelete(); + + expect(runtime.journaledGatewayName).toBe("nemoclaw-31818"); + expect(new Set(probedGateways)).toEqual(new Set(["nemoclaw-31818"])); + expect(() => + createSandboxRecreateRuntime( + sessionStore, + request, + "alpha", + "nemoclaw", + SOURCE_ENTRY, + observe, + () => undefined, + ), + ).toThrow(/does not match the requested replacement/i); + }); + it("recovers or rejects at every resumed-onboard mutation boundary (#6492)", () => { const session = createSession({ sandboxName: "alpha" }); beginSandboxRecreateTransaction( diff --git a/src/lib/onboard/sandbox-recreate-transaction.ts b/src/lib/onboard/sandbox-recreate-transaction.ts index c74cc3bcca0..dd61db06b78 100644 --- a/src/lib/onboard/sandbox-recreate-transaction.ts +++ b/src/lib/onboard/sandbox-recreate-transaction.ts @@ -368,6 +368,7 @@ export type SandboxRecreateSourcePresence = "missing" | "source"; export interface SandboxRecreateRuntime { readonly acceptedTarget: boolean; readonly targetGeneration: string | undefined; + readonly journaledGatewayName: string | null; readonly registrationFields: Pick< SandboxEntry, "lifecycleGeneration" | "lifecycleLiveIdentityFingerprint" @@ -381,6 +382,7 @@ export interface SandboxRecreateRuntime { const NO_SANDBOX_RECREATE: SandboxRecreateRuntime = { acceptedTarget: false, targetGeneration: undefined, + journaledGatewayName: null, registrationFields: {}, advance: () => undefined, beginDelete: () => "source", @@ -394,7 +396,7 @@ export function createSandboxRecreateRuntime( sandboxName: string, gatewayName: string, registryEntry: SandboxEntry | null, - observe: (sandboxName: string) => SandboxRecreateObservation, + observe: (sandboxName: string, gatewayName: string) => SandboxRecreateObservation, note: (message: string) => void, ): SandboxRecreateRuntime { if (!request) return NO_SANDBOX_RECREATE; @@ -413,7 +415,11 @@ export function createSandboxRecreateRuntime( }); }; let targetLiveIdentityFingerprint = transaction.targetLiveIdentityFingerprint; - const recovery = planSandboxRecreateRecovery(transaction, observe(sandboxName), registryEntry); + const recovery = planSandboxRecreateRecovery( + transaction, + observe(sandboxName, transaction.gatewayName), + registryEntry, + ); if (recovery.action === "reject") { throw new Error(`Cannot resume sandbox '${sandboxName}' recreation: ${recovery.reason}.`); } @@ -428,6 +434,7 @@ export function createSandboxRecreateRuntime( return { acceptedTarget: recovery.action === "accept_target", targetGeneration: transaction.targetGeneration, + journaledGatewayName: transaction.gatewayName, get registrationFields() { return { lifecycleGeneration: transaction.targetGeneration, @@ -438,7 +445,7 @@ export function createSandboxRecreateRuntime( }, advance, beginDelete: () => { - const live = observe(sandboxName); + const live = observe(sandboxName, transaction.gatewayName); if (live.state !== "missing") { if ( !transaction.sourceLiveIdentityFingerprint || @@ -453,7 +460,7 @@ export function createSandboxRecreateRuntime( return live.state === "missing" ? "missing" : "source"; }, confirmDeleted: () => { - if (observe(sandboxName).state !== "missing") { + if (observe(sandboxName, transaction.gatewayName).state !== "missing") { throw new Error( `Cannot continue sandbox '${sandboxName}' recreation: OpenShell still reports the journaled source after delete.`, ); @@ -461,7 +468,7 @@ export function createSandboxRecreateRuntime( advance("deleted"); }, recordCreated: () => { - const observation = observe(sandboxName); + const observation = observe(sandboxName, transaction.gatewayName); sessionStore.updateSession((current) => { targetLiveIdentityFingerprint = recordSandboxRecreateTargetCreated( current, diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index 2d545eb28df..f205475c4a3 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -566,7 +566,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): } const probedGateway = sourceSandboxGateway(argv, "get"); const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; - return probedGateway && probedGateway !== deletedSourceGateway + return probedGateway && !deletedSourceGateways.has(probedGateway) ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } : { status: 1, @@ -575,12 +575,12 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): stderr: "Error: sandbox alpha not found", }; }); - let deletedSourceGateway: string | null = null; + const deletedSourceGateways = new Set(); const runOpenshellSpy = vi.spyOn(openshellRuntime, "runOpenshell").mockImplementation((args) => { const argv = args as string[]; const deleteGateway = sourceSandboxGateway(argv, "delete"); if (deleteGateway) { - deletedSourceGateway = deleteGateway; + deletedSourceGateways.add(deleteGateway); return { status: 0, output: "" }; } if ( diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index a133c78e518..8eafd919c67 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -394,7 +394,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): failedFiles: [], })), ); - let deletedSourceGateway: string | null = null; + const deletedSourceGateways = new Set(); const runOpenshellSpy = vi .spyOn(openshellRuntime, "runOpenshell") .mockImplementation((args: unknown) => { @@ -403,7 +403,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): if (overrideResult) return overrideResult; const deleteGateway = sourceSandboxGateway(argv, "delete"); if (deleteGateway) { - deletedSourceGateway = deleteGateway; + deletedSourceGateways.add(deleteGateway); return { status: 0, output: "" }; } if ( @@ -428,7 +428,7 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): } const probedGateway = sourceSandboxGateway(argv, "get"); const liveSource = "Name: alpha\nId: sbx-alpha-source\nPhase: Ready\n"; - return probedGateway && probedGateway !== deletedSourceGateway + return probedGateway && !deletedSourceGateways.has(probedGateway) ? { status: 0, output: liveSource, stdout: liveSource, stderr: "" } : { status: 1, output: "", stderr: "Error: sandbox alpha not found" }; }); From 304893a2e88e244e14f16bfb59fd2320a664c86c Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 31 Jul 2026 09:31:29 -0700 Subject: [PATCH 21/25] test(rebuild): align refreshed journal fixtures Signed-off-by: Senthil Ravichandran --- .../sandbox/rebuild-destroy-phase.test.ts | 1 + .../rebuild-recreate-reasoning.test.ts | 19 +++++++++++++++++++ test/helpers/onboard-script-mocks.cjs | 17 +++++++++++++++++ test/helpers/rebuild-flow-harness.ts | 13 +++++++++++++ test/helpers/rebuild-flow-test-harness.ts | 13 +++++++++++++ test/onboard-installer-restore-intent.test.ts | 2 ++ test/onboard-reservation-recreate.test.ts | 1 + test/onboard-sandbox-recreation.test.ts | 10 ++++++++++ 8 files changed, 76 insertions(+) diff --git a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts index a53758fbc2e..411b9b9b17f 100644 --- a/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts +++ b/src/lib/actions/sandbox/rebuild-destroy-phase.test.ts @@ -210,6 +210,7 @@ describe("rebuild destroy phase", () => { sandboxName: "alpha", sandboxEntry: { name: "alpha", agent: "openclaw" }, staleRecovery: false, + recreateJournal: stubRecreateJournal(), backupManifest: null, log: vi.fn(), bail, diff --git a/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts b/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts index aa849a4c90d..cff587bff4d 100644 --- a/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts +++ b/src/lib/actions/sandbox/rebuild-recreate-reasoning.test.ts @@ -4,11 +4,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { restoreEnv } from "../../../../test/helpers/env-test-helpers"; +import * as shields from "../../shields"; import type { Session } from "../../state/onboard-session"; import * as onboardSession from "../../state/onboard-session"; import type { RebuildDurableConfig } from "./rebuild-durable-config"; import type { RebuildRecreateOnboardOpts } from "./rebuild-gpu-opt-out"; import { rebuildOnboardDependencies } from "./rebuild-onboard-dependencies"; +import type { RebuildRecreateJournal } from "./rebuild-recreate-journal"; import { type RebuildRecreatePhaseInput, runRebuildRecreatePhase } from "./rebuild-recreate-phase"; import type { RebuildResumeConfig } from "./rebuild-resume-config"; @@ -49,6 +51,9 @@ const recreateOptions: RebuildRecreateOnboardOpts = { authoritativeResumeConfig: true, acceptThirdPartySoftware: true, agent: "openclaw", + recreateProvider: "compatible-endpoint", + recreateModel: "mock/deepseek-compatible", + recreatePreferredInferenceApi: "openai-completions", fromDockerfile: null, sandboxGpu: null, sandboxGpuDevice: null, @@ -66,6 +71,18 @@ const recreateOptions: RebuildRecreateOnboardOpts = { baseImageResolutionHint: null, }; +const recreateJournal: RebuildRecreateJournal = { + id: "11111111-1111-4111-8111-111111111111", + acceptedTarget: false, + sourceConfirmedAbsent: true, + targetGeneration: "22222222-2222-4222-8222-222222222222", + targetIntentFingerprint: "rebuild-reasoning-target", + markDeleting: vi.fn(), + observeSourceForDelete: vi.fn((): "missing" => "missing"), + confirmDeleted: vi.fn(), + completeAcceptedTarget: vi.fn(), +}; + function makeInput(overrides: Partial = {}): RebuildRecreatePhaseInput { return { sandboxName: SANDBOX_NAME, @@ -75,6 +92,7 @@ function makeInput(overrides: Partial = {}): RebuildR durableConfig, resumeConfig: compatibleResumeConfig, recreateOptions, + recreateJournal, fromDockerfile: null, rebuildAgent: "openclaw", messagingPlan: null, @@ -110,6 +128,7 @@ describe("rebuild recreate compatible-endpoint reasoning handoff (#7940)", () => session = onboardSession.createSession({ sandboxName: SANDBOX_NAME }); vi.spyOn(console, "log").mockImplementation(() => undefined); vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(shields, "clearShieldsState").mockImplementation(() => undefined); vi.spyOn(onboardSession, "loadSession").mockImplementation(() => session); vi.spyOn(onboardSession, "updateSession").mockImplementation((mutator) => { session = mutator(session) ?? session; diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 730057aa899..e5e41405810 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -159,9 +159,26 @@ function mockOnboardRunCapture(command, options = {}) { return mockSandboxExecCurl(command, options); } +function mockStandaloneGatewayTeardownAuthority() { + const authority = require( + path.resolve(__dirname, "../../src/lib/onboard/gateway-teardown-authority.ts"), + ); + authority.resolveGatewayTeardownAuthority = ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }); +} + module.exports = { isOpenClawSecurityInventoryProbe, mockOnboardRunCapture, mockSandboxExecCurl, + mockStandaloneGatewayTeardownAuthority, normalizeCommand, }; diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index b5f148c3ecd..b6d0e6d8079 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -53,6 +53,7 @@ const rebuildFlowHelpers = requireDist("./rebuild-flow-helpers.js"); const rebuildManagedImage = requireDist("./rebuild-managed-image-preflight.js"); const rebuildMessagingConflict = requireDist("./rebuild-messaging-conflict-preflight.js"); const rebuildRoutePreflight = requireDist("./rebuild-preflight-guards.js"); +const gatewayTeardownAuthority = requireDist("../../onboard/gateway-teardown-authority.js"); const shields = requireDist("../../shields/index.js"); type RebuildFlowStep = { @@ -332,6 +333,18 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); + vi.spyOn(gatewayTeardownAuthority, "resolveGatewayTeardownAuthority").mockImplementation( + ({ gatewayName, gatewayPort }: { gatewayName: string; gatewayPort: number }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + ); vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ result: { status: 0, output: overrides.sandboxListOutput ?? "alpha Ready" }, }); diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index fc5f746e20a..d00aa7ebbd5 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -33,6 +33,7 @@ const openshellRuntime = requireDist("../../adapters/openshell/runtime.js"); const dockerInspect = requireDist("../../adapters/docker/inspect.js"); const sandboxList = requireDist("../../openshell-sandbox-list.js"); const resolve = requireDist("../../adapters/openshell/resolve.js"); +const gatewayTeardownAuthority = requireDist("../../onboard/gateway-teardown-authority.js"); const agentDefs = requireDist("../../agent/defs.js"); const agentRuntime = requireDist("../../agent/runtime.js"); const { rebuildOnboardDependencies } = requireDist("./rebuild-onboard-dependencies.js"); @@ -93,6 +94,18 @@ export function createRebuildFlowHarness(overrides: RebuildFlowOverrides = {}): vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null); vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null); + vi.spyOn(gatewayTeardownAuthority, "resolveGatewayTeardownAuthority").mockImplementation( + ({ gatewayName, gatewayPort }: { gatewayName: string; gatewayPort: number }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + ); vi.spyOn(sandboxList, "captureSandboxListWithGatewayRecovery").mockResolvedValue({ result: { status: 0, diff --git a/test/onboard-installer-restore-intent.test.ts b/test/onboard-installer-restore-intent.test.ts index 4e36a437dd0..240bba273f9 100644 --- a/test/onboard-installer-restore-intent.test.ts +++ b/test/onboard-installer-restore-intent.test.ts @@ -37,6 +37,7 @@ describe("createSandbox installer restore intent", () => { const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); @@ -247,6 +248,7 @@ const MARKER_SHA = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852 const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const sandboxState = require(${sandboxStatePath}); diff --git a/test/onboard-reservation-recreate.test.ts b/test/onboard-reservation-recreate.test.ts index b2ae4931658..e27b22b5d5a 100644 --- a/test/onboard-reservation-recreate.test.ts +++ b/test/onboard-reservation-recreate.test.ts @@ -50,6 +50,7 @@ describe("onboard sandbox recreate reservation safety", () => { const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); const registry = require(${registryPath}); const onboardSession = require(${onboardSessionPath}); diff --git a/test/onboard-sandbox-recreation.test.ts b/test/onboard-sandbox-recreation.test.ts index c22d5087e5b..dce0278f1be 100644 --- a/test/onboard-sandbox-recreation.test.ts +++ b/test/onboard-sandbox-recreation.test.ts @@ -30,6 +30,7 @@ describe("onboard helpers", () => { const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -110,6 +111,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -223,6 +225,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -373,6 +376,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -497,6 +501,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -645,6 +650,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -766,6 +772,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -898,6 +905,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -1043,6 +1051,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); @@ -1182,6 +1191,7 @@ const { createSandbox } = require(${onboardPath}); const script = String.raw` const runner = require(${runnerPath}); +require(${onboardScriptMocksPath}).mockStandaloneGatewayTeardownAuthority(); const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); let _deleted = false; const registry = require(${registryPath}); From 9e19967ad0049066ae2377a216c46011e127b6d8 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 31 Jul 2026 09:38:12 -0700 Subject: [PATCH 22/25] test(rebuild): type refreshed authority mocks Signed-off-by: Senthil Ravichandran --- test/helpers/rebuild-flow-harness.ts | 4 +++- test/helpers/rebuild-flow-test-harness.ts | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/helpers/rebuild-flow-harness.ts b/test/helpers/rebuild-flow-harness.ts index b6d0e6d8079..972081810e9 100644 --- a/test/helpers/rebuild-flow-harness.ts +++ b/test/helpers/rebuild-flow-harness.ts @@ -53,7 +53,9 @@ const rebuildFlowHelpers = requireDist("./rebuild-flow-helpers.js"); const rebuildManagedImage = requireDist("./rebuild-managed-image-preflight.js"); const rebuildMessagingConflict = requireDist("./rebuild-messaging-conflict-preflight.js"); const rebuildRoutePreflight = requireDist("./rebuild-preflight-guards.js"); -const gatewayTeardownAuthority = requireDist("../../onboard/gateway-teardown-authority.js"); +const gatewayTeardownAuthority = requireDist( + "../../onboard/gateway-teardown-authority.js", +) as typeof import("../../src/lib/onboard/gateway-teardown-authority"); const shields = requireDist("../../shields/index.js"); type RebuildFlowStep = { diff --git a/test/helpers/rebuild-flow-test-harness.ts b/test/helpers/rebuild-flow-test-harness.ts index d00aa7ebbd5..b12300593a1 100644 --- a/test/helpers/rebuild-flow-test-harness.ts +++ b/test/helpers/rebuild-flow-test-harness.ts @@ -33,7 +33,9 @@ const openshellRuntime = requireDist("../../adapters/openshell/runtime.js"); const dockerInspect = requireDist("../../adapters/docker/inspect.js"); const sandboxList = requireDist("../../openshell-sandbox-list.js"); const resolve = requireDist("../../adapters/openshell/resolve.js"); -const gatewayTeardownAuthority = requireDist("../../onboard/gateway-teardown-authority.js"); +const gatewayTeardownAuthority = requireDist( + "../../onboard/gateway-teardown-authority.js", +) as typeof import("../../src/lib/onboard/gateway-teardown-authority"); const agentDefs = requireDist("../../agent/defs.js"); const agentRuntime = requireDist("../../agent/runtime.js"); const { rebuildOnboardDependencies } = requireDist("./rebuild-onboard-dependencies.js"); From e3f5982d4d2ef0d15ec6b2e4d94db3583d0f6a58 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 31 Jul 2026 09:44:13 -0700 Subject: [PATCH 23/25] docs(rebuild): explain interrupted replacement recovery Signed-off-by: Senthil Ravichandran --- .../recover-rebuild-sandboxes.mdx | 27 +++++++++++++++++++ docs/reference/commands.mdx | 14 ++++++++++ 2 files changed, 41 insertions(+) diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 58c777f81d0..bd6aad4bf2a 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -164,6 +164,33 @@ Follow the printed recovery guidance, using `$$nemoclaw recover` The rebuild command preserves manifest-defined Deep Agents state, regenerates `config.toml`, reconstructs managed MCP projection state, and reapplies registered policies while recreating the container. +### Continue an Interrupted Replacement + +Before `rebuild` deletes the existing sandbox, NemoClaw records a replacement journal in the onboarding session. +The journal binds the operation to the sandbox name, recorded OpenShell gateway, source identity, and replacement settings. +It stores fingerprints instead of credential values or raw OpenShell sandbox IDs. + +If `rebuild` stops after recording the journal, rerun the command with the same replacement settings. +The rerun takes one of these actions: + +- It continues deletion when the live sandbox still has the journaled source identity. +- It continues creation when the recorded OpenShell gateway explicitly reports the source sandbox as absent. +- It accepts an existing replacement only when its live identity and sandbox registry generation match the journal. + +An accepted replacement is not deleted again. +The command reports `Sandbox '' already holds the replacement from the interrupted rebuild.` and preserves the state backup path when one exists. +Pass `--verbose` to include the replacement identifier, OpenShell gateway, and journal phase in rebuild diagnostics. + +NemoClaw fails closed when the selected gateway, replacement settings, durable source registry fields, or live source or target identity no longer matches the journal. +The error starts with `Cannot resume sandbox '' replacement` or `Cannot resume sandbox '' recreation` and includes the mismatch. +Do not delete a same-name sandbox to bypass this check. +Inspect the named OpenShell gateway and sandbox, correct the reported drift, and rerun the original command. + +A same-name recreation started by `$$nemoclaw onboard` uses the same replacement journal. +If that recreation is interrupted after the `Journaled replacement` message, rerun the original onboarding command with the same target settings. +The active replacement can continue without adding `--resume`. +Use `--resume` for interrupted onboarding steps that occur before a replacement journal exists. + If an archive command preserves at least one state directory, NemoClaw keeps the usable entries and reports the manifest-defined paths that could not be archived. If a manifest-declared state file fails, NemoClaw stops before deleting the original sandbox even when it preserved state directories, unless you explicitly pass `--force`. If every state directory fails, NemoClaw stops before deleting the original sandbox even when it captured loose files, unless you explicitly pass `--force`. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 423517c2e2d..dc2c58b7055 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -281,6 +281,11 @@ Use `--resume` only for resumable interrupted or failed sessions, not to change During resume, NemoClaw reruns preflight, gateway, provider, and sandbox repair checks even when the saved session has already reached a later nonterminal onboarding phase. If the recorded session conflicts with flags you pass on the recovery run, NemoClaw exits and tells you to either rerun with the original settings or start over. +An active same-name replacement is separate from ordinary onboarding-step resume. +If onboarding printed `Journaled replacement` before it stopped, rerun the original onboarding command with the same target settings. +The replacement can continue without an explicit `--resume` flag. +Refer to [Continue an Interrupted Replacement](../manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes#continue-an-interrupted-replacement) for the identity checks and failure conditions. + Use `--fresh` to discard the saved onboarding session and start the wizard from the beginning. This clears stale or failed session state before NemoClaw creates a new session record. It also bypasses locally recorded sandbox base-image resolution metadata and reruns normal candidate resolution. @@ -640,6 +645,11 @@ NemoClaw aborts the recreate when the backup cannot complete in full, including Set `NEMOCLAW_RECREATE_WITHOUT_BACKUP=1` to skip the pre-recreate backup. The destination sandbox starts with a fresh workspace. +Before deletion, onboarding prints a `Journaled replacement` diagnostic with the replacement identifier, recorded OpenShell gateway, and current phase. +If the process stops after this point, a later same-target onboarding run continues the active replacement without requiring `--resume`. +It accepts a ready same-name replacement only when the live identity and sandbox registry generation match the journal. +It fails closed if the gateway, source, target, durable source registry fields, or replacement settings changed. + For OpenClaw, the backed-up paths include agents, extensions, workspace, skills, hooks, identity, devices, canvas, cron, memory, telegram, wechat, credentials, and `/sandbox/.openclaw/workspace/`. @@ -2784,6 +2794,10 @@ After OpenShell accepts the sandbox deletion, `rebuild` waits until OpenShell ex Only then can NemoClaw perform any required local registry removal and begin creating the replacement. If OpenShell does not confirm absence within the bounded wait, including when gateway transport errors block the probes, `rebuild` exits nonzero before registry removal or replacement creation and preserves both the local registry entry and the state backup. Restore OpenShell connectivity and confirm the sandbox's live state before you retry, and keep the printed backup path for recovery. +Before deletion, rebuild records a replacement journal that binds the operation to the recorded gateway, source identity, and target settings. +Rerunning the same rebuild continues from the recorded boundary or accepts the proven replacement instead of deleting it again. +Use `--verbose` to print the replacement identifier, gateway, and journal phase. +Refer to [Continue an Interrupted Replacement](../manage-sandboxes/operate-sandboxes/recover-and-rebuild-sandboxes#continue-an-interrupted-replacement) for the recovery procedure and fail-closed conditions. When rebuild starts with shields up, NemoClaw opens a 30-minute shields-down window for backup and recreation. A detached auto-lock timer remains active until NemoClaw commits a successful shields-up state, so it can attempt to restore lockdown if the host rebuild process exits unexpectedly. From 09d2c9a103a6b6e7341025c85376e11eecb82396 Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 31 Jul 2026 09:47:07 -0700 Subject: [PATCH 24/25] docs(rebuild): correct recovery error guidance Signed-off-by: Senthil Ravichandran --- docs/manage-sandboxes/recover-rebuild-sandboxes.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index bd6aad4bf2a..40fb3ecba34 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -182,7 +182,7 @@ The command reports `Sandbox '' already holds the replacement from the int Pass `--verbose` to include the replacement identifier, OpenShell gateway, and journal phase in rebuild diagnostics. NemoClaw fails closed when the selected gateway, replacement settings, durable source registry fields, or live source or target identity no longer matches the journal. -The error starts with `Cannot resume sandbox '' replacement` or `Cannot resume sandbox '' recreation` and includes the mismatch. +The error names the sandbox and the mismatch that stopped recovery. Do not delete a same-name sandbox to bypass this check. Inspect the named OpenShell gateway and sandbox, correct the reported drift, and rerun the original command. From dfb14639cc9c87e48f5ab30c694f775949c52e0e Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Fri, 31 Jul 2026 09:50:12 -0700 Subject: [PATCH 25/25] test(rebuild): isolate resume authority fixture Signed-off-by: Senthil Ravichandran --- .../actions/sandbox/rebuild-resume-snapshot.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts index 83472557089..1fd184b2df7 100644 --- a/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts +++ b/src/lib/actions/sandbox/rebuild-resume-snapshot.test.ts @@ -10,6 +10,7 @@ import * as agentDefs from "../../agent/defs"; import * as agentRuntime from "../../agent/runtime"; import * as gatewayRuntime from "../../gateway-runtime-action"; import * as nim from "../../inference/nim"; +import * as gatewayTeardownAuthority from "../../onboard/gateway-teardown-authority"; import * as sessionRecovery from "../../onboard/session-recovery"; import * as sandboxList from "../../openshell-sandbox-list"; import * as sandboxVersion from "../../sandbox/version"; @@ -91,6 +92,18 @@ describe("rebuild resume snapshot repair", () => { spies.push( vi.spyOn(gatewayDrift, "detectOpenShellStateRpcPreflightIssue").mockReturnValue(null), vi.spyOn(gatewayDrift, "detectOpenShellStateRpcResultIssue").mockReturnValue(null), + vi + .spyOn(gatewayTeardownAuthority, "resolveGatewayTeardownAuthority") + .mockImplementation(({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + })), vi.spyOn(gatewayRuntime, "recoverNamedGatewayRuntime").mockResolvedValue({ recovered: true, before: { state: "healthy_named", status: "", gatewayInfo: "", activeGateway: null },