diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index ace4cb192a0..2594428916d 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -40,7 +40,7 @@ After a failure, inspect the artifacts and remove resources that target cleanup These credentials remain valid until they expire or an administrator revokes them in their issuing services. If cleanup fails, remove the recorded Brev workspace. Rotate or revoke each credential to remove later access. This Brev credential boundary applies to trusted `main` pushes and Launchable or full manual runs. It does not apply to manual PR runs, which keep `include_staging_brev_launchable=false`. -For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVIDIA_API_KEY` repository secret only to the trusted qualification step. Trusted host code uses it for NGC login and passes it as `NGC_API_KEY` and `NIM_NGC_API_KEY` to the temporary NIM container. Candidate managed sandboxes receive generated local route tokens instead of this key. The live fixture attempts to stop and remove `nemoclaw-managed-image-nim-e2e`, but Docker stop or removal errors do not fail the test. A surviving container can retain the API key until runner teardown. The final workflow step removes the job's isolated Docker credential directory and fails if that removal does not complete. The workflow does not revoke the NVIDIA API key. Rotate or revoke it in the issuing NVIDIA service to remove later access. +For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVIDIA_API_KEY` repository secret only to the trusted qualification step. Trusted host code uses it for NGC login and passes it as `NGC_API_KEY` and `NIM_NGC_API_KEY` to the temporary NIM container. Candidate managed sandboxes receive generated local route tokens instead of this key. The live fixture removes the temporary NIM container only if its exact ID, name, requested image, immutable image ID, cohort owner, and provider kind match the recorded authority. The test fails if evidence is missing or ambiguous, a name is reused, authority drifts, removal is indeterminate, or the exact ID or name remains. A cleanup refusal can leave the container and its API key in place until runner teardown. The final workflow step removes the job's isolated Docker credential directory and fails if that removal does not complete. The workflow does not revoke the NVIDIA API key. Revoke it, or rotate it and disable the old value, in the issuing NVIDIA service. Verify that the exposed key is no longer valid. Resolve the current PR and trusted workflow identities: diff --git a/scripts/checks/run-managed-image-openshell-e2e.ts b/scripts/checks/run-managed-image-openshell-e2e.ts index 12b21a80025..8cedf07c383 100644 --- a/scripts/checks/run-managed-image-openshell-e2e.ts +++ b/scripts/checks/run-managed-image-openshell-e2e.ts @@ -659,11 +659,15 @@ function assertProtectedLocalInference( } } -function failureInjectingAdapter(onboard: OnboardModule): ManagedBootstrapAdapter { +export function failureInjectingAdapter( + onboard: OnboardModule, + stateRoot: string, +): ManagedBootstrapAdapter { const adapter = createDockerManagedBootstrapAdapter({ runCaptureOpenshell: onboard.runCaptureOpenshell, runOpenshell: onboard.runOpenshell, sleep: onboard.sleepSeconds, + stateRoot, }); return { ...adapter, @@ -1002,7 +1006,8 @@ async function run failureInjectingAdapter(onboard!), + createManagedBootstrapAdapter: (stateRoot: string) => + failureInjectingAdapter(onboard!, stateRoot), } : {}), }, diff --git a/src/lib/onboard/sandbox-gpu-create-flow.test.ts b/src/lib/onboard/sandbox-gpu-create-flow.test.ts index 922afe3e17b..a76be485183 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.test.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.test.ts @@ -318,6 +318,8 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { expectedSupervisorArgv: ["/mxc/supervisor"], }; const deps = createDeps(); + const adapterOverride = {} as never; + deps.createManagedBootstrapAdapter = vi.fn(() => adapterOverride); vi.mocked(deps.runCaptureOpenshell).mockImplementation((args) => args[1] === "get" ? "ID: mxc-alpha\n" : "alpha Ready", ); @@ -340,8 +342,14 @@ describe("runSandboxGpuCreateFlow provider-owned managed create", () => { expect(result).toMatchObject({ route: "none", runtimePatch: patch }); expect(createLifecycle).toHaveBeenCalledWith( - expect.objectContaining({ providerId: "mxc", route: "none" }), + expect.objectContaining({ + providerId: "mxc", + route: "none", + stateRoot: "/tmp/nemoclaw-mxc-bootstrap", + adapterOverride, + }), ); + expect(deps.createManagedBootstrapAdapter).toHaveBeenCalledWith("/tmp/nemoclaw-mxc-bootstrap"); expect(mocks.streamSandboxCreate).toHaveBeenCalledWith( "mxc-launch", input.createArgv.slice(1), diff --git a/src/lib/onboard/sandbox-gpu-create-flow.ts b/src/lib/onboard/sandbox-gpu-create-flow.ts index 6f52aefdbdf..2a5b10d1399 100644 --- a/src/lib/onboard/sandbox-gpu-create-flow.ts +++ b/src/lib/onboard/sandbox-gpu-create-flow.ts @@ -111,7 +111,7 @@ export interface SandboxGpuCreateFlowDeps { /** Production callers configure the hidden portable lifecycle through the default implementation. */ installPortableDemoLifecycle?: typeof installPortableDemoSandboxLifecycle; /** Production callers omit this factory and use the runtime provider's adapter. */ - createManagedBootstrapAdapter?: () => ManagedBootstrapAdapter; + createManagedBootstrapAdapter?: (stateRoot: string) => ManagedBootstrapAdapter; } export interface SandboxGpuCreateFlowResult { diff --git a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts index 5b91caa7368..47861cc3c4c 100644 --- a/src/lib/onboard/sandbox-gpu-create-run-attempt.ts +++ b/src/lib/onboard/sandbox-gpu-create-run-attempt.ts @@ -108,7 +108,7 @@ export function createSandboxGpuCreateAttemptRunner( heldWorkloadArgv: input.sandboxStartupCommand, authorityStore: managedBootstrap.authorityStore, ...(deps.createManagedBootstrapAdapter - ? { adapterOverride: deps.createManagedBootstrapAdapter() } + ? { adapterOverride: deps.createManagedBootstrapAdapter(managedBootstrap.stateRoot) } : {}), route, persistStartupCommand: input.persistStartupCommand === true, diff --git a/test/e2e/README.md b/test/e2e/README.md index e2fb831bca8..c9f0c6b7384 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -837,7 +837,7 @@ The workflow does not rotate or revoke these API keys or messaging credentials. Live targets can create external resources. After a failure, inspect the workflow artifacts and remove resources that target cleanup did not remove. -For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVIDIA_API_KEY` repository secret only to the trusted qualification step. Trusted host code uses it for NGC login and passes it as `NGC_API_KEY` and `NIM_NGC_API_KEY` to the temporary NIM container. Candidate managed sandboxes receive generated local route tokens instead of this key. The live fixture attempts to stop and remove `nemoclaw-managed-image-nim-e2e`, but Docker stop or removal errors do not fail the test. A surviving container can retain the API key until runner teardown. The final workflow step removes the job's isolated Docker credential directory and fails if that removal does not complete. The workflow does not revoke the NVIDIA API key. Rotate or revoke it in the issuing NVIDIA service to remove later access. +For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVIDIA_API_KEY` repository secret only to the trusted qualification step. Trusted host code uses it for NGC login and passes it as `NGC_API_KEY` and `NIM_NGC_API_KEY` to the temporary, cohort-owned NIM container. Candidate managed sandboxes receive generated local route tokens instead of this key. Before starting NIM or vLLM, the live fixture rejects a pre-existing cohort container name. It records the full container ID, requested image, immutable image ID, cohort owner, and provider label, then removes only that exact container after revalidating every field. Missing, ambiguous, name-reused, drifted, or indeterminate cleanup evidence fails the test, as does any retained exact ID or name. A fail-closed refusal can leave the secret-bearing NIM container alive until runner teardown; inspect the redacted artifacts and remove only the verified container. The final workflow step removes the job's isolated Docker credential directory and fails if that removal does not complete. The workflow does not revoke the NVIDIA API key. Revoke it, or rotate it and disable the old value, in the issuing NVIDIA service. Verify that the exposed key is no longer valid. For a manual PR run, provide the current PR number, lowercase 40-character head SHA, head repository, lowercase 40-character base SHA, trusted `main` workflow SHA, and a review reason containing 10 to 500 printable characters. Leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false` to use this PR revision selection. diff --git a/test/e2e/live/managed-image-protected-runtime-helpers.ts b/test/e2e/live/managed-image-protected-runtime-helpers.ts index 3408c808a97..f0728070db3 100644 --- a/test/e2e/live/managed-image-protected-runtime-helpers.ts +++ b/test/e2e/live/managed-image-protected-runtime-helpers.ts @@ -13,13 +13,11 @@ import { type ProtectedManagedImageContract, parseProtectedManagedImageContracts, } from "../../../scripts/checks/managed-image-protected-runtime-contract.ts"; +import { PROTECTED_MANAGED_IMAGE_COHORT_PATTERN } from "../../../scripts/checks/protected-managed-image-contract.ts"; import { adoptServedModelId, dockerLoginNgc, pullNimImage, - startNimContainerByName, - stopNimContainerByName, - waitForNimHealth, } from "../../../src/lib/inference/nim.ts"; import { getOllamaProxyToken, @@ -44,12 +42,34 @@ const OLLAMA_MODEL = "qwen3.5:9b"; const VLLM_MODEL = "Qwen/Qwen2.5-0.5B-Instruct"; const VLLM_IMAGE = "vllm/vllm-openai@sha256:0fec7ec5f3e6bc168e54899935fb0557da908a4832a1dbc88e2debcf2f889416"; -const VLLM_CONTAINER = "nemoclaw-managed-image-vllm-e2e"; const NIM_CATALOG_MODEL = "nvidia/nemotron-3-nano-30b-a3b"; -const NIM_CONTAINER = "nemoclaw-managed-image-nim-e2e"; const AGENT_QUALIFICATION_TIMEOUT_MS = 10 * 60_000; const ROLLBACK_QUALIFICATION_TIMEOUT_MS = 10 * 60_000; const PROTECTED_READINESS_CAPTURE_LIMIT_BYTES = 4 * 1024 * 1024; +const PROTECTED_PROVIDER_CONTAINER_MAX_LENGTH = 63; + +export const PROTECTED_PROVIDER_OWNER_LABEL = "io.nvidia.nemoclaw.e2e-owner"; +export const PROTECTED_PROVIDER_KIND_LABEL = "io.nvidia.nemoclaw.e2e-provider"; + +export type ProtectedProviderKind = "nim" | "vllm"; + +export interface ProtectedProviderContainerAuthority { + readonly containerId: string; + readonly imageId: string; + readonly kind: ProtectedProviderKind; + readonly name: string; + readonly owner: string; + readonly requestedImage: string; +} + +interface ProtectedProviderContainerState { + authority: ProtectedProviderContainerAuthority | null; + readonly kind: ProtectedProviderKind; + readonly name: string; + readonly owner: string; + removed: boolean; + reportedContainerId: string | null; +} type RuntimeFixtures = Pick; @@ -59,6 +79,160 @@ interface ProtectedRuntimeReadinessCommand { readonly captureLimitBytes: number; } +export function protectedProviderContainerName( + kind: ProtectedProviderKind, + cohort = process.env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT ?? "", +): string { + if (!PROTECTED_MANAGED_IMAGE_COHORT_PATTERN.test(cohort)) { + throw new Error("NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT has an invalid protected cohort"); + } + const name = `nemoclaw-mi-${kind}-${cohort}`; + if ( + name.length > PROTECTED_PROVIDER_CONTAINER_MAX_LENGTH || + !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/u.test(name) + ) { + throw new Error("protected provider container name is outside the Docker name contract"); + } + return name; +} + +export function protectedProviderReportedContainerId(name: string, stdout: string): string { + if ( + name.length > PROTECTED_PROVIDER_CONTAINER_MAX_LENGTH || + !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/u.test(name) + ) { + throw new Error("protected provider container name is outside the Docker name contract"); + } + const candidate = stdout.trim(); + if (!/^[a-f0-9]{64}$/u.test(candidate)) { + throw new Error(`provider start did not report one full container ID for ${name}`); + } + return candidate; +} + +function assertProviderAuthority(authority: ProtectedProviderContainerAuthority): void { + if ( + !/^[a-f0-9]{64}$/u.test(authority.containerId) || + !/^sha256:[a-f0-9]{64}$/u.test(authority.imageId) || + !PROTECTED_MANAGED_IMAGE_COHORT_PATTERN.test(authority.owner) || + protectedProviderContainerName(authority.kind, authority.owner) !== authority.name || + !authority.requestedImage || + /[\0\r\n|]/u.test(authority.requestedImage) + ) { + throw new Error("protected provider container authority is invalid"); + } +} + +export function protectedProviderContainerPreflightCommand( + name: string, +): ProtectedRuntimeReadinessCommand { + if ( + name.length > PROTECTED_PROVIDER_CONTAINER_MAX_LENGTH || + !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/u.test(name) + ) { + throw new Error("protected provider container name is outside the Docker name contract"); + } + return { + command: "bash", + captureLimitBytes: PROTECTED_READINESS_CAPTURE_LIMIT_BYTES, + args: [ + "--noprofile", + "--norc", + "-c", + [ + "set -euo pipefail", + 'expected_name="$1"', + 'rows="$(docker ps -a --no-trunc --filter "name=^/${expected_name}$" --format \'{{.ID}}\')" || {', + " printf 'provider container preflight is indeterminate for %s\\n' \"$expected_name\" >&2", + " exit 70", + "}", + '[[ -z "$rows" ]] || {', + ' printf \'refusing to replace pre-existing provider container %s: %s\\n\' "$expected_name" "$rows" >&2', + " exit 70", + "}", + "printf 'provider-container-absent name=%s\\n' \"$expected_name\"", + ].join("\n"), + "protected-provider-preflight", + name, + ], + }; +} + +export function protectedProviderContainerCleanupCommand( + authority: ProtectedProviderContainerAuthority, +): ProtectedRuntimeReadinessCommand { + assertProviderAuthority(authority); + return { + command: "bash", + captureLimitBytes: PROTECTED_READINESS_CAPTURE_LIMIT_BYTES, + args: [ + "--noprofile", + "--norc", + "-c", + [ + "set -euo pipefail", + 'expected_name="$1"', + 'expected_id="$2"', + 'expected_image="$3"', + 'expected_image_id="$4"', + 'expected_owner="$5"', + 'expected_kind="$6"', + `owner_label=${JSON.stringify(PROTECTED_PROVIDER_OWNER_LABEL)}`, + `kind_label=${JSON.stringify(PROTECTED_PROVIDER_KIND_LABEL)}`, + 'name_rows="$(docker ps -a --no-trunc --filter "name=^/${expected_name}$" --format \'{{.ID}}\')" || {', + " printf 'provider cleanup inventory is indeterminate for %s\\n' \"$expected_name\" >&2", + " exit 70", + "}", + '[[ -n "$name_rows" ]] || {', + " printf 'provider cleanup evidence is missing for %s\\n' \"$expected_name\" >&2", + " exit 70", + "}", + "[[ \"$name_rows\" != *$'\\n'* ]] || {", + ' printf \'provider cleanup evidence is ambiguous for %s: %s\\n\' "$expected_name" "$name_rows" >&2', + " exit 70", + "}", + '[[ "$name_rows" == "$expected_id" ]] || {', + ' printf \'provider container name %s was reused: expected %s, got %s\\n\' "$expected_name" "$expected_id" "$name_rows" >&2', + " exit 70", + "}", + 'inspection="$(docker container inspect --format \'{{.Id}}|{{.Name}}|{{.Config.Image}}|{{.Image}}|{{ index .Config.Labels "io.nvidia.nemoclaw.e2e-owner" }}|{{ index .Config.Labels "io.nvidia.nemoclaw.e2e-provider" }}\' "$expected_id")" || {', + " printf 'provider authority inspection is indeterminate for %s\\n' \"$expected_name\" >&2", + " exit 70", + "}", + "IFS='|' read -r actual_id actual_name actual_image actual_image_id actual_owner actual_kind extra <<<\"$inspection\"", + '[[ -z "${extra:-}" && "$actual_id" == "$expected_id" && "$actual_name" == "/$expected_name" && "$actual_image" == "$expected_image" && "$actual_image_id" == "$expected_image_id" && "$actual_owner" == "$expected_owner" && "$actual_kind" == "$expected_kind" ]] || {', + " printf 'provider cleanup authority drifted for %s; refusing removal\\n' \"$expected_name\" >&2", + " exit 70", + "}", + 'docker rm -f "$expected_id" >/dev/null || {', + " printf 'provider cleanup removal is indeterminate for %s\\n' \"$expected_name\" >&2", + " exit 70", + "}", + 'id_rows="$(docker ps -a --no-trunc --filter "id=${expected_id}" --format \'{{.ID}}\')" || {', + " printf 'provider cleanup ID verification is indeterminate for %s\\n' \"$expected_name\" >&2", + " exit 70", + "}", + 'name_rows="$(docker ps -a --no-trunc --filter "name=^/${expected_name}$" --format \'{{.ID}}\')" || {', + " printf 'provider cleanup name verification is indeterminate for %s\\n' \"$expected_name\" >&2", + " exit 70", + "}", + '[[ -z "$id_rows" && -z "$name_rows" ]] || {', + ' printf \'provider cleanup retained or replaced %s: id=%s name=%s\\n\' "$expected_name" "$id_rows" "$name_rows" >&2', + " exit 70", + "}", + 'printf \'provider-container-removed name=%s id=%s image=%s image_id=%s %s=%s %s=%s\\n\' "$expected_name" "$expected_id" "$expected_image" "$expected_image_id" "$owner_label" "$expected_owner" "$kind_label" "$expected_kind"', + ].join("\n"), + "protected-provider-cleanup", + authority.name, + authority.containerId, + authority.requestedImage, + authority.imageId, + authority.owner, + authority.kind, + ], + }; +} + function imageContracts(): ProtectedManagedImageContract[] { const contractPath = process.env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT; if (!contractPath || !path.isAbsolute(contractPath)) { @@ -276,19 +450,139 @@ async function proveOllamaGpuPlacement(host: HostCliClient): Promise { expect(result.exitCode, resultText(result)).toBe(0); } -async function startProtectedVllm(host: HostCliClient): Promise { - await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { - artifactName: "pre-cleanup-vllm", +function protectedProviderContainerState( + kind: ProtectedProviderKind, + owner: string, +): ProtectedProviderContainerState { + return { + authority: null, + kind, + name: protectedProviderContainerName(kind, owner), + owner, + removed: false, + reportedContainerId: null, + }; +} + +async function assertProtectedProviderContainerAbsent( + host: HostCliClient, + state: ProtectedProviderContainerState, + artifactName: string, +): Promise { + const command = protectedProviderContainerPreflightCommand(state.name); + const result = await host.command(command.command, command.args, { + artifactName, + captureLimitBytes: command.captureLimitBytes, env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }); + expect(result.exitCode, resultText(result)).toBe(0); +} + +async function inspectProtectedProviderContainer( + host: HostCliClient, + state: ProtectedProviderContainerState, + requestedImage: string, + artifactName: string, +): Promise { + const reportedContainerId = state.reportedContainerId; + if (!reportedContainerId) { + throw new Error(`provider start did not report authority for ${state.name}`); + } + const result = await host.command( + "docker", + [ + "container", + "inspect", + "--format", + '{{.Id}}|{{.Name}}|{{.Config.Image}}|{{.Image}}|{{ index .Config.Labels "io.nvidia.nemoclaw.e2e-owner" }}|{{ index .Config.Labels "io.nvidia.nemoclaw.e2e-provider" }}', + reportedContainerId, + ], + { + artifactName, + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, + ); + if (result.exitCode !== 0) { + throw new Error( + `provider authority inspection failed for ${state.name}: ${resultText(result)}`, + ); + } + const lines = result.stdout.trim().split("\n"); + const fields = lines.length === 1 ? lines[0]!.split("|") : []; + if (fields.length !== 6) { + throw new Error(`provider authority inspection was ambiguous for ${state.name}`); + } + const [containerId, actualName, actualImage, imageId, owner, kind] = fields; + const authority: ProtectedProviderContainerAuthority = { + containerId: containerId!, + imageId: imageId!, + kind: state.kind, + name: state.name, + owner: state.owner, + requestedImage, + }; + if ( + containerId !== reportedContainerId || + actualName !== `/${state.name}` || + actualImage !== requestedImage || + owner !== state.owner || + kind !== state.kind + ) { + throw new Error(`provider authority drifted after start for ${state.name}`); + } + assertProviderAuthority(authority); + return authority; +} + +async function removeProtectedProviderContainer( + host: HostCliClient, + state: ProtectedProviderContainerState, + artifactName: string, +): Promise { + if (state.removed || !state.reportedContainerId) { + await assertProtectedProviderContainerAbsent(host, state, `${artifactName}-absent`); + return; + } + if (!state.authority) { + const command = protectedProviderContainerPreflightCommand(state.name); + const evidence = await host.command(command.command, command.args, { + artifactName: `${artifactName}-authority-missing`, + captureLimitBytes: command.captureLimitBytes, + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + throw new Error( + `provider cleanup authority is missing for ${state.name}; see ${evidence.artifacts.result}`, + ); + } + const command = protectedProviderContainerCleanupCommand(state.authority); + const result = await host.command(command.command, command.args, { + artifactName, + captureLimitBytes: command.captureLimitBytes, + env: buildAvailabilityProbeEnv(), + timeoutMs: 120_000, + }); + expect(result.exitCode, resultText(result)).toBe(0); + state.removed = true; +} + +async function startProtectedVllm( + host: HostCliClient, + state: ProtectedProviderContainerState, +): Promise { const start = await host.command( "docker", [ "run", "--detach", "--name", - VLLM_CONTAINER, + state.name, + "--label", + `${PROTECTED_PROVIDER_OWNER_LABEL}=${state.owner}`, + "--label", + `${PROTECTED_PROVIDER_KIND_LABEL}=${state.kind}`, "--gpus", "all", "--publish", @@ -310,7 +604,14 @@ async function startProtectedVllm(host: HostCliClient): Promise { }, ); expect(start.exitCode, resultText(start)).toBe(0); - const readiness = protectedVllmReadinessCommand(); + state.reportedContainerId = protectedProviderReportedContainerId(state.name, start.stdout); + state.authority = await inspectProtectedProviderContainer( + host, + state, + VLLM_IMAGE, + "inspect-vllm-authority", + ); + const readiness = protectedVllmReadinessCommand(state.name); const ready = await host.command(readiness.command, readiness.args, { artifactName: "wait-vllm", captureLimitBytes: readiness.captureLimitBytes, @@ -322,7 +623,7 @@ async function startProtectedVllm(host: HostCliClient): Promise { "docker", [ "exec", - VLLM_CONTAINER, + state.name, "python3", "-c", "import torch; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))", @@ -336,45 +637,143 @@ async function startProtectedVllm(host: HostCliClient): Promise { expect(cuda.exitCode, resultText(cuda)).toBe(0); } -export function protectedVllmReadinessCommand(): ProtectedRuntimeReadinessCommand { +export function protectedVllmReadinessCommand( + containerName: string, +): ProtectedRuntimeReadinessCommand { + if ( + containerName.length > PROTECTED_PROVIDER_CONTAINER_MAX_LENGTH || + !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/u.test(containerName) + ) { + throw new Error("protected provider container name is outside the Docker name contract"); + } return { command: "bash", captureLimitBytes: PROTECTED_READINESS_CAPTURE_LIMIT_BYTES, args: [ + "--noprofile", + "--norc", "-c", `set -euo pipefail attempt=0 -for attempt in $(seq 1 300); do - if curl -fsS --connect-timeout 2 http://127.0.0.1:8000/v1/models >/dev/null 2>&1; then +deadline=$((SECONDS + 600)) +while [ "$SECONDS" -lt "$deadline" ]; do + attempt=$((attempt + 1)) + if curl -fsS --connect-timeout 2 --max-time 5 http://127.0.0.1:8000/v1/models >/dev/null 2>&1; then printf 'managed-image-vllm-ready attempts=%s\n' "$attempt" exit 0 fi - if ! docker container inspect "${VLLM_CONTAINER}" --format '{{.State.Running}}' | grep -Fx true >/dev/null; then + if ! docker container inspect "${containerName}" --format '{{.State.Running}}' | grep -Fx true >/dev/null; then break fi sleep 2 done -docker logs --tail 200 "${VLLM_CONTAINER}" >&2 || true +docker logs --tail 200 "${containerName}" >&2 || true printf 'managed-image-vllm-not-ready attempts=%s\n' "$attempt" >&2 exit 1`, ], }; } -async function startProtectedNim(host: HostCliClient, apiKey: string): Promise { - stopNimContainerByName(NIM_CONTAINER, { silent: true }); +export function protectedNimReadinessCommand( + containerName: string, +): ProtectedRuntimeReadinessCommand { + if ( + containerName.length > PROTECTED_PROVIDER_CONTAINER_MAX_LENGTH || + !/^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/u.test(containerName) + ) { + throw new Error("protected provider container name is outside the Docker name contract"); + } + return { + command: "bash", + captureLimitBytes: PROTECTED_READINESS_CAPTURE_LIMIT_BYTES, + args: [ + "--noprofile", + "--norc", + "-c", + `set -euo pipefail +attempt=0 +deadline=$((SECONDS + 1200)) +while [ "$SECONDS" -lt "$deadline" ]; do + attempt=$((attempt + 1)) + if curl -fsS --connect-timeout 5 --max-time 5 http://127.0.0.1:8000/v1/models >/dev/null 2>&1; then + printf 'managed-image-nim-ready attempts=%s\n' "$attempt" + exit 0 + fi + if ! docker container inspect "${containerName}" --format '{{.State.Running}}' | grep -Fx true >/dev/null; then + break + fi + sleep 5 +done +docker logs --tail 200 "${containerName}" >&2 || true +printf 'managed-image-nim-not-ready attempts=%s\n' "$attempt" >&2 +exit 1`, + ], + }; +} + +async function startProtectedNim( + host: HostCliClient, + state: ProtectedProviderContainerState, + apiKey: string, +): Promise { expect(dockerLoginNgc(apiKey), "NGC login must succeed for protected NIM qualification").toBe( true, ); - pullNimImage(NIM_CATALOG_MODEL); - startNimContainerByName(NIM_CONTAINER, NIM_CATALOG_MODEL, 8000, { ngcApiKey: apiKey }); - expect( - waitForNimHealth(8000, 20 * 60, { container: NIM_CONTAINER }), - "NIM must become healthy", - ).toBe(true); + const image = pullNimImage(NIM_CATALOG_MODEL); + const start = await host.command( + "docker", + [ + "run", + "--detach", + "--name", + state.name, + "--label", + `${PROTECTED_PROVIDER_OWNER_LABEL}=${state.owner}`, + "--label", + `${PROTECTED_PROVIDER_KIND_LABEL}=${state.kind}`, + "--gpus", + "all", + "--publish", + "8000:8000", + "--shm-size", + "16g", + "--env", + "NGC_API_KEY", + "--env", + "NIM_NGC_API_KEY", + image, + ], + { + artifactName: "start-nim", + env: { + ...buildAvailabilityProbeEnv(), + NGC_API_KEY: apiKey, + NIM_NGC_API_KEY: apiKey, + }, + redactionValues: [apiKey], + timeoutMs: 20 * 60_000, + }, + ); + expect(start.exitCode, resultText(start)).toBe(0); + state.reportedContainerId = protectedProviderReportedContainerId(state.name, start.stdout); + state.authority = await inspectProtectedProviderContainer( + host, + state, + image, + "inspect-nim-authority", + ); + const readiness = protectedNimReadinessCommand(state.name); + const ready = await host.command(readiness.command, readiness.args, { + artifactName: "wait-nim", + captureLimitBytes: readiness.captureLimitBytes, + env: buildAvailabilityProbeEnv(), + redactionValues: [apiKey], + timeoutMs: 21 * 60_000, + }); + expect(ready.exitCode, resultText(ready)).toBe(0); const servedModel = adoptServedModelId(NIM_CATALOG_MODEL, 8000); expect(servedModel, "NIM must report one safe served model").toBeTruthy(); - const cuda = await host.command("docker", ["exec", NIM_CONTAINER, "nvidia-smi", "-L"], { + const cuda = await host.command("docker", ["exec", state.name, "nvidia-smi", "-L"], { artifactName: "nim-cuda-initialization", env: buildAvailabilityProbeEnv(), timeoutMs: 60_000, @@ -425,23 +824,88 @@ async function qualifyEveryRollback( for (const contract of contracts) await qualifyRollback(host, contract); } -async function proveOwnedRuntimeInventoryClean(host: HostCliClient): Promise { - const result = await host.command( - "bash", - [ - "-lc", - `set -euo pipefail -containers="$(docker ps -a --format '{{.Label "openshell.ai/sandbox-name"}}' --filter label=openshell.ai/managed-by=openshell | grep '^${MANAGED_IMAGE_PROTECTED_SANDBOX_PREFIX}' || true)" -networks="$(docker network ls --format '{{.Name}}' | grep '^nemoclaw-managed-pr-' || true)" -test -z "$containers" -test -z "$networks"`, +export function protectedProviderFinalInventoryCommand( + cohort: string, + vllmName: string, + nimName: string, +): ProtectedRuntimeReadinessCommand { + if ( + protectedProviderContainerName("vllm", cohort) !== vllmName || + protectedProviderContainerName("nim", cohort) !== nimName + ) { + throw new Error("protected provider inventory requires exact cohort container names"); + } + return { + command: "bash", + captureLimitBytes: PROTECTED_READINESS_CAPTURE_LIMIT_BYTES, + args: [ + "--noprofile", + "--norc", + "-c", + [ + "set -euo pipefail", + 'cohort="$1"', + 'vllm_name="$2"', + 'nim_name="$3"', + 'sandbox_prefix="$4"', + `owner_label=${JSON.stringify(PROTECTED_PROVIDER_OWNER_LABEL)}`, + `kind_label=${JSON.stringify(PROTECTED_PROVIDER_KIND_LABEL)}`, + 'printf \'expected-provider name=%s %s=%s %s=vllm\\n\' "$vllm_name" "$owner_label" "$cohort" "$kind_label"', + 'printf \'expected-provider name=%s %s=%s %s=nim\\n\' "$nim_name" "$owner_label" "$cohort" "$kind_label"', + 'provider_rows="$(docker ps -a --no-trunc --filter "label=${owner_label}=${cohort}" --filter "label=${kind_label}" --format \'{{.ID}}|{{.Names}}|{{.Label "io.nvidia.nemoclaw.e2e-owner"}}|{{.Label "io.nvidia.nemoclaw.e2e-provider"}}\')" || {', + " echo 'provider-owned container inventory is indeterminate' >&2", + " exit 70", + "}", + 'vllm_rows="$(docker ps -a --no-trunc --filter "name=^/${vllm_name}$" --format \'{{.ID}}\')" || {', + " echo 'vLLM container-name inventory is indeterminate' >&2", + " exit 70", + "}", + 'nim_rows="$(docker ps -a --no-trunc --filter "name=^/${nim_name}$" --format \'{{.ID}}\')" || {', + " echo 'NIM container-name inventory is indeterminate' >&2", + " exit 70", + "}", + 'sandbox_labels="$(docker ps -a --format \'{{.Label "openshell.ai/sandbox-name"}}\' --filter label=openshell.ai/managed-by=openshell)" || {', + " echo 'managed sandbox container inventory is indeterminate' >&2", + " exit 70", + "}", + 'containers="$(printf \'%s\\n\' "$sandbox_labels" | grep "^${sandbox_prefix}" || true)"', + "network_names=\"$(docker network ls --format '{{.Name}}')\" || {", + " echo 'managed network inventory is indeterminate' >&2", + " exit 70", + "}", + "networks=\"$(printf '%s\\n' \"$network_names\" | grep '^nemoclaw-managed-pr-' || true)\"", + '[[ -z "$provider_rows" && -z "$vllm_rows" && -z "$nim_rows" && -z "$containers" && -z "$networks" ]] || {', + ' printf \'protected runtime inventory retained state: providers=%s vllm=%s nim=%s sandboxes=%s networks=%s\\n\' "$provider_rows" "$vllm_rows" "$nim_rows" "$containers" "$networks" >&2', + " exit 70", + "}", + "printf 'protected-runtime-inventory-clean\\n'", + ].join("\n"), + "protected-provider-final-inventory", + cohort, + vllmName, + nimName, + MANAGED_IMAGE_PROTECTED_SANDBOX_PREFIX, ], - { - artifactName: "final-managed-image-owned-runtime-inventory", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }, - ); + }; +} + +async function proveOwnedRuntimeInventoryClean( + host: HostCliClient, + cohort: string, + providerStates: readonly ProtectedProviderContainerState[], +): Promise { + const vllmName = providerStates.find(({ kind }) => kind === "vllm")?.name; + const nimName = providerStates.find(({ kind }) => kind === "nim")?.name; + if (!vllmName || !nimName) { + throw new Error("protected provider inventory requires vLLM and NIM container names"); + } + const command = protectedProviderFinalInventoryCommand(cohort, vllmName, nimName); + const result = await host.command(command.command, command.args, { + artifactName: "final-managed-image-owned-runtime-inventory", + captureLimitBytes: command.captureLimitBytes, + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); expect(result.exitCode, resultText(result)).toBe(0); } @@ -452,16 +916,16 @@ export async function qualifyProtectedManagedImageRuntime( const { artifacts, cleanup, host, progress } = fixtures; const contracts = imageContracts(); const ngcApiKey = requiredNgcApiKey(ngcApiKeyInput); + const cohort = process.env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT ?? ""; + const vllmState = protectedProviderContainerState("vllm", cohort); + const nimState = protectedProviderContainerState("nim", cohort); + const providerStates = [vllmState, nimState] as const; cleanup.trackDisposable("remove protected vLLM container", async () => { - await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { - artifactName: "cleanup-vllm-container", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }); + await removeProtectedProviderContainer(host, vllmState, "cleanup-vllm-container"); }); - cleanup.trackDisposable("remove protected NIM container", () => { - stopNimContainerByName(NIM_CONTAINER, { silent: true }); + cleanup.trackDisposable("remove protected NIM container", async () => { + await removeProtectedProviderContainer(host, nimState, "cleanup-nim-container"); }); cleanup.trackDisposable("stop protected Ollama runtime", async () => { killStaleProxy(); @@ -485,6 +949,8 @@ export async function qualifyProtectedManagedImageRuntime( assertNvidiaAvailable(nvidia, (message) => { throw new Error(message ?? "protected GPU runner is unavailable"); }); + await assertProtectedProviderContainerAbsent(host, vllmState, "preflight-vllm-container"); + await assertProtectedProviderContainerAbsent(host, nimState, "preflight-nim-container"); activePhase = "qualify all managed agents with GPU-backed Ollama"; progress.phase("qualify all managed agents with GPU-backed Ollama"); @@ -499,30 +965,31 @@ export async function qualifyProtectedManagedImageRuntime( activePhase = "qualify all managed agents with GPU-backed vLLM"; progress.phase("qualify all managed agents with GPU-backed vLLM"); - await startProtectedVllm(host); + await startProtectedVllm(host, vllmState); await qualifyEveryAgent(host, contracts, "vllm", VLLM_MODEL, { NEMOCLAW_VLLM_LOCAL_TOKEN: randomBytes(24).toString("hex"), }); - await host.command("docker", ["rm", "-f", VLLM_CONTAINER], { - artifactName: "stop-vllm-before-nim", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }); + await removeProtectedProviderContainer(host, vllmState, "stop-vllm-before-nim"); activePhase = "qualify all managed agents with GPU-backed NVIDIA NIM"; progress.phase("qualify all managed agents with GPU-backed NVIDIA NIM"); - const nimModel = await startProtectedNim(host, ngcApiKey); + const nimModel = await startProtectedNim(host, nimState, ngcApiKey); await qualifyEveryAgent(host, contracts, "nim", nimModel, { NEMOCLAW_VLLM_LOCAL_TOKEN: randomBytes(24).toString("hex"), }); - stopNimContainerByName(NIM_CONTAINER, { silent: true }); + await removeProtectedProviderContainer(host, nimState, "stop-nim-before-rollback"); activePhase = "prove all-agent managed bootstrap rollback and exact cleanup"; progress.phase("prove all-agent managed bootstrap rollback and exact cleanup"); await qualifyEveryRollback(host, contracts); - await proveOwnedRuntimeInventoryClean(host); + await proveOwnedRuntimeInventoryClean(host, cohort, providerStates); + const providerContainers = providerStates.map(({ authority }) => { + if (!authority) throw new Error("protected provider authority is missing from the receipt"); + return authority; + }); await artifacts.writeJson("managed-image-protected-runtime-summary.json", { agents: PROTECTED_MANAGED_IMAGE_AGENTS, + providerContainers, providers: ["ollama", "vllm", "nim"], rollbackAgents: PROTECTED_MANAGED_IMAGE_AGENTS, }); diff --git a/test/e2e/support/managed-image-protected-runtime-readiness.test.ts b/test/e2e/support/managed-image-protected-runtime-readiness.test.ts index 7a332272c07..5f973a9d9ab 100644 --- a/test/e2e/support/managed-image-protected-runtime-readiness.test.ts +++ b/test/e2e/support/managed-image-protected-runtime-readiness.test.ts @@ -12,7 +12,16 @@ import { HostCliClient } from "../fixtures/clients/host.ts"; import { startTestProgress, type TestProgress } from "../fixtures/progress.ts"; import { ShellProbe } from "../fixtures/shell-probe.ts"; import { + PROTECTED_PROVIDER_KIND_LABEL, + PROTECTED_PROVIDER_OWNER_LABEL, + type ProtectedProviderContainerAuthority, + protectedNimReadinessCommand, protectedOllamaReadinessCommand, + protectedProviderContainerCleanupCommand, + protectedProviderContainerName, + protectedProviderContainerPreflightCommand, + protectedProviderFinalInventoryCommand, + protectedProviderReportedContainerId, protectedVllmReadinessCommand, } from "../live/managed-image-protected-runtime-helpers.ts"; @@ -22,9 +31,6 @@ vi.mock("../../../src/lib/inference/nim.ts", () => ({ adoptServedModelId: () => "", dockerLoginNgc: () => false, pullNimImage: () => undefined, - startNimContainerByName: () => undefined, - stopNimContainerByName: () => undefined, - waitForNimHealth: () => false, })); vi.mock("../../../src/lib/inference/ollama/proxy.ts", () => ({ getOllamaProxyToken: () => undefined, @@ -55,6 +61,11 @@ interface ReadinessFixture { const fixtureRoots: string[] = []; const fixtureProgress: TestProgress[] = []; +const PROVIDER_COHORT = "protected-123-1"; +const PROVIDER_CONTAINER_ID = "a".repeat(64); +const PROVIDER_IMAGE_ID = `sha256:${"b".repeat(64)}`; +const PROVIDER_IMAGE = `registry.example/vllm@sha256:${"c".repeat(64)}`; +const VLLM_PROVIDER_NAME = protectedProviderContainerName("vllm", PROVIDER_COHORT); afterEach(() => { for (const progress of fixtureProgress) progress.stop(); @@ -115,7 +126,363 @@ function writeCommand(binDir: string, name: string, body: string): void { fs.chmodSync(commandPath, 0o755); } -describe("protected managed-image readiness commands", () => { +function providerAuthority(): ProtectedProviderContainerAuthority { + return { + containerId: PROVIDER_CONTAINER_ID, + imageId: PROVIDER_IMAGE_ID, + kind: "vllm", + name: VLLM_PROVIDER_NAME, + owner: PROVIDER_COHORT, + requestedImage: PROVIDER_IMAGE, + }; +} + +function installProviderDocker( + fixture: ReadinessFixture, + scenario: + | "ambiguous" + | "authority-drift" + | "id-verification-indeterminate" + | "indeterminate" + | "inspect-indeterminate" + | "missing" + | "name-verification-indeterminate" + | "normal" + | "remove-indeterminate" + | "reused" + | "reuse-after-remove", + initialState: "absent" | "present" = "present", +): NodeJS.ProcessEnv { + const stateFile = path.join(fixture.root, "provider-container-state"); + const removeLog = path.join(fixture.root, "provider-container-remove.log"); + fs.writeFileSync(stateFile, `${initialState}\n`, "utf8"); + writeCommand( + fixture.binDir, + "docker", + `command_name="$1" +shift +state="$(/bin/cat "$FAKE_PROVIDER_STATE")" +if [ "$command_name" = ps ]; then + if [ "$FAKE_PROVIDER_SCENARIO" = indeterminate ]; then + echo 'docker daemon unavailable' >&2 + exit 125 + fi + filter='' + while [ "$#" -gt 0 ]; do + if [ "$1" = --filter ]; then + filter="$2" + shift 2 + else + shift + fi + done + if [ "$state" = absent ] && [ "$FAKE_PROVIDER_SCENARIO" = id-verification-indeterminate ] && printf '%s' "$filter" | /usr/bin/grep -q '^id='; then + echo 'docker daemon unavailable during ID verification' >&2 + exit 125 + fi + if [ "$state" = absent ] && [ "$FAKE_PROVIDER_SCENARIO" = name-verification-indeterminate ] && printf '%s' "$filter" | /usr/bin/grep -q '^name='; then + echo 'docker daemon unavailable during name verification' >&2 + exit 125 + fi + [ "$state" = present ] || { + if [ "$FAKE_PROVIDER_SCENARIO" = reuse-after-remove ] && printf '%s' "$filter" | /usr/bin/grep -q '^name='; then + printf '%s\\n' "$FAKE_PROVIDER_REPLACEMENT_ID" + fi + exit 0 + } + case "$FAKE_PROVIDER_SCENARIO" in + missing) exit 0 ;; + ambiguous) printf '%s\\n%s\\n' "$FAKE_PROVIDER_ID" "$FAKE_PROVIDER_REPLACEMENT_ID" ;; + reused) printf '%s\\n' "$FAKE_PROVIDER_REPLACEMENT_ID" ;; + *) printf '%s\\n' "$FAKE_PROVIDER_ID" ;; + esac + exit 0 +fi +if [ "$command_name" = container ] && [ "$1" = inspect ]; then + if [ "$FAKE_PROVIDER_SCENARIO" = inspect-indeterminate ]; then + echo 'docker inspect unavailable' >&2 + exit 125 + fi + owner="$FAKE_PROVIDER_OWNER" + [ "$FAKE_PROVIDER_SCENARIO" != authority-drift ] || owner=other-owner + printf '%s|/%s|%s|%s|%s|vllm\\n' \\ + "$FAKE_PROVIDER_ID" \\ + "$FAKE_PROVIDER_NAME" \\ + "$FAKE_PROVIDER_IMAGE" \\ + "$FAKE_PROVIDER_IMAGE_ID" \\ + "$owner" + exit 0 +fi +if [ "$command_name" = rm ] && [ "$1" = -f ] && [ "$2" = "$FAKE_PROVIDER_ID" ]; then + if [ "$FAKE_PROVIDER_SCENARIO" = remove-indeterminate ]; then + echo 'docker removal unavailable' >&2 + exit 125 + fi + printf '%s\\n' "$2" >>"$FAKE_PROVIDER_REMOVE_LOG" + printf 'absent\\n' >"$FAKE_PROVIDER_STATE" + exit 0 +fi +echo "unexpected fake docker command: $command_name $*" >&2 +exit 64`, + ); + return { + ...fixture.env, + FAKE_PROVIDER_ID: PROVIDER_CONTAINER_ID, + FAKE_PROVIDER_IMAGE: PROVIDER_IMAGE, + FAKE_PROVIDER_IMAGE_ID: PROVIDER_IMAGE_ID, + FAKE_PROVIDER_NAME: VLLM_PROVIDER_NAME, + FAKE_PROVIDER_OWNER: PROVIDER_COHORT, + FAKE_PROVIDER_REMOVE_LOG: removeLog, + FAKE_PROVIDER_REPLACEMENT_ID: "d".repeat(64), + FAKE_PROVIDER_SCENARIO: scenario, + FAKE_PROVIDER_STATE: stateFile, + }; +} + +function installProviderInventoryDocker( + fixture: ReadinessFixture, + scenario: + | "clean" + | "indeterminate-network" + | "indeterminate-provider" + | "retained-name" + | "retained-network" + | "retained-provider" + | "retained-sandbox" = "clean", +): NodeJS.ProcessEnv { + const commandLog = path.join(fixture.root, "provider-inventory-docker.log"); + writeCommand( + fixture.binDir, + "docker", + `command_name="$1" +shift +printf '%s %s\\n' "$command_name" "$*" >>"$FAKE_PROVIDER_INVENTORY_COMMAND_LOG" +if [ "$command_name" = ps ]; then + if [ "$FAKE_PROVIDER_INVENTORY_SCENARIO" = indeterminate-provider ]; then + echo 'provider inventory unavailable' >&2 + exit 125 + fi + case "$*" in + *"label=$FAKE_PROVIDER_OWNER_LABEL=$FAKE_PROVIDER_COHORT"*) + case "$*" in + *"label=$FAKE_PROVIDER_KIND_LABEL"*) + [ "$FAKE_PROVIDER_INVENTORY_SCENARIO" != retained-provider ] || printf '%s|provider|%s|vllm\\n' "$FAKE_PROVIDER_CONTAINER_ID" "$FAKE_PROVIDER_COHORT" + exit 0 + ;; + *) printf '%s|protected-registry|%s|\\n' "$FAKE_PROVIDER_REGISTRY_ID" "$FAKE_PROVIDER_COHORT" ;; + esac + ;; + *"name=^/$FAKE_PROVIDER_VLLM_NAME\\$"*) + [ "$FAKE_PROVIDER_INVENTORY_SCENARIO" != retained-name ] || printf '%s\\n' "$FAKE_PROVIDER_CONTAINER_ID" + exit 0 + ;; + *"label=openshell.ai/managed-by=openshell"*) + [ "$FAKE_PROVIDER_INVENTORY_SCENARIO" != retained-sandbox ] || printf 'nmc-mi-protected-retained\\n' + exit 0 + ;; + *) exit 0 ;; + esac +fi +if [ "$command_name" = network ] && [ "$1" = ls ]; then + if [ "$FAKE_PROVIDER_INVENTORY_SCENARIO" = indeterminate-network ]; then + echo 'network inventory unavailable' >&2 + exit 125 + fi + [ "$FAKE_PROVIDER_INVENTORY_SCENARIO" != retained-network ] || printf 'nemoclaw-managed-pr-retained\\n' + exit 0 +fi +echo "unexpected fake docker command: $command_name $*" >&2 +exit 64`, + ); + return { + ...fixture.env, + FAKE_PROVIDER_COHORT: PROVIDER_COHORT, + FAKE_PROVIDER_CONTAINER_ID: PROVIDER_CONTAINER_ID, + FAKE_PROVIDER_INVENTORY_COMMAND_LOG: commandLog, + FAKE_PROVIDER_INVENTORY_SCENARIO: scenario, + FAKE_PROVIDER_KIND_LABEL: PROTECTED_PROVIDER_KIND_LABEL, + FAKE_PROVIDER_OWNER_LABEL: PROTECTED_PROVIDER_OWNER_LABEL, + FAKE_PROVIDER_REGISTRY_ID: "e".repeat(64), + FAKE_PROVIDER_VLLM_NAME: VLLM_PROVIDER_NAME, + }; +} + +describe("protected managed-image runtime commands", () => { + it("derives bounded provider container names from the protected cohort", () => { + expect(VLLM_PROVIDER_NAME).toBe("nemoclaw-mi-vllm-protected-123-1"); + expect(protectedProviderContainerName("nim", PROVIDER_COHORT)).toBe( + "nemoclaw-mi-nim-protected-123-1", + ); + expect( + protectedProviderContainerName("vllm", "protected-99999999999999999999-9999999999"), + ).toHaveLength(58); + expect(() => protectedProviderContainerName("vllm", "other-123-1")).toThrow( + "invalid protected cohort", + ); + }); + + it("refuses a pre-existing provider container without deleting it", async () => { + const fixture = createReadinessFixture(); + const env = installProviderDocker(fixture, "normal"); + const command = protectedProviderContainerPreflightCommand(VLLM_PROVIDER_NAME); + const result = await fixture.host.command(command.command, command.args, { + artifactName: "provider-preflight-preexisting", + captureLimitBytes: command.captureLimitBytes, + env, + timeoutMs: 5_000, + }); + + expect(result.exitCode).toBe(70); + expect(result.stderr).toContain("refusing to replace pre-existing provider container"); + expect(fs.existsSync(env.FAKE_PROVIDER_REMOVE_LOG!)).toBe(false); + expect(fs.readFileSync(result.artifacts.result, "utf8")).toContain(VLLM_PROVIDER_NAME); + }); + + it.each([ + { case: "missing", stdout: "" }, + { case: "short", stdout: "abc123" }, + { case: "ambiguous", stdout: `${PROVIDER_CONTAINER_ID}\n${"d".repeat(64)}` }, + ])("rejects a $case provider run ID before authority capture", ({ stdout }) => { + expect(() => protectedProviderReportedContainerId(VLLM_PROVIDER_NAME, stdout)).toThrow( + "did not report one full container ID", + ); + }); + + it("removes a provider container by exact authority and verifies ID and name absence", async () => { + const fixture = createReadinessFixture(); + const env = installProviderDocker(fixture, "normal"); + const command = protectedProviderContainerCleanupCommand(providerAuthority()); + const result = await fixture.host.command(command.command, command.args, { + artifactName: "provider-cleanup-exact-authority", + captureLimitBytes: command.captureLimitBytes, + env, + timeoutMs: 5_000, + }); + + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(`name=${VLLM_PROVIDER_NAME}`); + expect(result.stdout).toContain(`id=${PROVIDER_CONTAINER_ID}`); + expect(result.stdout).toContain(`${PROTECTED_PROVIDER_OWNER_LABEL}=${PROVIDER_COHORT}`); + expect(result.stdout).toContain(`${PROTECTED_PROVIDER_KIND_LABEL}=vllm`); + expect(fs.readFileSync(env.FAKE_PROVIDER_STATE!, "utf8").trim()).toBe("absent"); + expect(fs.readFileSync(env.FAKE_PROVIDER_REMOVE_LOG!, "utf8").trim()).toBe( + PROVIDER_CONTAINER_ID, + ); + expect(fs.readFileSync(result.artifacts.result, "utf8")).toContain(PROVIDER_IMAGE_ID); + + const revalidation = protectedProviderContainerPreflightCommand(VLLM_PROVIDER_NAME); + const revalidated = await fixture.host.command(revalidation.command, revalidation.args, { + artifactName: "provider-cleanup-callback-revalidation", + captureLimitBytes: revalidation.captureLimitBytes, + env, + timeoutMs: 5_000, + }); + expect(revalidated.exitCode).toBe(0); + expect(revalidated.stdout).toContain(`name=${VLLM_PROVIDER_NAME}`); + }); + + it.each([ + ["missing", "cleanup evidence is missing"], + ["ambiguous", "cleanup evidence is ambiguous"], + ["reused", "container name"], + ["indeterminate", "inventory is indeterminate"], + ["inspect-indeterminate", "authority inspection is indeterminate"], + ["authority-drift", "cleanup authority drifted"], + ["remove-indeterminate", "cleanup removal is indeterminate"], + ["id-verification-indeterminate", "ID verification is indeterminate"], + ["name-verification-indeterminate", "name verification is indeterminate"], + ] as const)("fails closed for %s provider cleanup evidence", async (scenario, message) => { + const fixture = createReadinessFixture(); + const env = installProviderDocker(fixture, scenario); + const command = protectedProviderContainerCleanupCommand(providerAuthority()); + const result = await fixture.host.command(command.command, command.args, { + artifactName: `provider-cleanup-${scenario}`, + captureLimitBytes: command.captureLimitBytes, + env, + timeoutMs: 5_000, + }); + + expect(result.exitCode).toBe(70); + expect(result.stderr).toContain(message); + const removalEvidence = fs.existsSync(env.FAKE_PROVIDER_REMOVE_LOG!) + ? fs.readFileSync(env.FAKE_PROVIDER_REMOVE_LOG!, "utf8").trim() + : ""; + expect(removalEvidence).toBe( + scenario === "id-verification-indeterminate" || scenario === "name-verification-indeterminate" + ? PROVIDER_CONTAINER_ID + : "", + ); + }); + + it("fails when the provider name is reused after exact-ID removal", async () => { + const fixture = createReadinessFixture(); + const env = installProviderDocker(fixture, "reuse-after-remove"); + const command = protectedProviderContainerCleanupCommand(providerAuthority()); + const result = await fixture.host.command(command.command, command.args, { + artifactName: "provider-cleanup-name-reused-after-remove", + captureLimitBytes: command.captureLimitBytes, + env, + timeoutMs: 5_000, + }); + + expect(result.exitCode).toBe(70); + expect(result.stderr).toContain("retained or replaced"); + expect(fs.readFileSync(env.FAKE_PROVIDER_REMOVE_LOG!, "utf8").trim()).toBe( + PROVIDER_CONTAINER_ID, + ); + }); + + it("excludes a same-cohort registry from the provider cleanup inventory", async () => { + const fixture = createReadinessFixture(); + const env = installProviderInventoryDocker(fixture); + const nimName = protectedProviderContainerName("nim", PROVIDER_COHORT); + const command = protectedProviderFinalInventoryCommand( + PROVIDER_COHORT, + VLLM_PROVIDER_NAME, + nimName, + ); + const result = await fixture.host.command(command.command, command.args, { + artifactName: "provider-final-inventory-with-registry", + captureLimitBytes: command.captureLimitBytes, + env, + timeoutMs: 5_000, + }); + + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(`expected-provider name=${VLLM_PROVIDER_NAME}`); + expect(result.stdout).toContain(`expected-provider name=${nimName}`); + expect(result.stdout).toContain("protected-runtime-inventory-clean"); + const dockerCalls = fs.readFileSync(env.FAKE_PROVIDER_INVENTORY_COMMAND_LOG!, "utf8"); + expect(dockerCalls).toContain(`label=${PROTECTED_PROVIDER_OWNER_LABEL}=${PROVIDER_COHORT}`); + expect(dockerCalls).toContain(`label=${PROTECTED_PROVIDER_KIND_LABEL}`); + }); + + it.each([ + ["retained-provider", "retained state"], + ["retained-name", "retained state"], + ["retained-sandbox", "retained state"], + ["retained-network", "retained state"], + ["indeterminate-provider", "provider-owned container inventory is indeterminate"], + ["indeterminate-network", "managed network inventory is indeterminate"], + ] as const)("fails closed for %s final provider inventory", async (scenario, message) => { + const fixture = createReadinessFixture(); + const env = installProviderInventoryDocker(fixture, scenario); + const command = protectedProviderFinalInventoryCommand( + PROVIDER_COHORT, + VLLM_PROVIDER_NAME, + protectedProviderContainerName("nim", PROVIDER_COHORT), + ); + const result = await fixture.host.command(command.command, command.args, { + artifactName: `provider-final-inventory-${scenario}`, + captureLimitBytes: command.captureLimitBytes, + env, + timeoutMs: 5_000, + }); + + expect(result.exitCode).toBe(70); + expect(result.stderr).toContain(message); + }); + it.each([ { kind: "relative", logPath: "ollama.log" }, { kind: "multiline", logPath: "/tmp/ollama\r\nother.log" }, @@ -139,7 +506,7 @@ describe("protected managed-image readiness commands", () => { env: fixture.env, timeoutMs: 5_000, }); - const vllmCommand = protectedVllmReadinessCommand(); + const vllmCommand = protectedVllmReadinessCommand(VLLM_PROVIDER_NAME); const vllm = await fixture.host.command(vllmCommand.command, vllmCommand.args, { artifactName: "vllm-readiness-success", captureLimitBytes: vllmCommand.captureLimitBytes, @@ -150,7 +517,7 @@ describe("protected managed-image readiness commands", () => { expect(ollama.command.slice(0, 2)).toEqual(["bash", "-c"]); expect(ollama.exitCode).toBe(0); expect(ollama.stdout).toBe("restart_mode=manual\nmanaged-image-ollama-ready\n"); - expect(vllm.command.slice(0, 2)).toEqual(["bash", "-c"]); + expect(vllm.command.slice(0, 4)).toEqual(["bash", "--noprofile", "--norc", "-c"]); expect(vllm.exitCode).toBe(0); expect(vllm.stdout).toBe("managed-image-vllm-ready attempts=1\n"); }); @@ -239,7 +606,7 @@ describe("protected managed-image readiness commands", () => { it("retains the vLLM readiness failure after oversized failing Docker logs", async () => { const fixture = createReadinessFixture(); const sensitiveValue = "oversized-vllm-readiness-sensitive-value"; - const command = protectedVllmReadinessCommand(); + const command = protectedVllmReadinessCommand(VLLM_PROVIDER_NAME); const sourceLog = path.join(fixture.root, "vllm-oversized-source.log"); fs.writeFileSync( sourceLog, @@ -247,13 +614,12 @@ describe("protected managed-image readiness commands", () => { "utf8", ); writeCommand(fixture.binDir, "curl", "/bin/sleep 0.2\nexit 1"); - writeCommand(fixture.binDir, "seq", "printf '1\\n'"); writeCommand(fixture.binDir, "sleep", "exit 0"); writeCommand( fixture.binDir, "docker", `if [ "$1" = "container" ]; then - printf 'true\\n' + printf 'false\\n' exit 0 fi /bin/cat "$FAKE_VLLM_SOURCE_LOG" @@ -277,11 +643,46 @@ exit 42`, expect(Buffer.byteLength(stderrArtifact)).toBeLessThanOrEqual(command.captureLimitBytes + 256); }); - it("reports vLLM diagnostics when the container stops during readiness", async () => { + it("redacts provider-native NIM failure evidence in memory and artifacts", async () => { const fixture = createReadinessFixture(); - const command = protectedVllmReadinessCommand(); + const sensitiveValue = "protected-nim-readiness-sensitive-value"; + const command = protectedNimReadinessCommand( + protectedProviderContainerName("nim", PROVIDER_COHORT), + ); writeCommand(fixture.binDir, "curl", "exit 1"); writeCommand(fixture.binDir, "seq", "printf '1\\n'"); + writeCommand(fixture.binDir, "sleep", "exit 0"); + writeCommand( + fixture.binDir, + "docker", + `if [ "$1" = "container" ]; then + printf 'false\\n' + exit 0 +fi +printf 'provider-native-log %s\\n' "$FAKE_NIM_SECRET" +exit 42`, + ); + + const result = await fixture.host.command(command.command, command.args, { + artifactName: "nim-readiness-provider-failure", + captureLimitBytes: command.captureLimitBytes, + env: { ...fixture.env, FAKE_NIM_SECRET: sensitiveValue }, + redactionValues: [sensitiveValue], + timeoutMs: 5_000, + }); + + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain("provider-native-log [REDACTED]"); + expect(result.stderr).toContain("managed-image-nim-not-ready attempts=1"); + expect(result.stderr).not.toContain(sensitiveValue); + expect(fs.readFileSync(result.artifacts.stderr, "utf8")).not.toContain(sensitiveValue); + expect(fs.readFileSync(result.artifacts.result, "utf8")).not.toContain(sensitiveValue); + }); + + it("reports vLLM diagnostics when the container stops during readiness", async () => { + const fixture = createReadinessFixture(); + const command = protectedVllmReadinessCommand(VLLM_PROVIDER_NAME); + writeCommand(fixture.binDir, "curl", "exit 1"); writeCommand(fixture.binDir, "sleep", "exit 99"); writeCommand( fixture.binDir, @@ -304,4 +705,101 @@ printf 'vllm-stopped-diagnostic\\n'`, expect(result.stderr).toContain("vllm-stopped-diagnostic"); expect(result.stderr).toContain("managed-image-vllm-not-ready attempts=1"); }); + + it("bounds a connected-but-stalled vLLM probe before collecting diagnostics", async () => { + const fixture = createReadinessFixture(); + const command = protectedVllmReadinessCommand(VLLM_PROVIDER_NAME); + const curlArgvLog = path.join(fixture.root, "curl-argv.log"); + writeCommand( + fixture.binDir, + "curl", + `printf '%s\\n' "$@" >>"$FAKE_CURL_ARGV_LOG" +/bin/sleep 0.2 +exit 28`, + ); + writeCommand(fixture.binDir, "sleep", "exit 0"); + writeCommand( + fixture.binDir, + "docker", + `if [ "$1" = "container" ]; then + printf 'false\\n' + exit 0 +fi +printf 'vllm-stalled-probe-diagnostic\\n'`, + ); + + const result = await fixture.host.command(command.command, command.args, { + artifactName: "vllm-readiness-stalled-probe", + captureLimitBytes: command.captureLimitBytes, + env: { ...fixture.env, FAKE_CURL_ARGV_LOG: curlArgvLog }, + timeoutMs: 5_000, + }); + + expect(result.exitCode).toBe(1); + expect(result.timedOut).toBe(false); + expect(fs.readFileSync(curlArgvLog, "utf8").trim().split("\n")).toEqual([ + "-fsS", + "--connect-timeout", + "2", + "--max-time", + "5", + "http://127.0.0.1:8000/v1/models", + ]); + expect(result.stderr).toContain("vllm-stalled-probe-diagnostic"); + expect(result.stderr).toContain("managed-image-vllm-not-ready attempts=1"); + }); + + it("bounds a connected-but-stalled NIM probe and redacts its diagnostics", async () => { + const fixture = createReadinessFixture(); + const sensitiveValue = "protected-nim-stalled-probe-api-key"; + const command = protectedNimReadinessCommand( + protectedProviderContainerName("nim", PROVIDER_COHORT), + ); + const curlArgvLog = path.join(fixture.root, "nim-curl-argv.log"); + writeCommand( + fixture.binDir, + "curl", + `printf '%s\\n' "$@" >>"$FAKE_CURL_ARGV_LOG" +/bin/sleep 0.2 +exit 28`, + ); + writeCommand(fixture.binDir, "sleep", "exit 0"); + writeCommand( + fixture.binDir, + "docker", + `if [ "$1" = "container" ]; then + printf 'false\\n' + exit 0 +fi +printf 'nim-stalled-probe-diagnostic key=%s\\n' "$FAKE_NIM_API_KEY"`, + ); + + const result = await fixture.host.command(command.command, command.args, { + artifactName: "nim-readiness-stalled-probe", + captureLimitBytes: command.captureLimitBytes, + env: { + ...fixture.env, + FAKE_CURL_ARGV_LOG: curlArgvLog, + FAKE_NIM_API_KEY: sensitiveValue, + }, + redactionValues: [sensitiveValue], + timeoutMs: 5_000, + }); + + expect(result.exitCode).toBe(1); + expect(result.timedOut).toBe(false); + expect(fs.readFileSync(curlArgvLog, "utf8").trim().split("\n")).toEqual([ + "-fsS", + "--connect-timeout", + "5", + "--max-time", + "5", + "http://127.0.0.1:8000/v1/models", + ]); + expect(result.stderr).toContain("nim-stalled-probe-diagnostic key=[REDACTED]"); + expect(result.stderr).toContain("managed-image-nim-not-ready attempts=1"); + expect(result.stderr).not.toContain(sensitiveValue); + expect(fs.readFileSync(result.artifacts.stderr, "utf8")).not.toContain(sensitiveValue); + expect(fs.readFileSync(result.artifacts.result, "utf8")).not.toContain(sensitiveValue); + }); }); diff --git a/test/managed-image-protected-runtime-contract.test.ts b/test/managed-image-protected-runtime-contract.test.ts index 5a9a07238b9..c19be1b79ea 100644 --- a/test/managed-image-protected-runtime-contract.test.ts +++ b/test/managed-image-protected-runtime-contract.test.ts @@ -21,6 +21,7 @@ import { assertExactSandboxImage, assertFailedBootstrapContainerCleanup, createProtectedManagedImageBootstrapInput, + failureInjectingAdapter, MANAGED_IMAGE_OPENSHELL_SUPERVISOR_ARGV, type ManagedImageCommandResult, type ManagedImageCommandRunner, @@ -81,6 +82,32 @@ function createManagedImageCommandRunner( } describe("protected managed-image runtime contract", () => { + it("binds the rollback failure adapter to the canonical managed-bootstrap state root", async () => { + const stateRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-protected-rollback-")); + const journalRoot = path.join(stateRoot, "managed-bootstrap"); + try { + const adapter = failureInjectingAdapter( + { + runCaptureOpenshell: () => "", + runOpenshell: () => ({ status: 0, stdout: "", stderr: "" }), + sleepSeconds: () => undefined, + } as never, + stateRoot, + ); + + expect(adapter.awaitBootstrap).toEqual(expect.any(Function)); + expect(fs.statSync(stateRoot).isDirectory()).toBe(true); + expect(fs.existsSync(journalRoot)).toBe(false); + await expect(adapter.recoverUnfinishedTransactions()).resolves.toEqual({ + receipts: [], + failures: [], + }); + expect(fs.statSync(journalRoot).isDirectory()).toBe(true); + } finally { + fs.rmSync(stateRoot, { recursive: true, force: true }); + } + }); + it("binds the public and protected managed-image plans to one supervisor argv (#7744)", () => { const authorityStore = {}; const publicLaunch = resolveOnboardManagedBootstrapLaunch({