Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1268,10 +1268,13 @@ No PR E2E controller dispatches the risk plan.
The `full-e2e` target enforces a separate hard acceptance contract for the
first fresh onboarding path in that job. It measures from the onboard root span
(a conservative anchor before wizard step `[1/8]`) through the first non-empty
agent response, requires the local BuildKit prebuild for the NemoClaw-generated
context without a gateway-builder fallback, enforces the calibrated root and
phase limits in the budget file, and limits the longest onboard output gap to
60 seconds. A violation fails
agent response and reads the registered workload receipt. A `legacy-dockerfile`
receipt requires the local BuildKit prebuild without a gateway-builder fallback.
A `managed-image` receipt instead requires an exact digest that matches the
registered sandbox image tag, a non-empty publication cohort, and an exact
40-character source revision, and it forbids a local BuildKit prebuild. Both
paths enforce the calibrated root and phase limits in the budget file and limit
the longest onboard output gap to 60 seconds. A violation fails
`full-e2e`, and the target writes its evidence to `onboard-progress-budget.json`.
The artifact records the first-turn command wall clock and OpenClaw's internal
agent duration separately. Older or malformed OpenClaw output records an
Expand Down
39 changes: 39 additions & 0 deletions test/e2e/live/full-e2e-workload-evidence.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { readManagedWorkloadAuthority } from "../../../src/lib/onboard/workload/authority.ts";
import { load as loadSandboxRegistry } from "../../../src/lib/state/registry/persistence.ts";

export function readFullE2eColdWorkloadEvidence(
sandboxName: string,
usedBuildKitPrebuild: boolean,
) {
const entry = loadSandboxRegistry().sandboxes[sandboxName];
if (!entry) {
throw new Error(`full E2E sandbox '${sandboxName}' is missing from the registry`);
}

const managedAuthority = readManagedWorkloadAuthority(entry);
if (managedAuthority) {
if (usedBuildKitPrebuild) {
throw new Error("managed-image cold onboarding must not use a local BuildKit prebuild");
}
return {
kind: managedAuthority.receipt.kind,
reference: managedAuthority.receipt.reference,
sourceCohort: managedAuthority.receipt.sourceCohort,
sourceRevision: managedAuthority.receipt.sourceRevision,
} as const;
}

if (entry.workload?.kind !== "legacy-dockerfile") {
throw new Error("full E2E cold onboarding must register a supported workload receipt");
}
if (!usedBuildKitPrebuild) {
throw new Error("legacy Dockerfile cold onboarding must use the local BuildKit prebuild");
}
return {
kind: entry.workload.kind,
reference: entry.workload.reference,
} as const;
}
4 changes: 3 additions & 1 deletion test/e2e/live/full-e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import {
fullE2eInferenceProbeEvidence,
runFullE2eInferenceProbe,
} from "./full-e2e-inference-probe.ts";
import { readFullE2eColdWorkloadEvidence } from "./full-e2e-workload-evidence.ts";
import { runOpenClawLaunchReadinessLeaseTurns } from "./launch-agent-turn.ts";
import { bindApprovedPrBaseForBaseImageComparison } from "./pr-base-comparison.ts";

Expand Down Expand Up @@ -263,6 +264,7 @@ async function assertColdOnboardPerformance(input: {
const maxSilenceMs = maximumOutputSilenceMs(traceWindow, input.outputEvents);
const maxSilenceSecs = Math.ceil(maxSilenceMs / 1_000);
const rootEndToInstallCompletionMs = input.installCompletedAtMs - traceWindow.finishedAtMs;
const workload = readFullE2eColdWorkloadEvidence(SANDBOX_NAME, usedBuildKitPrebuild);

const firstTurnStartedAtMs = Date.now();
const turn = await input.sandbox.execShell(
Expand Down Expand Up @@ -333,13 +335,13 @@ async function assertColdOnboardPerformance(input: {
maxSilenceBudgetSecs: MAX_SILENCE_SECS,
buildKitFallback,
usedBuildKitPrebuild,
workload,
classicBuildSteps,
responseChars,
});

expect(plain, "expected literal wizard step [1/8] in installer output").toContain("[1/8]");
expect(buildKitFallback, "expected no fallback from BuildKit to the gateway builder").toBe(false);
expect(usedBuildKitPrebuild, "expected the cold install to use BuildKit").toBe(true);
expect(classicBuildSteps, "expected no classic per-instruction build steps").toBe(0);
expect(
maxSilenceSecs,
Expand Down
12 changes: 0 additions & 12 deletions test/e2e/live/hermes-gpu-startup-proof.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,6 @@ export async function assertHermesGpuStartupProof({
expect(installText).not.toContain("Reusing existing Docker-driver gateway");
expect(installText).not.toContain("[reuse] Skipping gateway (running)");
if (gpuRoute === "compatibility-only") {
expect(installText).toContain(
"Recreating OpenShell Docker sandbox container with NVIDIA GPU access",
);
expect(installText).toContain("Docker container mode selected:");
for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) {
expect(installText).not.toContain(fragment);
Expand All @@ -65,20 +62,11 @@ export async function assertHermesGpuStartupProof({
for (const fragment of HERMES_GPU_FALLBACK_DISCLOSURE_FRAGMENTS) {
expect(installText).toContain(fragment);
}
expect(installText).toContain(
"Recreating OpenShell Docker sandbox container with NVIDIA GPU access",
);
expect(installText).toContain("Docker container mode selected:");
} else {
expect(installText).toContain(
"Direct sandbox GPU enabled; allowing OpenShell GPU policy enrichment.",
);
expect(installText).not.toContain(
"Recreating OpenShell Docker sandbox container with NVIDIA GPU access",
);
expect(installText).toContain(
"Recreating OpenShell Docker sandbox container with restart-safe startup",
);
expect(installText).toContain(
"Docker container mode selected: persistent sandbox startup command",
);
Expand Down
Loading