feat(runtime-cf): route exec by the container handle, so a run can hold more than one

packages/core/src/fakes/sandbox-fake.ts

@@ -29,7 +29,9 @@ export type CannedProgram = Record<string, CannedExec>;
 
 /** Inspectable record of every call made to the fake. */
 export type SandboxFakeState = {
-  readonly acquired: { image?: string }[];
+  readonly acquired: { image?: string; key?: string }[];
+  /** Container ids a run destroyed explicitly, in order. */
+  readonly destroyed: string[];
   readonly clones: { repo: string; sha: string }[];
   /** every `exec` / `runDetached` call — `env` lets tests assert injection. */
   readonly execs: {
@@ -105,6 +107,7 @@ export const makeSandboxFake = (
 ): { layer: Layer.Layer<Sandbox>; state: SandboxFakeState } => {
   const state: SandboxFakeState = {
     acquired: [],
+    destroyed: [],
     clones: [],
     execs: [],
     exposed: [],
@@ -121,11 +124,29 @@ export const makeSandboxFake = (
   };
 
   const service: SandboxService = {
+    // One container per execution, a DISTINCT one per `key` — the live layer's
+    // rule, mirrored here because the two used to disagree about exactly that.
+    //
+    // This fake minted a fresh id on every `acquire`, so a run that acquired
+    // twice appeared to get two containers; the CF layer returned the
+    // execution's single id both times and the two acquisitions raced for one
+    // filesystem. A suite written against this fake could not have caught it,
+    // and did not: five isolated stages went green here and died in production
+    // with `CheckoutFailed`, having wiped each other's checkout.
     acquire: (opts) =>
       Effect.sync(() => {
-        state.acquired.push({ image: opts.image });
-        containerSeq += 1;
-        return { id: `fake-container-${containerSeq}` } satisfies Container;
+        state.acquired.push({
+          image: opts.image,
+          ...(opts.key !== undefined ? { key: opts.key } : {}),
+        });
+        return {
+          id: opts.key === undefined ? "fake-container" : `fake-container:${opts.key}`,
+        } satisfies Container;
+      }),
+
+    destroy: ({ container }) =>
+      Effect.sync(() => {
+        state.destroyed.push(container.id);
       }),
 
     gitClone: ({ repo, sha }) =>
@@ -206,7 +227,11 @@ export const makeSandboxFake = (
         detachedCommands.set(id, command);
         return Effect.succeed({
           id,
-          container: { id: `fake-container-${containerSeq}` },
+          // The container the process runs IN — the caller's when it named one,
+          // else this execution's. Not a fresh id per launch: a detached
+          // process does not get its own container, and pretending it did was
+          // the same fiction `acquire` used to tell.
+          container: opts.container ?? { id: "fake-container" },
         } satisfies DetachedHandle);
       }),
 

packages/core/src/primitives/primitives.test.ts

@@ -7,6 +7,7 @@
 // covers `workspace`, `installCached`, `sharded`, `bootApp`, `probeHttp`.
 
 import { Effect } from "effect";
+import { sandbox } from "../services/sandbox";
 import { describe, expect, it } from "vitest";
 import { bootApp } from "./boot-app";
 import { installCached, TOOLS } from "./install-cached";
@@ -16,14 +17,35 @@ import { makeCFRuntimeTest } from "../testing";
 import { workspace } from "./workspace";
 
 describe("workspace", () => {
+  it("a keyed acquire is a DIFFERENT container; an unkeyed one is always the same", async () => {
+    const { layer, handles } = makeCFRuntimeTest();
+
+    const [a, b, c] = await Effect.runPromise(
+      Effect.all([
+        sandbox.acquire({}),
+        sandbox.acquire({}),
+        sandbox.acquire({ key: "features" }),
+      ]).pipe(Effect.provide(layer)),
+    );
+
+    // The property the live layer enforces and this fake used to contradict:
+    // unkeyed is the execution's own container, every time. A fake that minted
+    // a fresh id per call let a run look isolated here and share one filesystem
+    // in production.
+    expect(a.id).toBe(b.id);
+    expect(c.id).not.toBe(a.id);
+    expect(handles.sandbox.acquired.map((x) => x.key)).toEqual([undefined, undefined, "features"]);
+  });
+
+
   it("acquires a container, clones the repo, returns { container, dir }", async () => {
     const { layer, handles } = makeCFRuntimeTest();
 
     const out = await Effect.runPromise(
       workspace({ repo: "owner/myrepo", sha: "abc123" }).pipe(Effect.provide(layer)),
     );
 
-    expect(out.container.id).toMatch(/^fake-container-/);
+    expect(out.container.id).toBe("fake-container");
     expect(out.dir).toBe("/workspace/myrepo");
     expect(handles.sandbox.acquired).toHaveLength(1);
     expect(handles.sandbox.clones).toEqual([{ repo: "owner/myrepo", sha: "abc123" }]);

packages/core/src/primitives/workspace.ts

@@ -23,9 +23,24 @@ export const workspace = (opts: {
   sha: string;
   image?: string; // container image override
   install?: boolean; // run installCached after the clone
+  /**
+   * Put this workspace in a container of its own, named by this key.
+   *
+   * Omitted, the checkout lands in the execution's single container, which is
+   * what one workspace per run wants. Named, it lands in a second container —
+   * the difference between two workspaces and two names for one, since
+   * `git clone` clears its target directory and a shared container means the
+   * second checkout deletes the first.
+   *
+   * The container is the caller's to `destroy`; see `sandbox.acquire`.
+   */
+  key?: string;
 }) =>
   Effect.gen(function* () {
-    const container = yield* sandbox.acquire({ image: opts.image });
+    const container = yield* sandbox.acquire({
+      ...(opts.image !== undefined ? { image: opts.image } : {}),
+      ...(opts.key !== undefined ? { key: opts.key } : {}),
+    });
     const dir = yield* sandbox.git.clone({
       repo: opts.repo,
       sha: opts.sha,
@@ -64,6 +79,7 @@ export const ensureWorkspace = (opts: {
   sha: string;
   image?: string;
   install?: boolean;
+  key?: string;
 }) =>
   Effect.gen(function* () {
     const probe = yield* sandbox.exec({

packages/core/src/services/sandbox.ts

@@ -115,7 +115,38 @@ export interface SandboxService {
     image?: string;
     memMB?: number;
     vCPU?: number;
+    /**
+     * A SECOND container for this execution, addressed by name.
+     *
+     * Omitted, `acquire` returns the execution's own container — one per
+     * execution, which is what every run wanted until one wanted several. A
+     * `key` derives a distinct container id from the execution id and this
+     * name, so a run that needs work isolated from its own other work (stages
+     * that would otherwise race for one checkout directory) can have it.
+     *
+     * The key is part of an id that must stay DNS-safe and short — the live
+     * layer runs it through the same normalisation as the execution id, so an
+     * arbitrary string is safe to pass, but a long one is truncated into a
+     * digest and stops being readable in logs. Keep it short and stable: a
+     * stage label, a shard index.
+     *
+     * Reaping is the caller's. A keyed container is not covered by the
+     * dispatcher's end-of-run teardown, which destroys the execution's own id
+     * and cannot know what a run named; call `destroy` when the work is done,
+     * or accept the `sleepAfter` window as the backstop.
+     */
+    key?: string;
   }) => Effect.Effect<Container, ContainerLaunchFailed | ContainerBusy>;
+  /**
+   * Destroy a container now rather than at `sleepAfter`.
+   *
+   * Best-effort and idempotent: destroying an already-dead container, or one
+   * the SDK never provisioned, succeeds. The point is the bill — a container
+   * idles for the full `sleepAfter` window after its last command and is
+   * charged wall-clock for it, which on a CI-shaped workload is most of the
+   * cost of the ones a run acquired and finished with early.
+   */
+  readonly destroy: (opts: { container: Container }) => Effect.Effect<void>;
   readonly gitClone: (opts: {
     repo: string;
     sha: string;
@@ -168,8 +199,9 @@ export class Sandbox extends Context.Tag("@fractalboxdev/flare-dispatch-core/San
  * `Effect.flatMap(Sandbox, (s) => s.exec(...))`.
  */
 export const sandbox = {
-  acquire: (opts: { image?: string; memMB?: number; vCPU?: number } = {}) =>
+  acquire: (opts: { image?: string; memMB?: number; vCPU?: number; key?: string } = {}) =>
     Effect.flatMap(Sandbox, (s) => s.acquire(opts)),
+  destroy: (opts: { container: Container }) => Effect.flatMap(Sandbox, (s) => s.destroy(opts)),
   git: {
     clone: (opts: { repo: string; sha: string; container?: Container }) =>
       Effect.flatMap(Sandbox, (s) => s.gitClone(opts)),

packages/runtime-cf/src/sandbox-cf.test.ts

@@ -88,8 +88,17 @@ const { FakeSessionTerminatedError } = vi.hoisted(() => ({
 // returns whatever the current test installed via `currentBox`. The error class
 // is needed for `execToResult`'s `instanceof` recovery path.
 let currentBox: ReturnType<typeof makeFakeBox>;
+// Every id the Layer routed by, in order. The mock used to ignore its `id`
+// argument entirely, which is why no test could see that `exec` was resolving
+// the execution's container rather than the handle it was passed — the defect
+// that made five "isolated" stages share one filesystem.
+const requestedSandboxIds: string[] = [];
+
 vi.mock("@cloudflare/sandbox", () => ({
-  getSandbox: () => currentBox,
+  getSandbox: (_ns: unknown, id: string) => {
+    requestedSandboxIds.push(id);
+    return currentBox;
+  },
   SessionTerminatedError: FakeSessionTerminatedError,
 }));
 
@@ -278,6 +287,63 @@ describe("makeSandboxCloudflareLive — exposePort (C)", () => {
 // shell then exited) is a result the run can report + upload artifacts for;
 // only a could-not-launch error is an Effect failure. This is what lets a
 // failing `playwright-demo` still surface its videoUri/logUri.
+describe("makeSandboxCloudflareLive — container routing", () => {
+  const routingLayer = () => makeSandboxCloudflareLive(ns, makeBucket().bucket, "route-1");
+
+  it.effect("acquire is one container per execution, and a distinct one per key", () =>
+    Effect.gen(function* () {
+      currentBox = makeFakeBox({ proc: null });
+      const [a, b, keyed, keyed2] = yield* Effect.flatMap(SandboxTag, (s) =>
+        Effect.all([
+          s.acquire({}),
+          s.acquire({}),
+          s.acquire({ key: "features" }),
+          s.acquire({ key: "workspace" }),
+        ]),
+      ).pipe(Effect.provide(routingLayer()));
+
+      expect(a.id).toBe(b.id);
+      expect(keyed.id).not.toBe(a.id);
+      expect(keyed2.id).not.toBe(keyed.id);
+      // Ids stay DNS-safe and inside the preview-URL budget — the constraint
+      // `previewSafeSandboxId` exists for, now applied to keyed ids too.
+      for (const c of [a, keyed, keyed2]) {
+        expect(c.id).toMatch(/^[a-z0-9][a-z0-9-]*[a-z0-9]$/);
+        expect(c.id.length).toBeLessThanOrEqual(40);
+      }
+    }),
+  );
+
+  it.effect("exec routes by the handle it is given, not by the execution", () =>
+    Effect.gen(function* () {
+      currentBox = makeFakeBox({ proc: null });
+      currentBox.exec = vi.fn(async () => ({ exitCode: 0, duration: 1, stdout: "", stderr: "" }));
+
+      yield* Effect.flatMap(SandboxTag, (s) =>
+        Effect.gen(function* () {
+          const own = yield* s.acquire({});
+          const keyed = yield* s.acquire({ key: "features" });
+          requestedSandboxIds.length = 0;
+          yield* s.exec({ command: "one", container: own });
+          yield* s.exec({ command: "two", container: keyed });
+          yield* s.exec({ command: "three" });
+          return { own, keyed };
+        }),
+      ).pipe(
+        Effect.provide(routingLayer()),
+        Effect.flatMap(({ own, keyed }) =>
+          Effect.sync(() => {
+            // The whole bug in one assertion: the second exec must reach the
+            // keyed container, and an exec with no handle still reaches the
+            // execution's own.
+            expect(requestedSandboxIds).toEqual([own.id, keyed.id, own.id]);
+          }),
+        ),
+      );
+    }),
+  );
+});
+
 describe("makeSandboxCloudflareLive — exec result folding (D)", () => {
   const execLayer = () => makeSandboxCloudflareLive(ns, makeBucket().bucket, "exec-1");