fix(runs): a lost checkout under a path named "timeout" was never retried

apps/dispatcher/src/sandbox.ts

@@ -52,7 +52,11 @@ import type { Env } from "./env";
  * `offload-test`) then mis-rendered as a red lint/test verdict. Because
  * `destroy()` is the real teardown, a longer idle window costs extra ONLY on the
  * rare paths that skip finalize, so 10m (the SDK default, and the `LEASE_TTL_MS`
- * run-scale) buys durability across normal inter-step gaps at negligible cost.
+ * run-scale) is close to free. It is a grace period, not a durability
+ * mechanism: it EXTENDS how long an idle container stays awake, and it
+ * guarantees nothing — container disk is ephemeral on every path (ADR-0001
+ * rule 5), so a run that needs its checkout between steps re-establishes it
+ * with `ensureWorkspace`.
  * `sandbox-cf.ts` `isWorkingDirFailure` is the honesty backstop for the residual
  * (eviction / replay beyond this window): it re-classifies a lost-workspace exec
  * as a retryable `ExecFailed`, never a phantom finding.

packages/core/src/primitives/workspace.ts

@@ -53,7 +53,9 @@ export const workspace = (opts: {
  * precondition the failure destroyed is not a retry.
  *
  * Call this INSIDE the retryable step, not before it. That placement is the
- * whole point: the rebuild has to be part of what a retry re-runs.
+ * whole point: the rebuild has to be part of what a retry re-runs. That step's
+ * `retryOn` must list `CheckoutFailed` too, or a clone that flakes mid-rebuild
+ * ends the step one call short of the repair.
  *
  * Costs one `test -d` on the happy path. On a recycled container it is a clone
  * and an install — which is what the step was about to need anyway.

packages/runtime-cf/src/sandbox-cf.test.ts

@@ -429,6 +429,24 @@ describe("makeSandboxCloudflareLive — exec result folding (D)", () => {
       }),
   );
 
+  it.effect("a checkout path containing 'timeout' still fails as ExecFailed, not ExecTimeout", () =>
+    Effect.gen(function* () {
+      currentBox = makeFakeBox({ proc: null });
+      currentBox.exec = vi.fn(async () => ({
+        exitCode: 1,
+        duration: 0,
+        stdout: "",
+        stderr: "Failed to change directory to '/workspace/request-timeout'",
+      }));
+      const exit = yield* Effect.flatMap(SandboxTag, (s) =>
+        s.exec({ command: "cargo test", cwd: "/workspace/request-timeout", env: {} }),
+      ).pipe(Effect.provide(execLayer()), Effect.exit);
+      const err = failureOf<{ _tag: string; stderrTail?: string }>(exit);
+      expect(err?._tag).toBe("ExecFailed");
+      expect(err?.stderrTail).toContain("was missing at exec time");
+    }),
+  );
+
   it("isWorkingDirFailure — fires only on a cwd-set, non-zero, no-stdout, cd-error result", () => {
     const cd = (over: Record<string, unknown> = {}) => ({
       exitCode: 1,

packages/runtime-cf/src/sandbox-cf.ts

@@ -565,6 +565,13 @@ export const makeSandboxCloudflareLive = (
               );
               const err = redactCloneFailure(cause, token);
               if (scrubFailure !== undefined) {
+                // Logged as well as appended: `CheckoutFailed` is retryable in
+                // the runs that call `ensureWorkspace`, so an attempt that fails
+                // here and a later one that succeeds would carry this notice out
+                // of the run's final result and into per-attempt history alone.
+                console.warn(
+                  `sandbox.clone: credential scrub failed for ${targetDir} — a token may remain in .git/config until the next attempt clears it`,
+                );
                 err.message = `${err.message}\n[post-failure credential scrub ALSO failed: ${redact(
                   scrubFailure,
                   [token],
@@ -615,12 +622,19 @@ export const makeSandboxCloudflareLive = (
           // never ran — the checkout did not survive to this exec (container
           // recycled between durable steps). Raise a real ExecFailed rather than
           // fold a phantom non-zero result a `failOnNonZeroExit` run would render
-          // as a lint/test verdict (see `isWorkingDirFailure`). The throw is
-          // classified by the `catch` below.
+          // as a lint/test verdict (see `isWorkingDirFailure`). Built here
+          // rather than in the `catch`, whose timeout regex would match the
+          // `cwd` this message embeds.
           if (isWorkingDirFailure(result, cwd)) {
-            throw new Error(
-              `working directory '${cwd}' was missing at exec time — the checkout did not survive to this step (container recycled). stderr: ${stderr.slice(0, 200)}`,
-            );
+            throw new ExecFailed({
+              exitCode: -1,
+              stderrTail: diagnosticTail(
+                new Error(
+                  `working directory '${cwd}' was missing at exec time — the checkout did not survive to this step (container recycled). stderr: ${stderr.slice(0, 200)}`,
+                ),
+                redactValues,
+              ),
+            });
           }
           // Only a bounded TAIL is inlined in the step's return value, so the
           // Workflow checkpoint stays small (see `inlineTail`). When a viewer
@@ -641,6 +655,7 @@ export const makeSandboxCloudflareLive = (
           // generic launch failure. Prefer any stdout/stderr the throw carried
           // (some SDK errors attach them); else the Error message — this is
           // what Workflows persists via ExecFailed.message (#88).
+          if (cause instanceof ExecFailed) return cause;
           const message = cause instanceof Error ? cause.message : String(cause);
           if (/timed?\s*out|timeout/i.test(message)) {
             return new ExecTimeout({

packages/runtime-cf/src/step-runner-cf.test.ts

@@ -5,10 +5,15 @@
 
 import { Cause, Effect } from "effect";
 import { describe, expect, it } from "vitest";
-import { ExecFailed, ExecTimeout, runEffect } from "@fractalboxdev/flare-dispatch-core";
+import {
+  CheckoutFailed,
+  ExecFailed,
+  ExecTimeout,
+  runEffect,
+} from "@fractalboxdev/flare-dispatch-core";
 import { buildStepConfig, rethrowForRetryPolicy } from "./step-runner-cf";
 
-const thrownFor = async (failure: ExecFailed | ExecTimeout): Promise<unknown> => {
+const thrownFor = async (failure: ExecFailed | ExecTimeout | CheckoutFailed): Promise<unknown> => {
   try {
     await runEffect(Effect.fail(failure));
     throw new Error("expected a throw");
@@ -67,6 +72,20 @@ describe("rethrowForRetryPolicy", () => {
     expect(Cause.isCause(out.cause)).toBe(true);
   });
 
+  it("CheckoutFailed passes through under the runs' policy, so the rebuild's own clone retries", async () => {
+    const thrown = await thrownFor(new CheckoutFailed({ repo: "o/r", sha: "abc", cause: "flake" }));
+    expect(rethrowForRetryPolicy(thrown, ["ExecFailed", "StepFailed", "CheckoutFailed"])).toBe(
+      thrown,
+    );
+  });
+
+  it("CheckoutFailed is non-retryable under a policy that omits it", async () => {
+    const thrown = await thrownFor(new CheckoutFailed({ repo: "o/r", sha: "abc", cause: "flake" }));
+    const out = rethrowForRetryPolicy(thrown, ["ExecFailed", "StepFailed"]) as Error;
+    expect(out).not.toBe(thrown);
+    expect(out.name).toBe("NonRetryableError");
+  });
+
   it("without retryOn every failure stays retryable — the platform default", async () => {
     const thrown = await thrownFor(new ExecTimeout({ timeoutSec: 600, command: "pnpm test" }));
     expect(rethrowForRetryPolicy(thrown, undefined)).toBe(thrown);

runs/README.md

@@ -276,13 +276,23 @@ command in the same missing directory three times and reported a failure about a
 missing directory rather than anything about the code. The retry could never
 have worked: the thing it needed was the thing that was gone.
 
+That classification is made where the failure is raised, not by reading the
+message afterwards. The message embeds the missing `cwd`, and the runtime also
+matches `/timeout/i` on it to spot an SDK timeout — so a checkout under a path
+like `/workspace/request-timeout` used to land as `ExecTimeout`, which `retryOn`
+excludes on purpose, and the step died without ever retrying.
+
 So **every** PR run and every path within it — `offload-test` staged and
 single-exec, `check`, and `oxlint` — calls the `ensureWorkspace` primitive
 inside its retryable step: it
 probes `test -d <dir>/.git` and re-clones when the probe fails. On the happy path
 that is one extra exec of about a second; on a recycled container it is a clone
 and an install, which is what the step was going to need anyway.
 
+Because that re-clone runs inside the step, `retryOn` covers `CheckoutFailed`
+alongside `ExecFailed` and `StepFailed`: a clone that flakes mid-recovery would
+otherwise end the step one call short of the repair it was there to perform.
+
 Isolated stages need no probe: they acquire a workspace inside the retryable
 step already, so a retry rebuilds it by construction.
 

runs/check.test.ts

@@ -121,7 +121,11 @@ describe("check", () => {
         const execStep = handles.executions.steps.find((s) => s.name === "exec");
         expect(execStep?.metadata?.["stepOpts.timeoutSec"]).toBe(1800 + 120);
         expect(execStep?.metadata?.["stepOpts.retries"]).toBe(3);
-        expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]);
+        expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual([
+          "ExecFailed",
+          "StepFailed",
+          "CheckoutFailed",
+        ]);
       }).pipe(Effect.provide(layer));
     },
   );

runs/check.ts

@@ -164,15 +164,20 @@ const STEP_TIMEOUT_HEADROOM_SEC = 120;
 
 /**
  * Platform-failure retries — see the `exec` step. A verdict is never retried:
- * a command that runs and exits non-zero is a normal `ExecResult`, so neither
- * class here can reach one.
+ * a command that runs and exits non-zero is a normal `ExecResult`, so no class
+ * here can reach one.
  *
  * `StepFailed` is included because a platform kill leaves no Effect `Cause` to
  * read a tag from, and the runner falls back to that name — so listing only
  * `ExecFailed` left the purely-platform failure as the one thing not retried.
+ *
+ * `CheckoutFailed` is included because `ensureWorkspace` re-clones inside this
+ * step, in the same weather that destroyed the checkout. A transient clone
+ * failure there would end the step non-retryably, one call short of the
+ * recovery it was invoked to perform.
  */
 const PLATFORM_RETRIES = 3;
-const RETRY_ON = ["ExecFailed", "StepFailed"] as const;
+const RETRY_ON = ["ExecFailed", "StepFailed", "CheckoutFailed"] as const;
 
 /** CONFIG_KV key — strictly per-repo (see header: no global fallback). */
 const commandKey = (repo: string): string => `check.command:${repo}`;

runs/offload-test.test.ts

@@ -126,7 +126,11 @@ describe("offload-test", () => {
       yield* offloadTest.run(baseInput);
       const execStep = handles.executions.steps.find((st) => st.name === "exec");
       expect(execStep?.metadata?.["stepOpts.retries"]).toBe(3);
-      expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]);
+      expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual([
+        "ExecFailed",
+        "StepFailed",
+        "CheckoutFailed",
+      ]);
     }).pipe(Effect.provide(layer));
   });
 
@@ -232,7 +236,11 @@ describe("offload-test", () => {
         // raised by the engine, so `retryOn` cannot gate it: a wedged exec is
         // replayed for the whole budget.
         expect(execStep?.metadata?.["stepOpts.retries"]).toBe(3);
-        expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]);
+        expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual([
+          "ExecFailed",
+          "StepFailed",
+          "CheckoutFailed",
+        ]);
       }).pipe(Effect.provide(layer));
     },
   );
@@ -751,7 +759,11 @@ describe("offload-test staged mode", () => {
         const execWorkspace = handles.executions.steps.find((s) => s.name === "exec-workspace");
         expect(execWorkspace?.metadata?.["stepOpts.timeoutSec"]).toBe(900 + 120);
         expect(execWorkspace?.metadata?.["stepOpts.retries"]).toBe(3);
-        expect(execWorkspace?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]);
+        expect(execWorkspace?.metadata?.["stepOpts.retryOn"]).toEqual([
+          "ExecFailed",
+          "StepFailed",
+          "CheckoutFailed",
+        ]);
         // The suspicious labelled-key-missing fallback is recorded on the
         // stage's step metadata — `workspace` resolved its own key, so only
         // `features` is flagged.
@@ -1155,7 +1167,11 @@ describe("offload-test isolated stages", () => {
         // The retry contract is unchanged — it is the UNIT that changed.
         const execA = handles.executions.steps.find((s) => s.name === "exec-a");
         expect(execA?.metadata?.["stepOpts.retries"]).toBe(3);
-        expect(execA?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]);
+        expect(execA?.metadata?.["stepOpts.retryOn"]).toEqual([
+          "ExecFailed",
+          "StepFailed",
+          "CheckoutFailed",
+        ]);
 
         expect(result.exitCode).toBe(0);
         expect(result.durationMs).toBe(300);

runs/offload-test.ts

@@ -251,16 +251,15 @@ const STEP_TIMEOUT_HEADROOM_SEC = 120;
 const PLATFORM_RETRIES = 3;
 
 /**
- * The classes a stage step retries — both of which are the PLATFORM, never a
- * verdict.
+ * The classes a stage step retries — never a verdict.
  *
  * `ExecFailed` is the obvious one: the command could not run. `StepFailed` is
  * the one that was missing, and its absence made the whole retry policy inert
  * on the failure it was written for.
  *
- * A step body here can fail in exactly two typed ways — `ExecFailed` and
- * `ExecTimeout` — because a command that RUNS and exits non-zero comes back as a
- * normal `ExecResult`. When the platform kills the step outright, no Effect
+ * A command that RUNS and exits non-zero comes back as a normal `ExecResult`,
+ * so no tag listed here can carry one. When the platform kills the step
+ * outright, no Effect
  * `Cause` survives the Workflow boundary, `errorTagOf` falls back to
  * `"StepFailed"`, and a `retryOn` listing only `ExecFailed` classified that as
  * non-retryable — so the one failure mode that is purely the platform's was the
@@ -271,11 +270,26 @@ const PLATFORM_RETRIES = 3;
  * consumer's heaviest stage peaks at 2.2 GiB of 11.9 GiB with 8.4 GB of disk
  * free, so this is not resource pressure being papered over.
  *
+ * `CheckoutFailed` is the third because `ensureWorkspace` now re-clones INSIDE
+ * this step. The clone runs in exactly the weather that made the rebuild
+ * necessary — a container the platform just replaced — so its own transient
+ * failure would otherwise end the step non-retryably, one call short of the
+ * recovery it was invoked to perform.
+ *
+ * `CheckoutFailed` is a catch-all over the whole clone body, so deterministic
+ * failures ride it too — a bad sha, a repo that is gone, and on the substrate
+ * backend the recipe-pin mismatch. On the isolated path a stage's clone is the
+ * run's FIRST, with no earlier `checkout` to have caught those, so this
+ * knowingly spends the retry budget on some failures that cannot succeed: four
+ * clone attempts plus backoff, times the stages running at once, each minting
+ * its own installation token on the container backend. The alternative loses
+ * the repair on every path, so it is the worse trade.
+ *
  * `ExecTimeout` stays OUT, deliberately. Its tag survives the boundary intact
  * whenever there is a Cause to read, so it lands here as itself rather than as
  * `StepFailed` — and a command that outran its ceiling will outrun it again.
  */
-const RETRY_ON = ["ExecFailed", "StepFailed"] as const;
+const RETRY_ON = ["ExecFailed", "StepFailed", "CheckoutFailed"] as const;
 
 const stepTimeoutFor = (execTimeoutSec: number): number =>
   execTimeoutSec + STEP_TIMEOUT_HEADROOM_SEC;
@@ -844,7 +858,8 @@ export const offloadTest = defineRun({
               // `printf '%s\n' '<line>'`, so free vendor text could close the
               // quote. A `[A-Za-z0-9_-]` id and a signed integer cannot.
               const rendered = Option.match(Cause.failureOption(exit.cause), {
-                onSome: (f) => `${String((f as { cause?: unknown }).cause ?? "")} ${String((f as { message?: unknown }).message ?? "")}`,
+                onSome: (f) =>
+                  `${String((f as { cause?: unknown }).cause ?? "")} ${String((f as { message?: unknown }).message ?? "")}`,
                 onNone: () => "",
               });
               const reference = /reference\s*=\s*([A-Za-z0-9_-]{1,64})/.exec(rendered)?.[1];

runs/oxlint.test.ts

@@ -289,7 +289,11 @@ describe("oxlint source determinism", () => {
       // oxlint exiting non-zero is a normal ExecResult decided by the run body,
       // so `retryOn: ExecFailed` can only ever cover the container.
       expect(execStep?.metadata?.["stepOpts.retries"]).toBe(3);
-      expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]);
+      expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual([
+        "ExecFailed",
+        "StepFailed",
+        "CheckoutFailed",
+      ]);
     }).pipe(Effect.provide(layer));
   });
 });

runs/oxlint.ts

@@ -97,15 +97,20 @@ const TIMEOUT_SEC_DEFAULT = 300;
 
 /**
  * Platform-failure retries — see the `exec` step. A verdict is never retried:
- * a command that runs and exits non-zero is a normal `ExecResult`, so neither
- * class here can reach one.
+ * a command that runs and exits non-zero is a normal `ExecResult`, so no class
+ * here can reach one.
  *
  * `StepFailed` is included because a platform kill leaves no Effect `Cause` to
  * read a tag from, and the runner falls back to that name — so listing only
  * `ExecFailed` left the purely-platform failure as the one thing not retried.
+ *
+ * `CheckoutFailed` is included because `ensureWorkspace` re-clones inside this
+ * step, in the same weather that destroyed the checkout. A transient clone
+ * failure there would end the step non-retryably, one call short of the
+ * recovery it was invoked to perform.
  */
 const PLATFORM_RETRIES = 3;
-const RETRY_ON = ["ExecFailed", "StepFailed"] as const;
+const RETRY_ON = ["ExecFailed", "StepFailed", "CheckoutFailed"] as const;
 
 export const oxlint = defineRun({
   name: "oxlint",