diff --git a/apps/dispatcher/src/sandbox.ts b/apps/dispatcher/src/sandbox.ts index e6bc6fa..f40cb5f 100644 --- a/apps/dispatcher/src/sandbox.ts +++ b/apps/dispatcher/src/sandbox.ts @@ -52,7 +52,11 @@ import type { Env } from "./env"; * `offload-test`) then mis-rendered as a red lint/test verdict. Because * `destroy()` is the real teardown, a longer idle window costs extra ONLY on the * rare paths that skip finalize, so 10m (the SDK default, and the `LEASE_TTL_MS` - * run-scale) buys durability across normal inter-step gaps at negligible cost. + * run-scale) is close to free. It is a grace period, not a durability + * mechanism: it EXTENDS how long an idle container stays awake, and it + * guarantees nothing — container disk is ephemeral on every path (ADR-0001 + * rule 5), so a run that needs its checkout between steps re-establishes it + * with `ensureWorkspace`. * `sandbox-cf.ts` `isWorkingDirFailure` is the honesty backstop for the residual * (eviction / replay beyond this window): it re-classifies a lost-workspace exec * as a retryable `ExecFailed`, never a phantom finding. diff --git a/packages/core/src/primitives/workspace.ts b/packages/core/src/primitives/workspace.ts index 726c7c6..ed236aa 100644 --- a/packages/core/src/primitives/workspace.ts +++ b/packages/core/src/primitives/workspace.ts @@ -53,7 +53,9 @@ export const workspace = (opts: { * precondition the failure destroyed is not a retry. * * Call this INSIDE the retryable step, not before it. That placement is the - * whole point: the rebuild has to be part of what a retry re-runs. + * whole point: the rebuild has to be part of what a retry re-runs. That step's + * `retryOn` must list `CheckoutFailed` too, or a clone that flakes mid-rebuild + * ends the step one call short of the repair. * * Costs one `test -d` on the happy path. On a recycled container it is a clone * and an install — which is what the step was about to need anyway. diff --git a/packages/runtime-cf/src/sandbox-cf.test.ts b/packages/runtime-cf/src/sandbox-cf.test.ts index b60165b..bb9b506 100644 --- a/packages/runtime-cf/src/sandbox-cf.test.ts +++ b/packages/runtime-cf/src/sandbox-cf.test.ts @@ -429,6 +429,24 @@ describe("makeSandboxCloudflareLive — exec result folding (D)", () => { }), ); + it.effect("a checkout path containing 'timeout' still fails as ExecFailed, not ExecTimeout", () => + Effect.gen(function* () { + currentBox = makeFakeBox({ proc: null }); + currentBox.exec = vi.fn(async () => ({ + exitCode: 1, + duration: 0, + stdout: "", + stderr: "Failed to change directory to '/workspace/request-timeout'", + })); + const exit = yield* Effect.flatMap(SandboxTag, (s) => + s.exec({ command: "cargo test", cwd: "/workspace/request-timeout", env: {} }), + ).pipe(Effect.provide(execLayer()), Effect.exit); + const err = failureOf<{ _tag: string; stderrTail?: string }>(exit); + expect(err?._tag).toBe("ExecFailed"); + expect(err?.stderrTail).toContain("was missing at exec time"); + }), + ); + it("isWorkingDirFailure — fires only on a cwd-set, non-zero, no-stdout, cd-error result", () => { const cd = (over: Record = {}) => ({ exitCode: 1, diff --git a/packages/runtime-cf/src/sandbox-cf.ts b/packages/runtime-cf/src/sandbox-cf.ts index 2666d89..31414fc 100644 --- a/packages/runtime-cf/src/sandbox-cf.ts +++ b/packages/runtime-cf/src/sandbox-cf.ts @@ -565,6 +565,13 @@ export const makeSandboxCloudflareLive = ( ); const err = redactCloneFailure(cause, token); if (scrubFailure !== undefined) { + // Logged as well as appended: `CheckoutFailed` is retryable in + // the runs that call `ensureWorkspace`, so an attempt that fails + // here and a later one that succeeds would carry this notice out + // of the run's final result and into per-attempt history alone. + console.warn( + `sandbox.clone: credential scrub failed for ${targetDir} — a token may remain in .git/config until the next attempt clears it`, + ); err.message = `${err.message}\n[post-failure credential scrub ALSO failed: ${redact( scrubFailure, [token], @@ -615,12 +622,19 @@ export const makeSandboxCloudflareLive = ( // never ran — the checkout did not survive to this exec (container // recycled between durable steps). Raise a real ExecFailed rather than // fold a phantom non-zero result a `failOnNonZeroExit` run would render - // as a lint/test verdict (see `isWorkingDirFailure`). The throw is - // classified by the `catch` below. + // as a lint/test verdict (see `isWorkingDirFailure`). Built here + // rather than in the `catch`, whose timeout regex would match the + // `cwd` this message embeds. if (isWorkingDirFailure(result, cwd)) { - throw new Error( - `working directory '${cwd}' was missing at exec time — the checkout did not survive to this step (container recycled). stderr: ${stderr.slice(0, 200)}`, - ); + throw new ExecFailed({ + exitCode: -1, + stderrTail: diagnosticTail( + new Error( + `working directory '${cwd}' was missing at exec time — the checkout did not survive to this step (container recycled). stderr: ${stderr.slice(0, 200)}`, + ), + redactValues, + ), + }); } // Only a bounded TAIL is inlined in the step's return value, so the // Workflow checkpoint stays small (see `inlineTail`). When a viewer @@ -641,6 +655,7 @@ export const makeSandboxCloudflareLive = ( // generic launch failure. Prefer any stdout/stderr the throw carried // (some SDK errors attach them); else the Error message — this is // what Workflows persists via ExecFailed.message (#88). + if (cause instanceof ExecFailed) return cause; const message = cause instanceof Error ? cause.message : String(cause); if (/timed?\s*out|timeout/i.test(message)) { return new ExecTimeout({ diff --git a/packages/runtime-cf/src/step-runner-cf.test.ts b/packages/runtime-cf/src/step-runner-cf.test.ts index e671afa..a7a3567 100644 --- a/packages/runtime-cf/src/step-runner-cf.test.ts +++ b/packages/runtime-cf/src/step-runner-cf.test.ts @@ -5,10 +5,15 @@ import { Cause, Effect } from "effect"; import { describe, expect, it } from "vitest"; -import { ExecFailed, ExecTimeout, runEffect } from "@fractalboxdev/flare-dispatch-core"; +import { + CheckoutFailed, + ExecFailed, + ExecTimeout, + runEffect, +} from "@fractalboxdev/flare-dispatch-core"; import { buildStepConfig, rethrowForRetryPolicy } from "./step-runner-cf"; -const thrownFor = async (failure: ExecFailed | ExecTimeout): Promise => { +const thrownFor = async (failure: ExecFailed | ExecTimeout | CheckoutFailed): Promise => { try { await runEffect(Effect.fail(failure)); throw new Error("expected a throw"); @@ -67,6 +72,20 @@ describe("rethrowForRetryPolicy", () => { expect(Cause.isCause(out.cause)).toBe(true); }); + it("CheckoutFailed passes through under the runs' policy, so the rebuild's own clone retries", async () => { + const thrown = await thrownFor(new CheckoutFailed({ repo: "o/r", sha: "abc", cause: "flake" })); + expect(rethrowForRetryPolicy(thrown, ["ExecFailed", "StepFailed", "CheckoutFailed"])).toBe( + thrown, + ); + }); + + it("CheckoutFailed is non-retryable under a policy that omits it", async () => { + const thrown = await thrownFor(new CheckoutFailed({ repo: "o/r", sha: "abc", cause: "flake" })); + const out = rethrowForRetryPolicy(thrown, ["ExecFailed", "StepFailed"]) as Error; + expect(out).not.toBe(thrown); + expect(out.name).toBe("NonRetryableError"); + }); + it("without retryOn every failure stays retryable — the platform default", async () => { const thrown = await thrownFor(new ExecTimeout({ timeoutSec: 600, command: "pnpm test" })); expect(rethrowForRetryPolicy(thrown, undefined)).toBe(thrown); diff --git a/runs/README.md b/runs/README.md index a1fbe95..092cbc1 100644 --- a/runs/README.md +++ b/runs/README.md @@ -276,6 +276,12 @@ command in the same missing directory three times and reported a failure about a missing directory rather than anything about the code. The retry could never have worked: the thing it needed was the thing that was gone. +That classification is made where the failure is raised, not by reading the +message afterwards. The message embeds the missing `cwd`, and the runtime also +matches `/timeout/i` on it to spot an SDK timeout — so a checkout under a path +like `/workspace/request-timeout` used to land as `ExecTimeout`, which `retryOn` +excludes on purpose, and the step died without ever retrying. + So **every** PR run and every path within it — `offload-test` staged and single-exec, `check`, and `oxlint` — calls the `ensureWorkspace` primitive inside its retryable step: it @@ -283,6 +289,10 @@ probes `test -d /.git` and re-clones when the probe fails. On the happy pat that is one extra exec of about a second; on a recycled container it is a clone and an install, which is what the step was going to need anyway. +Because that re-clone runs inside the step, `retryOn` covers `CheckoutFailed` +alongside `ExecFailed` and `StepFailed`: a clone that flakes mid-recovery would +otherwise end the step one call short of the repair it was there to perform. + Isolated stages need no probe: they acquire a workspace inside the retryable step already, so a retry rebuilds it by construction. diff --git a/runs/check.test.ts b/runs/check.test.ts index feb65cb..8b0ef1a 100644 --- a/runs/check.test.ts +++ b/runs/check.test.ts @@ -121,7 +121,11 @@ describe("check", () => { const execStep = handles.executions.steps.find((s) => s.name === "exec"); expect(execStep?.metadata?.["stepOpts.timeoutSec"]).toBe(1800 + 120); expect(execStep?.metadata?.["stepOpts.retries"]).toBe(3); - expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]); + expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual([ + "ExecFailed", + "StepFailed", + "CheckoutFailed", + ]); }).pipe(Effect.provide(layer)); }, ); diff --git a/runs/check.ts b/runs/check.ts index fd5fa29..484c0cf 100644 --- a/runs/check.ts +++ b/runs/check.ts @@ -164,15 +164,20 @@ const STEP_TIMEOUT_HEADROOM_SEC = 120; /** * Platform-failure retries — see the `exec` step. A verdict is never retried: - * a command that runs and exits non-zero is a normal `ExecResult`, so neither - * class here can reach one. + * a command that runs and exits non-zero is a normal `ExecResult`, so no class + * here can reach one. * * `StepFailed` is included because a platform kill leaves no Effect `Cause` to * read a tag from, and the runner falls back to that name — so listing only * `ExecFailed` left the purely-platform failure as the one thing not retried. + * + * `CheckoutFailed` is included because `ensureWorkspace` re-clones inside this + * step, in the same weather that destroyed the checkout. A transient clone + * failure there would end the step non-retryably, one call short of the + * recovery it was invoked to perform. */ const PLATFORM_RETRIES = 3; -const RETRY_ON = ["ExecFailed", "StepFailed"] as const; +const RETRY_ON = ["ExecFailed", "StepFailed", "CheckoutFailed"] as const; /** CONFIG_KV key — strictly per-repo (see header: no global fallback). */ const commandKey = (repo: string): string => `check.command:${repo}`; diff --git a/runs/offload-test.test.ts b/runs/offload-test.test.ts index 6101e57..ca003dc 100644 --- a/runs/offload-test.test.ts +++ b/runs/offload-test.test.ts @@ -126,7 +126,11 @@ describe("offload-test", () => { yield* offloadTest.run(baseInput); const execStep = handles.executions.steps.find((st) => st.name === "exec"); expect(execStep?.metadata?.["stepOpts.retries"]).toBe(3); - expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]); + expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual([ + "ExecFailed", + "StepFailed", + "CheckoutFailed", + ]); }).pipe(Effect.provide(layer)); }); @@ -232,7 +236,11 @@ describe("offload-test", () => { // raised by the engine, so `retryOn` cannot gate it: a wedged exec is // replayed for the whole budget. expect(execStep?.metadata?.["stepOpts.retries"]).toBe(3); - expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]); + expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual([ + "ExecFailed", + "StepFailed", + "CheckoutFailed", + ]); }).pipe(Effect.provide(layer)); }, ); @@ -751,7 +759,11 @@ describe("offload-test staged mode", () => { const execWorkspace = handles.executions.steps.find((s) => s.name === "exec-workspace"); expect(execWorkspace?.metadata?.["stepOpts.timeoutSec"]).toBe(900 + 120); expect(execWorkspace?.metadata?.["stepOpts.retries"]).toBe(3); - expect(execWorkspace?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]); + expect(execWorkspace?.metadata?.["stepOpts.retryOn"]).toEqual([ + "ExecFailed", + "StepFailed", + "CheckoutFailed", + ]); // The suspicious labelled-key-missing fallback is recorded on the // stage's step metadata — `workspace` resolved its own key, so only // `features` is flagged. @@ -1155,7 +1167,11 @@ describe("offload-test isolated stages", () => { // The retry contract is unchanged — it is the UNIT that changed. const execA = handles.executions.steps.find((s) => s.name === "exec-a"); expect(execA?.metadata?.["stepOpts.retries"]).toBe(3); - expect(execA?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]); + expect(execA?.metadata?.["stepOpts.retryOn"]).toEqual([ + "ExecFailed", + "StepFailed", + "CheckoutFailed", + ]); expect(result.exitCode).toBe(0); expect(result.durationMs).toBe(300); diff --git a/runs/offload-test.ts b/runs/offload-test.ts index afdb1ae..2d8d446 100644 --- a/runs/offload-test.ts +++ b/runs/offload-test.ts @@ -251,16 +251,15 @@ const STEP_TIMEOUT_HEADROOM_SEC = 120; const PLATFORM_RETRIES = 3; /** - * The classes a stage step retries — both of which are the PLATFORM, never a - * verdict. + * The classes a stage step retries — never a verdict. * * `ExecFailed` is the obvious one: the command could not run. `StepFailed` is * the one that was missing, and its absence made the whole retry policy inert * on the failure it was written for. * - * A step body here can fail in exactly two typed ways — `ExecFailed` and - * `ExecTimeout` — because a command that RUNS and exits non-zero comes back as a - * normal `ExecResult`. When the platform kills the step outright, no Effect + * A command that RUNS and exits non-zero comes back as a normal `ExecResult`, + * so no tag listed here can carry one. When the platform kills the step + * outright, no Effect * `Cause` survives the Workflow boundary, `errorTagOf` falls back to * `"StepFailed"`, and a `retryOn` listing only `ExecFailed` classified that as * non-retryable — so the one failure mode that is purely the platform's was the @@ -271,11 +270,26 @@ const PLATFORM_RETRIES = 3; * consumer's heaviest stage peaks at 2.2 GiB of 11.9 GiB with 8.4 GB of disk * free, so this is not resource pressure being papered over. * + * `CheckoutFailed` is the third because `ensureWorkspace` now re-clones INSIDE + * this step. The clone runs in exactly the weather that made the rebuild + * necessary — a container the platform just replaced — so its own transient + * failure would otherwise end the step non-retryably, one call short of the + * recovery it was invoked to perform. + * + * `CheckoutFailed` is a catch-all over the whole clone body, so deterministic + * failures ride it too — a bad sha, a repo that is gone, and on the substrate + * backend the recipe-pin mismatch. On the isolated path a stage's clone is the + * run's FIRST, with no earlier `checkout` to have caught those, so this + * knowingly spends the retry budget on some failures that cannot succeed: four + * clone attempts plus backoff, times the stages running at once, each minting + * its own installation token on the container backend. The alternative loses + * the repair on every path, so it is the worse trade. + * * `ExecTimeout` stays OUT, deliberately. Its tag survives the boundary intact * whenever there is a Cause to read, so it lands here as itself rather than as * `StepFailed` — and a command that outran its ceiling will outrun it again. */ -const RETRY_ON = ["ExecFailed", "StepFailed"] as const; +const RETRY_ON = ["ExecFailed", "StepFailed", "CheckoutFailed"] as const; const stepTimeoutFor = (execTimeoutSec: number): number => execTimeoutSec + STEP_TIMEOUT_HEADROOM_SEC; @@ -844,7 +858,8 @@ export const offloadTest = defineRun({ // `printf '%s\n' ''`, so free vendor text could close the // quote. A `[A-Za-z0-9_-]` id and a signed integer cannot. const rendered = Option.match(Cause.failureOption(exit.cause), { - onSome: (f) => `${String((f as { cause?: unknown }).cause ?? "")} ${String((f as { message?: unknown }).message ?? "")}`, + onSome: (f) => + `${String((f as { cause?: unknown }).cause ?? "")} ${String((f as { message?: unknown }).message ?? "")}`, onNone: () => "", }); const reference = /reference\s*=\s*([A-Za-z0-9_-]{1,64})/.exec(rendered)?.[1]; diff --git a/runs/oxlint.test.ts b/runs/oxlint.test.ts index 56750ea..166aee7 100644 --- a/runs/oxlint.test.ts +++ b/runs/oxlint.test.ts @@ -289,7 +289,11 @@ describe("oxlint source determinism", () => { // oxlint exiting non-zero is a normal ExecResult decided by the run body, // so `retryOn: ExecFailed` can only ever cover the container. expect(execStep?.metadata?.["stepOpts.retries"]).toBe(3); - expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual(["ExecFailed", "StepFailed"]); + expect(execStep?.metadata?.["stepOpts.retryOn"]).toEqual([ + "ExecFailed", + "StepFailed", + "CheckoutFailed", + ]); }).pipe(Effect.provide(layer)); }); }); diff --git a/runs/oxlint.ts b/runs/oxlint.ts index aa95c86..9524144 100644 --- a/runs/oxlint.ts +++ b/runs/oxlint.ts @@ -97,15 +97,20 @@ const TIMEOUT_SEC_DEFAULT = 300; /** * Platform-failure retries — see the `exec` step. A verdict is never retried: - * a command that runs and exits non-zero is a normal `ExecResult`, so neither - * class here can reach one. + * a command that runs and exits non-zero is a normal `ExecResult`, so no class + * here can reach one. * * `StepFailed` is included because a platform kill leaves no Effect `Cause` to * read a tag from, and the runner falls back to that name — so listing only * `ExecFailed` left the purely-platform failure as the one thing not retried. + * + * `CheckoutFailed` is included because `ensureWorkspace` re-clones inside this + * step, in the same weather that destroyed the checkout. A transient clone + * failure there would end the step non-retryably, one call short of the + * recovery it was invoked to perform. */ const PLATFORM_RETRIES = 3; -const RETRY_ON = ["ExecFailed", "StepFailed"] as const; +const RETRY_ON = ["ExecFailed", "StepFailed", "CheckoutFailed"] as const; export const oxlint = defineRun({ name: "oxlint", diff --git a/specs/adr/0001-cloudflare-workflows-scope.md b/specs/adr/0001-cloudflare-workflows-scope.md index f29125b..b0238e4 100644 --- a/specs/adr/0001-cloudflare-workflows-scope.md +++ b/specs/adr/0001-cloudflare-workflows-scope.md @@ -60,12 +60,13 @@ Three repo facts the decision has to account for: - **Replay re-executes anything not inside a completed step.** `self-heal-pr.ts:186-202` uses `runDetached` + `waitForExit` rather than one long `exec` precisely so a Worker eviction mid-agent does not re-spawn the agent and double-spend the model budget. The - container's `sleepAfter = "10m"` (`apps/dispatcher/src/sandbox.ts:60`) exists for the - same reason: the container filesystem is shared state *across* durable steps. + container's `sleepAfter = "10m"` (`apps/dispatcher/src/sandbox.ts:60`) narrows the idle + window in which a container is reclaimed between steps. It does not make the container + filesystem durable, and nothing does — see rule 5. ## Decision -Four rules govern how runs use Workflows. +Five rules govern how runs use Workflows. **1. One dispatch, one instance, and the instance id is the semantic idempotency key.** Every entry point (webhook, Action, schedule, child spawn) names its instance from the @@ -90,6 +91,30 @@ humans also act on — a GitHub issue, a PR, a deployment — the authoritative that entity's own status field (labels, PR state, deployment status), and each webhook event starts a short instance that reads state, acts, and writes state back. +**5. Container disk is not durable state, so a step re-establishes what it needs.** A +checkout does not survive between durable steps by right. Cloudflare states the +mechanism plainly: "All disk is ephemeral. When a Container instance goes to sleep, the +next time it is started, it will have a fresh disk as defined by its container image" +([Containers FAQ](https://developers.cloudflare.com/containers/faq/)). The same pages +give no guaranteed minimum runtime, restart an out-of-memory instance, and terminate +instances on host restarts. A `checkout` step that already completed is never re-run by +replay, so on the container backend every attempt after a container swap hits an empty +disk. `ensureWorkspace` (`packages/core/src/primitives/workspace.ts`) is the repair, and +it belongs INSIDE the retryable step — a rebuild the retry does not re-run is not a +rebuild. On the substrate backend (`SUBSTRATE_BACKEND=on`, `wrangler.jsonc` is `"off"` +today) `execUnderGrant` rebuilds inside the fence (`apps/substrate/src/facade.ts:210-212`), +so the probe is redundant there rather than wrong — but not free, and not a no-op: that +deploy still pays one fenced exec per retryable step, gains a retry on pool-admission +refusals, and pays a retried recipe-pin mismatch, because +`packages/runtime-cf/src/sandbox-facade.ts:274-287` folds both into `CheckoutFailed`. + +Scoped deliberately. A re-clone restores the tree the *spec* describes, which is right +for a suite, a lint or a build, and wrong for a step that reads a tree an earlier step +mutated. Applied to `self-heal-pr`'s verify step it would hand back a clean checkout and +pass on unmodified code, turning an infra failure into a wrong green — worse than the red +it replaces. Those steps need captured bytes restored, which is the `FileRef` chokepoint +still open in `REWRITE.md`. + Corollary to rules 3 and 4: wall-clock bounds must be written explicitly (`timeoutSec` on exec, `Effect.timeoutFail` around waits, as `waitForPort` already does at `sandbox-cf.ts:565-577`). Declaring `maxDurationSec` is documentation, not enforcement, @@ -149,6 +174,15 @@ anyway. Any new hibernating run cites this ADR and names its decider and timeout. - Step count and step-result size become design constraints runs are expected to respect, not incidental limits discovered in production. +- Rule 5 costs one `test -d` probe per retryable step on the happy path, and a clone plus + install when the probe misses. `CheckoutFailed` joins the retried set in those steps, + because the repair's own clone must survive the weather that made it necessary. +- Rule 5 is not yet repo-wide. `check`, `oxlint` and `offload-test` call `ensureWorkspace`; + `cdp-acceptance`, `matrix-fanout`, `playwright-e2e`, `playwright-demo`, `pr-review`, + `refresh-fixtures`, `spec-drift-pr`, `worker-deploy`, `org-spec-audit`, `vitest-shard` + and `release-notes` still call `workspace()` bare, as does `self-heal-pr`, which is in + the mutated-tree class the rule excludes. Each remaining one is either mechanically + portable or excluded, and that call is per run. ## Revisit triggers @@ -163,3 +197,8 @@ anyway. to keep entity state. - `maxDurationSec` becomes enforced, or Workflows introduces an instance-level wall-clock bound, changing what rule 2's corollary has to do by hand. +- The `FileRef` capture-and-restore chokepoint lands, making the mutated-tree case + serviceable. Rule 5's exclusion exists only because a re-clone is the wrong repair + there, and restoring captured bytes would let those steps recover too. +- Containers gain durable or reattachable disk, which would retire rule 5 rather than + extend it. diff --git a/specs/adr/README.md b/specs/adr/README.md index 73255bb..4526854 100644 --- a/specs/adr/README.md +++ b/specs/adr/README.md @@ -11,6 +11,6 @@ interfaces or assume they exist. | ADR | Status | Decision | | --- | --- | --- | -| [0001](./0001-cloudflare-workflows-scope.md) | proposed | How runs use Cloudflare Workflows: one instance per dispatch, hibernation reserved for bounded human decisions, entity lifecycles keep state in the system of record | +| [0001](./0001-cloudflare-workflows-scope.md) | proposed | How runs use Cloudflare Workflows: one instance per dispatch, hibernation reserved for bounded human decisions, entity lifecycles keep state in the system of record, container disk is never durable state | | [0002](./0002-memory-capability.md) | proposed | Org context is an optional, total `memory` capability with MCP-speaking adapters; runs consume, never produce | | [0003](./0003-no-context-relay.md) | proposed | Context backends ingest GitHub directly; FlareDispatch builds no relay, export, or pull endpoint for them |