diff --git a/.gitignore b/.gitignore index 21ee81c..be8d389 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ bin/ .DS_Store .corral/ /corral +/opencode.json diff --git a/.opencode/tools/corral.ts b/.opencode/tools/corral.ts index 37551b6..0c05691 100644 --- a/.opencode/tools/corral.ts +++ b/.opencode/tools/corral.ts @@ -18,7 +18,8 @@ async function loadKey(): Promise { } // Map the current OpenCode agent to a corral role for server-side -// enforcement. Anything unknown falls back to operator (human). +// enforcement. Unknown agents stay unprivileged; only non-model clients such +// as the CLI/TUI may claim the operator role directly. const ROLE_MAP: Record = { "corral-orchestrator": "orchestrator", "corral-planner": "planner", @@ -29,7 +30,7 @@ const ROLE_MAP: Record = { function roleFor(agent?: string): string { if (agent && agent in ROLE_MAP) return ROLE_MAP[agent] - return "operator" + return "unknown" } async function call(path: string, body?: unknown, role?: string) { @@ -40,7 +41,7 @@ async function call(path: string, body?: unknown, role?: string) { method: body === undefined ? "GET" : "POST", headers: { "Content-Type": "application/json", - "X-Corral-Role": role ?? "operator", + "X-Corral-Role": role ?? "unknown", ...(key ? { Authorization: `Bearer ${key}` } : {}), }, body: body === undefined ? undefined : JSON.stringify(body), @@ -67,14 +68,20 @@ export const plan = tool({ export const start = tool({ description: "Start a corral run from an approved graph.", - args: { graph: tool.schema.string().describe("Graph JSON (as returned by corral_plan)") }, + args: { + graph: tool.schema.string().describe("Graph JSON (as returned by corral_plan)"), + }, async execute(args, context) { - let graph: unknown + let parsed: unknown try { - graph = JSON.parse(args.graph) + parsed = JSON.parse(args.graph) } catch { return "error: graph is not valid JSON" } + const graph = + typeof parsed === "object" && parsed !== null && "graph" in parsed + ? (parsed as { graph: unknown }).graph + : parsed return call("/api/runs", { graph }, roleFor(context.agent)) }, }) @@ -89,6 +96,22 @@ export const status = tool({ }, }) +export const watch = tool({ + description: + "Watch a corral run and block until its state changes (new events, a human gate awaiting approval, or completion) or the timeout elapses. Drive the run loop by calling this repeatedly and passing the previous response's `since` cursor back. `gatesAwaitingApproval` lists human gates parked in running waiting for a decision: if the response's `autoApproveGates` is true the run is pre-authorized and you should approve each gate via corral_approve; otherwise never approve them yourself — report them to the user and keep watching until they resolve.", + args: { + runID: tool.schema.string(), + since: tool.schema.number().optional().describe("Event cursor; only return events after this"), + timeout: tool.schema.number().optional().describe("Block for up to this many seconds (default 60, max 120)"), + }, + async execute(args, context) { + const q = new URLSearchParams() + if (args.since !== undefined) q.set("since", String(args.since)) + if (args.timeout !== undefined) q.set("timeout", String(args.timeout)) + return call(`/api/runs/${args.runID}/watch?${q}`, undefined, roleFor(context.agent)) + }, +}) + export const approve = tool({ description: "Approve a human gate (or the run's merge) by node id.", args: { diff --git a/README.md b/README.md index 23a0537..13737fa 100644 --- a/README.md +++ b/README.md @@ -86,8 +86,10 @@ Inside OpenCode: 1. Switch to `corral-planner` and ask: `plan a graph to `. 2. Review the returned graph. 3. Switch to `corral-orchestrator` and ask it to start that graph. -4. Follow progress with `corral_status`; approve, reject, retry, cancel, or - steer nodes when needed. +4. Follow progress with `corral_status` / `corral_watch`; approve, reject, + retry, cancel, or steer nodes when needed. Trusted operator API clients may + set `autoApproveGates` when creating a run; model agents cannot grant that + authority to themselves. Or follow the same run from the terminal: @@ -99,22 +101,28 @@ corral tui Corral TUI inspecting a completed attempt, its worktree, command gate, and exit evidence -The TUI exposes the graph, node states, attempts, sessions, worktrees, evidence, -and operator actions. Worker edits stay in attempt worktrees; initialization -itself may add the OpenCode tool and agent config to your checkout. +The TUI follows durable server-sent run events, falls back to polling after a +stream failure, and exposes graph state, live transcript tails, budget usage, +attempts, sessions, worktrees, evidence, permissions, and operator actions. It +can raise desktop attention when a gate needs approval or a node fails. Worker +edits stay in attempt worktrees; initialization itself may add the OpenCode tool +and agent config to your checkout. ## What counts as evidence -Corral currently wires three completion paths: +Corral currently wires four completion paths: - **Command:** run an argv-style command in the attempt worktree and require exit code `0`. - **JSON Schema:** validate a declared JSON artifact against a schema. - **Default diff:** when no gate is declared, require at least one file diff reported by the driver. Prose alone fails. - -The graph schema also contains a reviewer-gate seam, but the production daemon -does not wire a reviewer implementation yet. +- **Reviewer:** a read-only OpenCode session reviews the attempt's evidence — + objective, prior feedback, transcript, the recorded diff artifact, and check + results — and must return exactly `APPROVED` or `CHANGES_REQUESTED`, followed + by a required `Note:` line. A change request returns its note as focused + retry feedback. Set `CORRAL_REVIEWER_MODEL` to a `provider/model` value to + use a specific model for review sessions. ## Proof, not promises @@ -156,8 +164,10 @@ evidence remain stored when an attempt retries. - **Landing:** merge nodes commit accepted worktree changes, merge branches with `--no-ff`, run their post-merge command, and prune consumed worktrees. -OpenCode is the implemented driver. The generic `adapter.Driver` interface is -the seam for future executors. +OpenCode is the production-wired driver. A self-contained Claude Code adapter +implements the same contract, including scoped permission mediation, but is not +yet selected by `corral daemon`. The generic `adapter.Driver` interface remains +the seam for additional executors. ## Operations @@ -168,15 +178,40 @@ the seam for future executors. | `corral doctor` | Check OpenCode, Git, daemon, plugin, and config | | `corral update` | Install a newer GitHub release after a sanity check | | `corral export ` | Print the full audit export | +| `corral worktrees` | List attempt worktrees; `--prune` removes clean merged/removed and stale ones | + +`status`, `tui`, `doctor`, and `export` read the repository key automatically. + +### Run-level safeguards + +The daemon ships with run-level safeguards enabled by default. Each is +overridable via an environment variable; a value of `0` disables it. These are +ceilings for runaway runs — normal runs should never hit them. -`status`, `tui`, and `doctor` read the repository key automatically. Until the -export command does the same, use: +| Variable | Default | Behavior | +|---|---|---| +| `CORRAL_BREAKER_MAX_FAILURES` | `5` | Circuit breaker: once `N` node failures occur within the window, the run stops starting new work; pending nodes are blocked (`reason: circuit breaker`) and an operator retry resets the breaker. | +| `CORRAL_BREAKER_WINDOW` | `900` (seconds, 15 min) | Failures are counted within this rolling window. | +| `CORRAL_RUN_MAX_TOKENS` | `1_000_000` | Run-level token budget, accumulated across all finished attempts; once exceeded, pending nodes are blocked (`reason: run budget exceeded`). | +| `CORRAL_RUN_MAX_COST` | `100` (USD) | Run-level cost budget, accumulated across all finished attempts; once exceeded, pending nodes are blocked. | + +Example: ```sh -CORRAL_DAEMON_KEY="$(cat .corral/api.key)" \ - corral export > audit.json +CORRAL_BREAKER_MAX_FAILURES=3 \ +CORRAL_BREAKER_WINDOW=600 \ +CORRAL_RUN_MAX_TOKENS=250000 \ +CORRAL_RUN_MAX_COST=50 \ +corral up ``` +`corral worktrees` works directly on git (no daemon, no key). It lists the +worktrees kept after failed attempts — path, branch, HEAD, last-activity time, +and dirty/locked markers — and with `--prune` removes clean ones that are safe +to drop: branches already merged into the main checkout, and (with `--stale +`, e.g. `24h`) worktrees idle longer than that. It never touches the +main checkout; dirty, locked, and detached worktrees are left alone. + ## Development ```sh @@ -194,9 +229,11 @@ The core packages are deliberately small: | `internal/graph` | graph schema, validation, states, ready computation | | `internal/sched` | leases, priority, retries, gates, merge orchestration | | `internal/store` | SQLite event log, materialized nodes, attempts, artifacts | -| `internal/verify` | command, JSON Schema, and diff evidence | +| `internal/verify` | command, JSON Schema, diff, and reviewer evidence | | `internal/worktree` | branch/worktree lifecycle and diff artifacts | | `internal/ocxadapter` | OpenCode sessions and completion reconciliation | +| `internal/claudeadapter` | standalone Claude Code sessions, usage, and permission mediation | +| `internal/ocxreviewer` | OpenCode reviewer sessions for the reviewer gate | | `internal/daemon` | control API, planning, role routing, audit export | | `internal/tui` | terminal dashboard and operator controls | @@ -205,9 +242,11 @@ Visual language, color roles, and asset rules live in the ## Scope -Corral is currently local, single-machine, single-repository software with one -implemented executor: OpenCode. Distributed workers, Codex/Claude drivers, -interactive graph editing, and a web dashboard remain roadmap work. +Corral is currently local, single-machine, single-repository software. OpenCode +is the production-wired executor; the Claude Code adapter is available as a +self-contained package but has no daemon selection/configuration path yet. +Distributed workers, a Codex driver, interactive graph editing, and a web +dashboard remain roadmap work. ## License diff --git a/cmd/corral/main.go b/cmd/corral/main.go index 255cf04..f848e0b 100644 --- a/cmd/corral/main.go +++ b/cmd/corral/main.go @@ -5,6 +5,7 @@ // corral init one-command local initialization // corral doctor environment and daemon checks // corral export full audit export of a run +// corral worktrees list attempt worktrees; --prune drops safe ones package main import ( @@ -35,6 +36,7 @@ import ( "corral/internal/daemon" "corral/internal/ocx" "corral/internal/ocxadapter" + "corral/internal/ocxreviewer" "corral/internal/sched" "corral/internal/spike" "corral/internal/store" @@ -61,7 +63,7 @@ func main() { func run(args []string) error { if len(args) < 1 { - return fmt.Errorf("usage: corral [flags]") + return fmt.Errorf("usage: corral [flags]") } switch args[0] { case "update": @@ -94,8 +96,14 @@ func run(args []string) error { out = args[3] } return exportCmd(args[1], out) + case "worktrees": + fs := flag.NewFlagSet("worktrees", flag.ExitOnError) + prune := fs.Bool("prune", false, "prune clean worktrees whose branch was merged or removed") + stale := fs.Duration("stale", 0, "with --prune, also prune worktrees idle longer than this (e.g. 24h, 72h)") + fs.Parse(args[1:]) + return worktreesCmd(*prune, *stale) default: - return fmt.Errorf("unknown command %q (try: daemon, tui, up, init, doctor, export, update)", args[0]) + return fmt.Errorf("unknown command %q (try: daemon, tui, up, init, doctor, export, update, worktrees)", args[0]) } } @@ -124,7 +132,7 @@ func daemonCmd(port int, apiKey string) error { // restart it on the same URL without breaking the adapter clients. servePort := freePort() var ocMu sync.Mutex - ocServer, err := spike.StartServer(ctx, dir, servePort, os.Stderr) + ocServer, err := spike.StartServerWithConfig(ctx, dir, servePort, os.Stderr, assets.OpenCodeConfigJSON) if err != nil { return fmt.Errorf("start opencode server: %w", err) } @@ -143,7 +151,7 @@ func daemonCmd(port int, apiKey string) error { return } time.Sleep(2 * time.Second) - ns, err := spike.StartServer(ctx, dir, servePort, os.Stderr) + ns, err := spike.StartServerWithConfig(ctx, dir, servePort, os.Stderr, assets.OpenCodeConfigJSON) if err != nil { log.Printf("opencode server restart failed: %v", err) return @@ -170,10 +178,13 @@ func daemonCmd(port int, apiKey string) error { wtm := worktree.NewManager(dir) eng := verify.New(dir) eng.Runner = verify.ExecRunner{} - s := sched.New(st, drv, &sched.EngineVerifier{Eng: eng}, clock.Real{}, sched.Options{ - Concurrency: 4, Worktrees: wtm, - }) + eng.Reviewer = ocxreviewer.New(oc, ocxreviewer.Options{Model: reviewerModel()}) + opts := schedOpts(wtm) + s := sched.New(st, drv, &sched.EngineVerifier{Eng: eng}, clock.Real{}, opts) + log.Printf("run safeguards: breaker %d failures per %s; run budget %d tokens / $%.2f", + opts.BreakerMaxFailures, opts.BreakerWindow, opts.RunMaxTokens, opts.RunMaxCost) d := daemon.New(st, s, daemon.NewOpenCodePlanner(oc, "", planTimeout()), dir, apiKey) + defer d.Close() if err := d.Resume(ctx); err != nil { log.Printf("resume: %v", err) } @@ -218,7 +229,9 @@ func tuiCmd() error { client := tui.NewClient(base, key) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - p := tea.NewProgram(tui.New(client, ctx), tea.WithAltScreen()) + model := tui.New(client, ctx) + model.EnableAttention() + p := tea.NewProgram(model, tea.WithAltScreen()) _, err := p.Run() return err } @@ -241,6 +254,48 @@ func planTimeout() time.Duration { return 5 * time.Minute } +// reviewerModel overrides the reviewer session model; override with +// CORRAL_REVIEWER_MODEL ("" = the OpenCode server default). +func reviewerModel() string { + return os.Getenv("CORRAL_REVIEWER_MODEL") +} + +// schedOpts returns the daemon scheduler options: the fixed +// concurrency/worktree wiring plus run-level safeguards with defaults +// that env vars override (a value of 0 disables a safeguard). +func schedOpts(wtm *worktree.Manager) sched.Options { + return sched.Options{ + Concurrency: 4, + Worktrees: wtm, + BreakerMaxFailures: intEnv("CORRAL_BREAKER_MAX_FAILURES", 5), + BreakerWindow: time.Duration(intEnv("CORRAL_BREAKER_WINDOW", 900)) * time.Second, + RunMaxTokens: intEnv("CORRAL_RUN_MAX_TOKENS", 1_000_000), + RunMaxCost: floatEnv("CORRAL_RUN_MAX_COST", 100), + } +} + +// intEnv returns the named env var as an int, or def when unset or +// unparsable. +func intEnv(name string, def int) int { + if v := os.Getenv(name); v != "" { + if n, err := strconv.Atoi(v); err == nil { + return n + } + } + return def +} + +// floatEnv returns the named env var as a float64, or def when unset or +// unparsable. +func floatEnv(name string, def float64) float64 { + if v := os.Getenv(name); v != "" { + if n, err := strconv.ParseFloat(v, 64); err == nil { + return n + } + } + return def +} + // statusCmd lists runs through the daemon (the TUI shows the same data). func statusCmd() error { return statusCmdWithDir(dirOf("")) @@ -401,7 +456,9 @@ func initCmd(wantDir string) error { } fmt.Println("api key written:", keyFile) } - cfg := map[string]any{"dir": dir, "apiKey": key, "daemonURL": daemonURL()} + // Keep the bearer token only in the mode-0600 api.key file. Config is + // project-readable metadata and must never duplicate authentication data. + cfg := map[string]any{"dir": dir, "daemonURL": daemonURL()} if err := writeJSONFile(filepath.Join(corralDir, "config.json"), cfg); err != nil { return err } @@ -434,7 +491,8 @@ func installPlugin(dir string) error { // installAgentConfig merges the corral agents (planner, orchestrator, // worker, reviewer, merger) into the project's opencode.json, preserving -// any existing configuration. Existing agent entries are left untouched. +// unrelated configuration. Managed agent definitions are refreshed on every +// init so an older permissive policy cannot bypass current role boundaries. func installAgentConfig(dir string) error { cfgPath := filepath.Join(dir, "opencode.json") existing := map[string]any{} @@ -458,9 +516,14 @@ func installAgentConfig(dir string) error { existingAgents = map[string]any{} } for name, def := range agents { - if _, ok := existingAgents[name]; !ok { - existingAgents[name] = def + // Model selection is a safe user customization. All authority-bearing + // fields (permission/tools/mode) come from the embedded definition. + if current, ok := existingAgents[name].(map[string]any); ok { + if model, ok := current["model"]; ok { + def.(map[string]any)["model"] = model + } } + existingAgents[name] = def } existing["agent"] = existingAgents return writeJSONFile(cfgPath, existing) @@ -519,7 +582,11 @@ func doctorWithURL(wantDir, url string) error { } func exportCmd(runID, outFile string) error { - client := tui.NewClient(daemonURL(), os.Getenv("CORRAL_DAEMON_KEY")) + key, err := readKey(dirOf("")) + if err != nil { + return err + } + client := tui.NewClient(daemonURL(), key) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() var payload json.RawMessage @@ -537,10 +604,68 @@ func exportCmd(runID, outFile string) error { fmt.Printf("audit export written to %s\n", outFile) return nil } - _, err := os.Stdout.Write(pretty.Bytes()) + _, err = os.Stdout.Write(pretty.Bytes()) return err } +// worktreesCmd lists the attempt worktrees kept after failed attempts +// and, with --prune, removes clean ones that are safe to drop (merged or +// removed branches, plus stale ones beyond --stale). Dirty worktrees and +// the main checkout are never touched. +func worktreesCmd(prune bool, stale time.Duration) error { + return worktreesCmdWithDir(dirOf(""), prune, stale) +} + +func worktreesCmdWithDir(dir string, prune bool, stale time.Duration) error { + wtm := worktree.NewManager(dir) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if prune { + removed, err := wtm.Prune(ctx, stale, time.Now()) + if err != nil { + return err + } + if len(removed) == 0 { + fmt.Println("nothing to prune") + return nil + } + for _, p := range removed { + fmt.Println("pruned", p) + } + return nil + } + infos, err := wtm.List(ctx) + if err != nil { + return err + } + if len(infos) == 0 { + fmt.Println("no attempt worktrees") + return nil + } + for _, info := range infos { + var marks []string + if info.Dirty { + marks = append(marks, "dirty") + } + if info.Locked { + marks = append(marks, "locked") + } + mark := "" + if len(marks) > 0 { + mark = " " + strings.Join(marks, ",") + } + fmt.Printf("%-42s %-28s %-7s %s%s\n", info.Path, info.Branch, shortHead(info.Head), info.Mtime.Format("2006-01-02 15:04"), mark) + } + return nil +} + +func shortHead(h string) string { + if len(h) > 7 { + return h[:7] + } + return h +} + func versionAtLeast(v, min string) bool { re := regexp.MustCompile(`\d+\.\d+\.\d+`) m := re.FindString(v) diff --git a/cmd/corral/main_test.go b/cmd/corral/main_test.go index ded942e..5b20cc8 100644 --- a/cmd/corral/main_test.go +++ b/cmd/corral/main_test.go @@ -16,6 +16,7 @@ import ( "corral/internal/livetest" "corral/internal/tui" + "corral/internal/worktree" ) // captureOutput runs fn with stdout redirected and returns its output. @@ -148,18 +149,39 @@ func TestDoctorPassesWithDaemonUp(t *testing.T) { } func TestExportCommand(t *testing.T) { - // Fake daemon serving a minimal export. + // Fake daemon serving a minimal export and recording the bearer key. payload := `{"runID":"run_1","status":"completed","events":[],"attempts":{},"artifacts":{}}` + var gotKey string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotKey = strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") w.Write([]byte(payload)) })) t.Cleanup(srv.Close) t.Setenv("CORRAL_DAEMON_URL", srv.URL) + t.Setenv("CORRAL_DAEMON_KEY", "") - outFile := filepath.Join(t.TempDir(), "audit.json") + // Serve the key from the repository, as status/tui/doctor do. + dir := t.TempDir() + os.MkdirAll(filepath.Join(dir, ".corral"), 0o755) + if err := os.WriteFile(filepath.Join(dir, ".corral", "api.key"), []byte("repo-key\n"), 0o600); err != nil { + t.Fatal(err) + } + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(cwd) }) + + outFile := filepath.Join(dir, "audit.json") if err := exportCmd("run_1", outFile); err != nil { t.Fatalf("export: %v", err) } + if gotKey != "repo-key" { + t.Fatalf("export used key %q, want repo key from .corral/api.key", gotKey) + } data, err := os.ReadFile(outFile) if err != nil { t.Fatal(err) @@ -169,12 +191,44 @@ func TestExportCommand(t *testing.T) { } } +func TestExportCommandEnvOverride(t *testing.T) { + // The env var must still override the repository key. + payload := `{"runID":"run_1","status":"completed","events":[],"attempts":{},"artifacts":{}}` + var gotKey string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotKey = strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + w.Write([]byte(payload)) + })) + t.Cleanup(srv.Close) + t.Setenv("CORRAL_DAEMON_URL", srv.URL) + t.Setenv("CORRAL_DAEMON_KEY", "env-key") + + dir := t.TempDir() + os.MkdirAll(filepath.Join(dir, ".corral"), 0o755) + os.WriteFile(filepath.Join(dir, ".corral", "api.key"), []byte("repo-key\n"), 0o600) + + outFile := filepath.Join(dir, "audit.json") + if err := exportCmd("run_1", outFile); err != nil { + t.Fatalf("export: %v", err) + } + if gotKey != "env-key" { + t.Fatalf("export used key %q, want env override", gotKey) + } +} + func TestInitMergesExistingOpenCodeConfig(t *testing.T) { dir := gitRepo(t) // Pre-existing opencode.json with custom settings must be preserved. existing := `{ "theme": "dark", - "agent": {"build": {"model": "custom/model"}} + "agent": { + "build": {"model": "custom/model"}, + "corral-orchestrator": { + "model": "custom/orchestrator", + "tools": {"bash": true}, + "permission": {"*": "allow"} + } + } }` if err := os.WriteFile(filepath.Join(dir, "opencode.json"), []byte(existing), 0o644); err != nil { t.Fatal(err) @@ -199,6 +253,17 @@ func TestInitMergesExistingOpenCodeConfig(t *testing.T) { t.Fatalf("corral agent %s missing: %s", name, data) } } + orchestrator := agents["corral-orchestrator"].(map[string]any) + if orchestrator["model"] != "custom/orchestrator" { + t.Fatalf("safe model customization lost: %s", data) + } + if _, ok := orchestrator["tools"]; ok { + t.Fatalf("legacy permissive tools survived managed-agent refresh: %s", data) + } + permission := orchestrator["permission"].(map[string]any) + if permission["*"] != "deny" || permission["corral_start"] != "allow" { + t.Fatalf("orchestrator policy was not hardened: %s", data) + } } func TestInitBacksUpInvalidConfig(t *testing.T) { @@ -259,6 +324,148 @@ func TestUpCmdSpawnsHealthyDaemon(t *testing.T) { _ = exec.Command("pkill", "-f", "corral daemon --port "+fmt.Sprint(port)).Run() } +func TestSchedOptsDefaultsAndOverrides(t *testing.T) { + for _, name := range []string{ + "CORRAL_BREAKER_MAX_FAILURES", "CORRAL_BREAKER_WINDOW", + "CORRAL_RUN_MAX_TOKENS", "CORRAL_RUN_MAX_COST", + } { + t.Setenv(name, "") + } + o := schedOpts(nil) + if o.Concurrency != 4 { + t.Fatalf("Concurrency = %d, want 4 (fixed wiring)", o.Concurrency) + } + if o.Worktrees != nil { + t.Fatalf("Worktrees = %v, want nil (as passed in)", o.Worktrees) + } + if o.BreakerMaxFailures != 5 { + t.Fatalf("BreakerMaxFailures = %d, want default 5", o.BreakerMaxFailures) + } + if o.BreakerWindow != 15*time.Minute { + t.Fatalf("BreakerWindow = %s, want default 15m", o.BreakerWindow) + } + if o.RunMaxTokens != 1_000_000 { + t.Fatalf("RunMaxTokens = %d, want default 1000000", o.RunMaxTokens) + } + if o.RunMaxCost != 100 { + t.Fatalf("RunMaxCost = %v, want default 100", o.RunMaxCost) + } + + // Overrides apply. + t.Setenv("CORRAL_BREAKER_MAX_FAILURES", "3") + t.Setenv("CORRAL_BREAKER_WINDOW", "60") + t.Setenv("CORRAL_RUN_MAX_TOKENS", "5000") + t.Setenv("CORRAL_RUN_MAX_COST", "0.25") + o = schedOpts(nil) + if o.BreakerMaxFailures != 3 { + t.Fatalf("BreakerMaxFailures = %d, want 3", o.BreakerMaxFailures) + } + if o.BreakerWindow != time.Minute { + t.Fatalf("BreakerWindow = %s, want 1m", o.BreakerWindow) + } + if o.RunMaxTokens != 5000 { + t.Fatalf("RunMaxTokens = %d, want 5000", o.RunMaxTokens) + } + if o.RunMaxCost != 0.25 { + t.Fatalf("RunMaxCost = %v, want 0.25", o.RunMaxCost) + } + + // 0 disables a safeguard rather than falling back to the default. + t.Setenv("CORRAL_RUN_MAX_TOKENS", "0") + t.Setenv("CORRAL_BREAKER_MAX_FAILURES", "0") + o = schedOpts(nil) + if o.RunMaxTokens != 0 || o.BreakerMaxFailures != 0 { + t.Fatalf("zero should disable safeguards: %+v", o) + } + + // Unparsable values fall back to the defaults. + t.Setenv("CORRAL_RUN_MAX_TOKENS", "banana") + t.Setenv("CORRAL_RUN_MAX_COST", "not-a-number") + t.Setenv("CORRAL_BREAKER_WINDOW", "soon") + o = schedOpts(nil) + if o.RunMaxTokens != 1_000_000 || o.RunMaxCost != 100 || o.BreakerWindow != 15*time.Minute { + t.Fatalf("unparsable env should fall back to defaults: %+v", o) + } +} + +func TestWorktreesCommand(t *testing.T) { + repo := gitRepo(t) + ctx := context.Background() + wtm := worktree.NewManager(repo) + path, err := wtm.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + + out := captureOutput(t, func() { err = worktreesCmdWithDir(repo, false, 0) }) + if err != nil { + t.Fatalf("worktrees: %v", err) + } + if !strings.Contains(out, "corral/r1/w1/1") { + t.Fatalf("listing missing branch:\n%s", out) + } + if strings.Contains(out, "pruned") { + t.Fatalf("listing pruned unexpectedly:\n%s", out) + } + + // Nothing is merged or stale, so --prune removes nothing. + out = captureOutput(t, func() { err = worktreesCmdWithDir(repo, true, 0) }) + if err != nil { + t.Fatalf("worktrees --prune: %v", err) + } + if !strings.Contains(out, "nothing to prune") { + t.Fatalf("prune output wrong:\n%s", out) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("worktree dir removed: %v", err) + } +} + +func TestWorktreesCommandPrunesMerged(t *testing.T) { + repo := gitRepo(t) + ctx := context.Background() + wtm := worktree.NewManager(repo) + path, err := wtm.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + if err := wtm.CommitWorktree(ctx, path); err != nil { + t.Fatal(err) + } + if err := wtm.MergeBranch(ctx, "corral/r1/w1/1"); err != nil { + t.Fatal(err) + } + + out := captureOutput(t, func() { err = worktreesCmdWithDir(repo, true, 0) }) + if err != nil { + t.Fatalf("worktrees --prune: %v", err) + } + if !strings.Contains(out, "pruned") || !strings.Contains(out, "corral/r1/w1/1") { + t.Fatalf("prune output wrong:\n%s", out) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("merged worktree dir not removed: %v", err) + } + // Main checkout untouched. + if data, err := os.ReadFile(filepath.Join(repo, "a.txt")); err != nil || string(data) != "hello" { + t.Fatalf("main checkout wrong: %v %q", err, data) + } +} + +func TestWorktreesCommandEmpty(t *testing.T) { + repo := gitRepo(t) + out := captureOutput(t, func() { _ = worktreesCmdWithDir(repo, false, 0) }) + if !strings.Contains(out, "no attempt worktrees") { + t.Fatalf("empty listing output wrong:\n%s", out) + } +} + func TestVersionAtLeast(t *testing.T) { cases := []struct { v, min string diff --git a/docs/task2-scheduler.md b/docs/task2-scheduler.md index 7095d45..58a705d 100644 --- a/docs/task2-scheduler.md +++ b/docs/task2-scheduler.md @@ -18,8 +18,8 @@ replay reproduces graph state, crash/restart test passes. `ClearLease` is for crash recovery. - `internal/sched` — the control loop: - One deterministic `Step()` per tick; all state mutation happens inside - it (cooperative simulation with the fake driver, async results channel - for real drivers). + it (cooperative simulation with the fake driver; real-driver completions + are drained and routed centrally by the scheduler). - Concurrency limit, priority ordering with **aging** (boost per saturated step, capped; only waiting nodes accrue). - Retry policy: `verifying → retry_wait → ready` on failed evidence while diff --git a/docs/task3-adapter.md b/docs/task3-adapter.md index 1c4213e..4fa8588 100644 --- a/docs/task3-adapter.md +++ b/docs/task3-adapter.md @@ -1,4 +1,4 @@ -# Task 3 — OpenCode adapter +# Task 3 — Provider adapters Status: **DONE** — the generic adapter contract is mapped onto real OpenCode sessions; integration test covers parallel execution and cancellation. @@ -8,29 +8,52 @@ sessions; integration test covers parallel execution and cancellation. - `internal/ocxadapter` — `Driver` implementing `adapter.Driver` + `adapter.Stepper`: - `Start`: `POST /session` (title `corral/`), async prompt with the - node objective (+ role prefix, optional model override), records the - OpenCode session; returns an `adapter.Session` handle. + node objective (+ role prefix). A per-attempt `provider/model` overrides + the driver default; records the OpenCode session and returns an + `adapter.Session` handle. - Completion detection: a shared `/global/event` SSE stream (opened once per driver) dispatches `session.idle` / `session.error` to the owning attempt's event channel; a per-attempt poller (`PollInterval`, default - 1s) queries `/session/status` + `/session/:id/message` as the - reconciliation fallback when events are missed or the stream drops. + 1s) queries `/session/:id/message` as the reconciliation fallback when + terminal events are missed or the stream drops. The first start waits at + most `StreamReadyTimeout` (default 1s) for SSE, then proceeds through REST + fallback instead of hanging when `/global/event` is unavailable. - Exactly-once completion: `attempt.completed` guard (checked before the transcript fetch and again under the driver mutex before emission); duplicate events (e.g. repeated `session.idle`) cannot emit twice, and the scheduler asserts exactly one attempt row per node. - Terminal classification from the transcript: aborted flag or `MessageAbortedError` → `StatusAborted`; other assistant error → - `StatusError`; `finish:"stop"` → `StatusIdle`; no finished message → - still running (poll again). + `StatusError`; `finish:"stop"` → `StatusIdle`; `finish:"tool-calls"` → + still running; any other non-empty finish reason → `StatusError`. - `Abort` → `POST /session/:id/abort`; `Messages` → adapter messages with text parts, costs/tokens, and user-summary diffs (patch format). + - Permission requests use `permission.asked` / `permission.v2.asked` for + low-latency notification plus one shared, bounded background + `GET /permission` reconciler; scheduler permission checks remain local and + non-blocking. Decisions use `POST /permission/:requestID/reply` with + `once` / `reject`. - Contract additions (adapter v0.2): `adapter.Completion`, `adapter.Stepper`, `Session.ServerID()`. -- `sched`: drains `adapter.Stepper` into the results channel; records +- `sched`: drains the shared `adapter.Stepper` through a central owner router + so concurrent runs receive only their own completions; records `ServerID` per attempt (`store.Attempt.ServerID`, new `server_id` column); new `RunHandle.CancelNode` (operator cancel → driver abort → `running → canceled`). +- `internal/claudeadapter` — a self-contained Claude Code implementation of + `adapter.Driver`, `adapter.Stepper`, and permission-aware sessions: + - launches one headless `claude -p --output-format stream-json` process per + attempt and reconciles its ordered transcript with process exit; + - records cumulative result usage/cost once per attempt and emits one + completion even when terminal signals repeat; + - mediates permission requests through a private local Unix socket and an + MCP stdio helper, validating attempt/session/request identity before a + decision is delivered; + - supports scoped read/write tool rules, model overrides, aborts, oversized + stream records, and high-volume event streams. + + This package is deliberately not wired into `cmd/corral`; OpenCode remains + the daemon's selected executor until provider selection/configuration lands. ## Acceptance verification @@ -38,9 +61,10 @@ sessions; integration test covers parallel execution and cancellation. |---|---| | Start, stream, message, inspect, cancel sessions | Integration test: sessions created + prompted; SSE stream dispatches terminal events; transcripts fetched via `Messages`; `CancelNode` aborts mid-run | | Record OpenCode server/session IDs per attempt | Asserted: `session_id` has `ses_` prefix, `server_id` == server base URL | -| Event stream + status polling fallback | Both paths implemented; polling is the completion path when events are dropped (same `maybeComplete` logic) | -| Duplicate/missing events cannot duplicate completion | Exactly 1 attempt row per node asserted; `completed` guard + scheduler drop of unknown/duplicate attempt results | +| Event stream + REST fallback | Startup is bounded when SSE is unavailable; transcript polling completes attempts, and durable permission polling recovers prompts missed during reconnects | +| Duplicate/missing events cannot duplicate completion | Exactly 1 attempt row per node asserted; `completed` guard + scheduler owner routing retains start-race completions until registration | | Two-node parallel run + cancellation test | Both sessions observed busy concurrently (peak ≥ 2); w1 done with real file output; w2 canceled with `aborted` attempt | +| Claude protocol and permission mediation | Protocol fixtures cover init/result/usage, large and high-volume streams, duplicate terminal events, scoped MCP decisions, aborts, and closed-driver behavior; race and cross-build checks cover the package | ## Notes @@ -49,3 +73,5 @@ sessions; integration test covers parallel execution and cancellation. - Watchers exit via attempt cancel / driver `Close`; the shared stream goroutine runs until `Close`. - Run: `go test ./internal/ocxadapter -run TestOpenCodeAdapterParallelAndCancel -v` +- Claude's package tests are deterministic; a live Claude run remains an + explicit provider-gated check rather than part of the default suite. diff --git a/docs/task4-verification.md b/docs/task4-verification.md index 68c9ed6..4f9c382 100644 --- a/docs/task4-verification.md +++ b/docs/task4-verification.md @@ -14,9 +14,13 @@ focused feedback; budgets bound retries; prose alone never completes work. (relative to the worktree) against the schema (santhosh-tekuri jsonschema); missing/invalid files and schema violations produce concrete feedback (first 5 validation errors). - - **reviewer**: injectable `Reviewer` session receives the attempt - transcript + worktree and must conclude APPROVED; rejection note is - the feedback. + - **reviewer**: `internal/ocxreviewer` implements the injectable + `Reviewer` seam on top of OpenCode: a read-only LLM session receives the + attempt's evidence (objective, prior feedback, transcript, recorded diff + artifact and check results), must return exactly `APPROVED` or + `CHANGES_REQUESTED` followed by a required `Note:` line; the change-request + note is the feedback. `CORRAL_REVIEWER_MODEL` overrides the session model + using OpenCode's `provider/model` format. - **default gate** (no method declared): an attempt must have produced at least one diff — agent prose alone cannot mark work complete. - **check nodes**: verdict derived from their own command run carried in @@ -39,6 +43,7 @@ focused feedback; budgets bound retries; prose alone never completes work. | Criterion | Evidence | |---|---| | Command, JSON-schema, reviewer checks | `verify` unit tests: pass/fail + feedback for each kind; check-node verdict from Meta | +| Reviewer approves / requests changes with a note | `ocxreviewer` tests: scripted fake LLM server covers exact APPROVED and CHANGES_REQUESTED verdicts with notes, malformed verdicts, session errors, missing verdicts, timeout; `TestOpenCodeReviewerLive` runs a real review session (gated on `CORRAL_LIVE`) | | Failed verification returns focused feedback | `TestFailThenPassAfterRetryWithFeedback`: gate 1 stderr reaches attempt 2 verbatim | | Retry count, timeout, budget bounded | retries from policy; time budget aborts (Task 2); `TestTokenBudgetBoundsRetries` stops retries after MaxTokens consumed | | Exhausted node becomes blocked or failed | `TestPermanentFailureBlocksDownstream`: failed, dependent blocked, run settles `waiting`, dependent never ran | @@ -52,3 +57,12 @@ focused feedback; budgets bound retries; prose alone never completes work. intervention (used by later tasks). - `TestOpenCodeEvidenceGates` is deterministic because the gates grep for fixed markers the prompt demands, independent of model behavior. +- Reviewer sessions run as the named `corral-reviewer` agent with an agent-level + wildcard deny at both named-agent and prompt permission layers, + review recorded diffs and command results from the evidence + prompt, and stay bound to the main OpenCode project where that named agent + is configured (they never access the attempt worktree). They poll the + transcript to idle and parse the verdict from the last assistant message. + The verdict must contain exactly two lines: + `APPROVED`/`CHANGES_REQUESTED` then a non-empty `Note:` line; anything else + fails the gate with a parse error. diff --git a/docs/task5-worktrees.md b/docs/task5-worktrees.md index 53ca431..bcce2bb 100644 --- a/docs/task5-worktrees.md +++ b/docs/task5-worktrees.md @@ -3,7 +3,9 @@ Status: **DONE** — every writing node runs in its own git worktree; write scopes serialize conflicting writers; diffs become content-addressed artifacts; failed worktrees are kept; merges require checks + human -approval. Verified deterministically and against a real OpenCode server. +approval. `corral worktrees` lists retained worktrees and prunes clean +merged/removed or stale ones without touching the main checkout. Verified +deterministically and against a real OpenCode server. ## Deliverables @@ -16,6 +18,17 @@ approval. Verified deterministically and against a real OpenCode server. `MergeBranch` (no-ff into main), `Remove`, `MainBranch`, `Repo`. - `ScopesOverlap`: path-boundary prefix semantics; empty/`*` scope collides with everything. + - `List`: enumerates attempt worktrees under `.corral-worktrees` via + `git worktree list --porcelain`, with branch, HEAD, last-activity mtime + (worktree dir + gitdir HEAD/index), and dirty state; the main checkout + is never listed. + - `BranchMerged`: reports whether a branch already has commits folded + into the main checkout branch (or no longer exists); a branch still at + the main tip is not "merged" so uncommitted work is never pruned. + - `Prune`: removes clean worktrees whose branch was merged/removed or + whose mtime exceeds a `staleAfter` threshold; sweeps orphaned git admin + entries via `git worktree prune`; skips dirty, locked, and detached + worktrees; never touches the main checkout. - `sched`: - Writing nodes (role `worker` or unlabeled agents) get a worktree per attempt; `Attempt.Cwd` and the attempt row record it (new @@ -62,6 +75,12 @@ approval. Verified deterministically and against a real OpenCode server. - Empty write scope on a writing node = whole repository (collides with everything); declare scopes to enable parallelism. - Worktrees are pruned only on successful merge; failed/abandoned - worktrees stay on disk for inspection (`git worktree prune` at run - cleanup is future work). + worktrees stay on disk for inspection. `corral worktrees` lists them + (path, branch, HEAD, mtime, dirty/locked state) and `corral worktrees + --prune` removes clean ones that are safe to drop — branches already + merged into main or removed — plus, with `--prune --stale ` + (e.g. `24h`), clean worktrees idle longer than the threshold. Dirty, + locked, and detached worktrees are never pruned, and the main checkout is + never touched. Stale-pruned branches are kept (their commits stay + recoverable) unless already merged. - Retries allocate a fresh worktree per attempt (deterministic redo). diff --git a/docs/task6-plugin.md b/docs/task6-plugin.md index 4d0444d..5ad0897 100644 --- a/docs/task6-plugin.md +++ b/docs/task6-plugin.md @@ -14,9 +14,17 @@ flow is verified end-to-end against a real OpenCode server. agent roles, default acceptance criteria) before `graph.Validate`. - `POST /api/runs` — start a run from a graph; run loops live on the daemon context (not request context — bug found and fixed) and - persist via SQLite. + persist via SQLite. An operator-only `autoApproveGates` option is stored + on the run and exposed by `GET /api/runs/{id}`; gates remain explicit, + and model agents cannot mint this authority themselves. - `GET /api/runs`, `GET /api/runs/{id}` — follow execution (states, attempts, event log). + - `GET /api/runs/{id}/watch` — bounded JSON long-poll of run deltas from a + `since` cursor (node transitions, gates awaiting approval, run done); + powers `corral_watch`. + - `GET /api/runs/{id}/events` — raw durable Server-Sent Events stream from + an `after` cursor, with replay, live delivery, heartbeat frames, and + terminal close; powers streaming clients such as the companion TUI. - `approve` / `reject` / `cancel` / `retry` / `steer` per node — including `RetryNode` (blocked→ready, retry_wait→ready, failed→ready operator override with retry budget reset) and run-loop restart after @@ -29,14 +37,18 @@ flow is verified end-to-end against a real OpenCode server. - `cmd/corral` — `corral daemon [--port 4519] [--key TOKEN]`: embeds `opencode serve`, wires store + adapter + worktrees + verifier. - `.opencode/tools/corral.ts` — the thin plugin: `corral_plan`, - `corral_start`, `corral_status`, `corral_approve`, `corral_reject`, - `corral_cancel`, `corral_retry`, `corral_steer`, calling the daemon and - mapping the session agent (`corral-*`) to a role. + `corral_start` (accepts the raw graph *or* the full `corral_plan` output, + unwrapping a leading `{"graph": ...}` wrapper), `corral_status`, + `corral_watch` (blocks on the + daemon long-poll endpoint and returns the first run delta — node transition, + gate awaiting approval, or run done — or times out), `corral_approve`, + `corral_reject`, `corral_cancel`, `corral_retry`, `corral_steer`, calling + the daemon and mapping the session agent (`corral-*`) to a role. - `example/opencode.json` — agent role configuration using OpenCode's - per-agent permissions: orchestrator (deny edit/bash, allow corral_*), - planner (read-only + corral_plan), worker (ask edits/bash), reviewer - (deny edit; bash allow only `git diff/status/log`, tests), merger (deny - edit; bash allow only `git status/log/diff`, ask merge/checkout/branch). + fail-closed per-agent permissions: every role starts with wildcard deny; + orchestrator allows only corral_*, planner only corral_plan, worker allows + read/glob and asks for edits/bash, reviewer denies all tools, and merger + allows only restricted git commands. ## Acceptance verification @@ -52,7 +64,8 @@ flow is verified end-to-end against a real OpenCode server. - Planner smoke is tolerant: if the model fails to emit a parseable graph it logs and skips (nondeterministic LLM output); normalization keeps drift recoverable. -- The plugin's role fallback is `operator` for unknown agents (human). +- The plugin's role fallback is unprivileged `unknown`; operator authority is + reserved for non-model CLI/TUI clients. - `steer` sends a message into the running session (agent sees it as a follow-up instruction). - Run: `go test ./internal/daemon -v` (includes the real-OpenCode E2E). diff --git a/docs/task7-tui.md b/docs/task7-tui.md index 935b82c..9068912 100644 --- a/docs/task7-tui.md +++ b/docs/task7-tui.md @@ -8,15 +8,17 @@ is the companion observability surface (no OpenCode fork needed). ## Deliverables - `internal/tui/api.go` — HTTP client + DTOs over the daemon endpoints - (`ListRuns`, `GetRun`, `Approve`, `Reject`, `Cancel`, `Retry`, `Steer`). + (`ListRuns`, `GetRun`, durable `StreamEvents`, live `Tail`, `Approve`, + `Reject`, `Cancel`, `Retry`, `Steer`, and permission responses). - `internal/tui/model.go` — bubbletea model. All state changes happen in `Update` (tick-driven fetch, keys), so it is fully testable without a terminal. Modes: list → detail → inspect → steer. Keys: - list: `↑/↓` (or j/k), `enter` detail, `q` quit - detail: `↑/↓` node, `a` approve, `r` reject, `c` cancel, `t` retry, - `s` steer (typed message, enter sends), `i` inspect, `esc` back + `p` allow permission, `d` deny permission, `s` steer (typed message, + enter sends), `i` inspect, `esc` back - inspect: attempts (status, session, worktree, elapsed, cost/tokens, - evidence), `esc` back + evidence) plus the active attempt's live transcript tail, `esc` back - `internal/tui/view.go` — lipgloss rendering: - runs list: id, status, per-node state chips (`w1:done gate:running`) - DAG view: nodes sorted with dependency arrows (`← dep (state)`), @@ -25,6 +27,8 @@ is the companion observability surface (no OpenCode fork needed). - inspect: objective, role, write scope, verification, attempt rows with session/worktree paths and evidence snippets - status line with keybinding help, last action, and errors +- `internal/tui/notify.go` — bounded, non-blocking terminal/desktop attention + for gates awaiting approval and node failures. - `cmd/corral tui` — connects to `CORRAL_DAEMON_URL` (default `http://127.0.0.1:4519`) with optional bearer key, alt-screen program. @@ -37,11 +41,15 @@ is the companion observability surface (no OpenCode fork needed). | Active agent/session/worktree | inspect shows session id + worktree path per attempt | | Attempts, elapsed time, budget, evidence | inspect rows: attempt #, status, elapsed, cost/tokens, evidence snippet (budget visible in run status) | | Inspect, steer, retry, cancel actions | `TestNodeActions` drives every key path through the model; `TestClientAgainstDaemon` round-trips all actions against a live daemon | +| Durable live updates and recovery | SSE client/model tests cover replay cursors, ordered incremental state, disconnect/gap fallback, terminal close, waiting-run continuation, and stale-response protection | +| Live output, budgets, attention | Tail endpoint/client/model tests, highest-utilization budget bars, and async bounded notification tests | ## Notes - The model executes commands like the tea runtime (`send` helper in tests) so no terminal is needed for coverage. -- Actions refresh immediately after execution; the view polls every 1s. +- Actions refresh immediately after execution. Detail views consume the + durable event stream; a 1s full-detail poll is used while that stream is + unavailable. Live tails refresh while inspecting an active attempt. - TUI talks only to the daemon API (role `operator`); the daemon enforces authorization. diff --git a/docs/task8-hardening.md b/docs/task8-hardening.md index 2ff40ae..1c7b7fd 100644 --- a/docs/task8-hardening.md +++ b/docs/task8-hardening.md @@ -7,8 +7,11 @@ Status: **DONE** — all acceptance criteria covered with tests. - **Permission requests → explicit blocked state** - `adapter.PermissionSession` (optional interface): `PendingPermission`, `RespondPermission`. - - `ocxadapter` tracks `permission.updated` events per session and answers - via `POST /session/:id/permissions/:permissionID`. + - `ocxadapter` tracks `permission.asked` / `permission.v2.asked`; one shared, + bounded background poll queries durable `GET /permission` across SSE + reconnect gaps, keeping scheduler `PendingPermission` checks local and + non-blocking. Answers use `POST /permission/:requestID/reply` + (`once` / `reject`). - Scheduler: a pending permission moves the node `running → blocked` (payload carries `permissionID`) and the session is *suspended* — its eventual completion still resolves the attempt (machine walk @@ -26,6 +29,17 @@ Status: **DONE** — all acceptance criteria covered with tests. new work starts (fixed ordering bug where new nodes could start the step after a budget/breaker trip). - Per-node time/token/cost budgets from Tasks 2/4 unchanged. +- **Daemon defaults**: the production daemon wires these safeguards with sane + defaults, each overridable via env vars (a value of `0` disables it): + - `CORRAL_BREAKER_MAX_FAILURES` (default `5`) and `CORRAL_BREAKER_WINDOW` + (default `900`s / 15 min) — the circuit breaker trips after that many node + failures within the window, blocking pending nodes until an operator retry + resets it. + - `CORRAL_RUN_MAX_TOKENS` (default `1_000_000`) and `CORRAL_RUN_MAX_COST` + (default `$100`) — run-level budgets accumulated across finished attempts; + pending nodes block once exceeded. Wired in `cmd/corral/main.go` via + `schedOpts`, keeping the fixed `Concurrency`/`Worktrees` wiring intact. + Covered by `TestSchedOptsDefaultsAndOverrides`. - **Secrets hygiene**: `store.Redact` strips bearer tokens, api keys, passwords/secrets/tokens and `sk-…` patterns at the persistence boundary (attempt evidence, event payloads, artifact content). Verified diff --git a/example/opencode.json b/example/opencode.json index 94e05f6..235e39f 100644 --- a/example/opencode.json +++ b/example/opencode.json @@ -4,13 +4,13 @@ "corral-orchestrator": { "description": "Corral orchestrator: controls runs and graphs, never edits files or runs arbitrary bash.", "mode": "primary", - "prompt": "You are the corral orchestrator. You plan runs, start them, monitor status, approve or reject gates, and steer workers. You never edit files and never run bash commands.", + "prompt": "You are the corral orchestrator. You start runs, watch them, approve or reject gates, and steer workers. You never edit files and never run bash commands.\n\nRun loop: start a run from an approved graph with corral_start, then repeatedly call corral_watch with the runID and the previous response's `since` cursor (use a timeout around 60) to follow it. Report milestones to the user as nodes progress.\n\nWhen corral_watch reports gatesAwaitingApproval:\n- If the response's autoApproveGates is true, the run is pre-authorized: call corral_approve for each waiting gate and continue watching.\n- If autoApproveGates is false, you are NOT pre-authorized: never call corral_approve. Tell the user the gate awaits their approval, and keep calling corral_watch until the gate resolves, then continue driving the run.\n\nKeep watching until the response shows done: true, then summarize the outcome for the user.", "permission": { - "edit": "deny", - "bash": "deny", + "*": "deny", "corral_plan": "allow", "corral_start": "allow", "corral_status": "allow", + "corral_watch": "allow", "corral_approve": "allow", "corral_reject": "allow", "corral_cancel": "allow", @@ -23,8 +23,13 @@ "mode": "primary", "prompt": "You are the corral planner. Analyze the codebase read-only and produce a corral task graph JSON for the user's goal using corral_plan. Never modify files or run commands.", "permission": { - "edit": "deny", - "bash": "deny", + "*": "deny", + "read": { + "*": "allow", + ".corral": "deny", + ".corral/*": "deny" + }, + "glob": "allow", "corral_plan": "allow" } }, @@ -33,25 +38,23 @@ "mode": "subagent", "prompt": "You are a corral worker. Implement the objective precisely, staying within your declared write scope. Write clean, minimal changes and verify your output.", "permission": { + "*": "deny", + "read": { + "*": "allow", + ".corral": "deny", + ".corral/*": "deny" + }, + "glob": "allow", "edit": "ask", "bash": "ask" } }, "corral-reviewer": { - "description": "Corral reviewer: read-only analysis plus tests and diffs.", + "description": "Corral reviewer: evaluates supplied evidence without tools.", "mode": "subagent", - "prompt": "You are a corral reviewer. Inspect the worker's output read-only; you may run tests and inspect diffs, but never modify files.", + "prompt": "You are a corral reviewer. Evaluate only the evidence supplied in the prompt. Never call tools or modify external state.", "permission": { - "edit": "deny", - "bash": { - "*": "deny", - "git diff*": "allow", - "git status*": "allow", - "git log*": "allow", - "go test*": "allow", - "npm test*": "allow", - "pytest*": "allow" - } + "*": "deny" } }, "corral-merger": { @@ -59,7 +62,7 @@ "mode": "subagent", "prompt": "You are the corral merger. You only perform git operations to merge accepted branches. Never edit source files.", "permission": { - "edit": "deny", + "*": "deny", "bash": { "*": "deny", "git status*": "allow", diff --git a/expense-report-workflow.excalidraw b/expense-report-workflow.excalidraw new file mode 100644 index 0000000..86d2a39 --- /dev/null +++ b/expense-report-workflow.excalidraw @@ -0,0 +1,2617 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://excalidraw.com", + "elements": [ + { + "id": "title", + "type": "text", + "x": 60, + "y": 40, + "width": 700, + "height": 36, + "text": "Agent-Prepared Monthly Expense Report", + "originalText": "Agent-Prepared Monthly Expense Report", + "fontSize": 28, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#111827", + "strokeColor": "#111827", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600001, + "version": 1, + "versionNonce": 600001, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "subtitle", + "type": "text", + "x": 60, + "y": 86, + "width": 1120, + "height": 22, + "text": "Every report line stays linked to its submission, card transaction, and receipt — from ingestion to approval.", + "originalText": "Every report line stays linked to its submission, card transaction, and receipt — from ingestion to approval.", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600002, + "version": 1, + "versionNonce": 600002, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r1_region", + "type": "rectangle", + "x": 40, + "y": 150, + "width": 390, + "height": 720, + "text": "", + "originalText": "", + "strokeColor": "#e5e7eb", + "backgroundColor": "#f9fafb", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100001, + "version": 1, + "versionNonce": 100001, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r2_region", + "type": "rectangle", + "x": 470, + "y": 150, + "width": 400, + "height": 720, + "text": "", + "originalText": "", + "strokeColor": "#e5e7eb", + "backgroundColor": "#f9fafb", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 200001, + "version": 1, + "versionNonce": 200001, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r1_label", + "type": "text", + "x": 60, + "y": 163, + "width": 140, + "height": 24, + "text": "1 · SOURCES", + "originalText": "1 · SOURCES", + "fontSize": 18, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#374151", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100002, + "version": 1, + "versionNonce": 100002, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r1_chip", + "type": "rectangle", + "x": 354, + "y": 165, + "width": 64, + "height": 24, + "text": "CONFIG", + "originalText": "CONFIG", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "#f3f4f6", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100003, + "version": 1, + "versionNonce": 100003, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "src_a_ellipse", + "type": "ellipse", + "x": 60, + "y": 200, + "width": 180, + "height": 70, + "text": "Online form\nexpenses → JSON API", + "originalText": "Online form\nexpenses → JSON API", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#3b82f6", + "backgroundColor": "#dbeafe", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100004, + "version": 1, + "versionNonce": 100004, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "src_a_json", + "type": "rectangle", + "x": 60, + "y": 290, + "width": 240, + "height": 140, + "text": "{\n \"employee\": \"A. Silva\",\n \"merchant\": \"Uber\",\n \"amount\": 82.40,\n \"date\": \"2026-08-03\"\n}", + "originalText": "{\n \"employee\": \"A. Silva\",\n \"merchant\": \"Uber\",\n \"amount\": 82.40,\n \"date\": \"2026-08-03\"\n}", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#cdd6f4", + "strokeColor": "#313244", + "backgroundColor": "#1e1e2e", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100005, + "version": 1, + "versionNonce": 100005, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "src_b_ellipse", + "type": "ellipse", + "x": 60, + "y": 460, + "width": 180, + "height": 70, + "text": "Corporate card\nCSV upload (Finance)", + "originalText": "Corporate card\nCSV upload (Finance)", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#3b82f6", + "backgroundColor": "#dbeafe", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100006, + "version": 1, + "versionNonce": 100006, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "src_b_csv", + "type": "rectangle", + "x": 60, + "y": 552, + "width": 240, + "height": 84, + "text": "date,merchant,amount\n2026-08-03,Uber,82.40\n2026-08-04,Delta,412.00", + "originalText": "date,merchant,amount\n2026-08-03,Uber,82.40\n2026-08-04,Delta,412.00", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#cdd6f4", + "strokeColor": "#313244", + "backgroundColor": "#1e1e2e", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100007, + "version": 1, + "versionNonce": 100007, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "src_c_ellipse", + "type": "ellipse", + "x": 60, + "y": 690, + "width": 180, + "height": 70, + "text": "Receipts attached\nPDF · images", + "originalText": "Receipts attached\nPDF · images", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#3b82f6", + "backgroundColor": "#dbeafe", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100008, + "version": 1, + "versionNonce": 100008, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "src_c_file", + "type": "rectangle", + "x": 60, + "y": 780, + "width": 240, + "height": 64, + "text": "receipt_2026-08-03.pdf\n→ OCR: \"Uber · $82.40\"", + "originalText": "receipt_2026-08-03.pdf\n→ OCR: \"Uber · $82.40\"", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#cdd6f4", + "strokeColor": "#313244", + "backgroundColor": "#1e1e2e", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100009, + "version": 1, + "versionNonce": 100009, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r2_label", + "type": "text", + "x": 490, + "y": 163, + "width": 250, + "height": 24, + "text": "2 · NORMALIZE & LINK", + "originalText": "2 · NORMALIZE & LINK", + "fontSize": 18, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#374151", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 200002, + "version": 1, + "versionNonce": 200002, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r2_chip", + "type": "rectangle", + "x": 794, + "y": 165, + "width": 64, + "height": 24, + "text": "CONFIG", + "originalText": "CONFIG", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "#f3f4f6", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 200003, + "version": 1, + "versionNonce": 200003, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "store_rect", + "type": "rectangle", + "x": 530, + "y": 320, + "width": 280, + "height": 100, + "text": "Linked expense records\n(one row per expense)", + "originalText": "Linked expense records\n(one row per expense)", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#0ea5e9", + "backgroundColor": "#e0f2fe", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 200004, + "version": 1, + "versionNonce": 200004, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "store_json", + "type": "rectangle", + "x": 510, + "y": 470, + "width": 300, + "height": 140, + "text": "{\n \"expense_id\": \"EXP-1042\",\n \"submission_id\": \"SUB-8810\",\n \"card_txn_id\": \"TXN-55217\",\n \"receipt_id\": \"RCPT-3091\"\n}", + "originalText": "{\n \"expense_id\": \"EXP-1042\",\n \"submission_id\": \"SUB-8810\",\n \"card_txn_id\": \"TXN-55217\",\n \"receipt_id\": \"RCPT-3091\"\n}", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#cdd6f4", + "strokeColor": "#313244", + "backgroundColor": "#1e1e2e", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 200005, + "version": 1, + "versionNonce": 200005, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "store_caption", + "type": "text", + "x": 510, + "y": 630, + "width": 355, + "height": 18, + "text": "Provenance preserved through every later step", + "originalText": "Provenance preserved through every later step", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#9ca3af", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 200006, + "version": 1, + "versionNonce": 200006, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "arr_a_store", + "type": "arrow", + "x": 246, + "y": 238, + "width": 278, + "height": 117, + "points": [[0, 0], [278, 117]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100010, + "version": 1, + "versionNonce": 100010, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "arr_b_store", + "type": "arrow", + "x": 246, + "y": 495, + "width": 278, + "height": 125, + "points": [[0, 0], [278, -125]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100011, + "version": 1, + "versionNonce": 100011, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "arr_c_store", + "type": "arrow", + "x": 246, + "y": 725, + "width": 278, + "height": 340, + "points": [[0, 0], [278, -340]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 100012, + "version": 1, + "versionNonce": 100012, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r3_region", + "type": "rectangle", + "x": 910, + "y": 150, + "width": 420, + "height": 720, + "text": "", + "originalText": "", + "strokeColor": "#e5e7eb", + "backgroundColor": "#f9fafb", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300001, + "version": 1, + "versionNonce": 300001, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r4_region", + "type": "rectangle", + "x": 1370, + "y": 150, + "width": 260, + "height": 720, + "text": "", + "originalText": "", + "strokeColor": "#e5e7eb", + "backgroundColor": "#f9fafb", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 400001, + "version": 1, + "versionNonce": 400001, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r3_label", + "type": "text", + "x": 930, + "y": 163, + "width": 370, + "height": 24, + "text": "3 · AGENT: MATCH · VERIFY · FLAG", + "originalText": "3 · AGENT: MATCH · VERIFY · FLAG", + "fontSize": 18, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#374151", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300002, + "version": 1, + "versionNonce": 300002, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r3_chip", + "type": "rectangle", + "x": 930, + "y": 193, + "width": 60, + "height": 24, + "text": "SCRIPT", + "originalText": "SCRIPT", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#8b5cf6", + "strokeColor": "#8b5cf6", + "backgroundColor": "#ede9fe", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300003, + "version": 1, + "versionNonce": 300003, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "match_rect", + "type": "rectangle", + "x": 970, + "y": 230, + "width": 300, + "height": 70, + "text": "Match expenses ↔ card txns\nby amount · date · merchant", + "originalText": "Match expenses ↔ card txns\nby amount · date · merchant", + "fontSize": 15, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#8b5cf6", + "backgroundColor": "#ede9fe", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300004, + "version": 1, + "versionNonce": 300004, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "arr_m_v", + "type": "arrow", + "x": 1120, + "y": 306, + "width": 0, + "height": 38, + "points": [[0, 0], [0, 38]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300005, + "version": 1, + "versionNonce": 300005, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "verify_rect", + "type": "rectangle", + "x": 970, + "y": 350, + "width": 300, + "height": 70, + "text": "Extract receipt fields (OCR)\n& verify against expense", + "originalText": "Extract receipt fields (OCR)\n& verify against expense", + "fontSize": 15, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#8b5cf6", + "backgroundColor": "#ede9fe", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300006, + "version": 1, + "versionNonce": 300006, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "arr_v_d", + "type": "arrow", + "x": 1120, + "y": 426, + "width": 0, + "height": 38, + "points": [[0, 0], [0, 38]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300007, + "version": 1, + "versionNonce": 300007, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "decide_diamond", + "type": "diamond", + "x": 990, + "y": 470, + "width": 260, + "height": 100, + "text": "Complete &\nconsistent?", + "originalText": "Complete &\nconsistent?", + "fontSize": 15, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#eab308", + "backgroundColor": "#fef9c3", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300008, + "version": 1, + "versionNonce": 300008, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "no_label", + "type": "text", + "x": 1132, + "y": 605, + "width": 25, + "height": 18, + "text": "no", + "originalText": "no", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#ef4444", + "strokeColor": "#ef4444", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300009, + "version": 1, + "versionNonce": 300009, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "arr_no_exc", + "type": "arrow", + "x": 1120, + "y": 576, + "width": 0, + "height": 78, + "points": [[0, 0], [0, 78]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#ef4444", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300010, + "version": 1, + "versionNonce": 300010, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "exc_rect", + "type": "rectangle", + "x": 970, + "y": 660, + "width": 300, + "height": 80, + "text": "Exception task — employee\nsupplies missing info", + "originalText": "Exception task — employee\nsupplies missing info", + "fontSize": 15, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#f97316", + "backgroundColor": "#ffedd5", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300011, + "version": 1, + "versionNonce": 300011, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "store_idmap", + "type": "text", + "x": 500, + "y": 780, + "width": 350, + "height": 40, + "text": "submission_id → form record · card_txn_id → CSV row\nreceipt_id → attached receipt file", + "originalText": "submission_id → form record · card_txn_id → CSV row\nreceipt_id → attached receipt file", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#9ca3af", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 200007, + "version": 1, + "versionNonce": 200007, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "arr_loop", + "type": "arrow", + "x": 964, + "y": 700, + "width": 514, + "height": 330, + "points": [[0, 0], [-514, 0], [-514, -330], [-440, -330]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300012, + "version": 1, + "versionNonce": 300012, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "loop_label", + "type": "text", + "x": 560, + "y": 708, + "width": 280, + "height": 16, + "text": "correction → re-matched & re-verified", + "originalText": "correction → re-matched & re-verified", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#9ca3af", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 300013, + "version": 1, + "versionNonce": 300013, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r4_label", + "type": "text", + "x": 1390, + "y": 163, + "width": 150, + "height": 24, + "text": "4 · ASSEMBLE", + "originalText": "4 · ASSEMBLE", + "fontSize": 18, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#374151", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 400002, + "version": 1, + "versionNonce": 400002, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r4_chip", + "type": "rectangle", + "x": 1548, + "y": 165, + "width": 60, + "height": 24, + "text": "SCRIPT", + "originalText": "SCRIPT", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#8b5cf6", + "strokeColor": "#8b5cf6", + "backgroundColor": "#ede9fe", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 400003, + "version": 1, + "versionNonce": 400003, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "asm_rect", + "type": "rectangle", + "x": 1410, + "y": 475, + "width": 190, + "height": 90, + "text": "Assemble monthly\nreport package", + "originalText": "Assemble monthly\nreport package", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#8b5cf6", + "backgroundColor": "#ede9fe", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 400004, + "version": 1, + "versionNonce": 400004, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "asm_caption", + "type": "text", + "x": 1395, + "y": 581, + "width": 230, + "height": 36, + "text": "Script: join linked records\ninto report lines", + "originalText": "Script: join linked records\ninto report lines", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 400005, + "version": 1, + "versionNonce": 400005, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "yes_label", + "type": "text", + "x": 1294, + "y": 490, + "width": 30, + "height": 18, + "text": "yes", + "originalText": "yes", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#22c55e", + "strokeColor": "#22c55e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 400006, + "version": 1, + "versionNonce": 400006, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "arr_yes_asm", + "type": "arrow", + "x": 1256, + "y": 520, + "width": 148, + "height": 0, + "points": [[0, 0], [148, 0]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 400007, + "version": 1, + "versionNonce": 400007, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "arr_asm_mock", + "type": "arrow", + "x": 1606, + "y": 520, + "width": 88, + "height": 190, + "points": [[0, 0], [88, -190]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 400008, + "version": 1, + "versionNonce": 400008, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r5_region", + "type": "rectangle", + "x": 1670, + "y": 150, + "width": 480, + "height": 720, + "text": "", + "originalText": "", + "strokeColor": "#e5e7eb", + "backgroundColor": "#f9fafb", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500001, + "version": 1, + "versionNonce": 500001, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r5_label", + "type": "text", + "x": 1690, + "y": 163, + "width": 140, + "height": 24, + "text": "5 · APPROVE", + "originalText": "5 · APPROVE", + "fontSize": 18, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#374151", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500002, + "version": 1, + "versionNonce": 500002, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r5_chip_config", + "type": "rectangle", + "x": 1970, + "y": 165, + "width": 64, + "height": 24, + "text": "CONFIG", + "originalText": "CONFIG", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "#f3f4f6", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500003, + "version": 1, + "versionNonce": 500003, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "r5_chip_core", + "type": "rectangle", + "x": 2042, + "y": 165, + "width": 84, + "height": 24, + "text": "CORE DEV", + "originalText": "CORE DEV", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#ef4444", + "strokeColor": "#ef4444", + "backgroundColor": "#fee2e2", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500004, + "version": 1, + "versionNonce": 500004, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_rect", + "type": "rectangle", + "x": 1700, + "y": 205, + "width": 430, + "height": 250, + "text": "", + "originalText": "", + "strokeColor": "#d1d5db", + "backgroundColor": "#ffffff", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500005, + "version": 1, + "versionNonce": 500005, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_title", + "type": "text", + "x": 1716, + "y": 218, + "width": 350, + "height": 20, + "text": "Approve — August 2026 expense report", + "originalText": "Approve — August 2026 expense report", + "fontSize": 15, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#111827", + "strokeColor": "#111827", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500006, + "version": 1, + "versionNonce": 500006, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_h1", + "type": "text", + "x": 1716, + "y": 250, + "width": 80, + "height": 14, + "text": "EXPENSE", + "originalText": "EXPENSE", + "fontSize": 11, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#9ca3af", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500007, + "version": 1, + "versionNonce": 500007, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_h2", + "type": "text", + "x": 1880, + "y": 250, + "width": 160, + "height": 14, + "text": "LINKED SOURCE RECORDS", + "originalText": "LINKED SOURCE RECORDS", + "fontSize": 11, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#9ca3af", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500008, + "version": 1, + "versionNonce": 500008, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_div1", + "type": "line", + "x": 1716, + "y": 270, + "width": 398, + "height": 0, + "points": [[0, 0], [398, 0]], + "strokeColor": "#e5e7eb", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500009, + "version": 1, + "versionNonce": 500009, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_r1a", + "type": "text", + "x": 1716, + "y": 278, + "width": 175, + "height": 18, + "text": "Uber · Aug 3 · $82.40", + "originalText": "Uber · Aug 3 · $82.40", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#111827", + "strokeColor": "#111827", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500010, + "version": 1, + "versionNonce": 500010, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_r1b", + "type": "text", + "x": 1880, + "y": 278, + "width": 240, + "height": 16, + "text": "SUB-8810 · TXN-55217 · RCPT-3091", + "originalText": "SUB-8810 · TXN-55217 · RCPT-3091", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#0ea5e9", + "strokeColor": "#0ea5e9", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500011, + "version": 1, + "versionNonce": 500011, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_div2", + "type": "line", + "x": 1716, + "y": 300, + "width": 398, + "height": 0, + "points": [[0, 0], [398, 0]], + "strokeColor": "#e5e7eb", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500012, + "version": 1, + "versionNonce": 500012, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_r2a", + "type": "text", + "x": 1716, + "y": 308, + "width": 190, + "height": 18, + "text": "Delta · Aug 4 · $412.00", + "originalText": "Delta · Aug 4 · $412.00", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#111827", + "strokeColor": "#111827", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500013, + "version": 1, + "versionNonce": 500013, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_r2b", + "type": "text", + "x": 1880, + "y": 308, + "width": 240, + "height": 16, + "text": "SUB-8811 · TXN-55231 · RCPT-3092", + "originalText": "SUB-8811 · TXN-55231 · RCPT-3092", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#0ea5e9", + "strokeColor": "#0ea5e9", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500014, + "version": 1, + "versionNonce": 500014, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_div3", + "type": "line", + "x": 1716, + "y": 330, + "width": 398, + "height": 0, + "points": [[0, 0], [398, 0]], + "strokeColor": "#e5e7eb", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500015, + "version": 1, + "versionNonce": 500015, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_r3a", + "type": "text", + "x": 1716, + "y": 338, + "width": 190, + "height": 18, + "text": "Hilton · Aug 5 · $268.90", + "originalText": "Hilton · Aug 5 · $268.90", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#111827", + "strokeColor": "#111827", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500016, + "version": 1, + "versionNonce": 500016, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_r3b", + "type": "text", + "x": 1880, + "y": 338, + "width": 240, + "height": 16, + "text": "SUB-8812 · TXN-55245 · RCPT-3093", + "originalText": "SUB-8812 · TXN-55245 · RCPT-3093", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#0ea5e9", + "strokeColor": "#0ea5e9", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500017, + "version": 1, + "versionNonce": 500017, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "btn_approve", + "type": "rectangle", + "x": 1716, + "y": 392, + "width": 100, + "height": 34, + "text": "Approve", + "originalText": "Approve", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#22c55e", + "backgroundColor": "#dcfce7", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500018, + "version": 1, + "versionNonce": 500018, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "btn_reject", + "type": "rectangle", + "x": 1828, + "y": 392, + "width": 100, + "height": 34, + "text": "Reject", + "originalText": "Reject", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#6b7280", + "strokeColor": "#d1d5db", + "backgroundColor": "#ffffff", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500019, + "version": 1, + "versionNonce": 500019, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "mock_caption", + "type": "text", + "x": 1700, + "y": 468, + "width": 445, + "height": 18, + "text": "Manager approves in-platform — human task, audit-logged", + "originalText": "Manager approves in-platform — human task, audit-logged", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500020, + "version": 1, + "versionNonce": 500020, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "gap_box", + "type": "rectangle", + "x": 1700, + "y": 520, + "width": 430, + "height": 205, + "text": "", + "originalText": "", + "strokeColor": "#ef4444", + "backgroundColor": "#fee2e2", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500021, + "version": 1, + "versionNonce": 500021, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "gap_title", + "type": "text", + "x": 1716, + "y": 532, + "width": 360, + "height": 20, + "text": "PLATFORM GAP → CORE PLATFORM DEVELOPMENT", + "originalText": "PLATFORM GAP → CORE PLATFORM DEVELOPMENT", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#ef4444", + "strokeColor": "#ef4444", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500022, + "version": 1, + "versionNonce": 500022, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "gap_body", + "type": "text", + "x": 1716, + "y": 558, + "width": 400, + "height": 158, + "text": "Requirement: render record-level provenance in the\napproval screen — each report line shows its linked\nsubmission, card transaction, and receipt as links\nto the original records.\nAcceptance criteria:\n• 100% of report lines display all three links\n• Links open the original submission, txn, receipt\n• Approval writes the linked IDs to the audit log", + "originalText": "Requirement: render record-level provenance in the\napproval screen — each report line shows its linked\nsubmission, card transaction, and receipt as links\nto the original records.\nAcceptance criteria:\n• 100% of report lines display all three links\n• Links open the original submission, txn, receipt\n• Approval writes the linked IDs to the audit log", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#374151", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500023, + "version": 1, + "versionNonce": 500023, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "out_ellipse", + "type": "ellipse", + "x": 2210, + "y": 320, + "width": 240, + "height": 80, + "text": "Approved monthly\nexpense report", + "originalText": "Approved monthly\nexpense report", + "fontSize": 16, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#22c55e", + "backgroundColor": "#dcfce7", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500024, + "version": 1, + "versionNonce": 500024, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "arr_mock_out", + "type": "arrow", + "x": 2136, + "y": 340, + "width": 68, + "height": 18, + "points": [[0, 0], [68, 18]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#374151", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 2, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 500025, + "version": 1, + "versionNonce": 500025, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "audit_bar", + "type": "rectangle", + "x": 60, + "y": 955, + "width": 2090, + "height": 55, + "text": "AUDIT LOG — every ingestion, match, flag, correction, and approval recorded with actor + timestamp", + "originalText": "AUDIT LOG — every ingestion, match, flag, correction, and approval recorded with actor + timestamp", + "fontSize": 15, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#111827", + "strokeColor": "#0ea5e9", + "backgroundColor": "#e0f2fe", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600010, + "version": 1, + "versionNonce": 600010, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "audit_arr_1", + "type": "arrow", + "x": 235, + "y": 876, + "width": 0, + "height": 73, + "points": [[0, 0], [0, 73]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600011, + "version": 1, + "versionNonce": 600011, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "audit_arr_2", + "type": "arrow", + "x": 670, + "y": 876, + "width": 0, + "height": 73, + "points": [[0, 0], [0, 73]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600012, + "version": 1, + "versionNonce": 600012, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "audit_arr_3", + "type": "arrow", + "x": 1120, + "y": 876, + "width": 0, + "height": 73, + "points": [[0, 0], [0, 73]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600013, + "version": 1, + "versionNonce": 600013, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "audit_arr_4", + "type": "arrow", + "x": 1500, + "y": 876, + "width": 0, + "height": 73, + "points": [[0, 0], [0, 73]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600014, + "version": 1, + "versionNonce": 600014, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "audit_arr_5", + "type": "arrow", + "x": 1910, + "y": 876, + "width": 0, + "height": 73, + "points": [[0, 0], [0, 73]], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "arrow", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600015, + "version": 1, + "versionNonce": 600015, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_title", + "type": "text", + "x": 60, + "y": 1035, + "width": 220, + "height": 18, + "text": "CLASSIFICATION KEY", + "originalText": "CLASSIFICATION KEY", + "fontSize": 13, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#9ca3af", + "strokeColor": "#9ca3af", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600016, + "version": 1, + "versionNonce": 600016, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_chip1", + "type": "rectangle", + "x": 60, + "y": 1065, + "width": 64, + "height": 24, + "text": "CONFIG", + "originalText": "CONFIG", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "#f3f4f6", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600017, + "version": 1, + "versionNonce": 600017, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_lab1", + "type": "text", + "x": 134, + "y": 1068, + "width": 195, + "height": 20, + "text": "Platform configuration", + "originalText": "Platform configuration", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600018, + "version": 1, + "versionNonce": 600018, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_chip2", + "type": "rectangle", + "x": 349, + "y": 1065, + "width": 60, + "height": 24, + "text": "SCRIPT", + "originalText": "SCRIPT", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#8b5cf6", + "strokeColor": "#8b5cf6", + "backgroundColor": "#ede9fe", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600019, + "version": 1, + "versionNonce": 600019, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_lab2", + "type": "text", + "x": 419, + "y": 1068, + "width": 300, + "height": 20, + "text": "Lightweight scripting (Python / JS)", + "originalText": "Lightweight scripting (Python / JS)", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600020, + "version": 1, + "versionNonce": 600020, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_chip3", + "type": "rectangle", + "x": 739, + "y": 1065, + "width": 84, + "height": 24, + "text": "CORE DEV", + "originalText": "CORE DEV", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#ef4444", + "strokeColor": "#ef4444", + "backgroundColor": "#fee2e2", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600021, + "version": 1, + "versionNonce": 600021, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_lab3", + "type": "text", + "x": 833, + "y": 1068, + "width": 220, + "height": 20, + "text": "Core platform development", + "originalText": "Core platform development", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600022, + "version": 1, + "versionNonce": 600022, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_chip4", + "type": "rectangle", + "x": 1073, + "y": 1065, + "width": 60, + "height": 24, + "text": "HUMAN", + "originalText": "HUMAN", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#f97316", + "strokeColor": "#f97316", + "backgroundColor": "#ffedd5", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600023, + "version": 1, + "versionNonce": 600023, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_lab4", + "type": "text", + "x": 1143, + "y": 1068, + "width": 90, + "height": 20, + "text": "Human step", + "originalText": "Human step", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600024, + "version": 1, + "versionNonce": 600024, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_chip5", + "type": "rectangle", + "x": 1253, + "y": 1065, + "width": 54, + "height": 24, + "text": "DATA", + "originalText": "DATA", + "fontSize": 12, + "fontFamily": 3, + "textAlign": "center", + "verticalAlign": "middle", + "color": "#0ea5e9", + "strokeColor": "#0ea5e9", + "backgroundColor": "#e0f2fe", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600025, + "version": 1, + "versionNonce": 600025, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + }, + { + "id": "leg_lab5", + "type": "text", + "x": 1317, + "y": 1068, + "width": 90, + "height": 20, + "text": "Data store", + "originalText": "Data store", + "fontSize": 14, + "fontFamily": 3, + "textAlign": "left", + "verticalAlign": "top", + "color": "#6b7280", + "strokeColor": "#6b7280", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "angle": 0, + "seed": 600026, + "version": 1, + "versionNonce": 600026, + "isDeleted": false, + "boundElements": [], + "updated": 1, + "link": null, + "locked": false, + "groupIds": [] + } + ], + "appState": { + "viewBackgroundColor": "#ffffff", + "gridSize": 20 + }, + "files": {} +} diff --git a/expense-report-workflow.png b/expense-report-workflow.png new file mode 100644 index 0000000..f998d41 Binary files /dev/null and b/expense-report-workflow.png differ diff --git a/go.mod b/go.mod index 62ec82a..e713e85 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,17 @@ module corral go 1.26.5 +require ( + github.com/charmbracelet/bubbletea v1.3.10 + github.com/charmbracelet/lipgloss v1.1.0 + github.com/charmbracelet/x/ansi v0.10.1 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 + modernc.org/sqlite v1.56.0 +) + require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect - github.com/charmbracelet/bubbletea v1.3.10 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect - github.com/charmbracelet/lipgloss v1.1.0 // indirect - github.com/charmbracelet/x/ansi v0.10.1 // indirect github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect github.com/charmbracelet/x/term v0.2.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -23,12 +28,10 @@ require ( github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect - github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.3.8 // indirect modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.56.0 // indirect ) diff --git a/go.sum b/go.sum index 406a95f..e2a4061 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,12 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= @@ -43,16 +47,44 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6Ng github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561 h1:MDc5xs78ZrZr3HMQugiXOAkSZtfTpbJLDr/lwfgO53E= +golang.org/x/exp v0.0.0-20220909182711-5c715a9e8561/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sqlite v1.56.0 h1:/D8e2RfFqoy/Zc6PuC76U28zFwmI/sYx1Kjm4yEn9e0= modernc.org/sqlite v1.56.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/adapter/adapter.go b/internal/adapter/adapter.go index 02aea2f..907ecfd 100644 --- a/internal/adapter/adapter.go +++ b/internal/adapter/adapter.go @@ -1,7 +1,7 @@ // Package adapter defines the generic executor contract that corral's -// scheduler uses to run nodes. OpenCode is the first implementation -// (Task 3); Codex, Claude and generic CLI drivers must satisfy the same -// interface. Nothing in this package depends on OpenCode. +// scheduler uses to run nodes. OpenCode is the production-wired +// implementation; the standalone Claude Code adapter satisfies the same +// contract. Nothing in this package depends on either provider. package adapter import ( @@ -110,6 +110,19 @@ type PermissionSession interface { RespondPermission(ctx context.Context, id string, allow bool) error } +// PermissionDetails carries the action an operator is about to authorize. +// Providers that can expose it should implement PermissionInfo; the scheduler +// includes it in blocked-state evidence so approvals are informed. +type PermissionDetails struct { + ID string + Tool string + Input string +} + +type PermissionInfo interface { + PendingPermissionDetails(context.Context) (PermissionDetails, bool, error) +} + // Event is a live stream item from a session, used for progress display // and for triggering verification once the attempt reaches idle. type Event struct { diff --git a/internal/assets/assets_test.go b/internal/assets/assets_test.go index 328a081..93f68fc 100644 --- a/internal/assets/assets_test.go +++ b/internal/assets/assets_test.go @@ -1,6 +1,7 @@ package assets import ( + "encoding/json" "os" "testing" ) @@ -33,6 +34,61 @@ func TestEmbeddedContentValid(t *testing.T) { if !contains(OpenCodeConfigJSON, `"corral-orchestrator"`) || !contains(OpenCodeConfigJSON, `"corral-planner"`) { t.Error("embedded config missing agents") } + if !contains(CorralPluginTS, `"graph" in parsed`) || !contains(CorralPluginTS, `(parsed as { graph: unknown }).graph`) { + t.Error("embedded start tool does not unwrap full corral_plan output") + } + if contains(CorralPluginTS, `return "operator"`) || contains(CorralPluginTS, `role ?? "operator"`) { + t.Error("embedded plugin lets unknown model agents mint operator authority") + } + if contains(CorralPluginTS, "autoApproveGates: tool.schema") { + t.Error("embedded plugin lets an orchestrator request its own gate pre-authorization") + } +} + +func TestEmbeddedAgentPoliciesFailClosed(t *testing.T) { + var cfg struct { + Agent map[string]struct { + Tools map[string]any `json:"tools"` + Permission map[string]any `json:"permission"` + } `json:"agent"` + } + if err := json.Unmarshal([]byte(OpenCodeConfigJSON), &cfg); err != nil { + t.Fatal(err) + } + for _, name := range []string{"corral-orchestrator", "corral-planner", "corral-worker", "corral-reviewer", "corral-merger"} { + agent, ok := cfg.Agent[name] + if !ok { + t.Fatalf("managed agent %q missing", name) + } + if len(agent.Tools) != 0 { + t.Errorf("%s has legacy tools override: %#v", name, agent.Tools) + } + if agent.Permission["*"] != "deny" { + t.Errorf("%s wildcard permission = %#v, want deny", name, agent.Permission["*"]) + } + if task, ok := agent.Permission["task"]; ok && task != "deny" { + t.Errorf("%s can delegate around its sandbox: %#v", name, task) + } + } + for name, action := range map[string]string{ + "corral_plan": "allow", "corral_start": "allow", "corral_status": "allow", + "corral_watch": "allow", "corral_approve": "allow", "corral_reject": "allow", + "corral_cancel": "allow", "corral_retry": "allow", "corral_steer": "allow", + } { + if cfg.Agent["corral-orchestrator"].Permission[name] != action { + t.Errorf("orchestrator %s = %#v, want %s", name, cfg.Agent["corral-orchestrator"].Permission[name], action) + } + } + if cfg.Agent["corral-planner"].Permission["corral_plan"] != "allow" { + t.Error("planner cannot call corral_plan") + } + if cfg.Agent["corral-planner"].Permission["glob"] != "allow" { + t.Error("planner cannot inspect repository structure") + } + worker := cfg.Agent["corral-worker"].Permission + if worker["edit"] != "ask" || worker["bash"] != "ask" || worker["glob"] != "allow" { + t.Errorf("worker policy = %#v", worker) + } } func contains(s, sub string) bool { diff --git a/internal/assets/corral.ts b/internal/assets/corral.ts index 37551b6..0c05691 100644 --- a/internal/assets/corral.ts +++ b/internal/assets/corral.ts @@ -18,7 +18,8 @@ async function loadKey(): Promise { } // Map the current OpenCode agent to a corral role for server-side -// enforcement. Anything unknown falls back to operator (human). +// enforcement. Unknown agents stay unprivileged; only non-model clients such +// as the CLI/TUI may claim the operator role directly. const ROLE_MAP: Record = { "corral-orchestrator": "orchestrator", "corral-planner": "planner", @@ -29,7 +30,7 @@ const ROLE_MAP: Record = { function roleFor(agent?: string): string { if (agent && agent in ROLE_MAP) return ROLE_MAP[agent] - return "operator" + return "unknown" } async function call(path: string, body?: unknown, role?: string) { @@ -40,7 +41,7 @@ async function call(path: string, body?: unknown, role?: string) { method: body === undefined ? "GET" : "POST", headers: { "Content-Type": "application/json", - "X-Corral-Role": role ?? "operator", + "X-Corral-Role": role ?? "unknown", ...(key ? { Authorization: `Bearer ${key}` } : {}), }, body: body === undefined ? undefined : JSON.stringify(body), @@ -67,14 +68,20 @@ export const plan = tool({ export const start = tool({ description: "Start a corral run from an approved graph.", - args: { graph: tool.schema.string().describe("Graph JSON (as returned by corral_plan)") }, + args: { + graph: tool.schema.string().describe("Graph JSON (as returned by corral_plan)"), + }, async execute(args, context) { - let graph: unknown + let parsed: unknown try { - graph = JSON.parse(args.graph) + parsed = JSON.parse(args.graph) } catch { return "error: graph is not valid JSON" } + const graph = + typeof parsed === "object" && parsed !== null && "graph" in parsed + ? (parsed as { graph: unknown }).graph + : parsed return call("/api/runs", { graph }, roleFor(context.agent)) }, }) @@ -89,6 +96,22 @@ export const status = tool({ }, }) +export const watch = tool({ + description: + "Watch a corral run and block until its state changes (new events, a human gate awaiting approval, or completion) or the timeout elapses. Drive the run loop by calling this repeatedly and passing the previous response's `since` cursor back. `gatesAwaitingApproval` lists human gates parked in running waiting for a decision: if the response's `autoApproveGates` is true the run is pre-authorized and you should approve each gate via corral_approve; otherwise never approve them yourself — report them to the user and keep watching until they resolve.", + args: { + runID: tool.schema.string(), + since: tool.schema.number().optional().describe("Event cursor; only return events after this"), + timeout: tool.schema.number().optional().describe("Block for up to this many seconds (default 60, max 120)"), + }, + async execute(args, context) { + const q = new URLSearchParams() + if (args.since !== undefined) q.set("since", String(args.since)) + if (args.timeout !== undefined) q.set("timeout", String(args.timeout)) + return call(`/api/runs/${args.runID}/watch?${q}`, undefined, roleFor(context.agent)) + }, +}) + export const approve = tool({ description: "Approve a human gate (or the run's merge) by node id.", args: { diff --git a/internal/assets/opencode.json b/internal/assets/opencode.json index 94e05f6..235e39f 100644 --- a/internal/assets/opencode.json +++ b/internal/assets/opencode.json @@ -4,13 +4,13 @@ "corral-orchestrator": { "description": "Corral orchestrator: controls runs and graphs, never edits files or runs arbitrary bash.", "mode": "primary", - "prompt": "You are the corral orchestrator. You plan runs, start them, monitor status, approve or reject gates, and steer workers. You never edit files and never run bash commands.", + "prompt": "You are the corral orchestrator. You start runs, watch them, approve or reject gates, and steer workers. You never edit files and never run bash commands.\n\nRun loop: start a run from an approved graph with corral_start, then repeatedly call corral_watch with the runID and the previous response's `since` cursor (use a timeout around 60) to follow it. Report milestones to the user as nodes progress.\n\nWhen corral_watch reports gatesAwaitingApproval:\n- If the response's autoApproveGates is true, the run is pre-authorized: call corral_approve for each waiting gate and continue watching.\n- If autoApproveGates is false, you are NOT pre-authorized: never call corral_approve. Tell the user the gate awaits their approval, and keep calling corral_watch until the gate resolves, then continue driving the run.\n\nKeep watching until the response shows done: true, then summarize the outcome for the user.", "permission": { - "edit": "deny", - "bash": "deny", + "*": "deny", "corral_plan": "allow", "corral_start": "allow", "corral_status": "allow", + "corral_watch": "allow", "corral_approve": "allow", "corral_reject": "allow", "corral_cancel": "allow", @@ -23,8 +23,13 @@ "mode": "primary", "prompt": "You are the corral planner. Analyze the codebase read-only and produce a corral task graph JSON for the user's goal using corral_plan. Never modify files or run commands.", "permission": { - "edit": "deny", - "bash": "deny", + "*": "deny", + "read": { + "*": "allow", + ".corral": "deny", + ".corral/*": "deny" + }, + "glob": "allow", "corral_plan": "allow" } }, @@ -33,25 +38,23 @@ "mode": "subagent", "prompt": "You are a corral worker. Implement the objective precisely, staying within your declared write scope. Write clean, minimal changes and verify your output.", "permission": { + "*": "deny", + "read": { + "*": "allow", + ".corral": "deny", + ".corral/*": "deny" + }, + "glob": "allow", "edit": "ask", "bash": "ask" } }, "corral-reviewer": { - "description": "Corral reviewer: read-only analysis plus tests and diffs.", + "description": "Corral reviewer: evaluates supplied evidence without tools.", "mode": "subagent", - "prompt": "You are a corral reviewer. Inspect the worker's output read-only; you may run tests and inspect diffs, but never modify files.", + "prompt": "You are a corral reviewer. Evaluate only the evidence supplied in the prompt. Never call tools or modify external state.", "permission": { - "edit": "deny", - "bash": { - "*": "deny", - "git diff*": "allow", - "git status*": "allow", - "git log*": "allow", - "go test*": "allow", - "npm test*": "allow", - "pytest*": "allow" - } + "*": "deny" } }, "corral-merger": { @@ -59,7 +62,7 @@ "mode": "subagent", "prompt": "You are the corral merger. You only perform git operations to merge accepted branches. Never edit source files.", "permission": { - "edit": "deny", + "*": "deny", "bash": { "*": "deny", "git status*": "allow", diff --git a/internal/claudeadapter/adapter.go b/internal/claudeadapter/adapter.go new file mode 100644 index 0000000..23ee928 --- /dev/null +++ b/internal/claudeadapter/adapter.go @@ -0,0 +1,1138 @@ +// Package claudeadapter maps the generic adapter contract onto Claude Code +// CLI/SDK sessions. Each attempt spawns a headless `claude -p` process +// (stream-json output) in the attempt's working directory and streams its +// transcript in; a watcher emits exactly one Completion per attempt, using +// the process exit as the reconciliation fallback when the terminal result +// event is missed. Permission prompts are forwarded through a local unix +// socket broker to the driver's --permission-prompt-tool MCP helper. +// +// The package is self-contained: it does not know about the scheduler and +// is not wired into cmd/corral. +package claudeadapter + +import ( + "bufio" + "bytes" + "context" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "corral/internal/adapter" +) + +// Options configures the Claude driver. +type Options struct { + // PollInterval is the fallback status-poll period. Completions are + // primarily driven by the process event stream; the poll re-checks + // terminal state in case an event was dropped. + PollInterval time.Duration + // Model overrides the default model for sessions ("" = driver default). + Model string + // Command is the claude binary to spawn ("" = "claude" on PATH). + Command string + // PermissionSocket overrides the unix socket path used by the + // permission broker ("" = auto in the temp dir). + PermissionSocket string + // DisablePermissions skips wiring the --permission-prompt-tool MCP + // helper and its broker socket. Permission prompts are then still + // tracked from the event stream but cannot be answered. + DisablePermissions bool +} + +func (o Options) poll() time.Duration { + if o.PollInterval <= 0 { + return time.Second + } + return o.PollInterval +} + +func (o Options) command() string { + if o.Command != "" { + return o.Command + } + return "claude" +} + +// Driver implements adapter.Driver and adapter.Stepper for Claude Code. +type Driver struct { + opts Options + + mu sync.Mutex + attempts map[string]*attempt // attemptID -> rec + bySession map[string]*attempt // sessionID -> rec + seen map[string]struct{} // all successfully started attempt IDs + completions []adapter.Completion + spawn spawnFunc + + brokerOnce sync.Once + broker *permissionBroker + brokerErr error + closed bool +} + +// attempt tracks one live claude session. +type attempt struct { + d *Driver + proc process + attemptID string + nodeID string + sessionID string // current id reported by Claude + launchID string // immutable --session-id used by the helper environment + cwd string + + aborted atomic.Bool + completed atomic.Bool + stopped bool // session closed by the driver; suppress completion + terminal bool // a terminal event (result/error/exit) was seen + subtype string + errMsg string + exited bool + exitErr error + readErr error + exitedCh chan processExit // stream reader -> watcher after stdout is drained + cancel context.CancelFunc + + mu sync.Mutex + transcript []adapter.Message + permission string // pending permission request id ("" = none); at most one request is admitted + permissionTool string + permissionInput string +} + +func New(opts Options) *Driver { + return &Driver{ + opts: opts, + attempts: map[string]*attempt{}, + bySession: map[string]*attempt{}, + seen: map[string]struct{}{}, + spawn: spawnCLI, + } +} + +// Close tears the driver down, killing any live sessions and the +// permission broker. +func (d *Driver) Close() { + d.mu.Lock() + if d.closed { + d.mu.Unlock() + return + } + d.closed = true + ats := make([]*attempt, 0, len(d.attempts)) + for _, at := range d.attempts { + ats = append(ats, at) + } + d.mu.Unlock() + for _, at := range ats { + at.cancel() + } + if d.broker != nil { + d.broker.close() + } +} + +// Start creates a headless claude session for the attempt, sends the +// objective, and starts a watcher that completes the attempt from the +// event stream with process-exit as fallback. +func (d *Driver) Start(ctx context.Context, a adapter.Attempt) (adapter.Session, error) { + d.mu.Lock() + if d.closed { + d.mu.Unlock() + return nil, fmt.Errorf("start claude: driver is closed") + } + if _, exists := d.seen[a.ID]; exists { + d.mu.Unlock() + return nil, fmt.Errorf("start claude: attempt %q already started", a.ID) + } + if !d.opts.DisablePermissions { + if err := d.startBroker(); err != nil { + d.mu.Unlock() + return nil, fmt.Errorf("start claude permission broker: %w", err) + } + } + spec := d.specFor(a) + proc, err := d.spawn(ctx, spec) + if err != nil { + d.mu.Unlock() + return nil, fmt.Errorf("start claude: %w", err) + } + atCtx, cancel := context.WithCancel(context.Background()) + at := &attempt{ + d: d, + proc: proc, + attemptID: a.ID, + nodeID: a.NodeID, + sessionID: spec.sessionID, + launchID: spec.sessionID, + cwd: a.Cwd, + exitedCh: make(chan processExit, 1), + cancel: cancel, + } + d.attempts[a.ID] = at + d.bySession[spec.sessionID] = at + d.seen[a.ID] = struct{}{} + d.mu.Unlock() + + go d.scan(at) + go at.watch(atCtx) + return &session{at: at}, nil +} + +// specFor builds the claude invocation for an attempt. +func (d *Driver) specFor(a adapter.Attempt) spawnSpec { + sid := newUUID() + args := []string{ + "-p", promptFor(a), + "--output-format", "stream-json", + "--verbose", + "--include-partial-messages", + "--session-id", sid, + } + if model := d.modelFor(a); model != "" { + args = append(args, "--model", model) + } + if tools := allowedTools(a); len(tools) > 0 { + if !d.opts.DisablePermissions && d.broker != nil { + tools = append(tools, permissionToolName) + } + args = append(args, "--allowedTools") + args = append(args, tools...) + } + env := os.Environ() + if !d.opts.DisablePermissions && d.broker != nil { + cfg, _ := json.Marshal(mcpConfig{ + Servers: map[string]mcpServerConfig{ + permissionServerName: { + Command: helperExecutable(), + Args: []string{helperFlag}, + Env: map[string]string{ + "CORRAL_CLAUDE_BROKER": d.broker.path, + "CORRAL_CLAUDE_ATTEMPT_ID": a.ID, + "CORRAL_CLAUDE_SESSION_ID": sid, + }, + }, + }, + }) + args = append(args, "--mcp-config", string(cfg)) + args = append(args, "--permission-prompt-tool", permissionToolName) + env = append(env, + "CORRAL_CLAUDE_BROKER="+d.broker.path, + "CORRAL_CLAUDE_ATTEMPT_ID="+a.ID, + "CORRAL_CLAUDE_SESSION_ID="+sid, + ) + } + return spawnSpec{ + command: d.opts.command(), + args: args, + env: env, + dir: a.Cwd, + sessionID: sid, + } +} + +func (d *Driver) modelFor(a adapter.Attempt) string { + if a.Model != "" { + return a.Model + } + return d.opts.Model +} + +// Step drains completed attempts (non-blocking). +func (d *Driver) Step(_ context.Context, _ time.Time) []adapter.Completion { + d.mu.Lock() + defer d.mu.Unlock() + out := d.completions + d.completions = nil + return out +} + +// attemptBySession returns the attempt owning a session id, if any. +func (d *Driver) attemptBySession(sessionID string) *attempt { + d.mu.Lock() + defer d.mu.Unlock() + return d.bySession[sessionID] +} + +func (d *Driver) attemptByID(attemptID string) *attempt { + d.mu.Lock() + defer d.mu.Unlock() + return d.attempts[attemptID] +} + +// updateSessionID atomically remaps an attempt when Claude's init event +// reports a session id different from the requested --session-id. +func (d *Driver) updateSessionID(at *attempt, sessionID string) { + if sessionID == "" { + return + } + d.mu.Lock() + defer d.mu.Unlock() + at.mu.Lock() + oldID := at.sessionID + at.sessionID = sessionID + at.mu.Unlock() + if oldID != sessionID { + delete(d.bySession, oldID) + } + d.bySession[sessionID] = at +} + +func (at *attempt) cleanup() { + at.d.mu.Lock() + delete(at.d.attempts, at.attemptID) + at.mu.Lock() + sessionID := at.sessionID + at.mu.Unlock() + delete(at.d.bySession, sessionID) + at.d.mu.Unlock() +} + +type processExit struct { + waitErr error + readErr error +} + +// scan reads arbitrary-size stream-json records and handles them inline. This +// keeps transcript/result processing ordered and lossless without a bounded +// event queue. On a read failure it continues draining stdout before Wait so a +// child blocked on a full pipe cannot deadlock shutdown. +func (d *Driver) scan(at *attempt) { + stdout := at.proc.stdout() + r := bufio.NewReader(stdout) + var readErr error + for { + line, err := r.ReadBytes('\n') + if len(bytes.TrimSpace(line)) > 0 { + var ev streamEvent + if json.Unmarshal(line, &ev) == nil { + at.handleEvent(ev) + } + } + if err == nil { + continue + } + if !errors.Is(err, io.EOF) { + readErr = fmt.Errorf("read claude stream: %w", err) + _, _ = io.Copy(io.Discard, r) + } + break + } + exit := processExit{waitErr: at.proc.wait(), readErr: readErr} + // Release the parent read descriptor only after all output is drained, but + // before publishing process exit so completion implies stdout is closed. + if closer, ok := stdout.(io.Closer); ok { + _ = closer.Close() + } + select { + case at.exitedCh <- exit: + default: // watcher already gone (driver closed) + } +} + +// watch drives an attempt to a terminal state. The event stream is the +// primary signal; a poll ticker is the reconciliation fallback. +func (at *attempt) watch(ctx context.Context) { + defer at.cleanup() + poll := time.NewTicker(at.d.opts.poll()) + defer poll.Stop() + for { + select { + case <-ctx.Done(): + at.terminate() + return + case <-poll.C: + at.d.maybeComplete(context.Background(), at) + case exit := <-at.exitedCh: + at.onExit(exit) + return + } + } +} + +func (at *attempt) handleEvent(ev streamEvent) { + switch ev.Type { + case "system": + if ev.Subtype != "init" { + return + } + if ev.SessionID != "" { + at.d.updateSessionID(at, ev.SessionID) + } + case "assistant": + at.appendMessage(ev.Message, "assistant") + case "user": + if rid := permissionRequest(ev.Message); rid != "" { + at.mu.Lock() + if at.permission == "" { + at.permission = rid + } + at.mu.Unlock() + return + } + at.appendMessage(ev.Message, "user") + case "result": + at.mu.Lock() + at.terminal = true + at.subtype = ev.Subtype + at.mu.Unlock() + at.applyResultAccounting(ev.TotalCostUSD, ev.Usage) + case "stream_error": + at.mu.Lock() + at.terminal = true + if ev.Error != "" { + at.errMsg = ev.Error + } + at.mu.Unlock() + } +} + +// onExit marks the process exit as the terminal event. It is the +// reconciliation fallback when the result event was dropped. +func (at *attempt) onExit(exit processExit) { + at.mu.Lock() + if at.stopped { + at.mu.Unlock() + return + } + at.terminal = true + at.exited = true + at.exitErr = exit.waitErr + at.readErr = exit.readErr + at.mu.Unlock() + at.d.maybeComplete(context.Background(), at) +} + +// terminate hard-kills a live process on driver shutdown. +func (at *attempt) terminate() { + at.mu.Lock() + at.stopped = true + at.mu.Unlock() + select { + case <-at.proc.done(): + return + default: + } + _ = at.proc.signal(os.Kill) + select { + case <-at.proc.done(): + case <-time.After(2 * time.Second): + } +} + +// appendMessage accumulates one transcript entry from a stream message. +func (at *attempt) appendMessage(raw json.RawMessage, role string) { + var m sdkMessage + if err := json.Unmarshal(raw, &m); err != nil { + return + } + am := adapter.Message{Role: role, Finish: m.Stop} + for _, c := range m.Content { + var b contentBlock + if json.Unmarshal(c, &b) != nil { + continue + } + switch b.Type { + case "text": + am.Text += b.Text + case "tool_result": + am.Text += toolResultText(b.Content) + } + } + at.mu.Lock() + at.transcript = append(at.transcript, am) + at.mu.Unlock() +} + +// applyResultAccounting stores Claude's cumulative result totals exactly once +// on the final assistant message. The scheduler sums message accounting, so +// copying per-turn assistant usage as well would double-count a session. +func (at *attempt) applyResultAccounting(cost float64, usage *sdkUsage) { + if cost == 0 && usage == nil { + return + } + at.mu.Lock() + defer at.mu.Unlock() + for i := len(at.transcript) - 1; i >= 0; i-- { + if at.transcript[i].Role == "assistant" { + at.transcript[i].Cost = cost + at.transcript[i].Tokens = usage.total() + return + } + } + at.transcript = append(at.transcript, adapter.Message{ + Role: "assistant", + Cost: cost, + Tokens: usage.total(), + }) +} + +func (at *attempt) snapshot() []adapter.Message { + at.mu.Lock() + defer at.mu.Unlock() + out := make([]adapter.Message, len(at.transcript)) + copy(out, at.transcript) + return out +} + +func (at *attempt) appendDiffs(diffs []adapter.Diff) { + if len(diffs) == 0 { + return + } + at.mu.Lock() + defer at.mu.Unlock() + // OpenCode exposes authoritative diff summaries on user messages. Keep + // the same adapter contract for Claude so default verification and + // reviewers consume provider-independent evidence. + at.transcript = append(at.transcript, adapter.Message{Role: "user", Diffs: diffs}) +} + +func (at *attempt) currentSessionID() string { + at.mu.Lock() + defer at.mu.Unlock() + return at.sessionID +} + +// terminalStatus decides whether the attempt reached a terminal state and +// what adapter.Status it maps to. Aborted attempts are terminal +// immediately; otherwise a result/error/exit event must have been seen. +func (at *attempt) terminalStatus() (adapter.Status, bool) { + at.mu.Lock() + defer at.mu.Unlock() + if !at.exited { + return "", false + } + if at.aborted.Load() { + return adapter.StatusAborted, true + } + if !at.terminal { + return "", false + } + if at.readErr != nil { + return adapter.StatusError, true + } + if at.exitErr != nil { + return adapter.StatusError, true + } + if at.subtype == "success" { + return adapter.StatusIdle, true + } + if at.errMsg != "" || at.subtype != "" { + return adapter.StatusError, true + } + if at.exited { + if at.exitErr == nil && len(at.transcript) > 0 { + return adapter.StatusIdle, true + } + return adapter.StatusError, true + } + return adapter.StatusIdle, true +} + +func (at *attempt) completionError() error { + at.mu.Lock() + defer at.mu.Unlock() + if at.readErr != nil { + return at.readErr + } + if at.errMsg != "" { + return errors.New(at.errMsg) + } + if at.exitErr != nil { + return at.exitErr + } + if at.subtype != "" && at.subtype != "success" { + return fmt.Errorf("claude result: %s", at.subtype) + } + return nil +} + +// maybeComplete emits a completion exactly once per attempt, guarded +// against duplicate or missing events. +func (d *Driver) maybeComplete(ctx context.Context, at *attempt) { + status, ok := at.terminalStatus() + if !ok { + return + } + if !at.completed.CompareAndSwap(false, true) { + return // duplicate event; already handled + } + completionErr := at.completionError() + if status == adapter.StatusIdle { + diffs, err := captureGitDiffs(ctx, at.cwd) + if err != nil { + // Verification must never receive a successful completion carrying + // partial or missing evidence for a Git worktree. A plain non-Git + // cwd is intentionally exempt and returns (nil, nil) below. + status = adapter.StatusError + completionErr = errors.Join(completionErr, fmt.Errorf("capture git diff evidence: %w", err)) + } else { + at.appendDiffs(diffs) + } + } + c := adapter.Completion{ + AttemptID: at.attemptID, + SessionID: at.currentSessionID(), + Status: status, + Messages: at.snapshot(), + Err: completionErr, + } + d.mu.Lock() + if !d.closed { + d.completions = append(d.completions, c) + } + d.mu.Unlock() +} + +// captureGitDiffs reads all tracked and untracked worktree changes against +// HEAD without touching the index. Attempts outside a Git worktree (or with +// no changes) simply provide no evidence. Once cwd is confirmed as a Git +// worktree, collection is atomic: any command or parse failure returns no +// diffs and an error so verification fails closed instead of seeing a partial +// change set. +func captureGitDiffs(ctx context.Context, cwd string) ([]adapter.Diff, error) { + if strings.TrimSpace(cwd) == "" { + return nil, nil + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + inside, err := gitOutput(ctx, cwd, false, "rev-parse", "--is-inside-work-tree") + if err != nil { + if isNotGitRepository(err) { + return nil, nil + } + return nil, err + } + if strings.TrimSpace(string(inside)) != "true" { + return nil, nil + } + statusOut, err := gitOutput(ctx, cwd, false, + "diff", "--no-ext-diff", "--no-textconv", "--no-renames", "--name-status", "-z", "HEAD", "--") + if err != nil { + return nil, err + } + + statuses := map[string]string{} + untrackedPaths := map[string]bool{} + fields := splitNUL(statusOut) + if len(fields)%2 != 0 { + return nil, fmt.Errorf("parse git name-status: odd field count %d", len(fields)) + } + for i := 0; i+1 < len(fields); i += 2 { + statuses[fields[i+1]] = diffStatus(fields[i]) + } + untracked, err := gitOutput(ctx, cwd, false, + "ls-files", "--others", "--exclude-standard", "-z", "--") + if err != nil { + return nil, err + } + for _, path := range splitNUL(untracked) { + statuses[path] = "added" + untrackedPaths[path] = true + } + + paths := make([]string, 0, len(statuses)) + for path := range statuses { + paths = append(paths, path) + } + sort.Strings(paths) + diffs := make([]adapter.Diff, 0, len(paths)) + for _, path := range paths { + untracked := untrackedPaths[path] + args := []string{"diff", "--no-ext-diff", "--no-textconv", "--binary", "--full-index", "--no-renames"} + if untracked { + args = append(args, "--no-index", "--", "/dev/null", path) + } else { + args = append(args, "HEAD", "--", path) + } + patch, err := gitOutput(ctx, cwd, untracked, args...) + if err != nil { + return nil, err + } + if len(patch) == 0 { + return nil, fmt.Errorf("git returned no patch for changed path %q", path) + } + + statArgs := []string{"diff", "--no-ext-diff", "--no-textconv", "--numstat", "-z", "--no-renames"} + if untracked { + statArgs = append(statArgs, "--no-index", "--", "/dev/null", path) + } else { + statArgs = append(statArgs, "HEAD", "--", path) + } + stat, err := gitOutput(ctx, cwd, untracked, statArgs...) + if err != nil { + return nil, err + } + additions, deletions, err := parseNumstat(stat) + if err != nil { + return nil, fmt.Errorf("parse git numstat for %q: %w", path, err) + } + diffs = append(diffs, adapter.Diff{ + File: path, Patch: string(patch), Additions: additions, + Deletions: deletions, Status: statuses[path], + }) + } + return diffs, nil +} + +func gitOutput(ctx context.Context, cwd string, allowDiffExit bool, args ...string) ([]byte, error) { + displayArgs := append([]string(nil), args...) + args = append([]string{"-c", "core.fsmonitor=false"}, args...) + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Dir = cwd + cmd.Env = append(os.Environ(), "LC_ALL=C") + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err == nil { + return out, nil + } + var exitErr *exec.ExitError + if allowDiffExit && errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return out, nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + detail := strings.TrimSpace(stderr.String()) + if detail != "" { + return nil, fmt.Errorf("git %s: %w: %s", strings.Join(displayArgs, " "), err, detail) + } + return nil, fmt.Errorf("git %s: %w", strings.Join(displayArgs, " "), err) +} + +func isNotGitRepository(err error) bool { + return strings.Contains(err.Error(), "not a git repository") +} + +func splitNUL(data []byte) []string { + raw := bytes.Split(data, []byte{0}) + out := make([]string, 0, len(raw)) + for _, field := range raw { + if len(field) > 0 { + out = append(out, string(field)) + } + } + return out +} + +func diffStatus(code string) string { + switch { + case strings.HasPrefix(code, "A"): + return "added" + case strings.HasPrefix(code, "D"): + return "deleted" + default: + return "modified" + } +} + +func parseNumstat(data []byte) (additions, deletions int, err error) { + line := data + if i := bytes.IndexByte(line, 0); i >= 0 { + line = line[:i] + } + first := bytes.IndexByte(line, '\t') + if first < 0 { + return 0, 0, fmt.Errorf("missing additions field") + } + secondRel := bytes.IndexByte(line[first+1:], '\t') + if secondRel < 0 { + return 0, 0, fmt.Errorf("missing deletions field") + } + second := first + 1 + secondRel + parse := func(field []byte) (int, error) { + if bytes.Equal(field, []byte("-")) { // binary diff + return 0, nil + } + return strconv.Atoi(string(field)) + } + additions, err = parse(line[:first]) + if err != nil { + return 0, 0, fmt.Errorf("invalid additions: %w", err) + } + deletions, err = parse(line[first+1 : second]) + if err != nil { + return 0, 0, fmt.Errorf("invalid deletions: %w", err) + } + return additions, deletions, nil +} + +// session implements adapter.Session and adapter.PermissionSession for a +// live claude session. +type session struct { + at *attempt +} + +func (s *session) ID() string { + s.at.mu.Lock() + defer s.at.mu.Unlock() + return s.at.sessionID +} +func (s *session) ServerID() string { return s.at.d.opts.command() } + +// Send is unsupported: each claude attempt is one headless process running +// its prompt to completion, so there is no live process to steer. +func (s *session) Send(_ context.Context, text string) error { + return fmt.Errorf("claudeadapter: cannot send %q to session %s: sessions run one prompt to completion", text, s.ID()) +} + +func (s *session) Abort(ctx context.Context) error { + // Serialize successful signal delivery with onExit/terminalStatus. This + // prevents a fast process exit from publishing an idle completion between + // signal delivery and marking the attempt aborted, while still ensuring + // failed signals never falsify the provider state. + s.at.mu.Lock() + select { + case <-s.at.proc.done(): + s.at.mu.Unlock() + return nil // already exited naturally; preserve its terminal status + default: + } + // SIGTERM is Claude Code's graceful stop (it aborts the turn, runs + // SessionEnd hooks and exits 143). Some platforms only implement Kill. + if termErr := s.at.proc.signal(syscall.SIGTERM); termErr != nil { + if killErr := s.at.proc.signal(os.Kill); killErr != nil { + select { + case <-s.at.proc.done(): + s.at.mu.Unlock() + return nil // raced with natural exit; neither signal succeeded + default: + } + s.at.mu.Unlock() + return errors.Join(termErr, killErr) + } + } + s.at.aborted.Store(true) + s.at.mu.Unlock() + select { + case <-s.at.proc.done(): + return nil + case <-ctx.Done(): + _ = s.at.proc.signal(os.Kill) + return ctx.Err() + case <-time.After(5 * time.Second): + return s.at.proc.signal(os.Kill) + } +} + +func (s *session) Status(_ context.Context) (adapter.Status, error) { + if s.at.aborted.Load() { + return adapter.StatusAborted, nil + } + if st, ok := s.at.terminalStatus(); ok { + return st, nil + } + return adapter.StatusRunning, nil +} + +func (s *session) Messages(_ context.Context) ([]adapter.Message, error) { + return s.at.snapshot(), nil +} + +func (s *session) Close(context.Context) error { + s.at.cancel() + return nil +} + +func (s *session) PendingPermission(_ context.Context) (string, bool, error) { + s.at.mu.Lock() + defer s.at.mu.Unlock() + if s.at.permission != "" { + return s.at.permission, true, nil + } + return "", false, nil +} + +func (s *session) RespondPermission(ctx context.Context, id string, allow bool) error { + s.at.mu.Lock() + if s.at.permission == "" { + s.at.mu.Unlock() + return fmt.Errorf("claudeadapter: no pending permission request") + } + if s.at.permission != id { + pending := s.at.permission + s.at.mu.Unlock() + return fmt.Errorf("claudeadapter: permission %q is not pending (want %q)", id, pending) + } + s.at.mu.Unlock() + if err := s.at.d.respondPermission(ctx, s.at, id, allow); err != nil { + return err + } + s.at.mu.Lock() + if s.at.permission == id { + s.at.permission = "" // resolved; the scheduler resumes automatically + } + s.at.mu.Unlock() + return nil +} + +// respondPermission delivers a decision through the broker. When no helper +// is waiting for the id (e.g. a stream-only tracked prompt) the request is +// treated as resolved. +func (d *Driver) respondPermission(ctx context.Context, at *attempt, id string, allow bool) error { + if d.broker == nil { + return nil + } + return d.broker.respond(ctx, at, id, allow) +} + +// promptFor builds the headless prompt for an attempt, mirroring ocx. +func promptFor(a adapter.Attempt) string { + var b strings.Builder + if a.Role != "" { + b.WriteString("(role: " + a.Role + ")\n") + } + b.WriteString(a.Objective) + if a.Feedback != "" { + b.WriteString("\n\nPrevious attempt was rejected. Fix these issues:\n" + a.Feedback) + } + return b.String() +} + +// allowedTools maps a writing attempt's scope onto Claude Code permission +// rules. Non-writing roles stay read-only even if a malformed graph supplies +// a write scope; role is the scheduler's authority boundary. +func allowedTools(a adapter.Attempt) []string { + tools := []string{"Read", "Glob", "Grep"} + if a.Role != "" && a.Role != "worker" { + return tools + } + seen := map[string]bool{"Read": true, "Glob": true, "Grep": true} + add := func(tool string) { + if !seen[tool] { + seen[tool] = true + tools = append(tools, tool) + } + } + if len(a.WriteScope) == 0 { + add("Edit") + add("Write") + } + for _, p := range a.WriteScope { + p = strings.TrimSpace(strings.ReplaceAll(p, `\`, "/")) + if p == "" { + continue + } + if p == "*" || p == "." { + add("Edit") + add("Write") + continue + } + p = strings.TrimSuffix(p, "/") + add("Edit(" + p + ")") + add("Write(" + p + ")") + // A declared directory scope covers descendants too. Adding this for + // a file scope is harmless and avoids guessing from filesystem state. + if !strings.ContainsAny(p, "*?[") { + add("Edit(" + p + "/**)") + add("Write(" + p + "/**)") + } + } + return tools +} + +func (s *session) PendingPermissionDetails(ctx context.Context) (adapter.PermissionDetails, bool, error) { + id, pending, err := s.PendingPermission(ctx) + if err != nil || !pending { + return adapter.PermissionDetails{}, false, err + } + s.at.mu.Lock() + defer s.at.mu.Unlock() + return adapter.PermissionDetails{ID: id, Tool: s.at.permissionTool, Input: s.at.permissionInput}, true, nil +} + +// newUUID returns a v4 UUID used as the claude --session-id. +func newUUID() string { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + // Non-crypto fallback (still RFC4122-shaped). + for i := range b { + b[i] = byte(i * 7) + } + } + b[6] = (b[6] & 0x0f) | 0x40 // version 4 + b[8] = (b[8] & 0x3f) | 0x80 // variant 10 + return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) +} + +// --------------------------------------------------------------------------- +// Claude Code stream-json protocol types. + +type streamEvent struct { + Type string `json:"type"` + Subtype string `json:"subtype"` + SessionID string `json:"session_id"` + Message json.RawMessage `json:"message"` + Result string `json:"result"` + Error string `json:"error"` + Event json.RawMessage `json:"event"` + TotalCostUSD float64 `json:"total_cost_usd"` + Usage *sdkUsage `json:"usage"` +} + +type sdkMessage struct { + Role string `json:"role"` + Content []json.RawMessage `json:"content"` + Stop string `json:"stop_reason"` +} + +type sdkUsage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + CacheCreationTokens int `json:"cache_creation_input_tokens"` + CacheReadTokens int `json:"cache_read_input_tokens"` +} + +func (u *sdkUsage) total() int { + if u == nil { + return 0 + } + return u.InputTokens + u.OutputTokens + u.CacheCreationTokens + u.CacheReadTokens +} + +type contentBlock struct { + Type string `json:"type"` + Text string `json:"text"` + ToolUseID string `json:"tool_use_id"` + Content json.RawMessage `json:"content"` + IsError bool `json:"is_error"` + State string `json:"state"` + ToolName string `json:"tool_name"` + RequestID string `json:"request_id"` + ID string `json:"id"` +} + +// toolResultText flattens a tool_result content field, which may be a plain +// string or an array of text blocks. +func toolResultText(raw json.RawMessage) string { + var s string + if json.Unmarshal(raw, &s) == nil { + return s + } + var blocks []struct { + Type string `json:"type"` + Text string `json:"text"` + } + if json.Unmarshal(raw, &blocks) != nil { + return "" + } + var b strings.Builder + for _, bl := range blocks { + if bl.Type == "text" { + b.WriteString(bl.Text) + } + } + return b.String() +} + +// permissionRequest returns a pending permission request id from a user +// message, or "" when the message carries none. +func permissionRequest(raw json.RawMessage) string { + var m sdkMessage + if err := json.Unmarshal(raw, &m); err != nil { + return "" + } + for _, c := range m.Content { + var b contentBlock + if json.Unmarshal(c, &b) != nil || b.Type != "permission_request" { + continue + } + if b.State != "needs_response" && b.State != "" { + continue + } + if b.RequestID != "" { + return b.RequestID + } + if b.ID != "" { + return b.ID + } + } + return "" +} + +// --------------------------------------------------------------------------- +// Process abstraction. The default implementation spawns the claude binary; +// tests substitute an in-process fake speaking the same stream-json protocol. + +type process interface { + stdout() io.Reader + done() <-chan struct{} // closed once the process has exited + wait() error // Wait result; valid once done() is closed + signal(os.Signal) error +} + +type spawnSpec struct { + command string + args []string + env []string + dir string + sessionID string +} + +type spawnFunc func(ctx context.Context, spec spawnSpec) (process, error) + +type cliProcess struct { + cmd *exec.Cmd + out io.ReadCloser + doneCh chan struct{} + waitMu sync.Mutex + waitErr error +} + +func (p *cliProcess) stdout() io.Reader { return p.out } +func (p *cliProcess) done() <-chan struct{} { + return p.doneCh +} +func (p *cliProcess) wait() error { + <-p.doneCh + p.waitMu.Lock() + defer p.waitMu.Unlock() + return p.waitErr +} +func (p *cliProcess) signal(sig os.Signal) error { + return p.cmd.Process.Signal(sig) +} + +func spawnCLI(ctx context.Context, spec spawnSpec) (process, error) { + cmd := exec.CommandContext(ctx, spec.command, spec.args...) + cmd.Dir = spec.dir + cmd.Env = spec.env + out, childOut, err := os.Pipe() + if err != nil { + return nil, err + } + cmd.Stdout = childOut + cmd.Stderr = io.Discard + if err := cmd.Start(); err != nil { + _ = out.Close() + _ = childOut.Close() + return nil, err + } + // The prompt comes from argv, so stdin stays closed. Close the parent's + // writer copy after Start; the child's inherited descriptor keeps the pipe + // open until exit. Unlike Cmd.StdoutPipe, this lets Wait run concurrently + // without closing unread buffered output. + _ = childOut.Close() + p := &cliProcess{cmd: cmd, out: out, doneCh: make(chan struct{})} + go func() { + err := cmd.Wait() + p.waitMu.Lock() + p.waitErr = err + p.waitMu.Unlock() + close(p.doneCh) + }() + return p, nil +} diff --git a/internal/claudeadapter/adapter_test.go b/internal/claudeadapter/adapter_test.go new file mode 100644 index 0000000..ed67ee1 --- /dev/null +++ b/internal/claudeadapter/adapter_test.go @@ -0,0 +1,1711 @@ +package claudeadapter + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" + + "corral/internal/adapter" + "corral/internal/clock" + "corral/internal/graph" + "corral/internal/livetest" + "corral/internal/sched" + "corral/internal/store" + "corral/internal/verify" +) + +// TestMain forwards the --corral-claude-permission-tool flag to the package's +// MCP permission helper so the live test's real claude can call back through +// the driver's broker (the same contract an embedding binary must honour). +func TestMain(m *testing.M) { + if len(os.Args) > 1 && os.Args[1] == helperFlag { + RunPermissionHelper() + os.Exit(0) + } + os.Exit(m.Run()) +} + +// fakeProc simulates a claude process: it emits stream-json events written by +// the test and records signals for abort assertions. +type fakeProc struct { + r *io.PipeReader + w *io.PipeWriter + doneCh chan struct{} + sigCh chan os.Signal + spec spawnSpec + mu sync.Mutex + exited bool + waitErr error + termErr error + killErr error + killExit bool + out io.Reader +} + +type trackingReadCloser struct { + io.ReadCloser + closed chan struct{} + once sync.Once +} + +func (r *trackingReadCloser) Close() error { + err := r.ReadCloser.Close() + r.once.Do(func() { close(r.closed) }) + return err +} + +// faultReader emits valid stream data, one synthetic read error, then more +// bytes. drained closes only after a caller keeps reading through the fault. +type faultReader struct { + before []byte + after []byte + fault error + stage int + drained chan struct{} + once sync.Once +} + +func (r *faultReader) Read(p []byte) (int, error) { + switch r.stage { + case 0: + if len(r.before) > 0 { + n := copy(p, r.before) + r.before = r.before[n:] + return n, nil + } + r.stage++ + return 0, r.fault + case 1: + if len(r.after) > 0 { + n := copy(p, r.after) + r.after = r.after[n:] + if len(r.after) == 0 { + r.stage++ + r.once.Do(func() { close(r.drained) }) + } + return n, nil + } + r.stage++ + r.once.Do(func() { close(r.drained) }) + return 0, io.EOF + default: + r.once.Do(func() { close(r.drained) }) + return 0, io.EOF + } +} + +type faultProc struct { + r *faultReader + doneCh chan struct{} + doneOnce sync.Once +} + +func newFaultProc(before, after []byte, fault error) *faultProc { + drained := make(chan struct{}) + p := &faultProc{ + r: &faultReader{before: before, after: after, fault: fault, drained: drained}, + doneCh: make(chan struct{}), + } + go func() { + <-drained + p.doneOnce.Do(func() { close(p.doneCh) }) + }() + return p +} + +func (p *faultProc) stdout() io.Reader { return p.r } +func (p *faultProc) done() <-chan struct{} { return p.doneCh } +func (p *faultProc) wait() error { + <-p.doneCh + return nil +} +func (p *faultProc) signal(os.Signal) error { + p.doneOnce.Do(func() { close(p.doneCh) }) + return nil +} + +func newFakeProc() (*fakeProc, *io.PipeReader) { + r, w := io.Pipe() + return &fakeProc{ + r: r, + w: w, + doneCh: make(chan struct{}), + sigCh: make(chan os.Signal, 8), + }, r +} + +func (f *fakeProc) stdout() io.Reader { + if f.out != nil { + return f.out + } + return f.r +} +func (f *fakeProc) done() <-chan struct{} { + return f.doneCh +} +func (f *fakeProc) wait() error { + f.mu.Lock() + defer f.mu.Unlock() + return f.waitErr +} +func (f *fakeProc) signal(sig os.Signal) error { + f.sigCh <- sig + if sig == syscall.SIGTERM && f.termErr != nil { + return f.termErr + } + if sig == os.Kill && f.killErr != nil { + return f.killErr + } + if sig == os.Kill && f.killExit { + f.finish(fmt.Errorf("killed")) + } + return nil +} + +// finish ends the fake process with a Wait result, closing the stream. +func (f *fakeProc) finish(err error) { + f.mu.Lock() + if !f.exited { + f.exited = true + f.waitErr = err + f.w.Close() + close(f.doneCh) + } + f.mu.Unlock() +} + +// write emits one stream-json event line. +func (f *fakeProc) write(ev map[string]any) { + b, err := json.Marshal(ev) + if err != nil { + panic(err) + } + f.w.Write(append(b, '\n')) +} + +// fakeSpawn returns a spawn func bound to a fake process, recording the +// spawn spec for assertions. +func fakeSpawn(fp *fakeProc) spawnFunc { + return func(_ context.Context, spec spawnSpec) (process, error) { + fp.spec = spec + return fp, nil + } +} + +func attemptFor(id, objective string) adapter.Attempt { + return adapter.Attempt{ + ID: id, + NodeID: id, + Objective: objective, + Role: "worker", + Model: "claude-sonnet-5", + WriteScope: []string{"src"}, + MaxDurationSeconds: 600, + } +} + +// drainSteps pulls one completion from the driver via Step. +func drainSteps(d *Driver, want int) []adapter.Completion { + deadline := time.Now().Add(10 * time.Second) + var out []adapter.Completion + for len(out) < want && time.Now().Before(deadline) { + cs := d.Step(context.Background(), time.Now()) + out = append(out, cs...) + if len(cs) == 0 { + time.Sleep(10 * time.Millisecond) + } + } + return out +} + +// drainFor polls Step for dur, returning as soon as any completion arrives. +// It is used to assert that no duplicate completion is ever emitted. +func drainFor(d *Driver, dur time.Duration) []adapter.Completion { + deadline := time.Now().Add(dur) + var out []adapter.Completion + for time.Now().Before(deadline) { + out = append(out, d.Step(context.Background(), time.Now())...) + if len(out) > 0 { + return out + } + time.Sleep(10 * time.Millisecond) + } + return out +} + +// waitPending polls PendingPermission until the request with the wanted id +// is pending (the broker and the event watcher populate it asynchronously). +func waitPending(t *testing.T, ps adapter.PermissionSession, want string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + pid, ok, err := ps.PendingPermission(context.Background()) + if err != nil { + t.Fatal(err) + } + if ok && pid == want { + return + } + time.Sleep(10 * time.Millisecond) + } + pid, ok, _ := ps.PendingPermission(context.Background()) + t.Fatalf("pending permission = %q, %v; want %q, true", pid, ok, want) +} + +func TestDriverImplementsAdapterInterfaces(t *testing.T) { + var _ adapter.Driver = (*Driver)(nil) + var _ adapter.Stepper = (*Driver)(nil) + var _ adapter.Session = (*session)(nil) + var _ adapter.PermissionSession = (*session)(nil) +} + +func TestAllowedToolsIncludeRecursiveWriteScope(t *testing.T) { + got := allowedTools(adapter.Attempt{WriteScope: []string{"src", "README.md", "*"}}) + want := []string{ + "Read", "Glob", "Grep", + "Edit(src)", "Write(src)", "Edit(src/**)", "Write(src/**)", + "Edit(README.md)", "Write(README.md)", "Edit(README.md/**)", "Write(README.md/**)", + "Edit", "Write", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("allowed tools = %v, want %v", got, want) + } +} + +func TestAllowedToolsEmptyScopeMeansWholeRepository(t *testing.T) { + got := allowedTools(adapter.Attempt{}) + want := []string{"Read", "Glob", "Grep", "Edit", "Write"} + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("allowed tools = %v, want %v", got, want) + } +} + +func TestAllowedToolsReviewerIsReadOnly(t *testing.T) { + for _, scope := range [][]string{nil, {"*"}, {"src"}} { + got := allowedTools(adapter.Attempt{Role: "reviewer", WriteScope: scope}) + want := []string{"Read", "Glob", "Grep"} + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("scope %v: allowed tools = %v, want %v", scope, got, want) + } + } +} + +func TestSystemInitUsesClaudeStreamShapeAndRemapsSession(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "task")) + if err != nil { + t.Fatal(err) + } + oldID := sess.ID() + actualID := newUUID() + sess.(*session).at.handleEvent(streamEvent{ + Type: "system", + Subtype: "init", + SessionID: actualID, + }) + if got := sess.ID(); got != actualID { + t.Fatalf("session id = %q, want init event id %q", got, actualID) + } + if got := drv.attemptBySession(actualID); got != sess.(*session).at { + t.Fatal("actual session id was not mapped to its attempt") + } + if got := drv.attemptBySession(oldID); got != nil { + t.Fatal("generated session id remained mapped after init remap") + } + fp.finish(nil) +} + +// TestStartCompletion drives the happy path: system subtype init, an assistant turn, +// tool callbacks, and a terminal result event. The completion must be emitted +// exactly once with the accumulated transcript, and the spawned claude must +// have received the expected headless flags. +func TestStartCompletion(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 50 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + a := attemptFor("w1/1", "create src/alpha.txt") + a.Cwd = t.TempDir() + sess, err := drv.Start(context.Background(), a) + if err != nil { + t.Fatal(err) + } + + // The session id is the UUID the driver generated and passed to claude. + sid := sess.ID() + if len(sid) != 36 || strings.Count(sid, "-") != 4 { + t.Fatalf("session id %q is not a UUID", sid) + } + + ev := map[string]any{"type": "system", "subtype": "init", "session_id": sid} + fp.write(ev) + fp.write(map[string]any{"type": "assistant", "message": map[string]any{ + "role": "assistant", "stop_reason": "end_turn", + "usage": map[string]any{"input_tokens": 10, "output_tokens": 20}, + "content": []map[string]any{{"type": "text", "text": "I will create the file."}}, + }}) + fp.write(map[string]any{"type": "user", "message": map[string]any{ + "role": "user", + "content": []map[string]any{{ + "type": "tool_result", "tool_use_id": "tu1", + "content": []map[string]any{{"type": "text", "text": "wrote src/alpha.txt"}}, + }}, + }}) + fp.write(map[string]any{"type": "assistant", "message": map[string]any{ + "role": "assistant", "stop_reason": "end_turn", + "usage": map[string]any{"input_tokens": 5, "output_tokens": 3}, + "content": []map[string]any{{"type": "text", "text": "Done. src/alpha.txt exists."}}, + }}) + fp.write(map[string]any{ + "type": "result", "subtype": "success", "session_id": sid, "result": "Done.", + "total_cost_usd": 0.004, + "usage": map[string]any{"input_tokens": 15, "output_tokens": 23}, + }) + fp.finish(nil) + + cs := drainSteps(drv, 1) + if len(cs) != 1 { + t.Fatalf("got %d completions, want 1", len(cs)) + } + c := cs[0] + if c.AttemptID != "w1/1" { + t.Errorf("attempt id = %q, want w1/1", c.AttemptID) + } + if c.SessionID != sid { + t.Errorf("session id = %q, want %q", c.SessionID, sid) + } + if c.Status != adapter.StatusIdle { + t.Errorf("status = %q, want idle", c.Status) + } + msgs := c.Messages + if len(msgs) != 3 { + t.Fatalf("messages = %d, want 3", len(msgs)) + } + if msgs[0].Role != "assistant" || !strings.Contains(msgs[0].Text, "I will create the file.") { + t.Errorf("msg[0] = %+v", msgs[0]) + } + if msgs[0].Tokens != 0 || msgs[0].Cost != 0 { + t.Errorf("msg[0] has per-turn accounting: %+v", msgs[0]) + } + if msgs[1].Role != "user" || !strings.Contains(msgs[1].Text, "wrote src/alpha.txt") { + t.Errorf("msg[1] = %+v", msgs[1]) + } + if !strings.Contains(msgs[2].Text, "Done.") { + t.Errorf("msg[2] = %+v", msgs[2]) + } + if msgs[2].Tokens != 38 || msgs[2].Cost != 0.004 { + t.Errorf("msg[2] cumulative accounting = %+v", msgs[2]) + } + + // The session view exposes the same transcript. + got, err := sess.Messages(context.Background()) + if err != nil || len(got) != 3 { + t.Fatalf("session messages: %v, %d", err, len(got)) + } + + // No duplicate completion when the process exits after the result event. + if extra := drainFor(drv, 300*time.Millisecond); len(extra) != 0 { + t.Fatalf("duplicate completions: %+v", extra) + } + + // The claude invocation carries the expected headless arguments. + args := strings.Join(fp.spec.args, " ") + for _, want := range []string{"-p", "--output-format", "stream-json", "--verbose", + "--include-partial-messages", "--session-id", sid, "--model", "claude-sonnet-5", + "--allowedTools"} { + if !strings.Contains(args, want) { + t.Errorf("args %q missing %q", args, want) + } + } + hasEdit := false + hasWrite := false + for _, arg := range fp.spec.args { + if strings.HasPrefix(arg, "Edit(") { + hasEdit = true + } + if strings.HasPrefix(arg, "Write(") { + hasWrite = true + } + } + if !hasEdit { + t.Errorf("write scope not mapped to an Edit rule: %v", fp.spec.args) + } + if !hasWrite { + t.Errorf("write scope not mapped to a Write rule: %v", fp.spec.args) + } + if fp.spec.dir != a.Cwd { + t.Errorf("cwd = %q, want %q", fp.spec.dir, a.Cwd) + } + if !strings.Contains(fp.spec.args[1], "(role: worker)") { + t.Errorf("prompt missing role header: %q", fp.spec.args[1]) + } + if !strings.Contains(fp.spec.args[1], a.Objective) { + t.Errorf("prompt missing objective: %q", fp.spec.args[1]) + } +} + +func TestCompletionCapturesWorktreeDiffsForDefaultVerification(t *testing.T) { + repo := t.TempDir() + git := func(args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = repo + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + } + git("init", "-q", "-b", "main") + if err := os.WriteFile(filepath.Join(repo, "tracked.txt"), []byte("before\n"), 0o644); err != nil { + t.Fatal(err) + } + git("add", "tracked.txt") + git("-c", "user.name=corral", "-c", "user.email=corral@local", "commit", "-qm", "init") + if err := os.WriteFile(filepath.Join(repo, "tracked.txt"), []byte("after\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "new file.txt"), []byte("new\n"), 0o644); err != nil { + t.Fatal(err) + } + + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 10 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + a := attemptFor("w1/1", "edit files") + a.Cwd = repo + sess, err := drv.Start(context.Background(), a) + if err != nil { + t.Fatal(err) + } + fp.write(map[string]any{"type": "result", "subtype": "success", "session_id": sess.ID()}) + fp.finish(nil) + + cs := drainSteps(drv, 1) + if len(cs) != 1 { + t.Fatalf("completions = %d, want 1", len(cs)) + } + var diffs []adapter.Diff + for _, msg := range cs[0].Messages { + if msg.Role == "user" { + diffs = append(diffs, msg.Diffs...) + } + } + if len(diffs) != 2 { + t.Fatalf("diffs = %+v, want tracked and untracked file", diffs) + } + byFile := make(map[string]adapter.Diff, len(diffs)) + for _, diff := range diffs { + byFile[diff.File] = diff + } + tracked := byFile["tracked.txt"] + if tracked.Status != "modified" || tracked.Additions != 1 || tracked.Deletions != 1 || + !strings.Contains(tracked.Patch, "+after") { + t.Errorf("tracked diff = %+v", tracked) + } + added := byFile["new file.txt"] + if added.Status != "added" || added.Additions != 1 || added.Deletions != 0 || + !strings.Contains(added.Patch, "+new") { + t.Errorf("untracked diff = %+v", added) + } + + verdict, err := verify.New(repo).Verify(context.Background(), &graph.Node{ + ID: "w1", Type: graph.NodeAgent, Objective: "edit files", + }, repo, 1, cs[0].Messages) + if err != nil { + t.Fatal(err) + } + if !verdict.Pass { + t.Fatalf("default verification rejected real file changes: %+v", verdict) + } +} + +func TestCaptureGitDiffsNonGitDirectoryIsBestEffort(t *testing.T) { + diffs, err := captureGitDiffs(context.Background(), t.TempDir()) + if err != nil { + t.Fatalf("non-Git directory returned error: %v", err) + } + if len(diffs) != 0 { + t.Fatalf("non-Git directory diffs = %+v, want none", diffs) + } +} + +func TestCaptureGitDiffsCancellationReturnsNoPartialEvidence(t *testing.T) { + repo := t.TempDir() + cmd := exec.Command("git", "init", "-q", "-b", "main") + cmd.Dir = repo + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, out) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + diffs, err := captureGitDiffs(ctx, repo) + if !errors.Is(err, context.Canceled) { + t.Fatalf("capture error = %v, want context canceled", err) + } + if len(diffs) != 0 { + t.Fatalf("canceled capture leaked partial diffs: %+v", diffs) + } +} + +func TestCompletionFailsClosedWhenGitEvidenceCannotBeCaptured(t *testing.T) { + repo := t.TempDir() + git := func(args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = repo + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + } + git("init", "-q", "-b", "main") + if err := os.WriteFile(filepath.Join(repo, "tracked.txt"), []byte("before\n"), 0o644); err != nil { + t.Fatal(err) + } + git("add", "tracked.txt") + git("-c", "user.name=corral", "-c", "user.email=corral@local", "commit", "-qm", "init") + headCmd := exec.Command("git", "rev-parse", "HEAD") + headCmd.Dir = repo + head, err := headCmd.Output() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, ".git", "refs", "heads", "main"), []byte(strings.Repeat("0", len(strings.TrimSpace(string(head))))+"\n"), 0o644); err != nil { + t.Fatal(err) + } + + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 10 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + a := attemptFor("w1/1", "edit files") + a.Cwd = repo + sess, err := drv.Start(context.Background(), a) + if err != nil { + t.Fatal(err) + } + fp.write(map[string]any{"type": "result", "subtype": "success", "session_id": sess.ID()}) + fp.finish(nil) + + cs := drainSteps(drv, 1) + if len(cs) != 1 { + t.Fatalf("completions = %d, want 1", len(cs)) + } + if cs[0].Status != adapter.StatusError { + t.Fatalf("status = %q, want error", cs[0].Status) + } + if cs[0].Err == nil || !strings.Contains(cs[0].Err.Error(), "capture git diff evidence") { + t.Fatalf("completion error = %v, want evidence capture error", cs[0].Err) + } + for _, msg := range cs[0].Messages { + if len(msg.Diffs) != 0 { + t.Fatalf("failed capture leaked partial diffs: %+v", msg.Diffs) + } + } +} + +func TestCompletedAttemptReleasesLiveRegistryAndRejectsReuse(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 10 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + a := attemptFor("w1/1", "task") + sess, err := drv.Start(context.Background(), a) + if err != nil { + t.Fatal(err) + } + sid := sess.ID() + fp.write(map[string]any{"type": "result", "subtype": "success", "session_id": sid}) + fp.finish(nil) + if got := drainSteps(drv, 1); len(got) != 1 { + t.Fatalf("completions = %d, want 1", len(got)) + } + + deadline := time.Now().Add(time.Second) + for drv.attemptByID(a.ID) != nil && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if drv.attemptByID(a.ID) != nil || drv.attemptBySession(sid) != nil { + t.Fatal("completed attempt retained in live driver registry") + } + spawned := false + drv.spawn = func(context.Context, spawnSpec) (process, error) { + spawned = true + return nil, errors.New("unexpected spawn") + } + if _, err := drv.Start(context.Background(), a); err == nil || !strings.Contains(err.Error(), "already started") { + t.Fatalf("reused attempt error = %v", err) + } + if spawned { + t.Fatal("duplicate completed attempt spawned another process") + } +} + +func TestLargeStreamJSONRecordIsNotLost(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 20 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "task")) + if err != nil { + t.Fatal(err) + } + want := strings.Repeat("x", 256<<10) + fp.write(map[string]any{"type": "assistant", "message": map[string]any{ + "role": "assistant", "content": []map[string]any{{"type": "text", "text": want}}, + }}) + fp.write(map[string]any{"type": "result", "subtype": "success", "session_id": sess.ID()}) + fp.finish(nil) + + cs := drainSteps(drv, 1) + if len(cs) != 1 || cs[0].Status != adapter.StatusIdle { + t.Fatalf("completion = %+v, want one idle", cs) + } + if len(cs[0].Messages) != 1 || cs[0].Messages[0].Text != want { + t.Fatalf("large record transcript length = %d, text bytes = %d", len(cs[0].Messages), func() int { + if len(cs[0].Messages) == 0 { + return 0 + } + return len(cs[0].Messages[0].Text) + }()) + } +} + +func TestFloodedStreamPreservesEveryEventInOrder(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: time.Hour}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "task")) + if err != nil { + t.Fatal(err) + } + const count = 2048 + for i := 0; i < count; i++ { + fp.write(map[string]any{"type": "assistant", "message": map[string]any{ + "role": "assistant", + "content": []map[string]any{{"type": "text", "text": fmt.Sprintf("%04d", i)}}, + }}) + } + fp.write(map[string]any{"type": "result", "subtype": "success", "session_id": sess.ID()}) + fp.finish(nil) + + cs := drainSteps(drv, 1) + if len(cs) != 1 { + t.Fatalf("completions = %d, want 1", len(cs)) + } + if got := len(cs[0].Messages); got != count { + t.Fatalf("messages = %d, want %d", got, count) + } + for i, msg := range cs[0].Messages { + if want := fmt.Sprintf("%04d", i); msg.Text != want { + t.Fatalf("message[%d] = %q, want %q", i, msg.Text, want) + } + } +} + +func TestCompletionBurstIsNeverDropped(t *testing.T) { + drv := New(Options{DisablePermissions: true}) + defer drv.Close() + + const count = 128 + for i := 0; i < count; i++ { + id := fmt.Sprintf("attempt-%03d", i) + at := &attempt{ + d: drv, attemptID: id, sessionID: "session-" + id, + terminal: true, exited: true, subtype: "success", + } + drv.maybeComplete(context.Background(), at) + } + + got := drv.Step(context.Background(), time.Now()) + if len(got) != count { + t.Fatalf("completion burst = %d, want %d", len(got), count) + } + for i, completion := range got { + if want := fmt.Sprintf("attempt-%03d", i); completion.AttemptID != want { + t.Fatalf("completion[%d] = %q, want %q", i, completion.AttemptID, want) + } + } +} + +func TestStreamReadErrorIsReportedAfterRemainingOutputIsDrained(t *testing.T) { + readErr := errors.New("synthetic stdout read failure") + first, _ := json.Marshal(map[string]any{"type": "assistant", "message": map[string]any{ + "role": "assistant", "content": []map[string]any{{"type": "text", "text": "before fault"}}, + }}) + remainder := bytes.Repeat([]byte("drain-me"), 64<<10) + fp := newFaultProc(append(first, '\n'), remainder, readErr) + drv := New(Options{DisablePermissions: true, PollInterval: 20 * time.Millisecond}) + drv.spawn = func(context.Context, spawnSpec) (process, error) { return fp, nil } + defer drv.Close() + + if _, err := drv.Start(context.Background(), attemptFor("w1/1", "task")); err != nil { + t.Fatal(err) + } + cs := drainSteps(drv, 1) + if len(cs) != 1 || cs[0].Status != adapter.StatusError { + t.Fatalf("completion = %+v, want one error", cs) + } + if !errors.Is(cs[0].Err, readErr) { + t.Fatalf("completion error = %v, want %v", cs[0].Err, readErr) + } + select { + case <-fp.r.drained: + default: + t.Fatal("stdout remainder was not drained after read error") + } +} + +func TestResultOwnsCumulativeCostAndUsageAccounting(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 20 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "task")) + if err != nil { + t.Fatal(err) + } + // Real assistant envelopes contain per-turn usage but no cost. Corral uses + // the result envelope's cumulative totals exactly once for budget accounting. + for _, turn := range []struct { + text string + input int + output int + }{{"working", 10, 2}, {"done", 30, 5}} { + fp.write(map[string]any{"type": "assistant", "message": map[string]any{ + "role": "assistant", "content": []map[string]any{{"type": "text", "text": turn.text}}, + "usage": map[string]any{"input_tokens": turn.input, "output_tokens": turn.output}, + }}) + } + fp.write(map[string]any{ + "type": "result", "subtype": "success", "session_id": sess.ID(), + "total_cost_usd": 0.125, + "usage": map[string]any{ + "input_tokens": 100, "output_tokens": 20, + "cache_creation_input_tokens": 30, "cache_read_input_tokens": 40, + }, + }) + fp.finish(nil) + + cs := drainSteps(drv, 1) + if len(cs) != 1 || len(cs[0].Messages) != 2 { + t.Fatalf("completion = %+v", cs) + } + var cost float64 + var tokens int + for _, msg := range cs[0].Messages { + cost += msg.Cost + tokens += msg.Tokens + } + if cost != 0.125 { + t.Errorf("summed cost = %v, want 0.125", cost) + } + if tokens != 190 { + t.Errorf("summed tokens = %d, want 190", tokens) + } + if cs[0].Messages[0].Cost != 0 || cs[0].Messages[0].Tokens != 0 { + t.Errorf("first assistant message duplicated cumulative accounting: %+v", cs[0].Messages[0]) + } + if cs[0].Messages[1].Cost != 0.125 || cs[0].Messages[1].Tokens != 190 { + t.Errorf("final assistant accounting = %+v", cs[0].Messages[1]) + } +} + +// TestExitFallbackCompletion verifies the reconciliation path: when the +// terminal result event is missed the process exit decides the completion, +// mapping exit 0 to idle and a non-zero exit to error. +func TestExitFallbackCompletion(t *testing.T) { + for _, tc := range []struct { + name string + err error + want adapter.Status + }{ + {"exit zero", nil, adapter.StatusIdle}, + {"exit nonzero", fmt.Errorf("exit status 1"), adapter.StatusError}, + } { + t.Run(tc.name, func(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 20 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "write a file")) + if err != nil { + t.Fatal(err) + } + sid := sess.ID() + fp.write(map[string]any{"type": "system", "subtype": "init", "session_id": sid}) + fp.write(map[string]any{"type": "assistant", "message": map[string]any{ + "role": "assistant", + "content": []map[string]any{{"type": "text", "text": "working..."}}, + }}) + // No result event: the completion must still arrive via exit. + fp.finish(tc.err) + + cs := drainSteps(drv, 1) + if len(cs) != 1 { + t.Fatalf("got %d completions, want 1", len(cs)) + } + if cs[0].Status != tc.want { + t.Errorf("status = %q, want %q", cs[0].Status, tc.want) + } + if len(cs[0].Messages) != 1 { + t.Errorf("messages = %d, want 1", len(cs[0].Messages)) + } + if extra := drainFor(drv, 300*time.Millisecond); len(extra) != 0 { + t.Fatalf("duplicate completions: %+v", extra) + } + }) + } +} + +// TestAbort verifies that aborting a running attempt yields exactly one +// completion with StatusAborted, and that the claude process receives a +// terminate signal. +func TestAbort(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 20 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "long task")) + if err != nil { + t.Fatal(err) + } + fp.write(map[string]any{"type": "system", "subtype": "init", "session_id": sess.ID()}) + fp.write(map[string]any{"type": "assistant", "message": map[string]any{ + "role": "assistant", + "content": []map[string]any{{"type": "text", "text": "starting..."}}, + }}) + + // Give the watcher time to record the events, then abort mid-run. + deadline := time.Now().Add(5 * time.Second) + for { + st, _ := sess.Status(context.Background()) + if st == adapter.StatusRunning { + break + } + if time.Now().After(deadline) { + t.Fatal("attempt never reached running") + } + time.Sleep(10 * time.Millisecond) + } + if err := sess.Abort(context.Background()); err != nil { + t.Fatal(err) + } + // The fake claude observes the SIGTERM and exits. + select { + case sig := <-fp.sigCh: + if sig != syscall.SIGTERM { + t.Errorf("signal = %v, want SIGTERM", sig) + } + case <-time.After(5 * time.Second): + t.Fatal("fake process never received the abort signal") + } + fp.finish(fmt.Errorf("signal: terminated")) + + cs := drainSteps(drv, 1) + if len(cs) != 1 { + t.Fatalf("got %d completions, want 1", len(cs)) + } + if cs[0].Status != adapter.StatusAborted { + t.Errorf("status = %q, want aborted", cs[0].Status) + } + if got, _ := sess.Status(context.Background()); got != adapter.StatusAborted { + t.Errorf("session status = %q, want aborted", got) + } +} + +func TestAbortWaitsForProcessExitBeforeCompletion(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 10 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "long task")) + if err != nil { + t.Fatal(err) + } + abortDone := make(chan error, 1) + go func() { abortDone <- sess.Abort(context.Background()) }() + select { + case sig := <-fp.sigCh: + if sig != syscall.SIGTERM { + t.Fatalf("signal = %v, want SIGTERM", sig) + } + case <-time.After(time.Second): + t.Fatal("abort did not signal process") + } + + time.Sleep(50 * time.Millisecond) + if got := drv.Step(context.Background(), time.Now()); len(got) != 0 { + t.Fatalf("completion emitted before process exit: %+v", got) + } + fp.finish(fmt.Errorf("signal: terminated")) + select { + case err := <-abortDone: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("abort did not return after process exit") + } + got := drainSteps(drv, 1) + if len(got) != 1 || got[0].Status != adapter.StatusAborted { + t.Fatalf("completion = %+v, want one aborted after exit", got) + } +} + +func TestAbortFallsBackToKillWhenTerminateIsUnsupported(t *testing.T) { + fp, _ := newFakeProc() + fp.termErr = errors.New("terminate unsupported") + fp.killExit = true + drv := New(Options{DisablePermissions: true}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "task")) + if err != nil { + t.Fatal(err) + } + if err := sess.Abort(context.Background()); err != nil { + t.Fatalf("Abort did not fall back to Kill: %v", err) + } + for _, want := range []os.Signal{syscall.SIGTERM, os.Kill} { + select { + case got := <-fp.sigCh: + if got != want { + t.Fatalf("signal = %v, want %v", got, want) + } + case <-time.After(time.Second): + t.Fatalf("missing signal %v", want) + } + } + got := drainSteps(drv, 1) + if len(got) != 1 || got[0].Status != adapter.StatusAborted { + t.Fatalf("completion = %+v, want aborted", got) + } +} + +func TestAbortSignalFailureDoesNotFalsifyLaterCompletion(t *testing.T) { + fp, _ := newFakeProc() + fp.termErr = errors.New("terminate failed") + fp.killErr = errors.New("kill failed") + drv := New(Options{DisablePermissions: true, PollInterval: 10 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "task")) + if err != nil { + t.Fatal(err) + } + + if err := sess.Abort(context.Background()); err == nil || + !strings.Contains(err.Error(), "terminate failed") || !strings.Contains(err.Error(), "kill failed") { + t.Fatalf("Abort error = %v, want both signal failures", err) + } + if status, err := sess.Status(context.Background()); err != nil || status != adapter.StatusRunning { + t.Fatalf("status after failed abort = %q, %v; want running", status, err) + } + fp.write(map[string]any{"type": "result", "subtype": "success", "session_id": sess.ID()}) + fp.finish(nil) + + got := drainSteps(drv, 1) + if len(got) != 1 || got[0].Status != adapter.StatusIdle || got[0].Err != nil { + t.Fatalf("completion after failed abort = %+v, want successful idle", got) + } +} + +func TestAbortAfterNaturalExitPreservesCompletion(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 10 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "task")) + if err != nil { + t.Fatal(err) + } + fp.write(map[string]any{"type": "result", "subtype": "success", "session_id": sess.ID()}) + fp.finish(nil) + if err := sess.Abort(context.Background()); err != nil { + t.Fatal(err) + } + got := drainSteps(drv, 1) + if len(got) != 1 || got[0].Status != adapter.StatusIdle || got[0].Err != nil { + t.Fatalf("completion after natural exit = %+v, want successful idle", got) + } + select { + case signal := <-fp.sigCh: + t.Fatalf("already-exited process received signal %v", signal) + default: + } +} + +// TestPermissionBroker exercises the real permission transport end to end: a +// helper goroutine acting as claude's MCP permission tool connects to the +// broker socket, parks a request, and waits; the scheduler-side session sees +// the pending permission and RespondPermission delivers the decision. +func TestPermissionBroker(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: false, PollInterval: 20 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "write src/alpha.txt")) + if err != nil { + t.Fatal(err) + } + ps, ok := sess.(adapter.PermissionSession) + if !ok { + t.Fatal("session does not implement PermissionSession") + } + sid := sess.ID() + if drv.broker == nil { + t.Fatal("permission broker not started") + } + // The spawned claude got the permission tool wiring. + args := strings.Join(fp.spec.args, " ") + for _, want := range []string{"--mcp-config", "--permission-prompt-tool", "mcp__corral__handle_permission_prompt"} { + if !strings.Contains(args, want) { + t.Errorf("args missing %q: %s", want, args) + } + } + if strings.Count(args, permissionToolName) < 2 { + t.Errorf("permission MCP tool not explicitly allowed and configured: %s", args) + } + configIndex := -1 + for i, arg := range fp.spec.args { + if arg == "--mcp-config" { + configIndex = i + 1 + break + } + } + if configIndex < 0 || configIndex >= len(fp.spec.args) { + t.Fatal("missing MCP config argument") + } + var config mcpConfig + if err := json.Unmarshal([]byte(fp.spec.args[configIndex]), &config); err != nil { + t.Fatalf("decode MCP config: %v", err) + } + helperEnv := config.Servers[permissionServerName].Env + if helperEnv["CORRAL_CLAUDE_ATTEMPT_ID"] != "w1/1" || helperEnv["CORRAL_CLAUDE_SESSION_ID"] != sid { + t.Fatalf("helper identity env = %v", helperEnv) + } + + fp.write(map[string]any{"type": "system", "subtype": "init", "session_id": sid}) + + // A helper (what claude spawns as --corral-claude-permission-tool) + // connects to the broker and parks a permission request. + helperDone := make(chan decision, 1) + helperErr := make(chan error, 1) + go func() { + conn, err := net.Dial("unix", drv.broker.path) + if err != nil { + helperErr <- err + return + } + defer conn.Close() + req := map[string]any{ + "attempt_id": "w1/1", "session_id": sid, "request_id": "req-1", "tool_name": "Write", + "prompt": "write file", "tool_input": map[string]any{"file_path": "src/alpha.txt"}, + } + if err := json.NewEncoder(conn).Encode(req); err != nil { + helperErr <- err + return + } + var reply decision + if err := json.NewDecoder(conn).Decode(&reply); err != nil { + helperErr <- err + return + } + helperDone <- reply + }() + + // The scheduler's view: the permission is pending and the session is + // permission-capable. + waitPending(t, ps, "req-1") + info, ok, err := sess.(adapter.PermissionInfo).PendingPermissionDetails(context.Background()) + if err != nil || !ok || info.Tool != "Write" || !strings.Contains(info.Input, "src/alpha.txt") { + t.Fatalf("permission details = %+v, %v, %v", info, ok, err) + } + + // Claude may issue tool calls concurrently. The adapter exposes one + // permission at a time, so a second request is denied immediately instead + // of replacing the first and leaving its helper stranded. + conn, err := net.Dial("unix", drv.broker.path) + if err != nil { + t.Fatal(err) + } + if err := json.NewEncoder(conn).Encode(map[string]any{ + "attempt_id": "w1/1", "session_id": sid, "request_id": "req-concurrent", "tool_name": "Bash", + }); err != nil { + t.Fatal(err) + } + var concurrent decision + if err := json.NewDecoder(conn).Decode(&concurrent); err != nil { + t.Fatal(err) + } + conn.Close() + if concurrent.Allow || !strings.Contains(concurrent.Message, "already pending") { + t.Fatalf("concurrent permission decision = %+v", concurrent) + } + if pid, ok, _ := ps.PendingPermission(context.Background()); !ok || pid != "req-1" { + t.Fatalf("concurrent request replaced pending permission: %q, %v", pid, ok) + } + + // Approve it; the parked helper must receive the allow decision. + if err := ps.RespondPermission(context.Background(), "req-1", true); err != nil { + t.Fatal(err) + } + select { + case reply := <-helperDone: + if !reply.Allow { + t.Errorf("helper got deny, want allow") + } + case err := <-helperErr: + t.Fatalf("helper error: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("helper never received the permission decision") + } + if pid, ok, _ := ps.PendingPermission(context.Background()); ok { + t.Errorf("permission still pending after response: %q", pid) + } + if err := ps.RespondPermission(context.Background(), "req-1", true); err == nil { + t.Fatal("responding twice to permission succeeded") + } + + // A denied request clears the pending state and returns deny. + go func() { + conn, err := net.Dial("unix", drv.broker.path) + if err != nil { + helperErr <- err + return + } + defer conn.Close() + json.NewEncoder(conn).Encode(map[string]any{ + "attempt_id": "w1/1", "session_id": sid, "request_id": "req-2", "tool_name": "Bash", + }) + var reply decision + if err := json.NewDecoder(conn).Decode(&reply); err != nil { + helperErr <- err + return + } + helperDone <- reply + }() + waitPending(t, ps, "req-2") + if err := ps.RespondPermission(context.Background(), "req-2", false); err != nil { + t.Fatal(err) + } + select { + case reply := <-helperDone: + if reply.Allow { + t.Errorf("helper got allow, want deny") + } + case err := <-helperErr: + t.Fatalf("helper error: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("helper never received the denied decision") + } + + // The attempt still completes normally afterwards. + fp.write(map[string]any{"type": "assistant", "message": map[string]any{ + "role": "assistant", + "content": []map[string]any{{"type": "text", "text": "done writing"}}, + }}) + fp.write(map[string]any{"type": "result", "subtype": "success", "session_id": sid}) + fp.finish(nil) + cs := drainSteps(drv, 1) + if len(cs) != 1 || cs[0].Status != adapter.StatusIdle { + t.Fatalf("completion = %+v, want a single idle", cs) + } +} + +func TestPermissionSocketOverridePreservesNonSocketPath(t *testing.T) { + path := filepath.Join(t.TempDir(), "broker.sock") + const content = "keep me" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + drv := New(Options{PermissionSocket: path}) + defer drv.Close() + spawned := false + drv.spawn = func(context.Context, spawnSpec) (process, error) { + spawned = true + return nil, errors.New("unexpected spawn") + } + + if _, err := drv.Start(context.Background(), attemptFor("w1/1", "task")); err == nil { + t.Fatal("Start replaced a non-socket permission path") + } + if spawned { + t.Error("spawn called after unsafe socket path") + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read preserved path: %v", err) + } + if string(got) != content { + t.Fatalf("preserved path = %q, want %q", got, content) + } +} + +// TestPermissionHelperClaude221226InputShape crosses the full MCP helper and +// broker boundary with Claude Code 2.1.226's permission-tool arguments: +// {tool_name,input,tool_use_id}. It spawns the actual exported helper path so +// attempt/session ownership must be obtained from the child environment. +func TestPermissionHelperClaude221226InputShape(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{PollInterval: 20 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + a := attemptFor("w1/1", "write src/alpha.txt") + sess, err := drv.Start(context.Background(), a) + if err != nil { + t.Fatal(err) + } + ps := sess.(adapter.PermissionSession) + + helper := exec.Command(os.Args[0], helperFlag) + helper.Env = append(os.Environ(), + "CORRAL_CLAUDE_BROKER="+drv.broker.path, + "CORRAL_CLAUDE_ATTEMPT_ID="+a.ID, + "CORRAL_CLAUDE_SESSION_ID="+sess.ID(), + ) + helperIn, err := helper.StdinPipe() + if err != nil { + t.Fatal(err) + } + helperOut, err := helper.StdoutPipe() + if err != nil { + t.Fatal(err) + } + var helperStderr strings.Builder + helper.Stderr = &helperStderr + if err := helper.Start(); err != nil { + t.Fatal(err) + } + waited := false + t.Cleanup(func() { + if waited { + return + } + _ = helperIn.Close() + _ = helper.Process.Kill() + _ = helper.Wait() + }) + enc := json.NewEncoder(helperIn) + dec := json.NewDecoder(helperOut) + + call := func(rpcID, toolUseID string, allow bool) string { + t.Helper() + arguments := map[string]any{ + "tool_name": "Write", + "input": map[string]any{"file_path": "src/alpha.txt", "content": "ok\n"}, + } + if toolUseID != "" { + arguments["tool_use_id"] = toolUseID + } + params, err := json.Marshal(map[string]any{ + "name": "handle_permission_prompt", + "arguments": arguments, + }) + if err != nil { + t.Fatal(err) + } + if err := enc.Encode(rpcRequest{ + ID: json.RawMessage(rpcID), + Method: "tools/call", + Params: params, + }); err != nil { + t.Fatal(err) + } + + var requestID string + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if id, ok, err := ps.PendingPermission(context.Background()); err != nil { + t.Fatal(err) + } else if ok { + requestID = id + break + } + time.Sleep(10 * time.Millisecond) + } + if requestID == "" { + t.Fatal("helper request never became pending") + } + if toolUseID != "" && requestID != toolUseID { + t.Fatalf("permission id = %q, want tool_use_id %q", requestID, toolUseID) + } + if toolUseID == "" && (len(requestID) != 36 || strings.Count(requestID, "-") != 4) { + t.Fatalf("generated permission id = %q, want UUID", requestID) + } + if err := ps.RespondPermission(context.Background(), requestID, allow); err != nil { + t.Fatal(err) + } + + var response struct { + Result struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } `json:"result"` + } + if err := dec.Decode(&response); err != nil { + t.Fatal(err) + } + if response.Result.IsError || len(response.Result.Content) != 1 || response.Result.Content[0].Type != "text" { + t.Fatalf("MCP response = %+v", response) + } + return response.Result.Content[0].Text + } + + allowText := call("1", "toolu_01ABC", true) + var allowed struct { + Behavior string `json:"behavior"` + UpdatedInput map[string]any `json:"updatedInput"` + } + if err := json.Unmarshal([]byte(allowText), &allowed); err != nil { + t.Fatalf("decode allow decision %q: %v", allowText, err) + } + if allowed.Behavior != "allow" || allowed.UpdatedInput["file_path"] != "src/alpha.txt" { + t.Fatalf("allow decision = %+v", allowed) + } + + denyText := call("2", "", false) + var denied struct { + Behavior string `json:"behavior"` + Message string `json:"message"` + } + if err := json.Unmarshal([]byte(denyText), &denied); err != nil { + t.Fatalf("decode deny decision %q: %v", denyText, err) + } + if denied.Behavior != "deny" || denied.Message == "" { + t.Fatalf("deny decision = %+v", denied) + } + + if err := helperIn.Close(); err != nil { + t.Fatal(err) + } + if err := helper.Wait(); err != nil { + t.Fatalf("helper exit: %v; stderr: %s", err, helperStderr.String()) + } + waited = true + fp.finish(nil) +} + +// TestStreamPermissionTracking covers the fallback path where the pending +// permission is observed from the event stream itself (no broker helper), as +// ocxadapter does with its permission.updated events. +func TestStreamPermissionTracking(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 20 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "task")) + if err != nil { + t.Fatal(err) + } + ps, ok := sess.(adapter.PermissionSession) + if !ok { + t.Fatal("session does not implement PermissionSession") + } + fp.write(map[string]any{"type": "system", "subtype": "init", "session_id": sess.ID()}) + fp.write(map[string]any{"type": "user", "message": map[string]any{ + "role": "user", + "content": []map[string]any{{ + "type": "permission_request", "state": "needs_response", + "request_id": "stream-req", "tool_name": "Write", + }}, + }}) + + waitPending(t, ps, "stream-req") + if err := ps.RespondPermission(context.Background(), "wrong-id", false); err == nil { + t.Fatal("responding to wrong permission id succeeded") + } + if pid, ok, _ := ps.PendingPermission(context.Background()); !ok || pid != "stream-req" { + t.Fatalf("wrong response changed pending permission: %q, %v", pid, ok) + } + // With permissions disabled there is no broker; responding still clears + // the pending state. + if err := ps.RespondPermission(context.Background(), "stream-req", false); err != nil { + t.Fatal(err) + } + if pid, ok, _ := ps.PendingPermission(context.Background()); ok { + t.Errorf("permission still pending: %q", pid) + } + if err := ps.RespondPermission(context.Background(), "stream-req", false); err == nil { + t.Fatal("responding to a resolved permission succeeded") + } + fp.finish(nil) +} + +func TestCLIProcessWaitBlocksUntilExitResultIsCached(t *testing.T) { + p := &cliProcess{doneCh: make(chan struct{})} + want := fmt.Errorf("exit status 7") + got := make(chan error, 1) + go func() { got <- p.wait() }() + + select { + case err := <-got: + t.Fatalf("wait returned before process exit: %v", err) + case <-time.After(50 * time.Millisecond): + } + + p.waitMu.Lock() + p.waitErr = want + p.waitMu.Unlock() + close(p.doneCh) + select { + case err := <-got: + if err != want { + t.Fatalf("wait error = %v, want %v", err, want) + } + case <-time.After(time.Second): + t.Fatal("wait did not return after process exit") + } +} + +func TestCLIProcessStdoutHelper(t *testing.T) { + if os.Getenv("CORRAL_CLAUDE_STDOUT_HELPER") != "1" { + return + } + _, _ = io.WriteString(os.Stdout, strings.Repeat("x", 4<<20)) +} + +func TestCLIProcessPreservesBufferedStdoutAfterFastExit(t *testing.T) { + proc, err := spawnCLI(context.Background(), spawnSpec{ + command: os.Args[0], + args: []string{"-test.run=^TestCLIProcessStdoutHelper$"}, + env: append(os.Environ(), "CORRAL_CLAUDE_STDOUT_HELPER=1"), + dir: t.TempDir(), + }) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(proc.stdout()) + if err != nil { + t.Fatal(err) + } + if err := proc.wait(); err != nil { + t.Fatal(err) + } + if got := bytes.Count(data, []byte("x")); got != 4<<20 { + t.Fatalf("stdout x bytes = %d, want %d", got, 4<<20) + } + if closer, ok := proc.stdout().(io.Closer); ok { + if err := closer.Close(); err != nil { + t.Fatal(err) + } + } +} + +func TestStreamScannerClosesStdoutAfterDrain(t *testing.T) { + fp, _ := newFakeProc() + tracked := &trackingReadCloser{ReadCloser: fp.r, closed: make(chan struct{})} + fp.out = tracked + drv := New(Options{DisablePermissions: true, PollInterval: 10 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "task")) + if err != nil { + t.Fatal(err) + } + fp.write(map[string]any{"type": "result", "subtype": "success", "session_id": sess.ID()}) + fp.finish(nil) + if got := drainSteps(drv, 1); len(got) != 1 { + t.Fatalf("completions = %d, want 1", len(got)) + } + select { + case <-tracked.closed: + case <-time.After(time.Second): + t.Fatal("stdout reader was not closed after stream drain") + } +} + +// TestStartError covers the failure path: when the claude binary cannot be +// spawned the driver returns an error so the scheduler fails the node. +func TestStartError(t *testing.T) { + drv := New(Options{DisablePermissions: true}) + defer drv.Close() + drv.spawn = func(context.Context, spawnSpec) (process, error) { + return nil, fmt.Errorf("claude: exec: not found") + } + if _, err := drv.Start(context.Background(), attemptFor("w1/1", "task")); err == nil { + t.Fatal("expected start error") + } +} + +func TestStartRejectsDuplicateAttemptID(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true}) + drv.spawn = fakeSpawn(fp) + defer drv.Close() + + a := attemptFor("w1/1", "task") + if _, err := drv.Start(context.Background(), a); err != nil { + t.Fatal(err) + } + if _, err := drv.Start(context.Background(), a); err == nil || !strings.Contains(err.Error(), "already started") { + t.Fatalf("duplicate Start error = %v", err) + } + fp.finish(nil) +} + +func TestStartRejectsClosedDriverWithoutSpawning(t *testing.T) { + drv := New(Options{DisablePermissions: true}) + drv.Close() + spawned := false + drv.spawn = func(context.Context, spawnSpec) (process, error) { + spawned = true + return nil, fmt.Errorf("unexpected spawn") + } + if _, err := drv.Start(context.Background(), attemptFor("w1/1", "task")); err == nil || !strings.Contains(err.Error(), "closed") { + t.Fatalf("Start after Close error = %v", err) + } + if spawned { + t.Fatal("closed driver spawned a process") + } +} + +// TestCloseKillsSessions verifies the driver tears down live sessions. +func TestCloseKillsSessions(t *testing.T) { + fp, _ := newFakeProc() + drv := New(Options{DisablePermissions: true, PollInterval: 20 * time.Millisecond}) + drv.spawn = fakeSpawn(fp) + + sess, err := drv.Start(context.Background(), attemptFor("w1/1", "task")) + if err != nil { + t.Fatal(err) + } + fp.write(map[string]any{"type": "system", "subtype": "init", "session_id": sess.ID()}) + fp.finish(nil) + + drv.Close() + // A second Close is a no-op. + drv.Close() + + // No completion may be emitted after the driver is closed (the scan + // path is suppressed by terminate). + if cs := drv.Step(context.Background(), time.Now()); len(cs) != 0 { + t.Fatalf("completions after close: %+v", cs) + } +} + +// skipLive gates tests that drive the real claude CLI. +func skipLive(t *testing.T) { + t.Helper() + livetest.SkipIfDisabled(t) + if os.Getenv("CORRAL_CLAUDE_LIVE") != "1" { + t.Skip("live claude test disabled (CORRAL_CLAUDE_LIVE=1 to run)") + } + if _, err := exec.LookPath("claude"); err != nil { + t.Skip("claude binary not found") + } +} + +// TestLiveClaudeSingleNode drives the real claude CLI through the scheduler: +// one worker writes a marker file in its own worktree, passes a command +// verification gate, and the run records exactly one attempt. +func TestLiveClaudeSingleNode(t *testing.T) { + skipLive(t) + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + defer cancel() + + proj, err := os.MkdirTemp("", "corral-claude-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(proj) }) + for _, args := range [][]string{{"init", "-q", "-b", "main"}, {"commit", "-q", "--allow-empty", "-m", "init"}} { + cmd := exec.Command("git", args...) + cmd.Dir = proj + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + } + + st, err := store.Open(filepath.Join(t.TempDir(), "claude.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + + drv := New(Options{PollInterval: 500 * time.Millisecond}) + t.Cleanup(func() { drv.Close() }) + + n := &graph.Node{ + ID: "w1", Type: graph.NodeAgent, Role: "worker", + Objective: "Create a file named claude-marker.txt containing exactly one line: CORRAL-CLAUDE-OK. Do not run any other commands.", + AcceptanceCriteria: []string{"marker file produced"}, + WriteScope: []string{"claude-marker.txt"}, + Verification: &graph.Verification{Kind: "command", Command: []string{"grep", "-q", "CORRAL-CLAUDE-OK", "claude-marker.txt"}}, + Priority: graph.PriorityNormal, + RetryPolicy: graph.RetryPolicy{MaxRetries: 1, Backoff: 5 * time.Second}, + Budget: graph.Budget{MaxDuration: 10 * time.Minute}, + } + + s := sched.New(st, drv, &sched.EngineVerifier{Eng: verify.New(proj)}, clock.Real{}, sched.Options{ + Concurrency: 1, + }) + h, err := s.Create(ctx, "run-claude", &graph.Graph{Nodes: []*graph.Node{n}}) + if err != nil { + t.Fatal(err) + } + if err := h.Run(ctx, 500*time.Millisecond); err != nil { + for _, id := range []string{"w1"} { + if st2, _ := h.State(graph.NodeID(id)); st2 != "" { + t.Logf("at timeout: %s -> %s", id, st2) + } + } + t.Fatalf("run: %v", err) + } + + if st1, _ := h.State("w1"); st1 != graph.StateDone { + t.Errorf("w1 state = %s, want done", st1) + } + atts, err := st.Attempts(ctx, "run-claude", "w1") + if err != nil { + t.Fatal(err) + } + if len(atts) != 1 { + t.Fatalf("w1 attempts = %d, want exactly 1", len(atts)) + } + if atts[0].Status != "done" { + t.Errorf("w1 attempt status = %s, want done", atts[0].Status) + } + if len(atts[0].SessionID) != 36 { + t.Errorf("w1 session id = %q, want a UUID (claude --session-id)", atts[0].SessionID) + } + data, err := os.ReadFile(filepath.Join(proj, "claude-marker.txt")) + if err != nil { + t.Fatalf("claude-marker.txt missing: %v", err) + } + if got := strings.TrimSpace(string(data)); got != "CORRAL-CLAUDE-OK" { + t.Errorf("claude-marker.txt = %q, want CORRAL-CLAUDE-OK", got) + } +} diff --git a/internal/claudeadapter/permission.go b/internal/claudeadapter/permission.go new file mode 100644 index 0000000..c95bdbb --- /dev/null +++ b/internal/claudeadapter/permission.go @@ -0,0 +1,509 @@ +package claudeadapter + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// Claude Code headless sessions surface permission prompts by invoking the +// --permission-prompt-tool MCP tool; the driver answers them through a local +// unix socket broker. +// +// claude --permission-prompt-tool mcp__corral__handle_permission_prompt +// │ +// (permission prompt) +// ▼ +// MCP helper child (--corral-claude-permission-tool, stdio JSON-RPC) +// │ net.Dial(unix) +// ▼ +// driver permission broker socket +// ▲ +// scheduler -> PermissionSession.RespondPermission +// +// The helper supplies its attempt and launch-session identity from the MCP +// process environment. The broker validates both, keeps the connection open +// until a decision arrives, and unblocks the helper (and Claude) exactly once. + +const ( + helperFlag = "--corral-claude-permission-tool" + permissionServerName = "corral" + permissionToolName = "mcp__corral__handle_permission_prompt" +) + +// mcpConfig is the --mcp-config document that wires the permission helper +// into claude as an MCP stdio server. +type mcpConfig struct { + Servers map[string]mcpServerConfig `json:"mcpServers"` +} + +type mcpServerConfig struct { + Command string `json:"command"` + Args []string `json:"args,omitempty"` + Env map[string]string `json:"env,omitempty"` +} + +// helperExecutable is the binary claude spawns as the MCP permission helper: +// the driver's own executable. The embedding binary must forward +// --corral-claude-permission-tool to RunPermissionHelper (see the live test +// for a TestMain example). +func helperExecutable() string { + if exe, err := os.Executable(); err == nil { + return exe + } + return os.Args[0] +} + +// brokerRequest is the wire message sent by the helper to the broker. +type brokerRequest struct { + AttemptID string `json:"attempt_id"` + SessionID string `json:"session_id"` + RequestID string `json:"request_id"` + ToolName string `json:"tool_name"` + ToolInput json.RawMessage `json:"tool_input"` +} + +// decision is the broker's reply to a permission request. +type decision struct { + Allow bool `json:"allow"` + Message string `json:"message,omitempty"` +} + +func denyDecision(msg string) decision { + return decision{Allow: false, Message: msg} +} + +// permissionBroker answers permission requests from claude's MCP helper over +// a unix socket, one connection per pending request. +type permissionBroker struct { + d *Driver + ln net.Listener + path string + dir string // auto-created private directory; empty for an override path + done chan struct{} + mu sync.Mutex + pending map[permissionKey]chan decision + closed bool +} + +type permissionKey struct { + attemptID string + requestID string +} + +func (d *Driver) startBroker() error { + d.brokerOnce.Do(func() { + path := d.opts.PermissionSocket + dir := "" + if path == "" { + var err error + dir, err = os.MkdirTemp("", "corral-claude-") + if err != nil { + d.brokerErr = err + return + } + path = filepath.Join(dir, "broker.sock") + } + if info, err := os.Lstat(path); err == nil { + if info.Mode()&os.ModeSocket == 0 { + if dir != "" { + _ = os.Remove(dir) + } + d.brokerErr = fmt.Errorf("refusing to replace non-socket permission path %q", path) + return + } + if err := os.Remove(path); err != nil { + if dir != "" { + _ = os.Remove(dir) + } + d.brokerErr = fmt.Errorf("remove stale socket: %w", err) + return + } + } else if !os.IsNotExist(err) { + if dir != "" { + _ = os.Remove(dir) + } + d.brokerErr = fmt.Errorf("inspect permission socket: %w", err) + return + } + ln, err := net.Listen("unix", path) + if err != nil { + if dir != "" { + _ = os.Remove(dir) + } + d.brokerErr = err + return + } + if err := os.Chmod(path, 0o600); err != nil { + _ = ln.Close() + _ = os.Remove(path) + if dir != "" { + _ = os.Remove(dir) + } + d.brokerErr = fmt.Errorf("secure socket mode: %w", err) + return + } + b := &permissionBroker{ + d: d, + ln: ln, + path: path, + dir: dir, + done: make(chan struct{}), + pending: map[permissionKey]chan decision{}, + } + d.broker = b + go b.serve() + }) + return d.brokerErr +} + +func (b *permissionBroker) serve() { + for { + conn, err := b.ln.Accept() + if err != nil { + select { + case <-b.done: + return + default: + } + continue + } + go b.handle(conn) + } +} + +// handle serves one helper connection: receive the request, park it as the +// attempt's pending permission, and block until a decision is delivered. +func (b *permissionBroker) handle(conn net.Conn) { + defer conn.Close() + enc := json.NewEncoder(conn) + dec := json.NewDecoder(bufio.NewReader(conn)) + var req brokerRequest + if err := dec.Decode(&req); err != nil || req.AttemptID == "" || req.SessionID == "" || req.RequestID == "" { + return + } + at := b.d.attemptByID(req.AttemptID) + if at == nil { + _ = enc.Encode(denyDecision("unknown attempt")) + return + } + at.mu.Lock() + validSession := req.SessionID == at.launchID || req.SessionID == at.sessionID + at.mu.Unlock() + if !validSession { + _ = enc.Encode(denyDecision("session does not belong to attempt")) + return + } + + ch := make(chan decision, 1) + key := permissionKey{attemptID: req.AttemptID, requestID: req.RequestID} + b.mu.Lock() + if b.closed { + b.mu.Unlock() + _ = enc.Encode(denyDecision("driver closed")) + return + } + at.mu.Lock() + if at.permission != "" && at.permission != req.RequestID { + at.mu.Unlock() + b.mu.Unlock() + _ = enc.Encode(denyDecision("another permission request is already pending")) + return + } + if _, exists := b.pending[key]; exists { + at.mu.Unlock() + b.mu.Unlock() + _ = enc.Encode(denyDecision("duplicate permission request")) + return + } + b.pending[key] = ch + at.permission = req.RequestID + at.permissionTool = sanitizePermissionText(req.ToolName, 200) + at.permissionInput = sanitizePermissionText(string(req.ToolInput), 2000) + at.mu.Unlock() + b.mu.Unlock() + + var reply decision + select { + case reply = <-ch: + case <-b.done: + reply = denyDecision("driver closed") + case <-time.After(10 * time.Minute): + reply = denyDecision("permission request timed out") + } + b.mu.Lock() + if b.pending[key] == ch { + delete(b.pending, key) + } + b.mu.Unlock() + at.mu.Lock() + if at.permission == req.RequestID { + at.permission = "" + at.permissionTool = "" + at.permissionInput = "" + } + at.mu.Unlock() + _ = enc.Encode(reply) +} + +func sanitizePermissionText(s string, max int) string { + s = strings.Map(func(r rune) rune { + if r == '\n' || r == '\r' || r == '\t' || (r >= 0x20 && r != 0x7f) { + return r + } + return -1 + }, s) + if len(s) > max { + return s[:max] + "..." + } + return s +} + +// respond claims and resolves the helper waiting on requestID. Missing and +// duplicate responses are rejected instead of being silently accepted. +func (b *permissionBroker) respond(ctx context.Context, at *attempt, requestID string, allow bool) error { + if err := ctx.Err(); err != nil { + return err + } + key := permissionKey{attemptID: at.attemptID, requestID: requestID} + b.mu.Lock() + ch := b.pending[key] + if ch != nil { + delete(b.pending, key) // claim the request; duplicate responses now fail + } + b.mu.Unlock() + if ch == nil { + return fmt.Errorf("claudeadapter: permission %q has no waiting helper", requestID) + } + reply := decision{Allow: allow} + if !allow { + reply.Message = "denied by operator" + } + ch <- reply // buffered channel; cannot block + return nil +} + +func (b *permissionBroker) close() { + b.mu.Lock() + if b.closed { + b.mu.Unlock() + return + } + b.closed = true + b.mu.Unlock() + close(b.done) + _ = b.ln.Close() + _ = os.Remove(b.path) + if b.dir != "" { + _ = os.Remove(b.dir) + } +} + +// --------------------------------------------------------------------------- +// MCP stdio server (the --corral-claude-permission-tool helper). The helper +// is spawned by claude as an MCP server; each tools/call for the permission +// tool is forwarded to the driver's broker socket and the decision is +// returned as the tool result. + +// RunPermissionHelper serves the MCP permission-prompt tool over stdio. The +// embedding binary must invoke it when os.Args[1] == +// "--corral-claude-permission-tool" (the helper spawns with that flag). It +// returns only on error; callers exit afterwards. +func RunPermissionHelper() { + broker := os.Getenv("CORRAL_CLAUDE_BROKER") + if broker == "" { + fmt.Fprintln(os.Stderr, "corral claude permission helper: CORRAL_CLAUDE_BROKER not set") + os.Exit(1) + } + attemptID := os.Getenv("CORRAL_CLAUDE_ATTEMPT_ID") + sessionID := os.Getenv("CORRAL_CLAUDE_SESSION_ID") + if attemptID == "" || sessionID == "" { + fmt.Fprintln(os.Stderr, "corral claude permission helper: attempt/session identity not set") + os.Exit(1) + } + s := &mcpServer{broker: broker, attemptID: attemptID, sessionID: sessionID} + if err := s.serve(os.Stdin, os.Stdout); err != nil && !errors.Is(err, io.EOF) { + fmt.Fprintf(os.Stderr, "corral claude permission helper: %v\n", err) + os.Exit(1) + } +} + +type rpcRequest struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` +} + +type rpcResponse struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Result any `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type mcpServer struct { + broker string + attemptID string + sessionID string + mu sync.Mutex +} + +func (s *mcpServer) serve(in io.Reader, out io.Writer) error { + enc := json.NewEncoder(out) + dec := json.NewDecoder(in) + for { + var req rpcRequest + if err := dec.Decode(&req); err != nil { + return err + } + switch req.Method { + case "initialize": + s.write(enc, req.ID, map[string]any{ + "protocolVersion": "2024-11-05", + "capabilities": map[string]any{"tools": map[string]any{}}, + "serverInfo": map[string]string{"name": "corral", "version": "0.0.1"}, + }) + case "notifications/initialized": + // no response expected + case "tools/list": + s.write(enc, req.ID, map[string]any{ + "tools": []map[string]any{{ + "name": "handle_permission_prompt", + "description": "Corral handles Claude Code permission prompts on behalf of the scheduler.", + "inputSchema": map[string]any{ + "type": "object", + "properties": map[string]any{ + "tool_name": map[string]string{"type": "string"}, + "input": map[string]string{"type": "object"}, + "tool_use_id": map[string]string{"type": "string"}, + }, + "required": []string{"tool_name", "input"}, + "additionalProperties": true, + }, + }}, + }) + case "tools/call": + go s.handleToolCall(enc, req.ID, req.Params) + case "ping": + s.write(enc, req.ID, map[string]any{}) + default: + s.writeErr(enc, req.ID, -32601, "unknown method "+req.Method) + } + } +} + +type toolCallParams struct { + Name string `json:"name"` + Arguments json.RawMessage `json:"arguments"` +} + +// handleToolCall blocks until the driver answers the permission prompt, so +// it runs in its own goroutine; writes are serialized by the encoder mutex. +func (s *mcpServer) handleToolCall(enc *json.Encoder, id json.RawMessage, params json.RawMessage) { + var call toolCallParams + var args struct { + ToolName string `json:"tool_name"` + ToolInput json.RawMessage `json:"input"` + ToolUseID string `json:"tool_use_id"` + } + if json.Unmarshal(params, &call) != nil || call.Name != "handle_permission_prompt" { + s.writeToolError(enc, id, "unexpected tool call") + return + } + if err := json.Unmarshal(call.Arguments, &args); err != nil || args.ToolName == "" || len(args.ToolInput) == 0 || string(args.ToolInput) == "null" { + s.writeToolError(enc, id, "invalid permission prompt arguments") + return + } + requestID := args.ToolUseID + if requestID == "" { + requestID = newUUID() + } + + reply, err := s.askBroker(brokerRequest{ + AttemptID: s.attemptID, + SessionID: s.sessionID, + RequestID: requestID, + ToolName: args.ToolName, + ToolInput: args.ToolInput, + }) + if err != nil { + s.writeToolError(enc, id, err.Error()) + return + } + text, err := json.Marshal(decisionJSON(reply, args.ToolInput)) + if err != nil { + s.writeToolError(enc, id, err.Error()) + return + } + s.write(enc, id, map[string]any{ + "content": []map[string]string{{"type": "text", "text": string(text)}}, + "isError": false, + }) +} + +// decisionJSON renders a broker decision as the permission tool result +// claude expects: an allow must echo the tool input back. +func decisionJSON(d decision, toolInput json.RawMessage) map[string]any { + if d.Allow { + return map[string]any{"behavior": "allow", "updatedInput": rawOrEmpty(toolInput)} + } + return map[string]any{"behavior": "deny", "message": d.Message} +} + +func rawOrEmpty(raw json.RawMessage) any { + if len(raw) == 0 || strings.EqualFold(string(raw), "null") { + return map[string]any{} + } + return raw +} + +// askBroker forwards a permission request to the driver's broker socket and +// waits for the decision. +func (s *mcpServer) askBroker(req brokerRequest) (decision, error) { + conn, err := net.DialTimeout("unix", s.broker, 10*time.Second) + if err != nil { + return denyDecision(err.Error()), err + } + defer conn.Close() + if err := json.NewEncoder(conn).Encode(req); err != nil { + return denyDecision(err.Error()), err + } + var reply decision + if err := json.NewDecoder(bufio.NewReader(conn)).Decode(&reply); err != nil { + return denyDecision(err.Error()), err + } + return reply, nil +} + +func (s *mcpServer) write(enc *json.Encoder, id json.RawMessage, result any) { + s.mu.Lock() + defer s.mu.Unlock() + _ = enc.Encode(rpcResponse{JSONRPC: "2.0", ID: id, Result: result}) +} + +func (s *mcpServer) writeErr(enc *json.Encoder, id json.RawMessage, code int, msg string) { + s.mu.Lock() + defer s.mu.Unlock() + _ = enc.Encode(rpcResponse{JSONRPC: "2.0", ID: id, Error: &rpcError{Code: code, Message: msg}}) +} + +func (s *mcpServer) writeToolError(enc *json.Encoder, id json.RawMessage, msg string) { + s.write(enc, id, map[string]any{ + "content": []map[string]string{{"type": "text", "text": msg}}, + "isError": true, + }) +} diff --git a/internal/daemon/broker.go b/internal/daemon/broker.go new file mode 100644 index 0000000..d63d99c --- /dev/null +++ b/internal/daemon/broker.go @@ -0,0 +1,110 @@ +package daemon + +import ( + "sync" + + "corral/internal/store" +) + +// broker is the in-daemon live-event hub. The store notifies it of every +// committed event; it fans those out to per-run subscribers (SSE clients). +// +// Subscribers are removed on disconnect via the returned unsubscribe func. +// A subscriber whose buffer overflows is dropped so the scheduler never +// blocks on a slow consumer; the client reconnects with an after cursor and +// the store replays the gap. +type broker struct { + mu sync.Mutex + subs map[string]map[*subscriber]struct{} + closed bool +} + +type subscriber struct { + runID string + ch chan store.Event +} + +const subscriberBuffer = 256 + +func newBroker() *broker { + return &broker{subs: map[string]map[*subscriber]struct{}{}} +} + +// Subscribe registers a subscriber for runID. Events for that run are +// delivered on the returned channel; the caller must call the returned +// func when done (e.g. on request context cancellation). +func (b *broker) Subscribe(runID string) (<-chan store.Event, func()) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + ch := make(chan store.Event) + close(ch) + return ch, func() {} + } + if b.subs[runID] == nil { + b.subs[runID] = map[*subscriber]struct{}{} + } + sub := &subscriber{runID: runID, ch: make(chan store.Event, subscriberBuffer)} + b.subs[runID][sub] = struct{}{} + return sub.ch, func() { b.unsubscribe(sub) } +} + +func (b *broker) unsubscribe(sub *subscriber) { + b.mu.Lock() + defer b.mu.Unlock() + subs := b.subs[sub.runID] + if _, ok := subs[sub]; ok { + delete(subs, sub) + close(sub.ch) + if len(subs) == 0 { + delete(b.subs, sub.runID) + } + } +} + +// Publish delivers an event to every subscriber of its run. It never +// blocks: slow consumers are dropped rather than stalling the writer. +func (b *broker) Publish(ev store.Event) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return + } + subs := b.subs[ev.RunID] + for sub := range subs { + select { + case sub.ch <- ev: + default: + delete(subs, sub) + close(sub.ch) + } + } + if len(subs) == 0 { + delete(b.subs, ev.RunID) + } +} + +// Close shuts the broker down, closing every subscriber channel. Publish +// after Close is a no-op. +func (b *broker) Close() { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return + } + b.closed = true + for _, subs := range b.subs { + for sub := range subs { + close(sub.ch) + } + } + b.subs = map[string]map[*subscriber]struct{}{} +} + +// SubscriberCount reports how many subscribers a run currently has +// (used by tests to assert cleanup on disconnect). +func (b *broker) SubscriberCount(runID string) int { + b.mu.Lock() + defer b.mu.Unlock() + return len(b.subs[runID]) +} diff --git a/internal/daemon/broker_test.go b/internal/daemon/broker_test.go new file mode 100644 index 0000000..4d93918 --- /dev/null +++ b/internal/daemon/broker_test.go @@ -0,0 +1,208 @@ +package daemon + +import ( + "context" + "net/http" + "net/http/httptest" + "os/exec" + "path/filepath" + "testing" + "time" + + "corral/internal/clock" + "corral/internal/graph" + "corral/internal/sched" + "corral/internal/store" + "corral/internal/verify" + "corral/internal/worktree" +) + +func TestBrokerRoutesByRun(t *testing.T) { + b := newBroker() + ch1, unsub1 := b.Subscribe("r1") + defer unsub1() + ch1b, unsub1b := b.Subscribe("r1") + defer unsub1b() + ch2, unsub2 := b.Subscribe("r2") + defer unsub2() + + b.Publish(store.Event{Seq: 1, RunID: "r1", Type: store.EventRun}) + + for _, ch := range []<-chan store.Event{ch1, ch1b} { + select { + case ev := <-ch: + if ev.Seq != 1 || ev.RunID != "r1" { + t.Fatalf("r1 subscriber got %+v", ev) + } + default: + t.Fatal("r1 subscriber missed event") + } + } + select { + case ev := <-ch2: + t.Fatalf("r2 subscriber got r1 event %+v", ev) + default: + } +} + +func TestBrokerUnsubscribeCleansUp(t *testing.T) { + b := newBroker() + _, unsub := b.Subscribe("r1") + if b.SubscriberCount("r1") != 1 { + t.Fatalf("count = %d, want 1", b.SubscriberCount("r1")) + } + unsub() + if b.SubscriberCount("r1") != 0 { + t.Fatalf("count after unsub = %d, want 0", b.SubscriberCount("r1")) + } + b.Publish(store.Event{Seq: 1, RunID: "r1"}) // must not panic or deliver +} + +func TestBrokerDropsSlowSubscriber(t *testing.T) { + b := newBroker() + ch, _ := b.Subscribe("r1") + for i := 0; i < subscriberBuffer; i++ { + b.Publish(store.Event{Seq: int64(i + 1), RunID: "r1"}) + } + b.Publish(store.Event{Seq: 1000, RunID: "r1"}) + if n := b.SubscriberCount("r1"); n != 0 { + t.Fatalf("slow subscriber not dropped: count = %d", n) + } + deadline := time.After(time.Second) + for { + select { + case _, ok := <-ch: + if !ok { + return + } + case <-deadline: + t.Fatal("dropped subscriber channel was not closed") + } + } +} + +func TestBrokerCloseClosesSubscribers(t *testing.T) { + b := newBroker() + ch, _ := b.Subscribe("r1") + b.Close() + if _, ok := <-ch; ok { + t.Fatal("subscriber channel not closed after Close") + } + b.Publish(store.Event{Seq: 1, RunID: "r1"}) // no-op after close + ch2, _ := b.Subscribe("r1") + if _, ok := <-ch2; ok { + t.Fatal("Subscribe after Close returned an open channel") + } +} + +func TestBrokerPublishUnsubscribeRace(t *testing.T) { + b := newBroker() + for i := 0; i < 500; i++ { + _, unsubscribe := b.Subscribe("r1") + done := make(chan struct{}) + go func(seq int64) { + defer close(done) + b.Publish(store.Event{Seq: seq, RunID: "r1"}) + }(int64(i + 1)) + unsubscribe() + <-done + } + if got := b.SubscriberCount("r1"); got != 0 { + t.Fatalf("subscriber count = %d, want 0", got) + } +} + +func TestClosingOneDaemonDoesNotDetachAnother(t *testing.T) { + st, err := store.Open(filepath.Join(t.TempDir(), "d.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + clk := clock.Real{} + s := sched.New(st, sched.NewFakeDriver(clk, nil), &sched.EngineVerifier{Eng: verify.New(t.TempDir())}, clk, sched.Options{}) + one := New(st, s, nil, t.TempDir(), "") + two := New(st, s, nil, t.TempDir(), "") + t.Cleanup(two.Close) + one.Close() + + events, unsubscribe := two.broker.Subscribe("r1") + defer unsubscribe() + if err := st.CreateRun(context.Background(), "r1", &graph.Graph{Nodes: []*graph.Node{{ + ID: "w1", Type: graph.NodeAgent, Objective: "o", AcceptanceCriteria: []string{"c"}, Priority: graph.PriorityNormal, + }}}, false, time.Now()); err != nil { + t.Fatal(err) + } + select { + case event := <-events: + if event.Seq != 1 || event.RunID != "r1" { + t.Fatalf("second daemon got %+v", event) + } + case <-time.After(time.Second): + t.Fatal("closing first daemon detached second daemon") + } +} + +func TestSSESubscriberCleanupOnDisconnect(t *testing.T) { + st, err := store.Open(filepath.Join(t.TempDir(), "d.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + repo := t.TempDir() + for _, args := range [][]string{ + {"init", "-q", "-b", "main"}, + {"-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "--allow-empty", "-m", "init"}, + } { + cmd := exec.Command("git", args...) + cmd.Dir = repo + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git: %v: %s", err, out) + } + } + clk := clock.Real{} + drv := sched.NewFakeDriver(clk, nil) + eng := verify.New(repo) + eng.Runner = verify.ExecRunner{} + s := sched.New(st, drv, &sched.EngineVerifier{Eng: eng}, clk, sched.Options{ + Concurrency: 4, Worktrees: worktree.NewManager(repo), + }) + d := New(st, s, nil, repo, "") + t.Cleanup(d.Close) + srv := httptest.NewServer(d.Handler()) + t.Cleanup(srv.Close) + + ctx := context.Background() + h, err := s.Create(ctx, "run_cleanup", &graph.Graph{Nodes: []*graph.Node{ + {ID: "w1", Type: graph.NodeAgent, Role: "worker", Objective: "o", + AcceptanceCriteria: []string{"c"}, Priority: graph.PriorityNormal, + Meta: map[string]string{"cwd": repo}, + Verification: &graph.Verification{Kind: "command", Command: []string{"true"}}}, + }}) + if err != nil { + t.Fatal(err) + } + go h.Run(ctx, 50*time.Millisecond) + + reqCtx, cancel := context.WithCancel(context.Background()) + req, _ := http.NewRequestWithContext(reqCtx, http.MethodGet, srv.URL+"/api/runs/run_cleanup/events", nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return d.broker.SubscriberCount("run_cleanup") == 1 }, "subscriber registered") + cancel() + waitFor(t, func() bool { return d.broker.SubscriberCount("run_cleanup") == 0 }, "subscriber cleaned up on disconnect") + resp.Body.Close() +} + +func waitFor(t *testing.T, cond func() bool, msg string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", msg) +} diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 5dce54e..826cb2d 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -11,6 +11,7 @@ import ( "fmt" "net/http" "sort" + "strconv" "strings" "sync" "time" @@ -53,8 +54,14 @@ type Daemon struct { apiKey string ctx context.Context - mu sync.Mutex - runs map[string]*sched.RunHandle + mu sync.RWMutex + runs map[string]*sched.RunHandle + broker *broker + eventHeartbeat time.Duration + eventHeartbeatMu sync.RWMutex + eventUnsubscribe func() + eventPumpDone chan struct{} + closeOnce sync.Once } // Dir returns the project directory the daemon manages. @@ -64,11 +71,36 @@ func (d *Daemon) Dir() string { return d.dir } func (d *Daemon) SetPlanner(p Planner) { d.plan = p } func New(st *store.Store, s *sched.Scheduler, plan Planner, dir, apiKey string) *Daemon { - return &Daemon{ + events, unsubscribe := st.SubscribeEvents() + d := &Daemon{ st: st, sched: s, plan: plan, dir: dir, apiKey: apiKey, - ctx: context.Background(), - runs: map[string]*sched.RunHandle{}, + ctx: context.Background(), + runs: map[string]*sched.RunHandle{}, + broker: newBroker(), + eventHeartbeat: defaultEventHeartbeat, + eventUnsubscribe: unsubscribe, + eventPumpDone: make(chan struct{}), + } + go d.forwardEvents(events) + return d +} + +func (d *Daemon) forwardEvents(events <-chan store.Event) { + defer close(d.eventPumpDone) + for ev := range events { + d.broker.Publish(ev) } + d.broker.Close() +} + +// Close detaches the daemon from the store and closes live event streams. +// Other daemons subscribed to the same store are unaffected. +func (d *Daemon) Close() { + d.closeOnce.Do(func() { + d.eventUnsubscribe() + d.broker.Close() + <-d.eventPumpDone + }) } // SetContext replaces the daemon's background context (its lifetime). @@ -106,6 +138,8 @@ func (d *Daemon) Handler() http.Handler { mux.HandleFunc("POST /api/runs", d.role(RoleOrchestrator, RoleOperator)(d.handleCreateRun)) mux.HandleFunc("GET /api/runs", d.handleListRuns) mux.HandleFunc("GET /api/runs/{id}", d.handleGetRun) + mux.HandleFunc("GET /api/runs/{id}/watch", d.handleWatchRun) + mux.HandleFunc("GET /api/runs/{id}/tail", d.handleTail) mux.HandleFunc("POST /api/runs/{id}/approve", d.role(RoleOperator, RoleOrchestrator)(d.handleApprove)) mux.HandleFunc("POST /api/runs/{id}/reject", d.role(RoleOperator, RoleOrchestrator)(d.handleReject)) mux.HandleFunc("POST /api/runs/{id}/cancel", d.role(RoleOperator, RoleOrchestrator)(d.handleCancel)) @@ -113,6 +147,7 @@ func (d *Daemon) Handler() http.Handler { mux.HandleFunc("POST /api/runs/{id}/steer", d.role(RoleOperator, RoleOrchestrator)(d.handleSteer)) mux.HandleFunc("POST /api/runs/{id}/permission", d.role(RoleOperator, RoleOrchestrator)(d.handlePermission)) mux.HandleFunc("GET /api/runs/{id}/export", d.handleExport) + mux.HandleFunc("GET /api/runs/{id}/events", d.handleEvents) mux.HandleFunc("GET /doc", d.handleOpenAPI) return d.auth(mux) } @@ -184,15 +219,23 @@ func (d *Daemon) handlePlan(w http.ResponseWriter, r *http.Request) { func (d *Daemon) handleCreateRun(w http.ResponseWriter, r *http.Request) { var req struct { - Graph *graph.Graph `json:"graph"` + Graph *graph.Graph `json:"graph"` + AutoApproveGates bool `json:"autoApproveGates"` } if err := readJSON(r, &req); err != nil || req.Graph == nil { http.Error(w, "graph required", http.StatusBadRequest) return } + if req.AutoApproveGates { + role, _ := parseRole(r.Header.Get("X-Corral-Role")) + if role != RoleOperator { + http.Error(w, "only an operator may pre-authorize human gates", http.StatusForbidden) + return + } + } ctx := r.Context() runID := "run_" + randID(6) - h, err := d.sched.Create(ctx, runID, req.Graph) + h, err := d.sched.Create(ctx, runID, req.Graph, sched.CreateOptions{AutoApproveGates: req.AutoApproveGates}) if err != nil { http.Error(w, "invalid graph: "+err.Error(), http.StatusUnprocessableEntity) return @@ -205,15 +248,20 @@ func (d *Daemon) handleCreateRun(w http.ResponseWriter, r *http.Request) { } func (d *Daemon) runHandle(id string) (*sched.RunHandle, error) { - d.mu.Lock() - defer d.mu.Unlock() - h, ok := d.runs[id] + h, ok := d.lookupRunHandle(id) if !ok { return nil, fmt.Errorf("unknown run %s", id) } return h, nil } +func (d *Daemon) lookupRunHandle(id string) (*sched.RunHandle, bool) { + d.mu.RLock() + defer d.mu.RUnlock() + h, ok := d.runs[id] + return h, ok +} + type runSummary struct { ID string `json:"id"` Status string `json:"status"` @@ -231,7 +279,7 @@ func (d *Daemon) handleListRuns(w http.ResponseWriter, r *http.Request) { var out []runSummary for _, ru := range runs { sum := runSummary{ID: ru.ID, Status: ru.Status, States: map[string]string{}} - if h, ok := d.runs[ru.ID]; ok { + if h, ok := d.lookupRunHandle(ru.ID); ok { sum.Done = h.Done() for _, n := range ru.Graph.Nodes { if st, ok := h.State(n.ID); ok { @@ -274,10 +322,14 @@ func (d *Daemon) handleGetRun(w http.ResponseWriter, r *http.Request) { attempts[string(n.ID)] = atts } resp := map[string]any{ - "runID": id, "status": ru.Status, "graph": ru.Graph, - "events": events, "attempts": attempts, - } - if h, ok := d.runs[id]; ok { + "runID": id, + "status": ru.Status, + "graph": ru.Graph, + "autoApproveGates": ru.AutoApproveGates, + "events": events, + "attempts": attempts, + } + if h, ok := d.lookupRunHandle(id); ok { states := map[string]string{} for _, n := range ru.Graph.Nodes { if st, ok := h.State(n.ID); ok { @@ -290,6 +342,119 @@ func (d *Daemon) handleGetRun(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, resp) } +// handleWatchRun long-polls a run for the orchestrator run loop. It +// returns as soon as the run produces new events (milestones, a gate +// awaiting approval, resolution, completion) or after the timeout, and +// always carries the current snapshot: node states, gates awaiting +// approval, whether the run is pre-authorized to auto-approve them, and +// the event cursor to pass back as `since`. +func (d *Daemon) handleWatchRun(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + q := r.URL.Query() + since, _ := strconv.ParseInt(q.Get("since"), 10, 64) + timeout := 60 + if v := q.Get("timeout"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + timeout = n + } + } + if timeout < 1 { + timeout = 1 + } + if timeout > 120 { + timeout = 120 + } + deadline := time.Now().Add(time.Duration(timeout) * time.Second) + + ctx := r.Context() + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + + for { + snap, changed, done, err := d.watchSnapshot(ctx, id, since) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + if changed || done || time.Now().After(deadline) { + writeJSON(w, http.StatusOK, snap) + return + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +// watchSnapshot builds the watch response for a run. changed reports +// whether any event happened after since; done whether the run settled. +func (d *Daemon) watchSnapshot(ctx context.Context, id string, since int64) (map[string]any, bool, bool, error) { + ru, err := d.st.Run(ctx, id) + if err != nil { + return nil, false, false, err + } + events, err := d.st.Events(ctx, id) + if err != nil { + return nil, false, false, err + } + maxSeq := since + var newEvents []store.Event + for _, e := range events { + if e.Seq > maxSeq { + maxSeq = e.Seq + } + if e.Seq > since { + newEvents = append(newEvents, e) + } + } + + states := map[string]string{} + done := false + if h, ok := d.lookupRunHandle(id); ok { + for _, n := range ru.Graph.Nodes { + if st, ok := h.State(n.ID); ok { + states[string(n.ID)] = string(st) + } + } + done = h.Done() + } else { + if ns, err := d.st.NodeStates(ctx, id); err == nil { + for nid, st := range ns { + states[string(nid)] = string(st) + } + } + done = ru.Status != "active" + } + // The run status above was read before done was resolved; the store is + // finalized before the handle reports done, so re-read it so the + // snapshot never carries done=true with a stale status. + if done && ru.Status == "active" { + if ru2, err := d.st.Run(ctx, id); err == nil { + ru.Status = ru2.Status + } + } + + var gates []string + for _, n := range ru.Graph.Nodes { + if n.Type == graph.NodeHuman && states[string(n.ID)] == string(graph.StateRunning) { + gates = append(gates, string(n.ID)) + } + } + + return map[string]any{ + "runID": id, + "status": ru.Status, + "done": done, + "autoApproveGates": ru.AutoApproveGates, + "states": states, + "gatesAwaitingApproval": gates, + "since": maxSeq, + "events": newEvents, + }, len(newEvents) > 0, done, nil +} + func (d *Daemon) nodeAction(w http.ResponseWriter, r *http.Request, fn func(ctx context.Context, id graph.NodeID) error) { var req struct { NodeID string `json:"nodeID"` @@ -310,12 +475,46 @@ func (d *Daemon) nodeAction(w http.ResponseWriter, r *http.Request, fn func(ctx writeJSON(w, http.StatusOK, map[string]any{"ok": true}) } +// handleTail returns the live transcript tail of a node's in-flight +// attempt (query params: node, lines). Used by the TUI's inspect view. +func (d *Daemon) handleTail(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + node := q.Get("node") + if node == "" || len(node) > 256 { + http.Error(w, "node required", http.StatusBadRequest) + return + } + lines := 40 + if v := q.Get("lines"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 || n > 500 { + http.Error(w, "lines must be between 1 and 500", http.StatusBadRequest) + return + } + lines = n + } + h, err := d.runHandle(r.PathValue("id")) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + tail, err := h.Tail(r.Context(), graph.NodeID(node), lines) + if err != nil { + http.Error(w, err.Error(), http.StatusConflict) + return + } + writeJSON(w, http.StatusOK, map[string]any{"node": node, "lines": tail}) +} + func (d *Daemon) handleApprove(w http.ResponseWriter, r *http.Request) { h, err := d.runHandle(r.PathValue("id")) if err != nil { http.Error(w, err.Error(), http.StatusNotFound) return } + if !d.gateActionAuthorized(w, r) { + return + } d.nodeAction(w, r, func(ctx context.Context, id graph.NodeID) error { return h.ApproveNode(ctx, id) }) } @@ -325,9 +524,32 @@ func (d *Daemon) handleReject(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusNotFound) return } + if !d.gateActionAuthorized(w, r) { + return + } d.nodeAction(w, r, func(ctx context.Context, id graph.NodeID) error { return h.RejectNode(ctx, id) }) } +// gateActionAuthorized enforces the run's persisted pre-authorization +// policy. Operators may always resolve human gates; orchestrators may do so +// only when the run was created with autoApproveGates enabled. +func (d *Daemon) gateActionAuthorized(w http.ResponseWriter, r *http.Request) bool { + role, _ := parseRole(r.Header.Get("X-Corral-Role")) + if role != RoleOrchestrator { + return true + } + ru, err := d.st.Run(r.Context(), r.PathValue("id")) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return false + } + if !ru.AutoApproveGates { + http.Error(w, "orchestrator is not pre-authorized to resolve human gates", http.StatusForbidden) + return false + } + return true +} + func (d *Daemon) handleCancel(w http.ResponseWriter, r *http.Request) { h, err := d.runHandle(r.PathValue("id")) if err != nil { diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go index 938e7f5..5d3b319 100644 --- a/internal/daemon/daemon_test.go +++ b/internal/daemon/daemon_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "net/http" "net/http/httptest" @@ -11,6 +12,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "testing" "time" @@ -78,6 +80,7 @@ func setupDaemon(t *testing.T, apiKey string) (*api, *daemon.Daemon, *store.Stor Concurrency: 4, Worktrees: wtm, }) d := daemon.New(st, s, nil, repo, apiKey) + t.Cleanup(d.Close) srv := httptest.NewServer(d.Handler()) t.Cleanup(srv.Close) return &api{t: t, cli: srv.Client(), base: srv.URL}, d, st, drv @@ -140,6 +143,11 @@ func TestRoleEnforcement(t *testing.T) { if code, _ := a.do("orchestrator", http.MethodPost, "/api/runs", map[string]any{"graph": g}); code != http.StatusCreated { t.Fatalf("orchestrator create run: %d, want 201", code) } + if code, body := a.do("orchestrator", http.MethodPost, "/api/runs", map[string]any{ + "graph": g, "autoApproveGates": true, + }); code != http.StatusForbidden || !strings.Contains(body, "only an operator") { + t.Fatalf("orchestrator self-authorization: %d %s, want 403", code, body) + } // Workers may not approve; operators may. (Run id from the created run.) if code, _ := a.do("worker", http.MethodPost, "/api/runs/whatever/approve", map[string]any{"nodeID": "w1"}); code != http.StatusForbidden { t.Fatalf("worker approve: %d, want 403", code) @@ -150,6 +158,89 @@ func TestRoleEnforcement(t *testing.T) { } } +func TestConcurrentWatchAndCreate(t *testing.T) { + a, _, _, _ := setupDaemon(t, "") + g := &graph.Graph{Nodes: []*graph.Node{gateNode("gate")}} + code, body := a.do("operator", http.MethodPost, "/api/runs", map[string]any{"graph": g}) + if code != http.StatusCreated { + t.Fatalf("create seed run: %d %s", code, body) + } + var created struct{ RunID string } + if err := json.Unmarshal([]byte(body), &created); err != nil { + t.Fatal(err) + } + + start := make(chan struct{}) + errs := make(chan error, 16) + request := func(method, path string, body any) error { + var rdr io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return err + } + rdr = bytes.NewReader(data) + } + req, err := http.NewRequest(method, a.base+path, rdr) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("X-Corral-Role", "operator") + resp, err := a.cli.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + respBody, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= http.StatusBadRequest { + return fmt.Errorf("%s %s: status %d: %s", method, path, resp.StatusCode, respBody) + } + return nil + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + <-start + created := 0 + for attempts := 0; attempts < 1000 && created < 50; attempts++ { + if err := request(http.MethodPost, "/api/runs", map[string]any{"graph": g}); err != nil { + if strings.Contains(err.Error(), "SQLITE_BUSY") { + time.Sleep(time.Millisecond) + continue + } + errs <- err + return + } + created++ + } + if created < 50 { + errs <- fmt.Errorf("created %d concurrent runs, want 50", created) + } + }() + for range 6 { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for range 200 { + if err := request(http.MethodGet, "/api/runs/"+created.RunID+"/watch?since=0&timeout=1", nil); err != nil { + errs <- err + return + } + } + }() + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + t.Error(err) + } +} + func TestAuthRequired(t *testing.T) { a, _, _, _ := setupDaemon(t, "sekret") req, _ := http.NewRequest(http.MethodGet, a.base+"/api/health", nil) @@ -260,6 +351,20 @@ func TestCancelAndRetryThroughAPI(t *testing.T) { if code != http.StatusOK { t.Fatalf("steer: %d %s", code, body) } + events, err := st.Events(context.Background(), created.RunID) + if err != nil { + t.Fatal(err) + } + var steer *store.Event + for i := range events { + if events[i].Type == store.EventSteer { + steer = &events[i] + break + } + } + if steer == nil || steer.NodeID != "w1" || !strings.Contains(string(steer.Payload), "wrap up") { + t.Fatalf("steer event missing or malformed: %+v", steer) + } code, body = a.do("operator", http.MethodPost, "/api/runs/"+created.RunID+"/cancel", map[string]any{"nodeID": "w1"}) if code != http.StatusOK { t.Fatalf("cancel: %d %s", code, body) @@ -268,6 +373,7 @@ func TestCancelAndRetryThroughAPI(t *testing.T) { // Failed node retried through the API. bad := workerNode("bad", "x.txt", "X") + bad.WriteScope = []string{"x.txt", "missing.txt"} bad.Verification = &graph.Verification{Kind: "command", Command: []string{"test", "-f", "missing.txt"}} bad.RetryPolicy = graph.RetryPolicy{MaxRetries: 0, Backoff: tick} drv.AppendScript("bad", sched.Script{Delay: 100 * time.Millisecond, Write: map[string]string{"x.txt": "X"}}) @@ -299,6 +405,193 @@ func (f *fakePlanner) Plan(_ context.Context, _ string) (*graph.Graph, error) { var _ = daemon.RoleOperator +// TestPreAuthorizedGateThroughAPI verifies that autoApproveGates is stored +// and exposed as authorization metadata while the gate still waits for an +// explicit orchestrator/operator API action. +func TestPreAuthorizedGateThroughAPI(t *testing.T) { + a, _, _, drv := setupDaemon(t, "") + drv.SetScript("w1", sched.Script{Delay: 100 * time.Millisecond, Write: map[string]string{"a.txt": "A1"}}) + g := &graph.Graph{Nodes: []*graph.Node{ + workerNode("w1", "a.txt", "A1"), + gateNode("gate", "w1"), + }} + code, body := a.do("operator", http.MethodPost, "/api/runs", map[string]any{"graph": g, "autoApproveGates": true}) + if code != http.StatusCreated { + t.Fatalf("create: %d %s", code, body) + } + var created struct{ RunID string } + json.Unmarshal([]byte(body), &created) + + // The flag is exposed on the run detail. + code, body = a.do("operator", http.MethodGet, "/api/runs/"+created.RunID, nil) + if code != http.StatusOK || !strings.Contains(body, `"autoApproveGates":true`) { + t.Fatalf("autoApproveGates not exposed: %d %s", code, body) + } + // The flag authorizes the orchestrator to act; scheduler still exposes + // the gate instead of silently bypassing it. + a.waitState(t, "", created.RunID, "gate", graph.StateRunning, 30*time.Second) + snap := a.watchUntil(t, created.RunID, "", func(m map[string]any) bool { + gates, _ := m["gatesAwaitingApproval"].([]any) + return len(gates) == 1 && gates[0] == "gate" + }) + if aa, _ := snap["autoApproveGates"].(bool); !aa { + t.Fatal("watch snapshot lost pre-authorization flag") + } + code, body = a.do("orchestrator", http.MethodPost, "/api/runs/"+created.RunID+"/approve", map[string]any{"nodeID": "gate"}) + if code != http.StatusOK { + t.Fatalf("orchestrator approve: %d %s", code, body) + } + a.waitState(t, "", created.RunID, "gate", graph.StateDone, 30*time.Second) + snap = a.watchUntil(t, created.RunID, "", func(m map[string]any) bool { + done, _ := m["done"].(bool) + return done + }) + if snap["status"] != "completed" { + t.Fatalf("run status = %v, want completed", snap["status"]) + } +} + +func TestOrchestratorGateActionsRequirePersistedPreAuthorization(t *testing.T) { + for _, tc := range []struct { + action string + want graph.State + }{ + {action: "approve", want: graph.StateDone}, + {action: "reject", want: graph.StateBlocked}, + } { + t.Run(tc.action, func(t *testing.T) { + a, _, st, _ := setupDaemon(t, "") + g := &graph.Graph{Nodes: []*graph.Node{gateNode("gate")}} + code, body := a.do("operator", http.MethodPost, "/api/runs", map[string]any{ + "graph": g, + "autoApproveGates": false, + }) + if code != http.StatusCreated { + t.Fatalf("create: %d %s", code, body) + } + var created struct{ RunID string } + if err := json.Unmarshal([]byte(body), &created); err != nil { + t.Fatal(err) + } + a.waitState(t, "", created.RunID, "gate", graph.StateRunning, 30*time.Second) + + ru, err := st.Run(context.Background(), created.RunID) + if err != nil { + t.Fatal(err) + } + if ru.AutoApproveGates { + t.Fatal("run unexpectedly persisted with autoApproveGates=true") + } + + code, body = a.do("orchestrator", http.MethodPost, "/api/runs/"+created.RunID+"/"+tc.action, map[string]any{"nodeID": "gate"}) + if code != http.StatusForbidden { + t.Fatalf("non-pre-authorized orchestrator %s: %d %s, want 403", tc.action, code, body) + } + a.waitState(t, "", created.RunID, "gate", graph.StateRunning, 5*time.Second) + + code, body = a.do("operator", http.MethodPost, "/api/runs/"+created.RunID+"/"+tc.action, map[string]any{"nodeID": "gate"}) + if code != http.StatusOK { + t.Fatalf("operator %s: %d %s", tc.action, code, body) + } + a.waitState(t, "", created.RunID, "gate", tc.want, 30*time.Second) + }) + } +} + +// TestWatchReportsGateAndDone drives the run through the watch endpoint +// (long-poll JSON): it must report the gate awaiting approval, and a done +// snapshot once the run completes after the operator approves. +func TestWatchReportsGateAndDone(t *testing.T) { + a, _, _, drv := setupDaemon(t, "") + drv.SetScript("w1", sched.Script{Delay: 100 * time.Millisecond, Write: map[string]string{"a.txt": "A1"}}) + g := &graph.Graph{Nodes: []*graph.Node{ + workerNode("w1", "a.txt", "A1"), + gateNode("gate", "w1"), + }} + code, body := a.do("operator", http.MethodPost, "/api/runs", map[string]any{"graph": g}) + if code != http.StatusCreated { + t.Fatalf("create: %d %s", code, body) + } + var created struct{ RunID string } + json.Unmarshal([]byte(body), &created) + + // The worker runs and the gate parks awaiting approval — the watch + // snapshot must flag it. + snap := a.watchUntil(t, created.RunID, "", func(m map[string]any) bool { + gates, _ := m["gatesAwaitingApproval"].([]any) + return len(gates) == 1 && gates[0] == "gate" + }) + if st, _ := snap["states"].(map[string]any)["gate"].(string); st != string(graph.StateRunning) { + t.Fatalf("gate state = %v, want running", st) + } + if aa, _ := snap["autoApproveGates"].(bool); aa { + t.Fatal("autoApproveGates set on a default run") + } + + // Approve via the API; the run completes and the watch reports done. + code, body = a.do("operator", http.MethodPost, "/api/runs/"+created.RunID+"/approve", map[string]any{"nodeID": "gate"}) + if code != http.StatusOK { + t.Fatalf("approve: %d %s", code, body) + } + snap = a.watchUntil(t, created.RunID, "", func(m map[string]any) bool { + d, _ := m["done"].(bool) + return d + }) + if s, _ := snap["status"].(string); s != "completed" { + t.Fatalf("done status = %v, want completed", snap["status"]) + } +} + +// TestWatchReportsDoneForSettledRun opens the watch stream after a run +// already settled: with no events after the cursor the endpoint still +// reports the done snapshot immediately. +func TestWatchReportsDoneForSettledRun(t *testing.T) { + a, _, _, drv := setupDaemon(t, "") + drv.SetScript("w1", sched.Script{Delay: 50 * time.Millisecond, Write: map[string]string{"a.txt": "A1"}}) + g := &graph.Graph{Nodes: []*graph.Node{workerNode("w1", "a.txt", "A1")}} + code, body := a.do("operator", http.MethodPost, "/api/runs", map[string]any{"graph": g}) + if code != http.StatusCreated { + t.Fatalf("create: %d %s", code, body) + } + var created struct{ RunID string } + json.Unmarshal([]byte(body), &created) + a.waitState(t, "", created.RunID, "w1", graph.StateDone, 30*time.Second) + + snap := a.watchUntil(t, created.RunID, "999999", func(m map[string]any) bool { + d, _ := m["done"].(bool) + return d + }) + if s, _ := snap["status"].(string); s != "completed" { + t.Fatalf("done status = %v, want completed", snap["status"]) + } +} + +// watchUntil long-polls /watch until a snapshot satisfies pred. +func (a *api) watchUntil(t *testing.T, runID, since string, pred func(map[string]any) bool) map[string]any { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + q := "timeout=1" + if since != "" { + q += "&since=" + since + } + code, body := a.do("operator", http.MethodGet, "/api/runs/"+runID+"/watch?"+q, nil) + if code != http.StatusOK { + t.Fatalf("watch: %d %s", code, body) + } + var m map[string]any + if err := json.Unmarshal([]byte(body), &m); err != nil { + t.Fatalf("watch decode: %v", err) + } + if pred(m) { + return m + } + if time.Now().After(deadline) { + t.Fatalf("watch timed out waiting for snapshot: %s", body) + } + } +} + func TestPermissionThroughAPI(t *testing.T) { a, d, _, drv := setupDaemon(t, "") workdir := d.Dir() diff --git a/internal/daemon/e2e_test.go b/internal/daemon/e2e_test.go index b7f5753..295af51 100644 --- a/internal/daemon/e2e_test.go +++ b/internal/daemon/e2e_test.go @@ -3,6 +3,8 @@ package daemon_test import ( "context" "encoding/json" + "fmt" + "net/http" "net/http/httptest" "os" "os/exec" @@ -11,6 +13,7 @@ import ( "testing" "time" + "corral/internal/assets" "corral/internal/clock" "corral/internal/daemon" "corral/internal/graph" @@ -68,6 +71,7 @@ func TestDaemonEndToEndRealOpenCode(t *testing.T) { Concurrency: 2, Worktrees: wtm, }) d := daemon.New(st, s, nil, proj, "") + t.Cleanup(d.Close) srvHTTP := httptest.NewServer(d.Handler()) t.Cleanup(srvHTTP.Close) api := &api{t: t, cli: srvHTTP.Client(), base: srvHTTP.URL} @@ -168,3 +172,328 @@ func ocNodeE2E(id graph.NodeID, prompt, file, marker string) *graph.Node { Budget: graph.Budget{MaxDuration: 12 * time.Minute}, } } + +// orchGraph builds the small approved graph the orchestrator drives in the +// run-loop E2E: a writing worker, a human gate, and an approval-gated merge. +func orchGraph() *graph.Graph { + return &graph.Graph{Version: 1, Nodes: []*graph.Node{ + ocNodeE2E("w1", "Create a file named alpha.txt containing one line: CORRAL. Do not run any other commands.", "alpha.txt", ""), + {ID: "gate", Type: graph.NodeHuman, Objective: "approve", Priority: graph.PriorityNormal, DependsOn: []graph.NodeID{"w1"}}, + {ID: "m", Type: graph.NodeMerge, Objective: "merge accepted work", Priority: graph.PriorityNormal, + DependsOn: []graph.NodeID{"gate"}, + Verification: &graph.Verification{Kind: "command", Command: []string{"test", "-s", "alpha.txt"}}}, + }} +} + +// orchEnv is the full real-OpenCode stack for the orchestrator run-loop +// test: a git project with the corral plugin + agents installed, an +// embedded opencode serve, the daemon, and its HTTP API. +type orchEnv struct { + api *api + oc *ocx.Client + proj string + srv *spike.Server +} + +func setupOrchestratorEnv(t *testing.T, ctx context.Context) *orchEnv { + t.Helper() + proj, err := os.MkdirTemp("", "corral-orch-e2e-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(proj) }) + for _, args := range [][]string{{"init", "-q", "-b", "main"}, {"commit", "-q", "--allow-empty", "-m", "init"}} { + cmd := exec.Command("git", args...) + cmd.Dir = proj + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git: %v: %s", err, out) + } + } + // Install the corral plugin + agent config so the real OpenCode server + // exposes the corral_* tools and the corral-orchestrator agent. + if err := os.MkdirAll(filepath.Join(proj, ".opencode", "tools"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proj, ".opencode", "tools", "corral.ts"), []byte(assets.CorralPluginTS), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proj, "opencode.json"), []byte(assets.OpenCodeConfigJSON), 0o644); err != nil { + t.Fatal(err) + } + + srv, err := spike.StartServer(ctx, proj, 0, os.Stderr) + if err != nil { + t.Fatal(err) + } + t.Cleanup(srv.Stop) + + st, err := store.Open(filepath.Join(t.TempDir(), "e2e.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + + oc := ocx.New(srv.Base, proj) + drv := ocxadapter.New(oc, ocxadapter.Options{PollInterval: 400 * time.Millisecond}) + t.Cleanup(func() { drv.Close() }) + wtm := worktree.NewManager(proj) + s := sched.New(st, drv, &sched.EngineVerifier{Eng: verify.New(proj)}, clock.Real{}, sched.Options{ + Concurrency: 2, Worktrees: wtm, + }) + d := daemon.New(st, s, nil, proj, "") + srvHTTP := httptest.NewServer(d.Handler()) + t.Cleanup(srvHTTP.Close) + t.Setenv("CORRAL_DAEMON_URL", srvHTTP.URL) + + return &orchEnv{api: &api{t: t, cli: srvHTTP.Client(), base: srvHTTP.URL}, oc: oc, proj: proj, srv: srv} +} + +// orchTools is the tool allowlist for the orchestrator session: only the +// corral_* control tools, so the agent can never wander into edits/bash. +var orchTools = map[string]bool{ + "corral_plan": true, "corral_start": true, "corral_status": true, + "corral_watch": true, "corral_approve": true, "corral_reject": true, + "corral_cancel": true, "corral_retry": true, "corral_steer": true, +} + +func orchestratorPrompt(autoApprove bool) string { + gj, _ := json.Marshal(orchGraph()) + return fmt.Sprintf(`You are driving a corral run as the orchestrator. + +First call corral_start with the EXACT graph JSON below and autoApproveGates set to %t to create the run: + +%s + +Then drive it to completion: +1. Call corral_watch with the runID from corral_start, passing a timeout around 60 and the previous response's "since" value on later calls. +2. Each time corral_watch returns, report the current progress to the user (nodes done or running, and any milestones). +3. When gatesAwaitingApproval is non-empty and autoApproveGates is true, you are pre-authorized: call corral_approve with the runID and each waiting gate nodeID, then keep watching. +4. When gatesAwaitingApproval is non-empty and autoApproveGates is false, you are NOT pre-authorized: never call corral_approve. Report to the user that the gate awaits their approval, and keep calling corral_watch until it resolves, then continue. +5. Keep watching until the response shows done: true, then report the final outcome. + +You are the corral-orchestrator agent: never edit files, never run bash, and use only the corral_* tools.`, autoApprove, gj) +} + +// waitForRun polls the daemon API until at least one run exists and +// returns its id (the orchestrator creates it via corral_start). +func waitForRun(t *testing.T, env *orchEnv, timeout time.Duration) string { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + code, body := env.api.do("operator", http.MethodGet, "/api/runs", nil) + if code == http.StatusOK { + var runs []struct { + ID string `json:"id"` + } + if json.Unmarshal([]byte(body), &runs) == nil && len(runs) > 0 { + return runs[0].ID + } + } + time.Sleep(500 * time.Millisecond) + } + t.Fatal("no run appeared; did the orchestrator call corral_start?") + return "" +} + +// runDetail returns the daemon's run detail as raw JSON. +func (env *orchEnv) runDetail(runID string) (int, string) { + return env.api.do("operator", http.MethodGet, "/api/runs/"+runID, nil) +} + +func (env *orchEnv) nodeState(runID, nodeID string) string { + code, body := env.runDetail(runID) + if code != http.StatusOK { + return "" + } + var r struct { + States map[string]string `json:"states"` + Done bool `json:"done"` + } + if json.Unmarshal([]byte(body), &r) != nil { + return "" + } + return r.States[nodeID] +} + +func (env *orchEnv) waitNodeState(runID, nodeID, want string, timeout time.Duration) { + t := env.api.t + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if env.nodeState(runID, nodeID) == want { + return + } + time.Sleep(200 * time.Millisecond) + } + t.Fatalf("node %s never reached %s", nodeID, want) +} + +func (env *orchEnv) waitRunDone(runID string, timeout time.Duration) { + t := env.api.t + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + code, body := env.runDetail(runID) + if code == http.StatusOK { + var r struct { + Done bool `json:"done"` + } + if json.Unmarshal([]byte(body), &r) == nil && r.Done { + return + } + } + time.Sleep(300 * time.Millisecond) + } + code, body := env.runDetail(runID) + t.Fatalf("run %s never completed (last: %d %s)", runID, code, body) +} + +// sessionTools returns the ordered completed tool calls of a session and +// the assistant's text output, used to prove the orchestrator drove the +// loop (start/watch/approve) the way the policy dictates. +func sessionTools(t *testing.T, oc *ocx.Client, sid string) (tools []string, text string) { + t.Helper() + msgs, err := oc.Messages(context.Background(), sid, 0) + if err != nil { + return nil, "" + } + var out strings.Builder + for _, m := range msgs { + if m.Info.Role != "assistant" { + continue + } + for _, p := range m.Parts { + var part struct { + Type string `json:"type"` + Text string `json:"text"` + Tool string `json:"tool"` + State string `json:"state"` + } + if json.Unmarshal(p, &part) != nil { + continue + } + switch part.Type { + case "text": + out.WriteString(part.Text) + out.WriteString("\n") + case "tool": + if part.State == "completed" { + tools = append(tools, part.Tool) + } + } + } + } + return tools, out.String() +} + +// TestOrchestratorRunLoopRealOpenCode exercises the corral-orchestrator +// agent prompt end to end against a real OpenCode install. The +// orchestrator session starts a run with corral_start and drives the watch +// loop, and the two subtests cover the approval policy: +// - pre-authorized (autoApproveGates=true): the orchestrator approves the +// gate itself and the run completes with no human action; +// - not pre-authorized: the run parks at the gate, the orchestrator +// reports it and never approves, and the run resumes only after an +// external (operator) API approve. +func TestOrchestratorRunLoopRealOpenCode(t *testing.T) { + livetest.SkipIfDisabled(t) + if _, err := exec.LookPath("opencode"); err != nil { + t.Skip("opencode binary not found") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + + for _, tc := range []struct { + name string + autoApprove bool + }{ + {"pre-authorized: run completes without any human", true}, + {"not pre-authorized: parks at the gate, resumes after external approve", false}, + } { + tc := tc + t.Run(tc.name, func(t *testing.T) { + env := setupOrchestratorEnv(t, ctx) + sess, err := env.oc.CreateSession(ctx, "corral/orchestrator") + if err != nil { + t.Fatalf("create orchestrator session: %v", err) + } + if err := env.oc.PromptAsyncAgentWithTools(ctx, sess.ID, orchestratorPrompt(tc.autoApprove), "", "corral-orchestrator", orchTools); err != nil { + t.Fatalf("prompt orchestrator: %v", err) + } + + runID := waitForRun(t, env, 5*time.Minute) + + if tc.autoApprove { + // The orchestrator must approve the gate itself: the run + // reaches the gate and completes with no operator action. + env.waitRunDone(runID, 12*time.Minute) + // Gate auto-approved and the merge folded the file into main. + if got := env.nodeState(runID, "gate"); got != string(graph.StateDone) { + t.Fatalf("gate state = %s, want done (orchestrator should have approved it)", got) + } + } else { + // The run parks at the gate: the orchestrator must NOT + // approve on its own. + env.waitNodeState(runID, "gate", string(graph.StateRunning), 8*time.Minute) + // Give the orchestrator a generous window to (incorrectly) + // approve; a policy violation would resolve the gate here. + time.Sleep(8 * time.Second) + if got := env.nodeState(runID, "gate"); got != string(graph.StateRunning) { + t.Fatalf("gate state = %s while not pre-authorized, want running (orchestrator must not approve)", got) + } + // The human approves externally, as an operator would. + code, body := env.api.do("operator", http.MethodPost, "/api/runs/"+runID+"/approve", map[string]any{"nodeID": "gate"}) + if code != 200 { + t.Fatalf("external approve: %d %s", code, body) + } + env.waitRunDone(runID, 12*time.Minute) + } + + // The merged artifact must be in the main checkout. + data, err := os.ReadFile(filepath.Join(env.proj, "alpha.txt")) + if err != nil || len(strings.TrimSpace(string(data))) == 0 { + t.Fatalf("merged content wrong: %v %q", err, data) + } + + // Wait for the orchestrator session to settle, then inspect its + // tool usage: it must have started the run, watched repeatedly, + // and honored the approval policy. + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + if msgs, err := env.oc.Messages(ctx, sess.ID, 4); err == nil && len(msgs) > 0 { + last := msgs[len(msgs)-1] + if last.Info.Finish != nil || last.Info.Error != nil { + break + } + } + time.Sleep(1 * time.Second) + } + tools, transcript := sessionTools(t, env.oc, sess.ID) + + started, watched := false, 0 + approved := false + for _, name := range tools { + switch name { + case "corral_start": + started = true + case "corral_watch": + watched++ + case "corral_approve": + approved = true + } + } + if !started || watched < 2 { + t.Errorf("orchestrator loop wrong: start=%v watch=%d; transcript:\n%s", started, watched, transcript) + } + if tc.autoApprove && !approved { + t.Errorf("pre-authorized orchestrator never called corral_approve; transcript:\n%s", transcript) + } + if !tc.autoApprove && approved { + t.Errorf("not-pre-authorized orchestrator called corral_approve; transcript:\n%s", transcript) + } + if !tc.autoApprove && !strings.Contains(strings.ToLower(transcript), "approval") && !strings.Contains(strings.ToLower(transcript), "gate") { + t.Errorf("orchestrator did not report the gate to the user; transcript:\n%s", transcript) + } + }) + } +} diff --git a/internal/daemon/events.go b/internal/daemon/events.go new file mode 100644 index 0000000..b58b9ce --- /dev/null +++ b/internal/daemon/events.go @@ -0,0 +1,207 @@ +package daemon + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "corral/internal/store" +) + +// defaultEventHeartbeat is the idle keep-alive interval for the SSE stream. +const defaultEventHeartbeat = 15 * time.Second + +// SetEventHeartbeat overrides the SSE heartbeat interval (default 15s). +func (d *Daemon) SetEventHeartbeat(interval time.Duration) { + if interval <= 0 { + interval = defaultEventHeartbeat + } + d.eventHeartbeatMu.Lock() + d.eventHeartbeat = interval + d.eventHeartbeatMu.Unlock() +} + +func (d *Daemon) eventHeartbeatInterval() time.Duration { + d.eventHeartbeatMu.RLock() + defer d.eventHeartbeatMu.RUnlock() + return d.eventHeartbeat +} + +// handleEvents streams the run's event log as server-sent events. The +// ?after= query cursor selects only events with seq greater than the +// cursor; persisted events are replayed first and live transitions (and +// heartbeats) follow on the broker. A terminal run event ends the stream after +// delivery; reconnecting with its sequence cursor is safe. +func (d *Daemon) handleEvents(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + runID := r.PathValue("id") + after, err := parseEventCursor(r.URL.Query().Get("after")) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + run, err := d.st.Run(ctx, runID) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + fl, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", http.StatusInternalServerError) + return + } + + // Subscribe before reading the durable snapshot. An event committed during + // the read queues a notification, and every notification is reconciled from + // the store by sequence, so overlap is deduplicated and gaps are impossible. + ch, unsubscribe := d.broker.Subscribe(runID) + defer unsubscribe() + replay, err := d.st.EventsAfter(ctx, runID, after) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + frames, err := encodeEventFrames(replay, after) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.Header().Set("X-Accel-Buffering", "no") + w.WriteHeader(http.StatusOK) + + last := after + terminal := run.Status == "completed" || run.Status == "canceled" + for _, frame := range frames { + if _, err := w.Write(frame.data); err != nil { + return + } + fl.Flush() + last = frame.event.Seq + terminal = terminal || terminalRunEvent(frame.event) + } + if len(frames) == 0 { + fl.Flush() + } + if terminal { + return + } + + hb := time.NewTicker(d.eventHeartbeatInterval()) + defer hb.Stop() + flushDurable := func() (bool, error) { + events, err := d.st.EventsAfter(ctx, runID, last) + if err != nil { + return false, err + } + for _, ev := range events { + if ev.Seq <= last { + continue + } + if err := writeEventSSE(w, fl, ev); err != nil { + return false, err + } + last = ev.Seq + if terminalRunEvent(ev) { + return true, nil + } + } + return false, nil + } + + for { + select { + case _, ok := <-ch: + if !ok { + return + } + // The broker is a wake-up path, not the source of truth. Reading the + // durable log here preserves order even when concurrent commits notify + // out of order. + terminal, err := flushDurable() + if err != nil || terminal { + return // headers are committed; reconnect replays from last id + } + case <-hb.C: + // A best-effort store or broker wakeup may be dropped when buffers + // overflow. Heartbeats also reconcile the durable cursor so a quiet + // stream never waits forever for another event. + terminal, err := flushDurable() + if err != nil || terminal { + return + } + if _, err := io.WriteString(w, ": ping\n\n"); err != nil { + return + } + fl.Flush() + case <-ctx.Done(): + return + } + } +} + +func parseEventCursor(raw string) (int64, error) { + if raw == "" { + return 0, nil + } + after, err := strconv.ParseInt(raw, 10, 64) + if err != nil || after < 0 { + return 0, fmt.Errorf("after must be a non-negative integer") + } + return after, nil +} + +type eventFrame struct { + event store.Event + data []byte +} + +func encodeEventFrames(events []store.Event, after int64) ([]eventFrame, error) { + frames := make([]eventFrame, 0, len(events)) + for _, ev := range events { + if ev.Seq <= after { + continue + } + var buf bytes.Buffer + if err := writeEventSSE(&buf, nil, ev); err != nil { + return nil, err + } + frames = append(frames, eventFrame{event: ev, data: buf.Bytes()}) + } + return frames, nil +} + +func terminalRunEvent(ev store.Event) bool { + if ev.Type != store.EventRun { + return false + } + var payload struct { + Status string `json:"status"` + } + if json.Unmarshal(ev.Payload, &payload) != nil { + return false + } + return payload.Status == "completed" || payload.Status == "canceled" +} + +// writeEventSSE frames one event as an SSE record with a resume id. +func writeEventSSE(w io.Writer, fl http.Flusher, ev store.Event) error { + b, err := json.Marshal(ev) + if err != nil { + return err + } + if _, err := fmt.Fprintf(w, "id: %d\ndata: %s\n\n", ev.Seq, b); err != nil { + return err + } + if fl != nil { + fl.Flush() + } + return nil +} diff --git a/internal/daemon/events_reconcile_test.go b/internal/daemon/events_reconcile_test.go new file mode 100644 index 0000000..a1a4528 --- /dev/null +++ b/internal/daemon/events_reconcile_test.go @@ -0,0 +1,65 @@ +package daemon + +import ( + "bufio" + "context" + "encoding/json" + "net/http/httptest" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "corral/internal/graph" + "corral/internal/store" +) + +func TestEventHeartbeatReconcilesDroppedWakeup(t *testing.T) { + st, err := store.Open(filepath.Join(t.TempDir(), "events.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + ctx := context.Background() + g := &graph.Graph{Nodes: []*graph.Node{{ + ID: "w", Type: graph.NodeAgent, Objective: "work", AcceptanceCriteria: []string{"done"}, + }}} + if err := st.CreateRun(ctx, "r", g, false, time.Now()); err != nil { + t.Fatal(err) + } + + // Deliberately omit the store-to-broker pump. The durable event below has + // no live wakeup, so only heartbeat cursor reconciliation can deliver it. + d := &Daemon{st: st, broker: newBroker(), eventHeartbeat: 20 * time.Millisecond} + srv := httptest.NewServer(d.Handler()) + defer srv.Close() + resp, err := srv.Client().Get(srv.URL + "/api/runs/r/events?after=1") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if _, err := st.AppendEvent(ctx, "r", "w", store.EventGraph, "", "", "", `{"version":2}`, time.Now()); err != nil { + t.Fatal(err) + } + + reader := bufio.NewReader(resp.Body) + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + line, err := reader.ReadString('\n') + if err != nil { + t.Fatal(err) + } + if strings.HasPrefix(line, "data: ") { + var event store.Event + if err := json.Unmarshal([]byte(strings.TrimSpace(strings.TrimPrefix(line, "data: "))), &event); err != nil { + t.Fatal(err) + } + if event.Seq != 2 { + t.Fatalf("event seq = %s, want 2", strconv.FormatInt(event.Seq, 10)) + } + return + } + } + t.Fatal("durable event was not reconciled on heartbeat") +} diff --git a/internal/daemon/events_test.go b/internal/daemon/events_test.go new file mode 100644 index 0000000..f4db31c --- /dev/null +++ b/internal/daemon/events_test.go @@ -0,0 +1,325 @@ +package daemon_test + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "corral/internal/graph" + "corral/internal/sched" + "corral/internal/store" +) + +// readSSE reads one server-sent event record from r, returning its id and +// data, or the heartbeat flag when the line is a comment. +func readSSE(t *testing.T, r *bufio.Reader) (id int64, data string, ping bool) { + t.Helper() + var recID string + var dataLines []string + for { + line, err := r.ReadString('\n') + if err != nil { + t.Fatalf("read sse: %v", err) + } + line = strings.TrimRight(line, "\r\n") + if line == "" { + if recID != "" || len(dataLines) > 0 { + n, _ := strconv.ParseInt(recID, 10, 64) + return n, strings.Join(dataLines, "\n"), false + } + continue + } + switch { + case strings.HasPrefix(line, ": "): + return 0, "", true + case strings.HasPrefix(line, "id: "): + recID = strings.TrimPrefix(line, "id: ") + case strings.HasPrefix(line, "data: "): + dataLines = append(dataLines, strings.TrimPrefix(line, "data: ")) + } + } +} + +func TestEventsSSEStreamsFullRun(t *testing.T) { + a, _, _, drv := setupDaemon(t, "") + drv.SetScript("w1", sched.Script{Delay: 200 * time.Millisecond, Write: map[string]string{"a.txt": "A1"}}) + g := &graph.Graph{Nodes: []*graph.Node{workerNode("w1", "a.txt", "A1")}} + code, body := a.do("operator", http.MethodPost, "/api/runs", map[string]any{"graph": g}) + if code != http.StatusCreated { + t.Fatalf("create: %d %s", code, body) + } + var created struct{ RunID string } + json.Unmarshal([]byte(body), &created) + + resp, err := http.Get(a.base + "/api/runs/" + created.RunID + "/events") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if ct := resp.Header.Get("Content-Type"); ct != "text/event-stream" { + t.Fatalf("content-type = %q, want text/event-stream", ct) + } + + r := bufio.NewReader(resp.Body) + var seqs []int64 + var types []string + toStates := map[string]bool{} + deadline := time.Now().Add(20 * time.Second) + completed := false + for time.Now().Before(deadline) { + id, data, ping := readSSE(t, r) + if ping { + continue + } + var ev store.Event + if err := json.Unmarshal([]byte(data), &ev); err != nil { + t.Fatalf("bad event json: %v: %s", err, data) + } + if ev.Seq != id { + t.Fatalf("id %d does not match event seq %d", id, ev.Seq) + } + seqs = append(seqs, ev.Seq) + types = append(types, string(ev.Type)) + if ev.To != "" { + toStates[string(ev.To)] = true + } + if ev.Type == store.EventRun && strings.Contains(data, "completed") { + completed = true + break + } + } + if !completed { + t.Fatalf("run completion never streamed; types = %v", types) + } + for i, s := range seqs { + if s != int64(i+1) { + t.Fatalf("seq[%d] = %d, want %d (contiguous from 1)", i, s, i+1) + } + } + for _, want := range []string{"ready", "leased", "running", "verifying", "done"} { + if !toStates[want] { + t.Fatalf("missing %s transition in stream (got %v)", want, toStates) + } + } + if types[0] != "run" { + t.Fatalf("first event type = %q, want run", types[0]) + } +} + +func TestEventsSSEAfterCursor(t *testing.T) { + a, _, _, drv := setupDaemon(t, "") + drv.SetScript("w1", sched.Script{Delay: 150 * time.Millisecond, Write: map[string]string{"a.txt": "A1"}}) + g := &graph.Graph{Nodes: []*graph.Node{workerNode("w1", "a.txt", "A1")}} + code, body := a.do("operator", http.MethodPost, "/api/runs", map[string]any{"graph": g}) + if code != http.StatusCreated { + t.Fatalf("create: %d %s", code, body) + } + var created struct{ RunID string } + json.Unmarshal([]byte(body), &created) + + // Replay-only: connect with a cursor and only the events past it must + // be emitted. + resp, err := http.Get(fmt.Sprintf("%s/api/runs/%s/events?after=5", a.base, created.RunID)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + r := bufio.NewReader(resp.Body) + first := true + var minSeq int64 + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + _, data, ping := readSSE(t, r) + if ping { + continue + } + var ev store.Event + if err := json.Unmarshal([]byte(data), &ev); err != nil { + t.Fatalf("bad event: %v", err) + } + if first { + minSeq = ev.Seq + first = false + } + if ev.Seq <= 5 { + t.Fatalf("event seq %d emitted despite after=5", ev.Seq) + } + if ev.Type == store.EventRun && strings.Contains(data, "completed") { + break + } + } + if first { + t.Fatal("no events received") + } + if minSeq != 6 { + t.Fatalf("first event seq = %d, want 6", minSeq) + } +} + +func TestEventsSSEHeartbeats(t *testing.T) { + a, d, st, _ := setupDaemon(t, "") + d.SetEventHeartbeat(100 * time.Millisecond) + const runID = "run_heartbeat" + if err := st.CreateRun(context.Background(), runID, &graph.Graph{Nodes: []*graph.Node{workerNode("w1", "a.txt", "A1")}}, false, time.Now()); err != nil { + t.Fatal(err) + } + + resp, err := http.Get(a.base + "/api/runs/" + runID + "/events?after=1") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + r := bufio.NewReader(resp.Body) + pings := 0 + deadline := time.Now().Add(800 * time.Millisecond) + for time.Now().Before(deadline) { + _, _, ping := readSSE(t, r) + if ping { + pings++ + } + } + if pings == 0 { + t.Fatal("no heartbeats received") + } +} + +func TestEventsSSEReconnectReplaysMissedEvents(t *testing.T) { + a, _, st, _ := setupDaemon(t, "") + ctx := context.Background() + const runID = "run_reconnect" + if err := st.CreateRun(ctx, runID, &graph.Graph{Nodes: []*graph.Node{workerNode("w1", "a.txt", "A1")}}, false, time.Now()); err != nil { + t.Fatal(err) + } + + firstCtx, cancel := context.WithCancel(ctx) + request, _ := http.NewRequestWithContext(firstCtx, http.MethodGet, a.base+"/api/runs/"+runID+"/events", nil) + response, err := a.cli.Do(request) + if err != nil { + t.Fatal(err) + } + id, _, ping := readSSE(t, bufio.NewReader(response.Body)) + if ping || id != 1 { + t.Fatalf("first stream record: id=%d ping=%v, want event 1", id, ping) + } + cancel() + response.Body.Close() + + for i := 2; i <= 3; i++ { + if _, err := st.AppendEvent(ctx, runID, "w1", store.EventGraph, "", "", "", fmt.Sprintf(`{"version":%d}`, i), time.Now()); err != nil { + t.Fatal(err) + } + } + + response, err = http.Get(a.base + "/api/runs/" + runID + "/events?after=1") + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + reader := bufio.NewReader(response.Body) + for want := int64(2); want <= 3; want++ { + id, data, ping := readSSE(t, reader) + if ping || id != want { + t.Fatalf("replayed record: id=%d ping=%v, want event %d", id, ping, want) + } + var event store.Event + if err := json.Unmarshal([]byte(data), &event); err != nil { + t.Fatal(err) + } + if event.Seq != want || event.RunID != runID { + t.Fatalf("replayed event = %+v", event) + } + } +} + +func TestEventsSSEUsesBearerAuthLikeOtherReadEndpoints(t *testing.T) { + a, _, st, _ := setupDaemon(t, "sekret") + const runID = "run_auth" + if err := st.CreateRun(context.Background(), runID, &graph.Graph{Nodes: []*graph.Node{workerNode("w1", "a.txt", "A1")}}, false, time.Now()); err != nil { + t.Fatal(err) + } + + request, _ := http.NewRequest(http.MethodGet, a.base+"/api/runs/"+runID+"/events", nil) + response, err := a.cli.Do(request) + if err != nil { + t.Fatal(err) + } + response.Body.Close() + if response.StatusCode != http.StatusUnauthorized { + t.Fatalf("without bearer: %d, want 401", response.StatusCode) + } + + request, _ = http.NewRequest(http.MethodGet, a.base+"/api/runs/"+runID+"/events", nil) + request.Header.Set("Authorization", "Bearer sekret") + response, err = a.cli.Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("with bearer: %d, want 200", response.StatusCode) + } + id, _, ping := readSSE(t, bufio.NewReader(response.Body)) + if ping || id != 1 { + t.Fatalf("authorized record: id=%d ping=%v, want event 1", id, ping) + } +} + +func TestEventsSSERejectsInvalidCursor(t *testing.T) { + a, _, st, _ := setupDaemon(t, "") + const runID = "run_cursor" + if err := st.CreateRun(context.Background(), runID, &graph.Graph{Nodes: []*graph.Node{workerNode("w1", "a.txt", "A1")}}, false, time.Now()); err != nil { + t.Fatal(err) + } + for _, after := range []string{"-1", "abc"} { + code, body := a.do("operator", http.MethodGet, "/api/runs/"+runID+"/events?after="+after, nil) + if code != http.StatusBadRequest { + t.Fatalf("after=%q: %d %s, want 400", after, code, body) + } + } +} + +func TestEventsSSEUnknownRun(t *testing.T) { + a, _, _, _ := setupDaemon(t, "") + code, body := a.do("operator", http.MethodGet, "/api/runs/nope/events", nil) + if code != http.StatusNotFound { + t.Fatalf("unknown run: %d %s, want 404", code, body) + } +} + +func TestEventsSSEDoesNotAffectJSONEndpoints(t *testing.T) { + a, _, st, drv := setupDaemon(t, "") + drv.SetScript("w1", sched.Script{Delay: 150 * time.Millisecond, Write: map[string]string{"a.txt": "A1"}}) + g := &graph.Graph{Nodes: []*graph.Node{workerNode("w1", "a.txt", "A1")}} + code, body := a.do("operator", http.MethodPost, "/api/runs", map[string]any{"graph": g}) + if code != http.StatusCreated { + t.Fatalf("create: %d %s", code, body) + } + var created struct{ RunID string } + json.Unmarshal([]byte(body), &created) + + // Open (and hold) an SSE connection while the run executes; the JSON + // endpoints must keep serving shape-stable responses regardless. + resp, err := http.Get(a.base + "/api/runs/" + created.RunID + "/events") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + // The run detail must still be served and shape-stable while the + // stream is open. + a.waitState(t, "", created.RunID, "w1", graph.StateDone, 20*time.Second) + code, body = a.do("operator", http.MethodGet, "/api/runs/"+created.RunID, nil) + if code != http.StatusOK || !strings.Contains(body, `"done":true`) { + t.Fatalf("run detail broken: %d %s", code, body) + } + if _, err := st.Run(context.Background(), created.RunID); err != nil { + t.Fatalf("store run read failed: %v", err) + } +} diff --git a/internal/daemon/hardening_test.go b/internal/daemon/hardening_test.go index 437b884..4c0d88a 100644 --- a/internal/daemon/hardening_test.go +++ b/internal/daemon/hardening_test.go @@ -71,6 +71,7 @@ func TestOpenAPIContract(t *testing.T) { t.Cleanup(func() { st.Close() }) s := sched.New(st, sched.NewFakeDriver(clock.Real{}, nil), &sched.EngineVerifier{Eng: verify.New(t.TempDir())}, clock.Real{}, sched.Options{}) d := daemon.New(st, s, nil, t.TempDir(), "") + t.Cleanup(d.Close) srv := httptest.NewServer(d.Handler()) t.Cleanup(srv.Close) @@ -82,6 +83,7 @@ func TestOpenAPIContract(t *testing.T) { paths, _ := doc["paths"].(map[string]any) registered := []string{ "/api/health", "/api/plan", "/api/runs", "/api/runs/{id}", + "/api/runs/{id}/watch", "/api/runs/{id}/events", "/api/runs/{id}/tail", "/api/runs/{id}/approve", "/api/runs/{id}/reject", "/api/runs/{id}/cancel", "/api/runs/{id}/retry", "/api/runs/{id}/steer", "/api/runs/{id}/permission", "/api/runs/{id}/export", "/doc", @@ -151,6 +153,7 @@ func TestAuditExport(t *testing.T) { eng := verify.New(workdir) s := sched.New(st, drv, &sched.EngineVerifier{Eng: eng}, clock.Real{}, sched.Options{Concurrency: 2}) d := daemon.New(st, s, nil, workdir, "") + t.Cleanup(d.Close) ctx := context.Background() d.SetContext(ctx) srv := httptest.NewServer(d.Handler()) diff --git a/internal/daemon/openapi.go b/internal/daemon/openapi.go index 56584f0..ff44d39 100644 --- a/internal/daemon/openapi.go +++ b/internal/daemon/openapi.go @@ -10,7 +10,7 @@ import ( // document and live responses against its schemas. const OpenAPI = `{ "openapi": "3.0.3", - "info": {"title": "corral daemon API", "version": "0.8.0"}, + "info": {"title": "corral daemon API", "version": "0.9.0"}, "paths": { "/api/health": { "get": {"responses": {"200": {"description": "ok", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Health"}}}}}} @@ -25,6 +25,12 @@ const OpenAPI = `{ "/api/runs/{id}": { "get": {"responses": {"200": {"description": "run detail", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/RunDetail"}}}}}} }, + "/api/runs/{id}/watch": { + "get": {"parameters": [{"name": "since", "in": "query", "schema": {"type": "integer"}}, {"name": "timeout", "in": "query", "schema": {"type": "integer"}}], "responses": {"200": {"description": "run snapshot", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/WatchResponse"}}}}}} + }, + "/api/runs/{id}/tail": { + "get": {"parameters": [{"name": "node", "in": "query", "required": true, "schema": {"type": "string", "maxLength": 256}}, {"name": "lines", "in": "query", "schema": {"type": "integer", "minimum": 1, "maximum": 500, "default": 40}}], "responses": {"200": {"description": "live attempt tail", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Tail"}}}}, "400": {"description": "invalid node or line count"}, "404": {"description": "run not found"}, "409": {"description": "node has no live attempt"}}} + }, "/api/runs/{id}/approve": {"post": {"responses": {"200": {"description": "ok", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Ok"}}}}}}}, "/api/runs/{id}/reject": {"post": {"responses": {"200": {"description": "ok", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Ok"}}}}}}}, "/api/runs/{id}/cancel": {"post": {"responses": {"200": {"description": "ok", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Ok"}}}}}}}, @@ -32,6 +38,17 @@ const OpenAPI = `{ "/api/runs/{id}/steer": {"post": {"responses": {"200": {"description": "ok", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Ok"}}}}}}}, "/api/runs/{id}/permission": {"post": {"responses": {"200": {"description": "ok", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Ok"}}}}}}}, "/api/runs/{id}/export": {"get": {"responses": {"200": {"description": "audit export", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/Export"}}}}}}}, + "/api/runs/{id}/events": { + "get": { + "parameters": [{"name": "after", "in": "query", "required": false, "schema": {"type": "integer", "minimum": 0}, "description": "only emit durable events with seq greater than this cursor"}], + "responses": { + "200": {"description": "server-sent stream whose data field is a durable Event and whose id is Event.seq", "content": {"text/event-stream": {"schema": {"type": "string"}}}}, + "400": {"description": "invalid cursor"}, + "404": {"description": "run not found"}, + "500": {"description": "event replay unavailable"} + } + } + }, "/doc": {"get": {"responses": {"200": {"description": "openapi document"}}}} }, "components": { @@ -93,11 +110,18 @@ const OpenAPI = `{ } }, "Event": { - "type": "object", "required": ["seq", "type"], + "type": "object", "required": ["seq", "runID", "type", "createdAt"], "properties": { - "seq": {"type": "integer"}, "nodeID": {"type": "string"}, + "seq": {"type": "integer"}, "runID": {"type": "string"}, "nodeID": {"type": "string"}, "type": {"type": "string"}, "from": {"type": "string"}, "to": {"type": "string"}, - "attemptID": {"type": "string"}, "createdAt": {"type": "integer"} + "attemptID": {"type": "string"}, "payload": {}, "createdAt": {"type": "integer"} + } + }, + "Tail": { + "type": "object", "required": ["node", "lines"], + "properties": { + "node": {"type": "string"}, + "lines": {"type": "array", "items": {"type": "string"}} } }, "Artifact": { @@ -113,11 +137,23 @@ const OpenAPI = `{ "properties": { "runID": {"type": "string"}, "status": {"type": "string"}, "done": {"type": "boolean"}, "graph": {"$ref": "#/components/schemas/Graph"}, + "autoApproveGates": {"type": "boolean"}, "states": {"type": "object", "additionalProperties": {"type": "string"}}, "attempts": {"type": "object", "additionalProperties": {"type": "array", "items": {"$ref": "#/components/schemas/Attempt"}}}, "events": {"type": "array", "items": {"$ref": "#/components/schemas/Event"}} } }, + "WatchResponse": { + "type": "object", "required": ["runID"], + "properties": { + "runID": {"type": "string"}, "status": {"type": "string"}, "done": {"type": "boolean"}, + "autoApproveGates": {"type": "boolean"}, + "states": {"type": "object", "additionalProperties": {"type": "string"}}, + "gatesAwaitingApproval": {"type": "array", "items": {"type": "string"}}, + "since": {"type": "integer"}, + "events": {"type": "array", "items": {"$ref": "#/components/schemas/Event"}} + } + }, "Export": { "type": "object", "required": ["runID", "graph", "events"], "properties": { diff --git a/internal/daemon/planner.go b/internal/daemon/planner.go index f306d51..8499ac6 100644 --- a/internal/daemon/planner.go +++ b/internal/daemon/planner.go @@ -32,17 +32,9 @@ func NewOpenCodePlanner(oc *ocx.Client, model string, timeout time.Duration) *Op // anything that executes or writes is disabled. This prevents tool loops // that stall planning. var planTools = map[string]bool{ - "bash": false, - "edit": false, - "write": false, - "apply_patch": false, - "webfetch": false, - "websearch": false, - "task": false, - "todowrite": false, - "question": false, - "skill": false, - "lsp": false, + "*": false, + "read": true, + "glob": true, } var errNoGraph = errors.New("planner produced no valid graph") diff --git a/internal/daemon/planner_test.go b/internal/daemon/planner_test.go index 9a81bbf..aa60b01 100644 --- a/internal/daemon/planner_test.go +++ b/internal/daemon/planner_test.go @@ -70,6 +70,12 @@ func TestParseGraphFromResponse(t *testing.T) { } } +func TestPlannerToolsFailClosed(t *testing.T) { + if planTools["*"] || !planTools["read"] || !planTools["glob"] || len(planTools) != 3 { + t.Fatalf("planner tools = %#v, want wildcard deny plus read/glob", planTools) + } +} + func TestFindJSONEnd(t *testing.T) { s := `{"a": {"b": [1, {"c": "} { "}]}} tail` if end := findJSONEnd(s, 0); end != len(s)-5 { diff --git a/internal/graph/graph_test.go b/internal/graph/graph_test.go index 599198c..5145419 100644 --- a/internal/graph/graph_test.go +++ b/internal/graph/graph_test.go @@ -75,6 +75,28 @@ func TestValidateRejectsDuplicateAndEmptyIDs(t *testing.T) { } } +func TestValidateRejectsUnknownAgentRoleAndUnsafeScope(t *testing.T) { + for _, tc := range []struct { + name string + role string + scope []string + }{ + {"unknown role", "admin", []string{"a.txt"}}, + {"absolute scope", "worker", []string{"/tmp/a.txt"}}, + {"parent scope", "worker", []string{"../a.txt"}}, + {"wildcard path", "worker", []string{"src/*.go"}}, + {"reviewer scope", "reviewer", []string{"a.txt"}}, + } { + t.Run(tc.name, func(t *testing.T) { + n := agent("a") + n.Role, n.WriteScope = tc.role, tc.scope + if err := Validate(&Graph{Nodes: []*Node{n}}); err == nil { + t.Fatal("unsafe agent accepted") + } + }) + } +} + func TestValidateRejectsMissingAcceptanceCriteria(t *testing.T) { n := agent("a") n.AcceptanceCriteria = nil diff --git a/internal/graph/validate.go b/internal/graph/validate.go index e914253..04425f3 100644 --- a/internal/graph/validate.go +++ b/internal/graph/validate.go @@ -2,6 +2,7 @@ package graph import ( "fmt" + "path/filepath" "sort" "strings" ) @@ -69,6 +70,19 @@ func validateNode(n *Node) error { return fmt.Errorf("objective exceeds %d chars", maxObjectiveLen) } if n.Type == NodeAgent { + switch n.Role { + case "", "worker", "reviewer": + default: + return fmt.Errorf("invalid agent role %q", n.Role) + } + if n.Role == "reviewer" && len(n.WriteScope) > 0 { + return fmt.Errorf("reviewer agent cannot declare write scope") + } + for _, scope := range n.WriteScope { + if err := validateWriteScope(scope); err != nil { + return err + } + } if len(n.AcceptanceCriteria) == 0 { return fmt.Errorf("agent node missing acceptance criteria") } @@ -122,6 +136,27 @@ func validateNode(n *Node) error { return nil } +func validateWriteScope(scope string) error { + scope = strings.TrimSpace(strings.ReplaceAll(scope, `\`, "/")) + if scope == "" { + return fmt.Errorf("empty write scope") + } + if scope == "*" { + return nil + } + if filepath.IsAbs(filepath.FromSlash(scope)) { + return fmt.Errorf("write scope %q must be relative", scope) + } + clean := filepath.ToSlash(filepath.Clean(filepath.FromSlash(scope))) + if clean == "." || clean == ".." || strings.HasPrefix(clean, "../") { + return fmt.Errorf("write scope %q escapes worktree", scope) + } + if strings.ContainsAny(clean, "*?[") { + return fmt.Errorf("write scope %q contains unsupported wildcard", scope) + } + return nil +} + func nodeLabel(n *Node, i int) string { if n == nil { return fmt.Sprintf("index %d", i) diff --git a/internal/ocx/client.go b/internal/ocx/client.go index c620af6..3c3ed92 100644 --- a/internal/ocx/client.go +++ b/internal/ocx/client.go @@ -73,7 +73,7 @@ func (c *Client) CreateSession(ctx context.Context, title string) (Session, erro } func (c *Client) PromptAsync(ctx context.Context, sid, text, model string) error { - return c.PromptAsyncWithTools(ctx, sid, text, model, nil) + return c.promptAsync(ctx, sid, text, model, "", nil) } // PromptAsyncWithTools sends an async prompt with an optional tool @@ -81,11 +81,39 @@ func (c *Client) PromptAsync(ctx context.Context, sid, text, model string) error // session so the agent cannot call them (e.g. planner sessions get no // bash/edit). func (c *Client) PromptAsyncWithTools(ctx context.Context, sid, text, model string, tools map[string]bool) error { + return c.promptAsync(ctx, sid, text, model, "", tools) +} + +// PromptAsyncAgent sends an async prompt running under a named agent +// (e.g. "corral-orchestrator"), so the session inherits that agent's +// system prompt and tool permissions. +func (c *Client) PromptAsyncAgent(ctx context.Context, sid, text, model, agent string) error { + return c.promptAsync(ctx, sid, text, model, agent, nil) +} + +// PromptAsyncAgentWithTools is PromptAsyncAgent plus a tool allowlist. +func (c *Client) PromptAsyncAgentWithTools(ctx context.Context, sid, text, model, agent string, tools map[string]bool) error { + return c.promptAsync(ctx, sid, text, model, agent, tools) +} + +// promptAsync is the shared prompt_async caller; agent and tools are +// optional session overrides. +func (c *Client) promptAsync(ctx context.Context, sid, text, model, agent string, tools map[string]bool) error { body := map[string]any{ "parts": []map[string]string{{"type": "text", "text": text}}, } if model != "" { - body["model"] = model + providerID, modelID, ok := strings.Cut(model, "/") + if !ok || strings.TrimSpace(providerID) == "" || strings.TrimSpace(modelID) == "" { + return fmt.Errorf("model %q must use provider/model format", model) + } + body["model"] = map[string]string{ + "providerID": providerID, + "modelID": modelID, + } + } + if agent != "" { + body["agent"] = agent } if tools != nil { body["tools"] = tools @@ -121,10 +149,21 @@ func (c *Client) Abort(ctx context.Context, sid string) error { return err } -// RespondPermission answers a pending permission request. -func (c *Client) RespondPermission(ctx context.Context, sid, permissionID, response string) error { - _, err := c.do(ctx, http.MethodPost, "/session/"+sid+"/permissions/"+permissionID, - url.Values{"directory": {c.dir}}, map[string]any{"response": response}, nil) +// PendingPermissions returns every unresolved permission request visible in +// this client's directory. Unlike the event stream, this endpoint can +// reconcile requests created during a disconnect. +func (c *Client) PendingPermissions(ctx context.Context) ([]PermissionRequest, error) { + var requests []PermissionRequest + _, err := c.do(ctx, http.MethodGet, "/permission", + url.Values{"directory": {c.dir}}, nil, &requests) + return requests, err +} + +// RespondPermission answers a pending permission request using OpenCode's +// current permission API. reply is one of "once", "always", or "reject". +func (c *Client) RespondPermission(ctx context.Context, permissionID, reply string) error { + _, err := c.do(ctx, http.MethodPost, "/permission/"+permissionID+"/reply", + url.Values{"directory": {c.dir}}, map[string]any{"reply": reply}, nil) return err } diff --git a/internal/ocx/client_test.go b/internal/ocx/client_test.go new file mode 100644 index 0000000..690e445 --- /dev/null +++ b/internal/ocx/client_test.go @@ -0,0 +1,72 @@ +package ocx + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func TestPromptAsyncEncodesModelReference(t *testing.T) { + var body map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + client := New(srv.URL, t.TempDir()) + if err := client.PromptAsync(context.Background(), "ses_1", "work", "openrouter/anthropic/claude-sonnet-4"); err != nil { + t.Fatal(err) + } + model, ok := body["model"].(map[string]any) + if !ok { + t.Fatalf("model = %#v, want object", body["model"]) + } + if model["providerID"] != "openrouter" || model["modelID"] != "anthropic/claude-sonnet-4" { + t.Fatalf("model = %#v", model) + } +} + +func TestPromptAsyncRejectsAmbiguousModel(t *testing.T) { + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + requests.Add(1) + })) + defer srv.Close() + + client := New(srv.URL, t.TempDir()) + err := client.PromptAsync(context.Background(), "ses_1", "work", "claude-sonnet-4") + if err == nil || !strings.Contains(err.Error(), "provider/model") { + t.Fatalf("error = %v", err) + } + if got := requests.Load(); got != 0 { + t.Fatalf("HTTP requests = %d, want 0", got) + } +} + +func TestMessagesDecodeRoleDependentSummary(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`[ + {"info":{"role":"user","summary":{"title":"change","body":"body","diffs":[{"file":"a.txt","patch":"+a","additions":1,"deletions":0,"status":"added"}]}},"parts":[]}, + {"info":{"role":"assistant","summary":true,"finish":"stop"},"parts":[]} + ]`)) + })) + defer srv.Close() + + messages, err := New(srv.URL, t.TempDir()).Messages(context.Background(), "ses_1", 0) + if err != nil { + t.Fatal(err) + } + if len(messages) != 2 || messages[0].Info.Summary == nil || len(messages[0].Info.Summary.Diffs) != 1 { + t.Fatalf("user summary = %#v", messages) + } + if messages[1].Info.Summary == nil || !messages[1].Info.Summary.Compacted { + t.Fatalf("assistant summary = %#v", messages[1].Info.Summary) + } +} diff --git a/internal/ocx/events.go b/internal/ocx/events.go index 708df4c..9e8e77d 100644 --- a/internal/ocx/events.go +++ b/internal/ocx/events.go @@ -18,8 +18,14 @@ type EventHandler func(Event) var streamClient = &http.Client{} func (c *Client) StreamEvents(ctx context.Context, handler EventHandler) error { + return c.StreamEventsReady(ctx, nil, handler) +} + +// StreamEventsReady is StreamEvents with a callback invoked after each +// successful HTTP subscription, before any event records are read. +func (c *Client) StreamEventsReady(ctx context.Context, ready func(), handler EventHandler) error { for { - err := c.streamOnce(ctx, handler) + err := c.streamOnce(ctx, ready, handler) if ctx.Err() != nil { return ctx.Err() } @@ -34,7 +40,7 @@ func (c *Client) StreamEvents(ctx context.Context, handler EventHandler) error { } } -func (c *Client) streamOnce(ctx context.Context, handler EventHandler) error { +func (c *Client) streamOnce(ctx context.Context, ready func(), handler EventHandler) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.base+"/global/event", nil) if err != nil { return err @@ -49,6 +55,9 @@ func (c *Client) streamOnce(ctx context.Context, handler EventHandler) error { if resp.StatusCode != http.StatusOK { return fmt.Errorf("event stream: %s", resp.Status) } + if ready != nil { + ready() + } br := bufio.NewReader(resp.Body) for { line, err := br.ReadString('\n') diff --git a/internal/ocx/types.go b/internal/ocx/types.go index a121084..9c6efa1 100644 --- a/internal/ocx/types.go +++ b/internal/ocx/types.go @@ -21,6 +21,14 @@ type SessionStatus struct { Next int `json:"next"` } +// PermissionRequest is an unresolved OpenCode permission request. The +// global permission endpoint is durable, so adapters can reconcile requests +// that were emitted while the SSE stream was disconnected. +type PermissionRequest struct { + ID string `json:"id"` + SessionID string `json:"sessionID"` +} + type FileDiff struct { File string `json:"file"` Patch string `json:"patch"` @@ -40,6 +48,35 @@ type TokenCount struct { } `json:"cache"` } +// MessageSummary is role-dependent on the OpenCode wire: user messages use +// an object containing diffs, while compacted assistant messages use a bool. +// Keeping both shapes decodable prevents one compacted assistant message from +// making the entire transcript unreadable. +type MessageSummary struct { + Title string `json:"title"` + Body string `json:"body"` + Diffs []FileDiff `json:"diffs"` + Compacted bool `json:"-"` +} + +func (s *MessageSummary) UnmarshalJSON(data []byte) error { + var compacted bool + if err := json.Unmarshal(data, &compacted); err == nil { + s.Compacted = compacted + s.Title = "" + s.Body = "" + s.Diffs = nil + return nil + } + type summary MessageSummary + var decoded summary + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *s = MessageSummary(decoded) + return nil +} + type MessageInfo struct { ID string `json:"id"` Role string `json:"role"` @@ -52,12 +89,8 @@ type MessageInfo struct { Cost float64 `json:"cost"` Tokens TokenCount Error *json.RawMessage `json:"error"` - Summary *struct { - Title string `json:"title"` - Body string `json:"body"` - Diffs []FileDiff `json:"diffs"` - } `json:"summary"` - Time struct { + Summary *MessageSummary `json:"summary"` + Time struct { Created int64 `json:"created"` Completed *int64 `json:"completed"` } `json:"time"` diff --git a/internal/ocxadapter/adapter.go b/internal/ocxadapter/adapter.go index a82ff55..197e37c 100644 --- a/internal/ocxadapter/adapter.go +++ b/internal/ocxadapter/adapter.go @@ -23,7 +23,11 @@ type Options struct { // PollInterval is the fallback status-poll period when events are // missed or the stream is down. PollInterval time.Duration - // Model overrides the default model for sessions ("" = server default). + // StreamReadyTimeout bounds how long the first Start waits for an SSE + // subscription before proceeding with REST reconciliation (default 1s). + StreamReadyTimeout time.Duration + // Model is the driver default when Attempt.Model is empty + // ("" leaves model selection to the server). Model string } @@ -34,6 +38,26 @@ func (o Options) poll() time.Duration { return o.PollInterval } +func (o Options) streamReadyTimeout() time.Duration { + if o.StreamReadyTimeout <= 0 { + return time.Second + } + return o.StreamReadyTimeout +} + +func (o Options) permissionPoll() time.Duration { + poll := o.poll() + if poll < 100*time.Millisecond { + return 100 * time.Millisecond + } + if poll > time.Second { + return time.Second + } + return poll +} + +const permissionRequestTimeout = 500 * time.Millisecond + // Driver implements adapter.Driver and adapter.Stepper for OpenCode. type Driver struct { oc *ocx.Client @@ -42,13 +66,17 @@ type Driver struct { mu sync.Mutex attempts map[string]*attempt // attemptID -> rec bySession map[string]*attempt // sessionID -> rec + seen map[string]struct{} // all successfully reserved attempt IDs clients map[string]*ocx.Client // cwd -> client (worktrees) - completions chan adapter.Completion + completions []adapter.Completion - streamOnce sync.Once - streamCtx context.Context - streamCancel context.CancelFunc - closed bool + streamOnce sync.Once + streamReady chan struct{} + readyOnce sync.Once + streamCtx context.Context + streamCancel context.CancelFunc + permissionWake chan struct{} + closed bool } type attempt struct { @@ -67,12 +95,14 @@ type attempt struct { func New(oc *ocx.Client, opts Options) *Driver { return &Driver{ - oc: oc, - opts: opts, - attempts: map[string]*attempt{}, - bySession: map[string]*attempt{}, - clients: map[string]*ocx.Client{}, - completions: make(chan adapter.Completion, 64), + oc: oc, + opts: opts, + attempts: map[string]*attempt{}, + bySession: map[string]*attempt{}, + seen: map[string]struct{}{}, + clients: map[string]*ocx.Client{}, + streamReady: make(chan struct{}), + permissionWake: make(chan struct{}, 1), } } @@ -83,10 +113,29 @@ func (d *Driver) Close() { return } d.closed = true + ats := make([]*attempt, 0, len(d.attempts)) + for _, at := range d.attempts { + ats = append(ats, at) + } d.mu.Unlock() if d.streamCancel != nil { d.streamCancel() } + d.readyOnce.Do(func() { close(d.streamReady) }) + for _, at := range ats { + at.cancel() + } + var wg sync.WaitGroup + for _, at := range ats { + wg.Add(1) + go func(at *attempt) { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = at.oc.Abort(ctx, at.sessionID) + }(at) + } + wg.Wait() } // clientFor returns the client bound to a directory (the attempt's @@ -108,17 +157,40 @@ func (d *Driver) clientFor(cwd string) *ocx.Client { // Start creates an OpenCode session, sends the objective, and starts a // watcher that completes the attempt via events + polling fallback. func (d *Driver) Start(ctx context.Context, a adapter.Attempt) (adapter.Session, error) { + role := a.Role + if role == "" { + role = "worker" + } + if role != "worker" && role != "reviewer" { + return nil, fmt.Errorf("start OpenCode: unsupported agent role %q", a.Role) + } + d.mu.Lock() + if d.closed { + d.mu.Unlock() + return nil, fmt.Errorf("start OpenCode: driver is closed") + } + if _, exists := d.seen[a.ID]; exists { + d.mu.Unlock() + return nil, fmt.Errorf("start OpenCode: attempt %q already started", a.ID) + } + d.seen[a.ID] = struct{}{} + d.mu.Unlock() + reserved := true + defer func() { + if !reserved { + return + } + d.mu.Lock() + delete(d.seen, a.ID) + d.mu.Unlock() + }() + client := d.clientFor(a.Cwd) title := "corral/" + a.NodeID sess, err := client.CreateSession(ctx, title) if err != nil { return nil, fmt.Errorf("create session: %w", err) } - prompt := promptFor(a) - if err := client.PromptAsync(ctx, sess.ID, prompt, d.opts.Model); err != nil { - return nil, fmt.Errorf("prompt: %w", err) - } - atCtx, atCancel := context.WithCancel(context.Background()) at := &attempt{ d: d, @@ -130,11 +202,84 @@ func (d *Driver) Start(ctx context.Context, a adapter.Attempt) (adapter.Session, cancel: atCancel, } d.mu.Lock() + if d.closed { + d.mu.Unlock() + cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = client.Abort(cleanupCtx, sess.ID) + cancel() + return nil, fmt.Errorf("start OpenCode: driver is closed") + } d.attempts[a.ID] = at d.bySession[sess.ID] = at + d.startStream(context.Background()) + d.mu.Unlock() + readyTimer := time.NewTimer(d.opts.streamReadyTimeout()) + select { + case <-d.streamReady: + case <-readyTimer.C: + // REST transcript/status polling completes attempts, while the durable + // permission poll recovers prompts missed before a later SSE reconnect. + // Open the shared gate so an unavailable stream delays only the first + // Start instead of serially delaying every attempt. + d.readyOnce.Do(func() { close(d.streamReady) }) + case <-ctx.Done(): + if !readyTimer.Stop() { + select { + case <-readyTimer.C: + default: + } + } + atCancel() + at.cleanup() + cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = client.Abort(cleanupCtx, sess.ID) + cancel() + return nil, fmt.Errorf("start OpenCode event stream: %w", ctx.Err()) + } + if !readyTimer.Stop() { + select { + case <-readyTimer.C: + default: + } + } + d.mu.Lock() + closed := d.closed d.mu.Unlock() + if closed { + atCancel() + at.cleanup() + cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = client.Abort(cleanupCtx, sess.ID) + cancel() + return nil, fmt.Errorf("start OpenCode: driver is closed") + } - d.startStream(atCtx) + prompt := promptFor(a) + model := a.Model + if model == "" { + model = d.opts.Model + } + if err := client.PromptAsyncAgent(ctx, sess.ID, prompt, model, "corral-"+role); err != nil { + atCancel() + at.cleanup() + cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = client.Abort(cleanupCtx, sess.ID) + cancel() + return nil, fmt.Errorf("prompt: %w", err) + } + d.wakePermissionReconcile() + d.mu.Lock() + closed = d.closed + d.mu.Unlock() + if closed { + atCancel() + at.cleanup() + cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = client.Abort(cleanupCtx, sess.ID) + cancel() + return nil, fmt.Errorf("start OpenCode: driver closed during prompt") + } + reserved = false go at.watch(atCtx) return &session{oc: client, at: at}, nil } @@ -153,15 +298,11 @@ func promptFor(a adapter.Attempt) string { // Step drains completed attempts (non-blocking). func (d *Driver) Step(_ context.Context, _ time.Time) []adapter.Completion { - var out []adapter.Completion - for { - select { - case c := <-d.completions: - out = append(out, c) - default: - return out - } - } + d.mu.Lock() + defer d.mu.Unlock() + out := d.completions + d.completions = nil + return out } // startStream opens the shared SSE stream exactly once and dispatches @@ -171,8 +312,11 @@ func (d *Driver) startStream(ctx context.Context) { sc, cancel := context.WithCancel(ctx) d.streamCtx = sc d.streamCancel = cancel + go d.reconcilePermissions(sc) go func() { - err := d.oc.StreamEvents(sc, func(ev ocx.Event) { + err := d.oc.StreamEventsReady(sc, func() { + d.readyOnce.Do(func() { close(d.streamReady) }) + }, func(ev ocx.Event) { var p struct { SessionID string `json:"sessionID"` } @@ -180,13 +324,22 @@ func (d *Driver) startStream(ctx context.Context) { return } switch ev.Type { - case "session.idle", "session.error", "permission.updated": + case "session.idle", "session.error", + "permission.asked", "permission.v2.asked", + "permission.replied", "permission.v2.replied", + "permission.updated": // pre-1.18 compatibility d.mu.Lock() at := d.bySession[p.SessionID] d.mu.Unlock() if at == nil { return } + if strings.HasPrefix(ev.Type, "permission.") { + // Record permission events inline. PendingPermission also polls the + // durable endpoint, covering reconnect gaps and dropped events. + at.handleEvent(ev) + return + } select { case at.events <- ev: default: // dropped; the poll fallback covers it @@ -200,15 +353,81 @@ func (d *Driver) startStream(ctx context.Context) { }) } +// wakePermissionReconcile asks the shared background poller to query pending +// permissions without putting provider I/O on the scheduler's hot path. +func (d *Driver) wakePermissionReconcile() { + select { + case d.permissionWake <- struct{}{}: + default: + } +} + +// reconcilePermissions continuously polls OpenCode's durable permission list. +// It is shared by all attempts and clients, so PendingPermission remains a +// local, non-blocking scheduler query even while the SSE stream reconnects. +func (d *Driver) reconcilePermissions(ctx context.Context) { + d.reconcilePermissionsOnce(ctx) + ticker := time.NewTicker(d.opts.permissionPoll()) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + case <-d.permissionWake: + } + d.reconcilePermissionsOnce(ctx) + } +} + +func (d *Driver) reconcilePermissionsOnce(ctx context.Context) { + d.mu.Lock() + groups := make(map[*ocx.Client][]*attempt) + for _, at := range d.attempts { + groups[at.oc] = append(groups[at.oc], at) + } + d.mu.Unlock() + + var wg sync.WaitGroup + for client, attempts := range groups { + client, attempts := client, attempts + wg.Add(1) + go func() { + defer wg.Done() + pollCtx, cancel := context.WithTimeout(ctx, permissionRequestTimeout) + defer cancel() + requests, err := client.PendingPermissions(pollCtx) + if err != nil { + return // keep last-known state; retry on the next shared poll + } + pending := make(map[string]string) + for _, request := range requests { + if request.SessionID == "" || request.ID == "" { + continue + } + if current := pending[request.SessionID]; current == "" || request.ID < current { + pending[request.SessionID] = request.ID + } + } + for _, at := range attempts { + at.mu.Lock() + at.permission = pending[at.sessionID] + at.mu.Unlock() + } + }() + } + wg.Wait() +} + // watch completes the attempt when the session reaches a terminal state, // using the event stream as primary signal and polling as fallback. func (at *attempt) watch(ctx context.Context) { + defer at.cleanup() poll := time.NewTicker(at.d.opts.poll()) defer poll.Stop() for { select { case <-ctx.Done(): - at.cleanup() return case <-poll.C: at.d.maybeComplete(ctx, at) @@ -219,10 +438,10 @@ func (at *attempt) watch(ctx context.Context) { } func (at *attempt) handleEvent(ev ocx.Event) { - if ev.Type == "permission.updated" { + switch ev.Type { + case "permission.asked", "permission.v2.asked", "permission.updated": var p struct { - ID string `json:"id"` - Type string `json:"type"` + ID string `json:"id"` } if err := ev.UnmarshalProps(&p); err == nil && p.ID != "" { at.mu.Lock() @@ -230,6 +449,18 @@ func (at *attempt) handleEvent(ev ocx.Event) { at.mu.Unlock() return } + case "permission.replied", "permission.v2.replied": + var p struct { + RequestID string `json:"requestID"` + } + if err := ev.UnmarshalProps(&p); err == nil && p.RequestID != "" { + at.mu.Lock() + if at.permission == p.RequestID { + at.permission = "" + } + at.mu.Unlock() + return + } } at.d.maybeComplete(context.Background(), at) } @@ -273,11 +504,31 @@ func (d *Driver) maybeComplete(ctx context.Context, at *attempt) { Status: status, Messages: toAdapterMessages(msgs), } - select { - case d.completions <- c: - default: - log.Printf("ocxadapter: completion channel full; dropping %s", at.attemptID) + if status == adapter.StatusError { + c.Err = terminalError(msgs) } + d.completions = append(d.completions, c) + at.cancel() +} + +func terminalError(msgs []ocx.Message) error { + for i := len(msgs) - 1; i >= 0; i-- { + message := msgs[i] + if message.Info.Role != "assistant" { + continue + } + if message.Info.Error != nil { + if name := errorName(message.Info.Error); name != "" { + return fmt.Errorf("OpenCode assistant error: %s", name) + } + return fmt.Errorf("OpenCode assistant error") + } + if message.Info.Finish != nil && *message.Info.Finish != "" { + return fmt.Errorf("OpenCode finish reason: %s", *message.Info.Finish) + } + break + } + return fmt.Errorf("OpenCode attempt failed") } // terminalStatus decides whether the transcript shows a terminal attempt @@ -301,7 +552,14 @@ func terminalStatus(msgs []ocx.Message, aborted bool) (bool, adapter.Status) { return true, adapter.StatusError } if m.Info.Finish != nil { - return true, adapter.StatusIdle + switch *m.Info.Finish { + case "", "tool-calls": + return false, "" // still streaming or entering a tool step + case "stop": + return true, adapter.StatusIdle + default: + return true, adapter.StatusError + } } return false, "" // still streaming } @@ -374,8 +632,11 @@ func (s *session) Send(ctx context.Context, text string) error { return s.oc.PromptAsync(ctx, s.at.sessionID, text, "") } func (s *session) Abort(ctx context.Context) error { + if err := s.oc.Abort(ctx, s.at.sessionID); err != nil { + return err + } s.at.aborted.Store(true) - return s.oc.Abort(ctx, s.at.sessionID) + return nil } func (s *session) Status(ctx context.Context) (adapter.Status, error) { statuses, err := s.oc.SessionStatus(ctx) @@ -401,21 +662,31 @@ func (s *session) Messages(ctx context.Context) ([]adapter.Message, error) { func (s *session) PendingPermission(_ context.Context) (string, bool, error) { s.at.mu.Lock() defer s.at.mu.Unlock() - if s.at.permission != "" { - return s.at.permission, true, nil - } - return "", false, nil + return s.at.permission, s.at.permission != "", nil } func (s *session) RespondPermission(ctx context.Context, id string, allow bool) error { - response := "deny" + pending, ok, err := s.PendingPermission(ctx) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("ocxadapter: no pending permission request") + } + if pending != id { + return fmt.Errorf("ocxadapter: permission %q is not pending (want %q)", id, pending) + } + reply := "reject" if allow { - response = "allow" + reply = "once" + } + if err := s.oc.RespondPermission(ctx, id, reply); err != nil { + return err } s.at.mu.Lock() if s.at.permission == id { s.at.permission = "" // resolved; resume continues automatically } s.at.mu.Unlock() - return s.oc.RespondPermission(ctx, s.at.sessionID, id, response) + return nil } diff --git a/internal/ocxadapter/adapter_test.go b/internal/ocxadapter/adapter_test.go index c27d67b..0701748 100644 --- a/internal/ocxadapter/adapter_test.go +++ b/internal/ocxadapter/adapter_test.go @@ -2,6 +2,10 @@ package ocxadapter_test import ( "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" @@ -22,6 +26,548 @@ import ( "corral/internal/verify" ) +func TestStartRejectsDuplicateAndClosedDriver(t *testing.T) { + var mu sync.Mutex + created := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/session": + mu.Lock() + created++ + mu.Unlock() + _ = json.NewEncoder(w).Encode(map[string]string{"id": "ses_unit"}) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_unit/prompt_async": + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_unit/abort": + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodGet && r.URL.Path == "/global/event": + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + <-r.Context().Done() + default: + http.Error(w, "unexpected "+r.Method+" "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + drv := ocxadapter.New(ocx.New(srv.URL, t.TempDir()), ocxadapter.Options{}) + t.Cleanup(drv.Close) + a := adapter.Attempt{ID: "run/n/1", NodeID: "n", Objective: "work"} + if _, err := drv.Start(context.Background(), a); err != nil { + t.Fatal(err) + } + if _, err := drv.Start(context.Background(), a); err == nil || !strings.Contains(err.Error(), "already started") { + t.Fatalf("duplicate Start error = %v", err) + } + mu.Lock() + gotCreated := created + mu.Unlock() + if gotCreated != 1 { + t.Fatalf("created sessions = %d, want 1", gotCreated) + } + drv.Close() + if _, err := drv.Start(context.Background(), adapter.Attempt{ID: "run/n/2", NodeID: "n", Objective: "work"}); err == nil || !strings.Contains(err.Error(), "closed") { + t.Fatalf("Start after Close error = %v", err) + } +} + +func TestStartUsesAttemptModelOverrideOnPromptWire(t *testing.T) { + tests := []struct { + name string + driverModel string + attemptModel string + wantProvider string + wantModel string + }{ + { + name: "attempt override", + driverModel: "anthropic/claude-sonnet-4", + attemptModel: "openai/gpt-5", + wantProvider: "openai", + wantModel: "gpt-5", + }, + { + name: "driver fallback", + driverModel: "anthropic/claude-sonnet-4", + wantProvider: "anthropic", + wantModel: "claude-sonnet-4", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got struct { + Model struct { + ProviderID string `json:"providerID"` + ModelID string `json:"modelID"` + } `json:"model"` + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/session": + _ = json.NewEncoder(w).Encode(map[string]string{"id": "ses_model"}) + case r.Method == http.MethodGet && r.URL.Path == "/global/event": + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + <-r.Context().Done() + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_model/prompt_async": + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Errorf("decode prompt: %v", err) + } + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodGet && r.URL.Path == "/permission": + _ = json.NewEncoder(w).Encode([]any{}) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_model/abort": + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "unexpected "+r.Method+" "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + drv := ocxadapter.New(ocx.New(srv.URL, t.TempDir()), ocxadapter.Options{Model: tt.driverModel}) + t.Cleanup(drv.Close) + _, err := drv.Start(context.Background(), adapter.Attempt{ + ID: "run/n/1", NodeID: "n", Objective: "work", Model: tt.attemptModel, + }) + if err != nil { + t.Fatal(err) + } + if got.Model.ProviderID != tt.wantProvider || got.Model.ModelID != tt.wantModel { + t.Fatalf("prompt model = %#v, want providerID=%q modelID=%q", got.Model, tt.wantProvider, tt.wantModel) + } + }) + } +} + +func TestStartSubscribesBeforePromptCanEmitPermission(t *testing.T) { + streamReady := make(chan struct{}) + permissionSent := make(chan struct{}) + var sendOnce sync.Once + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/session": + _ = json.NewEncoder(w).Encode(map[string]string{"id": "ses_fast"}) + case r.Method == http.MethodGet && r.URL.Path == "/global/event": + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + f, _ := w.(http.Flusher) + f.Flush() + close(streamReady) + <-permissionSent + payload, _ := json.Marshal(map[string]any{ + "type": "permission.asked", + "properties": map[string]any{"sessionID": "ses_fast", "id": "perm-fast"}, + }) + frame, _ := json.Marshal(map[string]any{"payload": json.RawMessage(payload)}) + _, _ = fmt.Fprintf(w, "data: %s\n\n", frame) + f.Flush() + <-r.Context().Done() + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_fast/prompt_async": + select { + case <-streamReady: + default: + t.Error("prompt arrived before event subscription was ready") + } + sendOnce.Do(func() { close(permissionSent) }) + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodGet && r.URL.Path == "/permission": + select { + case <-permissionSent: + _ = json.NewEncoder(w).Encode([]map[string]string{{"id": "perm-fast", "sessionID": "ses_fast"}}) + default: + _ = json.NewEncoder(w).Encode([]any{}) + } + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_fast/abort": + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "unexpected "+r.Method+" "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + drv := ocxadapter.New(ocx.New(srv.URL, t.TempDir()), ocxadapter.Options{PollInterval: time.Hour}) + t.Cleanup(drv.Close) + sess, err := drv.Start(context.Background(), adapter.Attempt{ID: "run/n/1", NodeID: "n", Objective: "work"}) + if err != nil { + t.Fatal(err) + } + ps := sess.(adapter.PermissionSession) + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if id, ok, _ := ps.PendingPermission(context.Background()); ok && id == "perm-fast" { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("permission emitted during prompt was lost") +} + +func TestStartFallsBackToRESTWhenEventStreamIsUnavailable(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/session": + _ = json.NewEncoder(w).Encode(map[string]string{"id": "ses_rest"}) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_rest/prompt_async": + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodGet && r.URL.Path == "/session/ses_rest/message": + _ = json.NewEncoder(w).Encode([]map[string]any{{ + "info": map[string]any{ + "id": "msg-rest", "role": "assistant", + "sessionID": "ses_rest", "finish": "stop", + }, + "parts": []any{}, + }}) + case r.Method == http.MethodGet && r.URL.Path == "/permission": + _ = json.NewEncoder(w).Encode([]any{}) + case r.Method == http.MethodGet && r.URL.Path == "/global/event": + http.Error(w, "events unavailable", http.StatusServiceUnavailable) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_rest/abort": + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "unexpected "+r.Method+" "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + drv := ocxadapter.New(ocx.New(srv.URL, t.TempDir()), ocxadapter.Options{ + PollInterval: 5 * time.Millisecond, + StreamReadyTimeout: 20 * time.Millisecond, + }) + t.Cleanup(drv.Close) + if _, err := drv.Start(context.Background(), adapter.Attempt{ + ID: "run/n/1", NodeID: "n", Objective: "work", + }); err != nil { + t.Fatalf("Start with unavailable SSE: %v", err) + } + + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if got := drv.Step(context.Background(), time.Now()); len(got) > 0 { + if got[0].Status != adapter.StatusIdle { + t.Fatalf("REST completion status = %q, want idle", got[0].Status) + } + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("REST polling did not complete attempt while SSE was unavailable") +} + +func TestPermissionReconcilesAfterEventGapWithoutBlockingCallers(t *testing.T) { + streamReady := make(chan struct{}) + permissionPolled := make(chan struct{}) + var pollOnce sync.Once + var mu sync.Mutex + pending := false + blockPermission := false + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/session": + _ = json.NewEncoder(w).Encode(map[string]string{"id": "ses_gap"}) + case r.Method == http.MethodGet && r.URL.Path == "/global/event": + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + select { + case <-streamReady: + default: + close(streamReady) + } + // End this subscription without a permission event. The durable + // permission poll must recover the missed request. + return + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_gap/prompt_async": + <-streamReady + mu.Lock() + pending = true + mu.Unlock() + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodGet && r.URL.Path == "/permission": + mu.Lock() + isPending := pending + block := blockPermission + mu.Unlock() + if block { + <-r.Context().Done() + return + } + if isPending { + pollOnce.Do(func() { close(permissionPolled) }) + _ = json.NewEncoder(w).Encode([]map[string]string{{ + "id": "perm-gap", "sessionID": "ses_gap", + }}) + return + } + _ = json.NewEncoder(w).Encode([]any{}) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_gap/abort": + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "unexpected "+r.Method+" "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + drv := ocxadapter.New(ocx.New(srv.URL, t.TempDir()), ocxadapter.Options{ + PollInterval: 5 * time.Millisecond, + StreamReadyTimeout: 100 * time.Millisecond, + }) + t.Cleanup(drv.Close) + sess, err := drv.Start(context.Background(), adapter.Attempt{ + ID: "run/n/1", NodeID: "n", Objective: "work", + }) + if err != nil { + t.Fatal(err) + } + ps := sess.(adapter.PermissionSession) + + select { + case <-permissionPolled: + case <-time.After(time.Second): + t.Fatal("background reconciliation never polled durable permissions") + } + deadline := time.Now().Add(time.Second) + for { + if id, ok, err := ps.PendingPermission(context.Background()); err != nil { + t.Fatal(err) + } else if ok && id == "perm-gap" { + break + } + if time.Now().After(deadline) { + t.Fatal("missed permission was not reconciled into local state") + } + time.Sleep(time.Millisecond) + } + + mu.Lock() + blockPermission = true + mu.Unlock() + returned := make(chan struct{}) + go func() { + _, _, _ = ps.PendingPermission(context.Background()) + close(returned) + }() + select { + case <-returned: + case <-time.After(50 * time.Millisecond): + t.Fatal("PendingPermission performed blocking network I/O") + } +} + +func TestAbortFailureDoesNotReportLocalAbort(t *testing.T) { + var abortCalls int + var mu sync.Mutex + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/session": + _ = json.NewEncoder(w).Encode(map[string]string{"id": "ses_abort"}) + case r.Method == http.MethodGet && r.URL.Path == "/global/event": + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + <-r.Context().Done() + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_abort/prompt_async": + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_abort/abort": + mu.Lock() + abortCalls++ + call := abortCalls + mu.Unlock() + if call == 1 { + http.Error(w, "abort failed", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + default: + http.Error(w, "unexpected "+r.Method+" "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + drv := ocxadapter.New(ocx.New(srv.URL, t.TempDir()), ocxadapter.Options{}) + t.Cleanup(drv.Close) + sess, err := drv.Start(context.Background(), adapter.Attempt{ID: "run/n/1", NodeID: "n", Objective: "work"}) + if err != nil { + t.Fatal(err) + } + if err := sess.Abort(context.Background()); err == nil { + t.Fatal("provider abort failure was hidden") + } + if status, err := sess.Status(context.Background()); err == nil && status == adapter.StatusAborted { + t.Fatal("failed provider abort was reported as locally aborted") + } +} + +func TestPermissionResponseValidatesIDAndRetainsFailedDecision(t *testing.T) { + var mu sync.Mutex + permissionCalls := 0 + failDecision := true + pending := true + gotReply := "" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/session": + _ = json.NewEncoder(w).Encode(map[string]string{"id": "ses_perm"}) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_perm/prompt_async": + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_perm/abort": + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodGet && r.URL.Path == "/permission": + mu.Lock() + isPending := pending + mu.Unlock() + if isPending { + _ = json.NewEncoder(w).Encode([]map[string]string{{"id": "perm-1", "sessionID": "ses_perm"}}) + } else { + _ = json.NewEncoder(w).Encode([]any{}) + } + case r.Method == http.MethodPost && r.URL.Path == "/permission/perm-1/reply": + var body struct { + Reply string `json:"reply"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("decode permission reply: %v", err) + } + mu.Lock() + permissionCalls++ + fail := failDecision + gotReply = body.Reply + if !fail { + pending = false + } + mu.Unlock() + if fail { + http.Error(w, "try again", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodGet && r.URL.Path == "/global/event": + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + f, _ := w.(http.Flusher) + f.Flush() + // Deliberately emit no permission event. The durable /permission + // endpoint must reconcile a request missed during an SSE gap. + <-r.Context().Done() + default: + http.Error(w, "unexpected "+r.Method+" "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + drv := ocxadapter.New(ocx.New(srv.URL, t.TempDir()), ocxadapter.Options{PollInterval: time.Hour}) + t.Cleanup(drv.Close) + sess, err := drv.Start(context.Background(), adapter.Attempt{ID: "run/n/1", NodeID: "n", Objective: "work"}) + if err != nil { + t.Fatal(err) + } + ps := sess.(adapter.PermissionSession) + deadline := time.Now().Add(time.Second) + for { + if id, ok, _ := ps.PendingPermission(context.Background()); ok && id == "perm-1" { + break + } + if time.Now().After(deadline) { + t.Fatal("permission event was not tracked") + } + time.Sleep(time.Millisecond) + } + if err := ps.RespondPermission(context.Background(), "wrong", true); err == nil { + t.Fatal("wrong permission ID was accepted") + } + mu.Lock() + gotCalls := permissionCalls + mu.Unlock() + if gotCalls != 0 { + t.Fatalf("wrong permission reached provider %d times", gotCalls) + } + if err := ps.RespondPermission(context.Background(), "perm-1", true); err == nil { + t.Fatal("provider failure was hidden") + } + if id, ok, _ := ps.PendingPermission(context.Background()); !ok || id != "perm-1" { + t.Fatalf("failed response cleared pending permission: %q, %v", id, ok) + } + mu.Lock() + failDecision = false + mu.Unlock() + if err := ps.RespondPermission(context.Background(), "perm-1", true); err != nil { + t.Fatal(err) + } + mu.Lock() + reply := gotReply + mu.Unlock() + if reply != "once" { + t.Fatalf("allow reply = %q, want once", reply) + } + if id, ok, _ := ps.PendingPermission(context.Background()); ok { + t.Fatalf("successful response left permission pending: %q", id) + } +} + +func TestIntermediateFinishDoesNotComplete(t *testing.T) { + var mu sync.Mutex + finish := "tool-calls" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/session": + _ = json.NewEncoder(w).Encode(map[string]string{"id": "ses_finish"}) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_finish/prompt_async": + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodGet && r.URL.Path == "/session/ses_finish/message": + mu.Lock() + current := finish + mu.Unlock() + _ = json.NewEncoder(w).Encode([]map[string]any{{ + "info": map[string]any{"id": "msg-1", "role": "assistant", "sessionID": "ses_finish", "finish": current}, + "parts": []any{}, + }}) + case r.Method == http.MethodPost && r.URL.Path == "/session/ses_finish/abort": + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodGet && r.URL.Path == "/global/event": + w.Header().Set("Content-Type", "text/event-stream") + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + <-r.Context().Done() + default: + http.Error(w, "unexpected "+r.Method+" "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + drv := ocxadapter.New(ocx.New(srv.URL, t.TempDir()), ocxadapter.Options{PollInterval: 5 * time.Millisecond}) + t.Cleanup(drv.Close) + if _, err := drv.Start(context.Background(), adapter.Attempt{ID: "run/n/1", NodeID: "n", Objective: "work"}); err != nil { + t.Fatal(err) + } + time.Sleep(40 * time.Millisecond) + if got := drv.Step(context.Background(), time.Now()); len(got) != 0 { + t.Fatalf("intermediate finish emitted completion: %+v", got) + } + mu.Lock() + finish = "stop" + mu.Unlock() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if got := drv.Step(context.Background(), time.Now()); len(got) > 0 { + if got[0].Status != adapter.StatusIdle { + t.Fatalf("terminal status = %q, want idle", got[0].Status) + } + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("finish=stop did not emit completion") +} + const ( w1Prompt = "Create a file named alpha.txt containing exactly one line: CORRAL-OC1. Do not run any other commands." w2Prompt = "Append one line to beta.txt every second, 30 lines total, numbered 1 to 30, using bash. Keep going until the loop finishes. Do not stop early." diff --git a/internal/ocxadapter/terminal_test.go b/internal/ocxadapter/terminal_test.go new file mode 100644 index 0000000..17534fd --- /dev/null +++ b/internal/ocxadapter/terminal_test.go @@ -0,0 +1,48 @@ +package ocxadapter + +import ( + "encoding/json" + "strings" + "testing" + + "corral/internal/adapter" + "corral/internal/ocx" +) + +func TestTerminalStatusClassifiesFinishReasons(t *testing.T) { + tests := []struct { + name string + finish string + terminal bool + status adapter.Status + }{ + {name: "stop", finish: "stop", terminal: true, status: adapter.StatusIdle}, + {name: "tool calls", finish: "tool-calls", terminal: false}, + {name: "length", finish: "length", terminal: true, status: adapter.StatusError}, + {name: "content filter", finish: "content-filter", terminal: true, status: adapter.StatusError}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + finish := test.finish + messages := []ocx.Message{{Info: ocx.MessageInfo{ + Role: "assistant", Finish: &finish, + }}} + terminal, status := terminalStatus(messages, false) + if terminal != test.terminal || status != test.status { + t.Fatalf("terminalStatus(%q) = (%v, %q), want (%v, %q)", + test.finish, terminal, status, test.terminal, test.status) + } + }) + } +} + +func TestTerminalErrorDescribesProviderFailure(t *testing.T) { + finish := "length" + if got := terminalError([]ocx.Message{{Info: ocx.MessageInfo{Role: "assistant", Finish: &finish}}}); got == nil || !strings.Contains(got.Error(), "length") { + t.Fatalf("finish error = %v", got) + } + raw := json.RawMessage(`{"name":"ProviderAuthError"}`) + if got := terminalError([]ocx.Message{{Info: ocx.MessageInfo{Role: "assistant", Error: &raw}}}); got == nil || !strings.Contains(got.Error(), "ProviderAuthError") { + t.Fatalf("assistant error = %v", got) + } +} diff --git a/internal/ocxreviewer/reviewer.go b/internal/ocxreviewer/reviewer.go new file mode 100644 index 0000000..929a453 --- /dev/null +++ b/internal/ocxreviewer/reviewer.go @@ -0,0 +1,327 @@ +// Package ocxreviewer implements the verify.Reviewer seam on top of +// OpenCode sessions (Task 4's reviewer gate). A reviewer session receives +// the completed attempt's evidence — objective, prior feedback, transcript, +// the recorded diff artifact and any check results — and must conclude with +// an explicit verdict: APPROVED or CHANGES_REQUESTED plus a note. The note +// becomes the gate feedback when the verdict is not approved, so the worker +// knows exactly what to fix. Sessions use the named reviewer agent with all +// tools denied and evaluate only the supplied evidence. +package ocxreviewer + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "corral/internal/adapter" + "corral/internal/ocx" + "corral/internal/verify" +) + +// Options tunes reviewer sessions. Zero values select sane defaults. +type Options struct { + // Model overrides the default model for reviewer sessions + // ("" = the OpenCode server default). + Model string + // Timeout bounds a single review session (default 5m). + Timeout time.Duration + // PollInterval is the transcript poll period while the session runs + // (default 400ms). + PollInterval time.Duration +} + +func (o Options) timeout() time.Duration { + if o.Timeout <= 0 { + return 5 * time.Minute + } + return o.Timeout +} + +func (o Options) poll() time.Duration { + if o.PollInterval <= 0 { + return 400 * time.Millisecond + } + return o.PollInterval +} + +const reviewerAgent = "corral-reviewer" + +// denyTools returns OpenCode's prompt-level wildcard deny. OpenCode converts +// this entry into a wildcard permission rule, covering built-in, plugin, MCP, +// and future tools without relying on the incomplete experimental ID list. +func denyTools() map[string]bool { + return map[string]bool{"*": false} +} + +// Driver implements verify.Reviewer for OpenCode sessions. +type Driver struct { + oc *ocx.Client + opts Options +} + +func New(oc *ocx.Client, opts Options) *Driver { + return &Driver{oc: oc, opts: opts} +} + +// Review runs a reviewer session for the attempt's evidence, waits for the +// session to reach idle, and parses the verdict from the transcript. +func (d *Driver) Review(ctx context.Context, req verify.ReviewRequest) (bool, string, error) { + // Reviewer agent configuration belongs to the daemon's main OpenCode + // project and is not guaranteed to exist in generated Git worktrees. The + // reviewer is evidence-only with all tools denied, so it does not need a + // session bound to the attempt worktree. + client := d.oc + reviewTools := denyTools() + + title := "corral/review/" + req.Attempt.NodeID + sess, err := client.CreateSession(ctx, title) + if err != nil { + return false, "", fmt.Errorf("review session: %w", err) + } + prompt := promptFor(req) + if err := client.PromptAsyncAgentWithTools(ctx, sess.ID, prompt, d.opts.Model, reviewerAgent, reviewTools); err != nil { + abortReview(client, sess.ID) + return false, "", fmt.Errorf("review prompt: %w", err) + } + + deadline := time.Now().Add(d.opts.timeout()) + var lastPollErr error + for { + select { + case <-ctx.Done(): + abortReview(client, sess.ID) + return false, "", ctx.Err() + default: + } + msgs, err := client.Messages(ctx, sess.ID, 0) + if err != nil { + lastPollErr = err + } else if term, errName := terminal(msgs); term { + if errName != "" { + return false, "", fmt.Errorf("review session error: %s", errName) + } + return verdict(msgs) + } + if time.Now().After(deadline) { + break + } + select { + case <-time.After(d.opts.poll()): + case <-ctx.Done(): + abortReview(client, sess.ID) + return false, "", ctx.Err() + } + } + + // Timed out: kill the session so it stops generating, and fail fast. + abortReview(client, sess.ID) + if lastPollErr != nil { + return false, "", fmt.Errorf("review timed out after %s (last poll: %v)", d.opts.timeout(), lastPollErr) + } + return false, "", fmt.Errorf("review timed out after %s (no terminal response)", d.opts.timeout()) +} + +func abortReview(client *ocx.Client, sessionID string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = client.Abort(ctx, sessionID) +} + +// terminal reports whether the newest assistant message ended the session, +// returning the error name when it terminated with a session error. +func terminal(msgs []ocx.Message) (bool, string) { + for i := len(msgs) - 1; i >= 0; i-- { + m := msgs[i] + if m.Info.Role != "assistant" { + continue + } + if m.Info.Error != nil { + return true, errorName(m.Info.Error) + } + if m.Info.Finish != nil { + switch *m.Info.Finish { + case "", "tool-calls": + return false, "" + case "stop": + return true, "" + default: + return true, "finish reason: " + *m.Info.Finish + } + } + return false, "" + } + return false, "" +} + +// verdict parses only the newest assistant response. Text parts are joined in +// their original order because OpenCode may split one continuous response +// across parts; older assistant messages can never supply a stale verdict. +func verdict(msgs []ocx.Message) (bool, string, error) { + for i := len(msgs) - 1; i >= 0; i-- { + m := msgs[i] + if m.Info.Role != "assistant" { + continue + } + var text strings.Builder + for _, part := range m.Parts { + var p struct { + Type string `json:"type"` + Text string `json:"text"` + } + if json.Unmarshal(part, &p) != nil || p.Type != "text" { + continue + } + text.WriteString(p.Text) + } + if approved, note, ok := parseVerdict(text.String()); ok { + return approved, note, nil + } + return false, "", fmt.Errorf("reviewer produced no explicit verdict") + } + return false, "", fmt.Errorf("reviewer produced no explicit verdict") +} + +// parseVerdict accepts exactly two lines: a case-sensitive verdict followed +// by a non-empty "Note: ..." line. Rejecting surrounding prose, legacy +// spellings, and extra lines prevents a verdict word inside commentary from +// being mistaken for the reviewer's decision. +func parseVerdict(text string) (approved bool, note string, ok bool) { + text = strings.TrimSuffix(strings.ReplaceAll(text, "\r\n", "\n"), "\n") + lines := strings.Split(text, "\n") + if len(lines) != 2 { + return false, "", false + } + switch lines[0] { + case "APPROVED": + approved = true + case "CHANGES_REQUESTED": + approved = false + default: + return false, "", false + } + const notePrefix = "Note: " + if !strings.HasPrefix(lines[1], notePrefix) { + return false, "", false + } + note = strings.TrimSpace(strings.TrimPrefix(lines[1], notePrefix)) + if note == "" { + return false, "", false + } + note = truncate(note, 2000) + return approved, note, true +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "..." +} + +// promptFor builds the reviewer prompt from the attempt's evidence. +func promptFor(req verify.ReviewRequest) string { + var b strings.Builder + b.WriteString("You are a corral reviewer. Review the completed attempt's evidence below and return a verdict.\n") + b.WriteString("\nOBJECTIVE:\n" + req.Attempt.Objective) + if req.Attempt.Role != "" { + b.WriteString("\n\nATTEMPT ROLE: " + req.Attempt.Role) + } + if req.Worktree != "" { + b.WriteString("\n\nWORKTREE: " + req.Worktree) + } + if req.Feedback != "" { + b.WriteString("\n\nPRIOR FEEDBACK (from a rejected attempt; confirm it is addressed):\n" + req.Feedback) + } + if d := diffArtifact(req.Messages); d != "" { + b.WriteString("\n\nEVIDENCE — RECORDED DIFF ARTIFACT (file changes from the attempt):\n" + d) + } + if c := checkResults(req.Messages); c != "" { + b.WriteString("\n\nEVIDENCE — CHECK RESULTS (command gates that already ran):\n" + c) + } + if t := transcript(req.Messages); t != "" { + b.WriteString("\n\nEVIDENCE — ATTEMPT TRANSCRIPT:\n" + t) + } + b.WriteString(` +VERDICT: +Reply with EXACTLY these two lines and nothing else. + +APPROVED +Note: + +or + +CHANGES_REQUESTED +Note: +`) + return b.String() +} + +// diffArtifact renders the file changes the attempt recorded (the diff +// artifact evidence) into a compact text block. +func diffArtifact(msgs []adapter.Message) string { + var b strings.Builder + for _, m := range msgs { + for _, d := range m.Diffs { + fmt.Fprintf(&b, "--- %s (%s) +%d -%d\n", d.File, d.Status, d.Additions, d.Deletions) + if d.Patch != "" { + b.WriteString(d.Patch) + b.WriteString("\n") + } + } + } + return strings.TrimSpace(b.String()) +} + +// checkResults renders command-gate results carried in message Meta +// (exit/stdout/stderr) as check-result evidence. +func checkResults(msgs []adapter.Message) string { + var b strings.Builder + for i, m := range msgs { + if m.Meta["exit"] == "" { + continue + } + fmt.Fprintf(&b, "# check %d: exit=%s\n", i, m.Meta["exit"]) + if out := m.Meta["stdout"]; out != "" { + b.WriteString("stdout:\n" + tail(out, 2000) + "\n") + } + if errOut := m.Meta["stderr"]; errOut != "" { + b.WriteString("stderr:\n" + tail(errOut, 2000) + "\n") + } + } + return strings.TrimSpace(b.String()) +} + +// transcript renders the attempt's user/assistant messages. +func transcript(msgs []adapter.Message) string { + var b strings.Builder + for _, m := range msgs { + text := strings.TrimSpace(m.Text) + if text == "" { + continue + } + fmt.Fprintf(&b, "%s: %s\n\n", m.Role, text) + } + return strings.TrimSpace(b.String()) +} + +func tail(s string, n int) string { + if len(s) <= n { + return s + } + return "..." + s[len(s)-n:] +} + +func errorName(raw *json.RawMessage) string { + if raw == nil { + return "" + } + var e struct { + Name string `json:"name"` + } + if err := json.Unmarshal(*raw, &e); err != nil { + return "" + } + return e.Name +} diff --git a/internal/ocxreviewer/reviewer_test.go b/internal/ocxreviewer/reviewer_test.go new file mode 100644 index 0000000..c906df3 --- /dev/null +++ b/internal/ocxreviewer/reviewer_test.go @@ -0,0 +1,359 @@ +package ocxreviewer + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "corral/internal/adapter" + "corral/internal/assets" + "corral/internal/livetest" + "corral/internal/ocx" + "corral/internal/spike" + "corral/internal/verify" +) + +// fakeLLM is a scripted OpenCode server: it records review prompts and +// serves a fixed sequence of transcripts, mirroring the fake-reviewer +// script surface. The last transcript repeats, so a reviewer that keeps +// polling settles on it deterministically. +type fakeLLM struct { + mu sync.Mutex + steps [][]ocx.Message + step int + prompts []string + promptAgents []string + promptTools []map[string]bool + directories []string + sessions int +} + +func newFakeLLM(steps ...[]ocx.Message) *fakeLLM { + return &fakeLLM{steps: steps} +} + +func (f *fakeLLM) serve() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.directories = append(f.directories, r.URL.Query().Get("directory")) + switch { + case r.Method == http.MethodPost && r.URL.Path == "/session": + f.sessions++ + _ = json.NewEncoder(w).Encode(ocx.Session{ID: "ses_1", Directory: "proj", Title: "review"}) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/prompt_async"): + var body struct { + Agent string `json:"agent"` + Tools map[string]bool `json:"tools"` + Parts []struct { + Text string `json:"text"` + } `json:"parts"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + f.promptAgents = append(f.promptAgents, body.Agent) + f.promptTools = append(f.promptTools, body.Tools) + for _, p := range body.Parts { + f.prompts = append(f.prompts, p.Text) + } + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/message"): + // Serve the next scripted transcript; the last one repeats so + // a reviewer that keeps polling settles on it deterministically. + if f.step < len(f.steps)-1 { + f.step++ + } + _ = json.NewEncoder(w).Encode(f.steps[f.step]) + default: + w.WriteHeader(http.StatusNotFound) + } + })) +} + +func llmBusy() []ocx.Message { + return []ocx.Message{{Info: ocx.MessageInfo{Role: "assistant"}}} +} + +func llmText(text string) ocx.Message { + return llmTextParts(text) +} + +func llmTextParts(texts ...string) ocx.Message { + finish := "stop" + parts := make([]json.RawMessage, 0, len(texts)) + for _, text := range texts { + part, _ := json.Marshal(map[string]string{"type": "text", "text": text}) + parts = append(parts, part) + } + return ocx.Message{ + Info: ocx.MessageInfo{Role: "assistant", Finish: &finish}, + Parts: parts, + } +} + +func llmError(name string) []ocx.Message { + raw, _ := json.Marshal(map[string]string{"name": name}) + rm := json.RawMessage(raw) + return []ocx.Message{{Info: ocx.MessageInfo{Role: "assistant", Error: &rm}}} +} + +func reviewReq(worktree string) verify.ReviewRequest { + return verify.ReviewRequest{ + Attempt: adapter.Attempt{ + ID: "run_1/w1/1", + NodeID: "w1", + Objective: "create manifest.json with a name field", + Role: "worker", + Cwd: worktree, + }, + Worktree: worktree, + Feedback: "manifest.json is missing the name field", + Messages: []adapter.Message{ + {Role: "user", Text: "create manifest.json", Diffs: []adapter.Diff{ + {File: "manifest.json", Patch: "+name: x", Additions: 1, Status: "added"}, + }}, + {Role: "assistant", Finish: "stop", Text: "created manifest.json", Meta: map[string]string{ + "exit": "0", "stdout": "ok", + }}, + }, + } +} + +func TestReviewerApproved(t *testing.T) { + llm := newFakeLLM( + llmBusy(), // session starts streaming: not terminal yet + []ocx.Message{llmText("APPROVED\nNote: manifest.json now has a name field and matches the schema.")}, + ) + srv := llm.serve() + defer srv.Close() + + mainDir := t.TempDir() + worktree := t.TempDir() + drv := New(ocx.New(srv.URL, mainDir), Options{PollInterval: time.Millisecond, Timeout: 5 * time.Second}) + approved, note, err := drv.Review(context.Background(), reviewReq(worktree)) + if err != nil { + t.Fatal(err) + } + if !approved { + t.Fatal("approved = false, want true") + } + if !strings.Contains(note, "name field") { + t.Fatalf("note = %q, want the reviewer's justification", note) + } + + if len(llm.prompts) != 1 { + t.Fatalf("review prompts = %d, want 1", len(llm.prompts)) + } + if got := llm.promptAgents[0]; got != "corral-reviewer" { + t.Errorf("review agent = %q, want %q", got, "corral-reviewer") + } + if got := llm.promptTools[0]; len(got) != 1 || got["*"] { + t.Errorf("review tools = %#v, want wildcard deny", got) + } + for _, directory := range llm.directories { + if directory != mainDir { + t.Errorf("review request directory = %q, want main project %q", directory, mainDir) + } + } + for _, want := range []string{"OBJECTIVE", "create manifest.json", "PRIOR FEEDBACK", "DIFF ARTIFACT", "manifest.json", "CHECK RESULTS", "exit=0"} { + if !strings.Contains(llm.prompts[0], want) { + t.Errorf("prompt missing %q", want) + } + } +} + +func TestReviewerNotApproved(t *testing.T) { + llm := newFakeLLM([]ocx.Message{llmText("CHANGES_REQUESTED\nNote: the manifest is still missing the required count field.")}) + srv := llm.serve() + defer srv.Close() + + drv := New(ocx.New(srv.URL, t.TempDir()), Options{PollInterval: time.Millisecond, Timeout: 5 * time.Second}) + approved, note, err := drv.Review(context.Background(), reviewReq(t.TempDir())) + if err != nil { + t.Fatal(err) + } + if approved { + t.Fatal("approved = true, want false") + } + if !strings.Contains(note, "count field") { + t.Fatalf("note = %q, want rejection feedback", note) + } +} + +func TestReviewerSessionError(t *testing.T) { + llm := newFakeLLM(llmError("MessageAbortedError")) + srv := llm.serve() + defer srv.Close() + + drv := New(ocx.New(srv.URL, t.TempDir()), Options{PollInterval: time.Millisecond, Timeout: 5 * time.Second}) + if _, _, err := drv.Review(context.Background(), reviewReq(t.TempDir())); err == nil { + t.Fatal("session error not surfaced") + } else if !strings.Contains(err.Error(), "MessageAbortedError") { + t.Fatalf("error = %q, want session error name", err) + } +} + +func TestReviewerTerminalFinishReasons(t *testing.T) { + toolCalls := "tool-calls" + if terminal, errName := terminal([]ocx.Message{{Info: ocx.MessageInfo{Role: "assistant", Finish: &toolCalls}}}); terminal || errName != "" { + t.Fatalf("tool-calls = terminal %v error %q, want still running", terminal, errName) + } + length := "length" + if terminal, errName := terminal([]ocx.Message{{Info: ocx.MessageInfo{Role: "assistant", Finish: &length}}}); !terminal || !strings.Contains(errName, "length") { + t.Fatalf("length = terminal %v error %q, want provider error", terminal, errName) + } +} + +func TestReviewerNoVerdict(t *testing.T) { + llm := newFakeLLM([]ocx.Message{llmText("The changes look fine to me; ship it.")}) + srv := llm.serve() + defer srv.Close() + + drv := New(ocx.New(srv.URL, t.TempDir()), Options{PollInterval: time.Millisecond, Timeout: 5 * time.Second}) + if _, _, err := drv.Review(context.Background(), reviewReq(t.TempDir())); err == nil { + t.Fatal("missing verdict not surfaced") + } else if !strings.Contains(err.Error(), "no explicit verdict") { + t.Fatalf("error = %q, want missing-verdict error", err) + } +} + +func TestReviewerTimeout(t *testing.T) { + llm := newFakeLLM(llmBusy()) // never reaches a terminal message + srv := llm.serve() + defer srv.Close() + + drv := New(ocx.New(srv.URL, t.TempDir()), Options{PollInterval: time.Millisecond, Timeout: 50 * time.Millisecond}) + if _, _, err := drv.Review(context.Background(), reviewReq(t.TempDir())); err == nil { + t.Fatal("timeout not surfaced") + } else if !strings.Contains(err.Error(), "timed out") { + t.Fatalf("error = %q, want timeout error", err) + } +} + +func TestParseVerdict(t *testing.T) { + cases := []struct { + text string + approved bool + note string + ok bool + }{ + {"APPROVED\nNote: good work", true, "good work", true}, + {"APPROVED\nNote: good work\n", true, "good work", true}, + {"APPROVED\r\nNote: good work\r\n", true, "good work", true}, + {"CHANGES_REQUESTED\nNote: missing tests", false, "missing tests", true}, + {"no verdict anywhere", false, "", false}, + {"NOT_APPROVED\nNote: legacy verdict", false, "", false}, + {"NOT APPROVED\nNote: legacy verdict", false, "", false}, + {"We approve.\nAPPROVED\nNote: embedded verdict", false, "", false}, + {"APPROVED\nNote:", false, "", false}, + {"APPROVED\nNote: valid\nextra", false, "", false}, + {"approved\nNote: wrong case", false, "", false}, + {"APPROVED Note: one line", false, "", false}, + } + for _, c := range cases { + approved, note, ok := parseVerdict(c.text) + if ok != c.ok || approved != c.approved || note != c.note { + t.Errorf("parseVerdict(%q) = (%v, %q, %v), want (%v, %q, %v)", + c.text, approved, note, ok, c.approved, c.note, c.ok) + } + } +} + +func TestVerdictRejectsTextOutsideExactResponse(t *testing.T) { + t.Run("extra text part", func(t *testing.T) { + msgs := []ocx.Message{llmTextParts("Here is my decision:\n", "APPROVED\nNote: verified")} + if _, _, err := verdict(msgs); err == nil { + t.Fatal("verdict accepted response with extra text part") + } + }) + + t.Run("stale prior verdict", func(t *testing.T) { + msgs := []ocx.Message{ + llmText("APPROVED\nNote: stale"), + llmText("I cannot decide."), + } + if _, _, err := verdict(msgs); err == nil { + t.Fatal("verdict accepted stale prior assistant response") + } + }) +} + +func TestPromptForIncludesEvidence(t *testing.T) { + req := reviewReq("/tmp/worktree") + p := promptFor(req) + for _, want := range []string{ + "OBJECTIVE", + "ATTEMPT ROLE: worker", + "WORKTREE: /tmp/worktree", + "PRIOR FEEDBACK", + "DIFF ARTIFACT", + "manifest.json", + "CHECK RESULTS", + "exit=0", + "TRANSCRIPT", + "VERDICT", + "CHANGES_REQUESTED", + } { + if !strings.Contains(p, want) { + t.Errorf("prompt missing %q", want) + } + } +} + +func TestReviewToolsAreReadOnly(t *testing.T) { + reviewTools := denyTools() + if len(reviewTools) != 1 || reviewTools["*"] { + t.Fatalf("review tools = %#v, want wildcard deny", reviewTools) + } +} + +func TestOpenCodeReviewerLive(t *testing.T) { + livetest.SkipIfDisabled(t) + if _, err := exec.LookPath("opencode"); err != nil { + t.Skip("opencode binary not found") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + proj, err := os.MkdirTemp("", "corral-review-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(proj) }) + for _, args := range [][]string{{"init", "-q", "-b", "main"}, {"commit", "-q", "--allow-empty", "-m", "init"}} { + cmd := exec.Command("git", args...) + cmd.Dir = proj + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v: %s", args, err, out) + } + } + if err := os.WriteFile(filepath.Join(proj, "manifest.json"), []byte(`{"name":"x"}`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(proj, "opencode.json"), []byte(assets.OpenCodeConfigJSON), 0o644); err != nil { + t.Fatal(err) + } + + srv, err := spike.StartServer(ctx, proj, 0, os.Stderr) + if err != nil { + t.Fatalf("start server: %v", err) + } + t.Cleanup(srv.Stop) + + drv := New(ocx.New(srv.Base, proj), Options{PollInterval: 400 * time.Millisecond, Timeout: 8 * time.Minute}) + approved, note, err := drv.Review(ctx, reviewReq(proj)) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(note) == "" { + t.Error("reviewer returned an empty note") + } + t.Logf("verdict: approved=%v note=%q", approved, note) +} diff --git a/internal/sched/gates_test.go b/internal/sched/gates_test.go index 8f36f70..528105e 100644 --- a/internal/sched/gates_test.go +++ b/internal/sched/gates_test.go @@ -134,6 +134,43 @@ func TestTokenBudgetBoundsRetries(t *testing.T) { } } +func TestPreAuthorizedGateStillRequiresApprovalAction(t *testing.T) { + st := newStore(t) + clk := fakeClock() + drv := sched.NewFakeDriver(clk, scriptsFor("w")) + ver := sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}) + s := newSched(t, st, drv, ver, clk, sched.Options{Concurrency: 1}) + g := &graph.Graph{Nodes: []*graph.Node{ + agent("w"), + {ID: "gate", Type: graph.NodeHuman, Objective: "approve", Priority: graph.PriorityNormal, DependsOn: []graph.NodeID{"w"}}, + }} + h, err := s.Create(context.Background(), "run-autoapprove", g, sched.CreateOptions{AutoApproveGates: true}) + if err != nil { + t.Fatal(err) + } + drive(t, h, clk, 100) + if h.Done() { + t.Fatal("pre-authorized run bypassed its human gate") + } + if st2, _ := h.State("gate"); st2 != graph.StateRunning { + t.Fatalf("gate state = %s, want running until orchestrator approval", st2) + } + if err := h.ApproveNode(context.Background(), "gate"); err != nil { + t.Fatal(err) + } + drive(t, h, clk, 100) + if !h.Done() { + t.Fatal("pre-authorized run did not settle after approval action") + } + r, err := st.Run(context.Background(), "run-autoapprove") + if err != nil { + t.Fatal(err) + } + if r.Status != "completed" { + t.Fatalf("run status = %s, want completed", r.Status) + } +} + func TestCheckNodeRunsCommandAndRetries(t *testing.T) { st := newStore(t) clk := fakeClock() diff --git a/internal/sched/hardening_test.go b/internal/sched/hardening_test.go index 16354c1..9c828f5 100644 --- a/internal/sched/hardening_test.go +++ b/internal/sched/hardening_test.go @@ -2,14 +2,253 @@ package sched_test import ( "context" + "fmt" + "sync" "testing" + "time" "corral/internal/adapter" "corral/internal/graph" "corral/internal/sched" + "corral/internal/store" "corral/internal/verify" ) +type flakyAbortDriver struct { + mu sync.Mutex + session *flakyAbortSession + emitted bool + attempt adapter.Attempt +} + +type flakyAbortSession struct { + driver *flakyAbortDriver + abortCalls int + aborted bool +} + +func (d *flakyAbortDriver) Start(_ context.Context, attempt adapter.Attempt) (adapter.Session, error) { + d.mu.Lock() + defer d.mu.Unlock() + d.attempt = attempt + d.session = &flakyAbortSession{driver: d} + return d.session, nil +} + +func (d *flakyAbortDriver) Step(context.Context, time.Time) []adapter.Completion { + d.mu.Lock() + defer d.mu.Unlock() + if d.session == nil || !d.session.aborted || d.emitted { + return nil + } + d.emitted = true + return []adapter.Completion{{AttemptID: d.attempt.ID, SessionID: d.session.ID(), Status: adapter.StatusAborted}} +} + +func (s *flakyAbortSession) ID() string { return "ses-flaky-abort" } +func (s *flakyAbortSession) ServerID() string { return "fake" } +func (s *flakyAbortSession) Send(context.Context, string) error { return nil } +func (s *flakyAbortSession) Abort(context.Context) error { + s.driver.mu.Lock() + defer s.driver.mu.Unlock() + s.abortCalls++ + if s.abortCalls == 1 { + return fmt.Errorf("transient abort failure") + } + s.aborted = true + return nil +} +func (s *flakyAbortSession) Status(context.Context) (adapter.Status, error) { + s.driver.mu.Lock() + defer s.driver.mu.Unlock() + if s.aborted { + return adapter.StatusAborted, nil + } + return adapter.StatusRunning, nil +} +func (s *flakyAbortSession) Messages(context.Context) ([]adapter.Message, error) { return nil, nil } +func (s *flakyAbortSession) Close(context.Context) error { return nil } + +func TestBudgetAbortFailureRetriesUntilProviderStops(t *testing.T) { + st := newStore(t) + clk := fakeClock() + drv := &flakyAbortDriver{} + n := agent("w1") + n.Budget.MaxDuration = tick + s := sched.New(st, drv, sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}), clk, sched.Options{Concurrency: 1}) + h, err := s.Create(context.Background(), "run-abort-retry", &graph.Graph{Nodes: []*graph.Node{n}}) + if err != nil { + t.Fatal(err) + } + if err := h.Step(context.Background()); err != nil { + t.Fatal(err) + } + clk.Advance(2 * tick) + if err := h.Step(context.Background()); err != nil { + t.Fatal(err) + } + if state, _ := h.State("w1"); state != graph.StateRunning { + t.Fatalf("state after failed abort = %s, want running for retry", state) + } + if err := h.Step(context.Background()); err != nil { + t.Fatal(err) + } + if state, _ := h.State("w1"); state != graph.StateFailed { + t.Fatalf("state after retried abort = %s, want failed", state) + } + drv.mu.Lock() + calls := drv.session.abortCalls + drv.mu.Unlock() + if calls != 2 { + t.Fatalf("abort calls = %d, want 2", calls) + } +} + +func TestPermissionWaitPausesAttemptTimeBudget(t *testing.T) { + st := newStore(t) + clk := fakeClock() + n := agent("w1") + n.Budget.MaxDuration = 10 * tick + + drv := sched.NewFakeDriver(clk, map[string][]sched.Script{ + "w1": {{Delay: time.Hour, Permission: "perm-1"}}, + }) + s := newSched(t, st, drv, &sched.EngineVerifier{Eng: verify.New(t.TempDir())}, clk, sched.Options{Concurrency: 1}) + h, err := s.Create(context.Background(), "run-budget-pause", &graph.Graph{Nodes: []*graph.Node{n}}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + for i := 0; i < 10; i++ { + step(h, clk, ctx) + if state, _ := h.State("w1"); state == graph.StateBlocked { + break + } + } + if state, _ := h.State("w1"); state != graph.StateBlocked { + t.Fatalf("w1 = %s, want blocked on permission", state) + } + + // Time spent waiting for an operator must not consume attempt runtime. + clk.Advance(100 * tick) + ps, err := h.PermissionSession(ctx, "w1") + if err != nil { + t.Fatal(err) + } + if err := ps.RespondPermission(ctx, "perm-1", true); err != nil { + t.Fatal(err) + } + if err := h.Resume(ctx); err != nil { + t.Fatal(err) + } + step(h, clk, ctx) // resumes and re-arms the saved runtime budget + + clk.Advance(8 * tick) + if err := h.Step(ctx); err != nil { + t.Fatal(err) + } + if state, _ := h.State("w1"); state != graph.StateRunning { + t.Fatalf("w1 = %s before saved budget elapsed, want running", state) + } + + clk.Advance(2 * tick) + if err := h.Step(ctx); err != nil { + t.Fatal(err) + } + if state, _ := h.State("w1"); state != graph.StateFailed { + t.Fatalf("w1 = %s after saved budget elapsed, want failed", state) + } +} + +func TestSharedDriverRoutesCompletionsToOwningRun(t *testing.T) { + st := newStore(t) + clk := fakeClock() + drv := sched.NewFakeDriver(clk, map[string][]sched.Script{ + "a": {{Delay: tick, Messages: []adapter.Message{{Role: "assistant", Finish: "stop", Text: "a"}}}}, + "b": {{Delay: tick, Messages: []adapter.Message{{Role: "assistant", Finish: "stop", Text: "b"}}}}, + }) + engine := &sched.EngineVerifier{Eng: verify.New(t.TempDir())} + s := newSched(t, st, drv, engine, clk, sched.Options{Concurrency: 1}) + n1, n2 := agent("a"), agent("b") + n1.Verification = &graph.Verification{Kind: "command", Command: []string{"true"}} + n2.Verification = &graph.Verification{Kind: "command", Command: []string{"true"}} + h1, err := s.Create(context.Background(), "run-one", &graph.Graph{Nodes: []*graph.Node{n1}}) + if err != nil { + t.Fatal(err) + } + h2, err := s.Create(context.Background(), "run-two", &graph.Graph{Nodes: []*graph.Node{n2}}) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + if err := h1.Step(ctx); err != nil { + t.Fatal(err) + } + if err := h2.Step(ctx); err != nil { + t.Fatal(err) + } + clk.Advance(tick) + + // One Stepper call drains both completions. Each must be retained for its + // owning handle instead of making whichever run stepped first fail. + if err := h1.Step(ctx); err != nil { + t.Fatalf("run one consumed another run's completion: %v", err) + } + if state, _ := h1.State("a"); state != graph.StateDone { + t.Fatalf("run one node = %s, want done", state) + } + if state, _ := h2.State("b"); state != graph.StateRunning { + t.Fatalf("run two node changed before its handle stepped: %s", state) + } + if err := h2.Step(ctx); err != nil { + t.Fatalf("run two lost its routed completion: %v", err) + } + if state, _ := h2.State("b"); state != graph.StateDone { + t.Fatalf("run two node = %s, want done", state) + } +} + +func TestCompletionBurstBeyondLegacyBuffer(t *testing.T) { + st := newStore(t) + clk := fakeClock() + const count = 40 + nodes := make([]*graph.Node, 0, count) + scripts := make(map[string][]sched.Script, count) + for i := 0; i < count; i++ { + id := graph.NodeID(fmt.Sprintf("n%02d", i)) + n := agent(id) + n.Verification = &graph.Verification{Kind: "command", Command: []string{"true"}} + nodes = append(nodes, n) + scripts[string(id)] = []sched.Script{{Delay: tick}} + } + drv := sched.NewFakeDriver(clk, scripts) + s := newSched(t, st, drv, &sched.EngineVerifier{Eng: verify.New(t.TempDir())}, clk, sched.Options{Concurrency: count}) + h, err := s.Create(context.Background(), "run-burst", &graph.Graph{Nodes: nodes}) + if err != nil { + t.Fatal(err) + } + if err := h.Step(context.Background()); err != nil { + t.Fatal(err) + } + clk.Advance(tick) + done := make(chan error, 1) + go func() { done <- h.Step(context.Background()) }() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("completion burst blocked while routing more than 32 results") + } + for _, n := range nodes { + if state, _ := h.State(n.ID); state != graph.StateDone { + t.Fatalf("%s = %s, want done", n.ID, state) + } + } +} + // TestPermissionRequestBlocksExplicitly drives the permission flow: the // node moves to an explicit blocked state while the session waits, then // resumes automatically after the operator answers, and completes. @@ -174,4 +413,235 @@ func TestRunBudgetBlocksNewWork(t *testing.T) { } } +func TestLoadRestoresRunBudgetAndRetryCannotBypassIt(t *testing.T) { + st := newStore(t) + clk := fakeClock() + ctx := context.Background() + a, z := agent("a"), agent("z") + g := &graph.Graph{Nodes: []*graph.Node{a, z}} + seed := newSched(t, st, sched.NewFakeDriver(clk, nil), sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}), clk, sched.Options{}) + if _, err := seed.Create(ctx, "run-budget-reload", g); err != nil { + t.Fatal(err) + } + persistTerminalAttempt(t, st, clk.Now(), "run-budget-reload", "a", graph.StateDone, 1.25, 100) + + drv := sched.NewFakeDriver(clk, scriptsFor("z")) + s := newSched(t, st, drv, sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}), clk, sched.Options{ + Concurrency: 1, RunMaxTokens: 50, + }) + h, err := s.Load(ctx, "run-budget-reload") + if err != nil { + t.Fatal(err) + } + if err := h.Step(ctx); err != nil { + t.Fatal(err) + } + if got, _ := h.State("z"); got != graph.StateBlocked { + t.Fatalf("z = %s after reload, want blocked by restored run budget", got) + } + if err := h.RetryNode(ctx, "z"); err != nil { + t.Fatal(err) + } + if err := h.Step(ctx); err != nil { + t.Fatal(err) + } + if got, _ := h.State("z"); got != graph.StateBlocked { + t.Fatalf("z = %s after retry, want run budget to block it again", got) + } + if attempts, _ := st.CountAttempts(ctx, "run-budget-reload", "z"); attempts != 0 { + t.Fatalf("z started despite restored run budget: %d attempts", attempts) + } +} + +func TestLoadRestoresBreakerAndOperatorRetryResetsHistory(t *testing.T) { + st := newStore(t) + clk := fakeClock() + ctx := context.Background() + a, z := agent("a"), agent("z") + g := &graph.Graph{Nodes: []*graph.Node{a, z}} + seed := newSched(t, st, sched.NewFakeDriver(clk, nil), sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}), clk, sched.Options{}) + if _, err := seed.Create(ctx, "run-breaker-reload", g); err != nil { + t.Fatal(err) + } + persistTerminalAttempt(t, st, clk.Now(), "run-breaker-reload", "a", graph.StateFailed, 0, 0) + + opts := sched.Options{Concurrency: 1, BreakerMaxFailures: 1, BreakerWindow: time.Hour} + drv := sched.NewFakeDriver(clk, scriptsFor("z")) + s := newSched(t, st, drv, sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}), clk, opts) + h, err := s.Load(ctx, "run-breaker-reload") + if err != nil { + t.Fatal(err) + } + if err := h.Step(ctx); err != nil { + t.Fatal(err) + } + if got, _ := h.State("z"); got != graph.StateBlocked { + t.Fatalf("z = %s after reload, want blocked by restored breaker", got) + } + if err := h.RetryNode(ctx, "z"); err != nil { + t.Fatal(err) + } + + // Retry reset is durable: a fresh handle must not reconstruct failures + // from before the operator override. + drv2 := sched.NewFakeDriver(clk, scriptsFor("z")) + s2 := newSched(t, st, drv2, sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}), clk, opts) + h2, err := s2.Load(ctx, "run-breaker-reload") + if err != nil { + t.Fatal(err) + } + if err := h2.Step(ctx); err != nil { + t.Fatal(err) + } + if got, _ := h2.State("z"); got != graph.StateRunning { + t.Fatalf("z = %s after durable breaker reset, want running", got) + } +} + +func TestLoadIgnoresHistoricalRetryForTerminalNode(t *testing.T) { + st := newStore(t) + clk := fakeClock() + ctx := context.Background() + n := agent("w1") + seed := newSched(t, st, sched.NewFakeDriver(clk, nil), sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}), clk, sched.Options{}) + if _, err := seed.Create(ctx, "run-terminal-retry", &graph.Graph{Nodes: []*graph.Node{n}}); err != nil { + t.Fatal(err) + } + now := clk.Now() + for _, edge := range [][2]graph.State{ + {graph.StatePending, graph.StateReady}, + {graph.StateReady, graph.StateLeased}, + {graph.StateLeased, graph.StateRunning}, + {graph.StateRunning, graph.StateVerifying}, + {graph.StateVerifying, graph.StateRetryWait}, + } { + if _, err := st.AppendTransition(ctx, "run-terminal-retry", "w1", edge[0], edge[1], "", now); err != nil { + t.Fatal(err) + } + } + oldReady := now.Add(-time.Hour).UnixMilli() + if _, err := st.AppendEvent(ctx, "run-terminal-retry", "w1", store.EventRetry, "", "", "", fmt.Sprintf(`{"readyAt":%d}`, oldReady), now); err != nil { + t.Fatal(err) + } + for _, edge := range [][2]graph.State{ + {graph.StateRetryWait, graph.StateReady}, + {graph.StateReady, graph.StateLeased}, + {graph.StateLeased, graph.StateRunning}, + {graph.StateRunning, graph.StateVerifying}, + {graph.StateVerifying, graph.StateDone}, + } { + if _, err := st.AppendTransition(ctx, "run-terminal-retry", "w1", edge[0], edge[1], "", now); err != nil { + t.Fatal(err) + } + } + + s := newSched(t, st, sched.NewFakeDriver(clk, nil), sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}), clk, sched.Options{}) + h, err := s.Load(ctx, "run-terminal-retry") + if err != nil { + t.Fatal(err) + } + if err := h.Step(ctx); err != nil { + t.Fatalf("historical retry corrupted terminal replay: %v", err) + } + if state, _ := h.State("w1"); state != graph.StateDone { + t.Fatalf("w1 = %s, want done", state) + } +} + +func TestLoadUsesLatestRetryDeadline(t *testing.T) { + st := newStore(t) + clk := fakeClock() + ctx := context.Background() + n := agent("w1") + seed := newSched(t, st, sched.NewFakeDriver(clk, nil), sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}), clk, sched.Options{}) + if _, err := seed.Create(ctx, "run-latest-retry", &graph.Graph{Nodes: []*graph.Node{n}}); err != nil { + t.Fatal(err) + } + now := clk.Now() + for _, edge := range [][2]graph.State{ + {graph.StatePending, graph.StateReady}, + {graph.StateReady, graph.StateLeased}, + {graph.StateLeased, graph.StateRunning}, + {graph.StateRunning, graph.StateVerifying}, + {graph.StateVerifying, graph.StateRetryWait}, + } { + if _, err := st.AppendTransition(ctx, "run-latest-retry", "w1", edge[0], edge[1], "", now); err != nil { + t.Fatal(err) + } + } + for _, readyAt := range []int64{now.Add(-time.Hour).UnixMilli(), now.Add(time.Hour).UnixMilli()} { + if _, err := st.AppendEvent(ctx, "run-latest-retry", "w1", store.EventRetry, "", "", "", fmt.Sprintf(`{"readyAt":%d}`, readyAt), now); err != nil { + t.Fatal(err) + } + } + s := newSched(t, st, sched.NewFakeDriver(clk, nil), sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}), clk, sched.Options{}) + h, err := s.Load(ctx, "run-latest-retry") + if err != nil { + t.Fatal(err) + } + if err := h.Step(ctx); err != nil { + t.Fatal(err) + } + if state, _ := h.State("w1"); state != graph.StateRetryWait { + t.Fatalf("w1 = %s, want retry_wait until latest deadline", state) + } +} + +func TestRetryDependentWithFailedDependencyReblocksCleanly(t *testing.T) { + st := newStore(t) + clk := fakeClock() + a := agent("a") + a.RetryPolicy.MaxRetries = 0 + b := agent("b", "a") + drv := sched.NewFakeDriver(clk, scriptsFor("a")) + ver := sched.NewFakeVerifier(map[string][]sched.Verdict{ + "a": {{Pass: false, Feedback: "fail"}}, + }, sched.Verdict{Pass: true}) + s := newSched(t, st, drv, ver, clk, sched.Options{Concurrency: 1}) + h, err := s.Create(context.Background(), "run-dependent-retry", &graph.Graph{Nodes: []*graph.Node{a, b}}) + if err != nil { + t.Fatal(err) + } + drive(t, h, clk, 20) + if got, _ := h.State("b"); got != graph.StateBlocked { + t.Fatalf("b = %s, want blocked", got) + } + if err := h.RetryNode(context.Background(), "b"); err != nil { + t.Fatal(err) + } + if err := h.Step(context.Background()); err != nil { + t.Fatalf("reblock retried dependent: %v", err) + } + if got, _ := h.State("b"); got != graph.StateBlocked { + t.Fatalf("b = %s after retry, want blocked while dependency failed", got) + } +} + +func persistTerminalAttempt(t *testing.T, st *store.Store, now time.Time, runID string, nodeID graph.NodeID, terminal graph.State, cost float64, tokens int) { + t.Helper() + ctx := context.Background() + states := []graph.State{graph.StateReady, graph.StateLeased, graph.StateRunning, graph.StateVerifying, terminal} + from := graph.StatePending + for _, to := range states { + if _, err := st.AppendEvent(ctx, runID, string(nodeID), store.EventTransition, from, to, "", "", now); err != nil { + t.Fatal(err) + } + from = to + } + if err := st.SetNodeState(ctx, runID, string(nodeID), terminal, now); err != nil { + t.Fatal(err) + } + started, finished := now.Add(-tick).UnixMilli(), now.UnixMilli() + status := "done" + if terminal == graph.StateFailed { + status = "failed" + } + if err := st.RecordAttempt(ctx, store.Attempt{ + ID: runID + "/" + string(nodeID) + "/1", RunID: runID, NodeID: string(nodeID), No: 1, + Status: status, StartedAt: &started, FinishedAt: &finished, Cost: cost, Tokens: tokens, + }); err != nil { + t.Fatal(err) + } +} + var _ = sched.Verdict{} diff --git a/internal/sched/sched.go b/internal/sched/sched.go index 6fa4e4b..e847fb5 100644 --- a/internal/sched/sched.go +++ b/internal/sched/sched.go @@ -10,6 +10,7 @@ import ( "encoding/json" "fmt" "sort" + "strings" "sync" "time" @@ -22,7 +23,6 @@ import ( ) const ( - resultsBuffer = 32 runStatusDone = "completed" runStatusWaiting = "waiting" ) @@ -85,6 +85,11 @@ type Scheduler struct { ver Verifier clk clock.Clock opts Options + + completionMu sync.Mutex + owners map[string]*RunHandle + pending map[*RunHandle][]Result + orphans map[string][]Result } func New(st *store.Store, drv adapter.Driver, ver Verifier, clk clock.Clock, opts Options) *Scheduler { @@ -94,7 +99,11 @@ func New(st *store.Store, drv adapter.Driver, ver Verifier, clk clock.Clock, opt if opts.LeaseTTL <= 0 { opts.LeaseTTL = 30 * time.Second } - return &Scheduler{store: st, drv: drv, ver: ver, clk: clk, opts: opts} + return &Scheduler{ + store: st, drv: drv, ver: ver, clk: clk, opts: opts, + owners: map[string]*RunHandle{}, pending: map[*RunHandle][]Result{}, + orphans: map[string][]Result{}, + } } type sessionRec struct { @@ -104,6 +113,8 @@ type sessionRec struct { sess adapter.Session deadline time.Time budgeted bool // time budget active + budgetPaused bool + budgetRemain time.Duration abortIsBudget bool // abort was initiated by the scheduler (budget), not operator worktree string branch string @@ -126,20 +137,34 @@ type RunHandle struct { runCost float64 runTokens int breaker bool - results chan Result + results []Result holder string done bool stepCount int64 started time.Time } +// CreateOptions carries per-run creation policies (extended without +// breaking existing callers, which keep compiling with no options). +type CreateOptions struct { + // AutoApproveGates marks the run as pre-authorized: the orchestrator + // agent approves human gates itself as they are reached, without + // waiting for the operator. When false the orchestrator must never + // approve gates on its own. + AutoApproveGates bool +} + // Create starts a new run for g. -func (s *Scheduler) Create(ctx context.Context, runID string, g *graph.Graph) (*RunHandle, error) { +func (s *Scheduler) Create(ctx context.Context, runID string, g *graph.Graph, opts ...CreateOptions) (*RunHandle, error) { if err := graph.Validate(g); err != nil { return nil, err } + var o CreateOptions + if len(opts) > 0 { + o = opts[0] + } now := s.clk.Now() - if err := s.store.CreateRun(ctx, runID, g, now); err != nil { + if err := s.store.CreateRun(ctx, runID, g, o.AutoApproveGates, now); err != nil { return nil, err } return s.newHandle(ctx, runID, g, now) @@ -220,26 +245,49 @@ func (s *Scheduler) Load(ctx context.Context, runID string) (*RunHandle, error) retryAt: map[graph.NodeID]time.Time{}, age: map[graph.NodeID]int{}, feedback: map[graph.NodeID]string{}, - results: make(chan Result, resultsBuffer), holder: fmt.Sprintf("corral-%d", now.UnixNano()), started: now, } + for _, n := range r.Graph.Nodes { + cost, tokens, err := s.store.NodeCost(ctx, runID, string(n.ID)) + if err != nil { + return nil, err + } + h.runCost += cost + h.runTokens += tokens + } for _, ev := range events { - if ev.Type == store.EventRetry { - var p struct { - ReadyAt int64 `json:"readyAt"` - } - if json.Unmarshal(ev.Payload, &p) == nil && p.ReadyAt > 0 { - h.retryAt[graph.NodeID(ev.NodeID)] = time.UnixMilli(p.ReadyAt) - } + if operatorRetryEvent(ev) { + // Operator retry is the durable circuit-breaker reset marker. + h.failures = nil + } + if ev.Type == store.EventTransition && ev.To == graph.StateFailed { + h.failures = append(h.failures, time.UnixMilli(ev.CreatedAt)) + } + } + for _, n := range r.Graph.Nodes { + state, _ := tr.State(n.ID) + if state != graph.StateRetryWait { + continue + } + if readyAt, ok := retryReadyAt(events, n.ID); ok { + h.retryAt[n.ID] = readyAt } } return h, nil } -// retryReadyAt finds the scheduled retry time from retry events. +func operatorRetryEvent(ev store.Event) bool { + var payload struct { + Reason string `json:"reason"` + } + return json.Unmarshal(ev.Payload, &payload) == nil && payload.Reason == "operator retry" +} + +// retryReadyAt finds the latest scheduled retry time from retry events. func retryReadyAt(events []store.Event, nodeID graph.NodeID) (time.Time, bool) { - for _, ev := range events { + for i := len(events) - 1; i >= 0; i-- { + ev := events[i] if graph.NodeID(ev.NodeID) != nodeID || ev.Type != store.EventRetry { continue } @@ -268,12 +316,66 @@ func (s *Scheduler) newHandle(ctx context.Context, runID string, g *graph.Graph, retryAt: map[graph.NodeID]time.Time{}, age: map[graph.NodeID]int{}, feedback: map[graph.NodeID]string{}, - results: make(chan Result, resultsBuffer), holder: fmt.Sprintf("corral-%d", now.UnixNano()), started: now, }, nil } +func completionResult(c adapter.Completion) Result { + return Result{ + AttemptID: c.AttemptID, SessionID: c.SessionID, Status: c.Status, + Messages: c.Messages, Err: c.Err, Budget: c.Budget, + } +} + +// registerAttempt binds a provider completion to its owning run. A different +// run may drain the shared driver between Start returning and this call, so +// completions observed in that window are retained as orphans and claimed now. +func (s *Scheduler) registerAttempt(h *RunHandle, attemptID string) { + s.completionMu.Lock() + defer s.completionMu.Unlock() + s.owners[attemptID] = h + if queued := s.orphans[attemptID]; len(queued) > 0 { + s.pending[h] = append(s.pending[h], queued...) + delete(s.orphans, attemptID) + } +} + +func (s *Scheduler) unregisterAttempt(h *RunHandle, attemptID string) { + s.completionMu.Lock() + defer s.completionMu.Unlock() + if s.owners[attemptID] == h { + delete(s.owners, attemptID) + } +} + +// collectCompletions drains the shared provider once and routes every result +// to its owning RunHandle. Calls from concurrent run loops are serialized. +func (s *Scheduler) collectCompletions(ctx context.Context, now time.Time) { + stepper, ok := s.drv.(adapter.Stepper) + if !ok { + return + } + s.completionMu.Lock() + defer s.completionMu.Unlock() + for _, completion := range stepper.Step(ctx, now) { + result := completionResult(completion) + if owner := s.owners[completion.AttemptID]; owner != nil { + s.pending[owner] = append(s.pending[owner], result) + } else { + s.orphans[completion.AttemptID] = append(s.orphans[completion.AttemptID], result) + } + } +} + +func (s *Scheduler) takeCompletions(h *RunHandle) []Result { + s.completionMu.Lock() + defer s.completionMu.Unlock() + results := s.pending[h] + delete(s.pending, h) + return results +} + // Step advances the run by one deterministic step. func (h *RunHandle) Step(ctx context.Context) error { h.mu.Lock() @@ -296,9 +398,15 @@ func (h *RunHandle) Step(ctx context.Context) error { // 2. Budget deadlines: abort attempts that exceeded their time budget. for _, rec := range h.sessions { if rec.budgeted && !rec.deadline.IsZero() && now.After(rec.deadline) { - _ = rec.sess.Abort(ctx) - rec.budgeted = false // abort already requested; completion arrives via results rec.abortIsBudget = true + if err := rec.sess.Abort(ctx); err != nil { + // Keep the deadline armed. Provider abort failures must be retried; + // otherwise one transient error lets an over-budget attempt run + // indefinitely. A natural completion racing this failure is still + // rejected below because abortIsBudget remains set. + continue + } + rec.budgeted = false // abort accepted; completion arrives via results } } @@ -316,8 +424,19 @@ func (h *RunHandle) Step(ctx context.Context) error { } delete(h.sessions, id) h.suspended[id] = rec + if rec.budgeted && !rec.deadline.IsZero() { + rec.budgetRemain = max(rec.deadline.Sub(now), 0) + rec.budgetPaused = true + } rec.budgeted = false - payload, _ := json.Marshal(map[string]any{"reason": "permission", "permissionID": pid}) + permission := map[string]any{"reason": "permission", "permissionID": pid} + if info, ok := rec.sess.(adapter.PermissionInfo); ok { + if detail, detailOK, detailErr := info.PendingPermissionDetails(ctx); detailErr == nil && detailOK { + permission["tool"] = detail.Tool + permission["input"] = detail.Input + } + } + payload, _ := json.Marshal(permission) if err := h.transit(ctx, id, graph.StateRunning, graph.StateBlocked, string(payload)); err != nil { return err } @@ -335,10 +454,13 @@ func (h *RunHandle) Step(ctx context.Context) error { } delete(h.suspended, id) h.sessions[id] = rec - if n := h.nodeByID(id); n != nil && n.Budget.MaxDuration > 0 { - // Re-arm the time budget with the remaining time. - rec.deadline = now.Add(time.Until(rec.deadline).Round(0)) + if rec.budgetPaused { + // Permission waits do not consume provider runtime. Re-arm against + // the scheduler clock with the budget left when the node blocked. + rec.deadline = now.Add(rec.budgetRemain) rec.budgeted = true + rec.budgetPaused = false + rec.budgetRemain = 0 } if h.done { h.done = false @@ -349,34 +471,19 @@ func (h *RunHandle) Step(ctx context.Context) error { } } - // 3. Cooperative driver completions. - if stepper, ok := h.s.drv.(adapter.Stepper); ok { - for _, c := range stepper.Step(ctx, now) { - h.results <- Result{ - AttemptID: c.AttemptID, - SessionID: c.SessionID, - Status: c.Status, - Messages: c.Messages, - Err: c.Err, - Budget: c.Budget, - } - } - } + // 3. Cooperative driver completions. The driver is shared by all runs; + // route results centrally so one run cannot consume another's attempt. + h.s.collectCompletions(ctx, now) + h.results = append(h.results, h.s.takeCompletions(h)...) // 4. Drain and handle results (completions from any source). - handled := 0 - for { - select { - case res := <-h.results: - handled++ - if err := h.finishAttempt(ctx, res); err != nil { - return err - } - default: - goto drained + for len(h.results) > 0 { + res := h.results[0] + h.results = h.results[1:] + if err := h.finishAttempt(ctx, res); err != nil { + return err } } -drained: // 5. Block permanently-unrunnable nodes. @@ -398,8 +505,8 @@ drained: if h.breaker { for _, n := range h.g.Nodes { st, _ := h.tr.State(n.ID) - if st == graph.StatePending { - if err := h.transit(ctx, n.ID, graph.StatePending, graph.StateBlocked, `{"reason":"circuit breaker"}`); err != nil { + if st == graph.StatePending || st == graph.StateReady { + if err := h.transit(ctx, n.ID, st, graph.StateBlocked, `{"reason":"circuit breaker"}`); err != nil { return err } } @@ -411,8 +518,8 @@ drained: h.s.opts.RunMaxCost > 0 && h.runCost >= h.s.opts.RunMaxCost { for _, n := range h.g.Nodes { st, _ := h.tr.State(n.ID) - if st == graph.StatePending { - if err := h.transit(ctx, n.ID, graph.StatePending, graph.StateBlocked, `{"reason":"run budget exceeded"}`); err != nil { + if st == graph.StatePending || st == graph.StateReady { + if err := h.transit(ctx, n.ID, st, graph.StateBlocked, `{"reason":"run budget exceeded"}`); err != nil { return err } } @@ -421,7 +528,8 @@ drained: ready, blocked := graph.ComputeReady(h.g, h.tr) for _, n := range blocked { - if err := h.transit(ctx, n.ID, graph.StatePending, graph.StateBlocked, ""); err != nil { + state, _ := h.tr.State(n.ID) + if err := h.transit(ctx, n.ID, state, graph.StateBlocked, ""); err != nil { return err } } @@ -488,6 +596,9 @@ func (h *RunHandle) transit(ctx context.Context, id graph.NodeID, from, to graph if _, err := h.s.store.AppendTransition(ctx, h.runID, string(id), from, to, payload, h.s.clk.Now()); err != nil { return err } + if to == graph.StateFailed { + h.failures = append(h.failures, h.s.clk.Now()) + } return nil } @@ -514,7 +625,7 @@ func (h *RunHandle) startAttempt(ctx context.Context, n *graph.Node) error { return err } no++ - attemptID := fmt.Sprintf("%s/%d", n.ID, no) + attemptID := fmt.Sprintf("%s/%s/%d", h.runID, n.ID, no) switch n.Type { case graph.NodeCheck: return h.startCheck(ctx, n, attemptID, no, now) @@ -562,6 +673,7 @@ func (h *RunHandle) startAttempt(ctx context.Context, n *graph.Node) error { } return nil } + h.s.registerAttempt(h, attemptID) started := now.UnixMilli() h.sessions[n.ID] = &sessionRec{ nodeID: n.ID, @@ -653,7 +765,7 @@ func (h *RunHandle) startCheck(ctx context.Context, n *graph.Node, attemptID str "stderr": stderr, }, } - h.results <- Result{AttemptID: attemptID, SessionID: sess.ID(), Status: adapter.StatusIdle, Messages: []adapter.Message{msg}} + h.results = append(h.results, Result{AttemptID: attemptID, SessionID: sess.ID(), Status: adapter.StatusIdle, Messages: []adapter.Message{msg}}) return h.emitEvent(ctx, store.EventAttempt, n.ID, graph.State(""), graph.State(""), attemptID, `{"phase":"start","sessionID":"`+sess.ID()+`"}`) } @@ -787,8 +899,9 @@ func (h *RunHandle) startMerge(ctx context.Context, n *graph.Node, attemptID str return h.completeInline(ctx, n, attemptID, no, now, exit, stdout, stderr, merged) } -// startGate parks a human gate node in running until an operator approves -// or rejects it. No driver session is involved. +// startGate parks a human gate node in running until an operator or an +// explicitly pre-authorized orchestrator approves or rejects it. No driver +// session is involved; pre-authorization is policy metadata, not a bypass. func (h *RunHandle) startGate(ctx context.Context, n *graph.Node, attemptID string, no int, now time.Time) error { sess := &gateSession{id: "gate:" + string(n.ID)} started := now.UnixMilli() @@ -842,7 +955,7 @@ func (h *RunHandle) completeInline(ctx context.Context, n *graph.Node, attemptID "stderr": stderr, }, } - h.results <- Result{AttemptID: attemptID, SessionID: sess.ID(), Status: adapter.StatusIdle, Messages: []adapter.Message{msg}} + h.results = append(h.results, Result{AttemptID: attemptID, SessionID: sess.ID(), Status: adapter.StatusIdle, Messages: []adapter.Message{msg}}) return h.emitEvent(ctx, store.EventAttempt, n.ID, graph.State(""), graph.State(""), attemptID, `{"phase":"start","sessionID":"`+sess.ID()+`"}`) } @@ -883,7 +996,6 @@ func (h *RunHandle) RetryNode(ctx context.Context, id graph.NodeID) error { return fmt.Errorf("unknown node %s", id) } now := h.s.clk.Now() - h.breaker = false switch st { case graph.StateBlocked: if err := h.transit(ctx, id, graph.StateBlocked, graph.StateReady, `{"reason":"operator retry"}`); err != nil { @@ -917,6 +1029,8 @@ func (h *RunHandle) RetryNode(ctx context.Context, id graph.NodeID) error { default: return fmt.Errorf("node %s is in state %s and cannot be retried", id, st) } + h.breaker = false + h.failures = nil // A settled run must be re-activated. if h.done { h.done = false @@ -945,6 +1059,8 @@ func (h *RunHandle) PermissionSession(_ context.Context, id graph.NodeID) (adapt } // Steer sends a message to the in-flight attempt of a node (agent steer). +// The steering action is recorded in the event log so it participates in +// the monotonic per-run sequence streamed by the live-event endpoint. func (h *RunHandle) Steer(ctx context.Context, id graph.NodeID, message string) error { h.mu.Lock() defer h.mu.Unlock() @@ -952,7 +1068,61 @@ func (h *RunHandle) Steer(ctx context.Context, id graph.NodeID, message string) if !ok { return fmt.Errorf("node %s has no in-flight attempt", id) } - return rec.sess.Send(ctx, message) + if err := rec.sess.Send(ctx, message); err != nil { + return err + } + payload, _ := json.Marshal(map[string]any{"message": message}) + _, err := h.s.store.AppendEvent(ctx, h.runID, string(id), store.EventSteer, graph.State(""), graph.State(""), rec.attemptID, string(payload), h.s.clk.Now()) + return err +} + +// Tail returns the last n transcript lines of the in-flight attempt of a +// node, for live output in the companion TUI. It reports an error when the +// node has no live session (e.g. inline check/gate nodes). +func (h *RunHandle) Tail(ctx context.Context, id graph.NodeID, n int) ([]string, error) { + if id == "" { + return nil, fmt.Errorf("node required") + } + if n < 1 || n > 500 { + return nil, fmt.Errorf("lines must be between 1 and 500") + } + h.mu.Lock() + rec := h.sessions[id] + if rec == nil { + rec = h.suspended[id] + } + if rec == nil { + h.mu.Unlock() + return nil, fmt.Errorf("node %s has no in-flight attempt", id) + } + // Session calls may perform network I/O. Copy the stable interface + // reference while protected, then release the scheduler mutex before + // waiting on the provider. + sess := rec.sess + h.mu.Unlock() + msgs, err := sess.Messages(ctx) + if err != nil { + return nil, err + } + return transcriptLines(msgs, n), nil +} + +// transcriptLines flattens a session transcript into text lines and keeps +// only the last n of them (a "tail"). +func transcriptLines(msgs []adapter.Message, n int) []string { + lines := make([]string, 0) + for _, m := range msgs { + for _, ln := range strings.Split(m.Text, "\n") { + if ln == "" { + continue + } + lines = append(lines, ln) + } + } + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return lines } func (h *RunHandle) decideGate(ctx context.Context, id graph.NodeID, approve bool) error { @@ -995,7 +1165,17 @@ func (h *RunHandle) finishAttempt(ctx context.Context, res Result) error { if rec == nil { return fmt.Errorf("result for unknown attempt %s", res.AttemptID) } + h.s.unregisterAttempt(h, res.AttemptID) res.Budget = res.Budget || rec.abortIsBudget + if res.Budget && res.Status == adapter.StatusIdle { + // The provider may finish naturally while a budget abort is being + // retried. It is now safe to stop tracking, but work completed after the + // deadline cannot pass verification as an on-time success. + res.Status = adapter.StatusError + if res.Err == nil { + res.Err = fmt.Errorf("time budget exceeded") + } + } now := h.s.clk.Now() delete(h.sessions, rec.nodeID) delete(h.suspended, rec.nodeID) @@ -1027,6 +1207,9 @@ func (h *RunHandle) finishAttempt(ctx context.Context, res Result) error { case adapter.StatusAborted, adapter.StatusError: to := graph.StateFailed reason := `"aborted"` + if res.Status == adapter.StatusError { + reason = `"provider error"` + } if res.Err != nil { reason = `"` + jsonEscape(res.Err.Error()) + `"` } @@ -1057,9 +1240,29 @@ func (h *RunHandle) finishAttempt(ctx context.Context, res Result) error { return err } node := h.nodeByID(rec.nodeID) - verdict, err := h.s.ver.Verdict(ctx, node, rec.no, rec.worktree, res.Messages) - if err != nil { - verdict = Verdict{Pass: false, Feedback: "verifier error: " + err.Error()} + var verdict Verdict + if rec.worktree != "" && h.s.opts.Worktrees != nil && worktree.NodeIsWriting(string(node.Type), node.Role) { + files, scopeErr := h.s.opts.Worktrees.ChangedFiles(ctx, rec.worktree) + if scopeErr != nil { + verdict = Verdict{Pass: false, Feedback: "scope inspection failed: " + scopeErr.Error()} + } else { + var outside []string + for _, file := range files { + if !worktree.ScopeContains(node.WriteScope, file) { + outside = append(outside, file) + } + } + if len(outside) > 0 { + verdict = Verdict{Pass: false, Feedback: "attempt changed paths outside write scope: " + strings.Join(outside, ", ")} + } + } + } + if verdict.Feedback == "" { + var err error + verdict, err = h.s.ver.Verdict(ctx, node, rec.no, rec.worktree, res.Messages) + if err != nil { + verdict = Verdict{Pass: false, Feedback: "verifier error: " + err.Error()} + } } ev, _ := json.Marshal(map[string]any{"pass": verdict.Pass, "feedback": verdict.Feedback, "evidence": verdict.Evidence}) if err := h.emitEvent(ctx, store.EventVerdict, rec.nodeID, graph.State(""), graph.State(""), rec.attemptID, string(ev)); err != nil { @@ -1123,7 +1326,6 @@ func (h *RunHandle) finishAttempt(ctx context.Context, res Result) error { FinishedAt: &finished, Evidence: verdict.Feedback, Cost: cost, Tokens: tokens, }) } - h.failures = append(h.failures, now) if err := h.transit(ctx, rec.nodeID, graph.StateVerifying, graph.StateFailed, ""); err != nil { return err } diff --git a/internal/sched/sched_test.go b/internal/sched/sched_test.go index 0b54b35..f6a52b6 100644 --- a/internal/sched/sched_test.go +++ b/internal/sched/sched_test.go @@ -302,6 +302,40 @@ func TestRetryExhaustedFails(t *testing.T) { } } +func TestAttemptIDsAreUniqueAcrossRuns(t *testing.T) { + st := newStore(t) + clk := fakeClock() + ctx := context.Background() + + for _, runID := range []string{"run-first", "run-second"} { + drv := sched.NewFakeDriver(clk, scriptsFor("worker")) + ver := sched.NewFakeVerifier(nil, sched.Verdict{Pass: true}) + s := newSched(t, st, drv, ver, clk, sched.Options{Concurrency: 1}) + h, err := s.Create(ctx, runID, &graph.Graph{Nodes: []*graph.Node{agent("worker")}}) + if err != nil { + t.Fatal(err) + } + drive(t, h, clk, 10) + if !h.Done() { + t.Fatalf("%s did not complete", runID) + } + } + + for _, runID := range []string{"run-first", "run-second"} { + attempts, err := st.Attempts(ctx, runID, "worker") + if err != nil { + t.Fatal(err) + } + if len(attempts) != 1 { + t.Fatalf("%s attempts = %d, want 1", runID, len(attempts)) + } + wantID := runID + "/worker/1" + if attempts[0].ID != wantID { + t.Fatalf("%s attempt ID = %q, want %q", runID, attempts[0].ID, wantID) + } + } +} + func TestBudgetAbortFails(t *testing.T) { st := newStore(t) clk := fakeClock() diff --git a/internal/sched/tail_test.go b/internal/sched/tail_test.go new file mode 100644 index 0000000..b2f2918 --- /dev/null +++ b/internal/sched/tail_test.go @@ -0,0 +1,84 @@ +package sched + +import ( + "context" + "testing" + "time" + + "corral/internal/adapter" + "corral/internal/graph" +) + +type blockingMessagesSession struct { + entered chan struct{} + release chan struct{} +} + +func (s *blockingMessagesSession) ID() string { return "session" } +func (s *blockingMessagesSession) ServerID() string { return "server" } +func (s *blockingMessagesSession) Send(context.Context, string) error { + return nil +} +func (s *blockingMessagesSession) Abort(context.Context) error { return nil } +func (s *blockingMessagesSession) Status(context.Context) (adapter.Status, error) { + return adapter.StatusRunning, nil +} +func (s *blockingMessagesSession) Messages(context.Context) ([]adapter.Message, error) { + close(s.entered) + <-s.release + return []adapter.Message{{Text: "tail"}}, nil +} +func (s *blockingMessagesSession) Close(context.Context) error { return nil } + +func TestTailDoesNotHoldRunMutexWhileFetchingMessages(t *testing.T) { + sess := &blockingMessagesSession{entered: make(chan struct{}), release: make(chan struct{})} + h := &RunHandle{sessions: map[graph.NodeID]*sessionRec{ + "node": {nodeID: "node", sess: sess}, + }} + tailDone := make(chan error, 1) + go func() { + _, err := h.Tail(context.Background(), "node", 10) + tailDone <- err + }() + select { + case <-sess.entered: + case <-time.After(time.Second): + t.Fatal("Tail did not call Messages") + } + + mutexAvailable := make(chan struct{}) + go func() { + _ = h.ActiveSessions() + close(mutexAvailable) + }() + select { + case <-mutexAvailable: + // Good: Messages is still blocked, but scheduler mutex is free. + case <-time.After(100 * time.Millisecond): + close(sess.release) + t.Fatal("Tail held RunHandle mutex across Messages I/O") + } + close(sess.release) + if err := <-tailDone; err != nil { + t.Fatal(err) + } +} + +func TestTailRejectsInvalidLineCount(t *testing.T) { + h := &RunHandle{} + if _, err := h.Tail(context.Background(), "", 100); err == nil { + t.Fatal("Tail accepted empty node ID") + } + for _, lines := range []int{-1, 0, 501} { + if _, err := h.Tail(context.Background(), "node", lines); err == nil { + t.Fatalf("Tail accepted lines=%d", lines) + } + } +} + +func TestTranscriptLinesEmptyReturnsEmptySlice(t *testing.T) { + lines := transcriptLines(nil, 10) + if lines == nil || len(lines) != 0 { + t.Fatalf("transcriptLines(nil) = %#v, want non-nil empty slice", lines) + } +} diff --git a/internal/sched/worktree_test.go b/internal/sched/worktree_test.go index a1b176e..20b222e 100644 --- a/internal/sched/worktree_test.go +++ b/internal/sched/worktree_test.go @@ -236,6 +236,27 @@ func TestFailedWorktreeRetainedForInspection(t *testing.T) { } } +func TestOutOfScopeWorktreeChangesCannotPass(t *testing.T) { + n := workerNode("bad-scope", "allowed.txt", "ok", "allowed.txt") + n.RetryPolicy.MaxRetries = 0 + st, h, _, clk := setupIsolated(t, &graph.Graph{Nodes: []*graph.Node{n}}, map[string][]sched.Script{ + "bad-scope": {{Delay: tick, Write: map[string]string{"outside.txt": "escape"}}}, + }) + ctx := context.Background() + for i := 0; i < 50 && !h.Done(); i++ { + if err := step(h, clk, ctx); err != nil { + t.Fatal(err) + } + } + if state, _ := h.State("bad-scope"); state != graph.StateFailed { + t.Fatalf("out-of-scope state = %s, want failed", state) + } + atts := attemptsOf(t, st, "run-wt", "bad-scope") + if len(atts) == 0 || !strings.Contains(atts[len(atts)-1].Evidence, "outside write scope") { + t.Fatalf("scope failure evidence = %+v", atts) + } +} + func TestMergeRunsOnlyAfterApproval(t *testing.T) { g := &graph.Graph{Nodes: []*graph.Node{ workerNode("w1", "a.txt", "A1", "a.txt"), diff --git a/internal/spike/server.go b/internal/spike/server.go index 633a092..a4b0a75 100644 --- a/internal/spike/server.go +++ b/internal/spike/server.go @@ -6,6 +6,7 @@ import ( "io" "net" "net/http" + "os" "os/exec" "time" ) @@ -19,6 +20,14 @@ type Server struct { // StartServer starts an opencode serve process. port 0 picks a free port; // a fixed port allows restarts on the same URL (daemon watchdog). func StartServer(ctx context.Context, workdir string, port int, stderr io.Writer) (*Server, error) { + return StartServerWithConfig(ctx, workdir, port, stderr, "") +} + +// StartServerWithConfig starts OpenCode with trusted managed agent policy +// injected through OPENCODE_CONFIG_CONTENT. The environment-level config is +// available for sessions whose directory is a generated worktree, where the +// main checkout's ignored opencode.json is intentionally absent. +func StartServerWithConfig(ctx context.Context, workdir string, port int, stderr io.Writer, configContent string) (*Server, error) { bin := "opencode" var cmd *exec.Cmd var base string @@ -34,6 +43,9 @@ func StartServer(ctx context.Context, workdir string, port int, stderr io.Writer cmd = exec.CommandContext(ctx, bin, "serve", "--port", fmt.Sprint(port), "--hostname", "127.0.0.1") cmd.Dir = workdir + if configContent != "" { + cmd.Env = append(os.Environ(), "OPENCODE_CONFIG_CONTENT="+configContent) + } if stderr != nil { cmd.Stderr = stderr } diff --git a/internal/store/redact_test.go b/internal/store/redact_test.go index 3984e02..abb108a 100644 --- a/internal/store/redact_test.go +++ b/internal/store/redact_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "encoding/json" "path/filepath" "strings" "testing" @@ -10,6 +11,34 @@ import ( "corral/internal/graph" ) +func TestRedactPreservesJSON(t *testing.T) { + in := `{"apiKey":"abcdef","nested":{"access_token":"tokensecret1"},"items":[{"client_secret":"hunter2hunter2"}],"authorization":"Bearer abcdefgh","aws_secret_access_key":"aws-secret-value","private_key":"private-key-value","accessKeySecret":"access-key-value","large":9007199254740993,"maxTokens":123,"tokenCount":4,"promptTokens":5,"completionTokens":6,"totalTokens":11,"cachedTokens":2,"cacheCreationTokens":1,"cacheReadTokens":2}` + got := Redact(in) + if !json.Valid([]byte(got)) { + t.Fatalf("Redact returned invalid JSON: %q", got) + } + for _, secret := range []string{"abcdef", "tokensecret1", "hunter2hunter2", "abcdefgh", "aws-secret-value", "private-key-value", "access-key-value"} { + if strings.Contains(got, secret) { + t.Fatalf("Redact(%q) still contains %q: %q", in, secret, got) + } + } + if !strings.Contains(got, "9007199254740993") { + t.Fatalf("Redact changed exact JSON number: %q", got) + } + for _, accounting := range []string{`"maxTokens":123`, `"tokenCount":4`, `"promptTokens":5`, `"completionTokens":6`, `"totalTokens":11`, `"cachedTokens":2`, `"cacheCreationTokens":1`, `"cacheReadTokens":2`} { + if !strings.Contains(got, accounting) { + t.Fatalf("Redact changed non-secret accounting key %s: %q", accounting, got) + } + } +} + +func TestRedactJSONScalarString(t *testing.T) { + got := Redact(`"Bearer sk-verysecretkey1234567890"`) + if strings.Contains(got, "verysecretkey") || !json.Valid([]byte(got)) { + t.Fatalf("Redact leaked scalar JSON secret or broke JSON: %q", got) + } +} + func TestRedact(t *testing.T) { cases := []struct{ in, wantAbsent string }{ {"Authorization: Bearer sk-secret1234567890abc", "sk-secret1234567890abc"}, @@ -39,24 +68,24 @@ func TestSecretsNeverPersisted(t *testing.T) { g := &graph.Graph{Nodes: []*graph.Node{{ ID: "w1", Type: graph.NodeAgent, Objective: "o", AcceptanceCriteria: []string{"c"}, }}} - if err := st.CreateRun(ctx, "r1", g, time.Now()); err != nil { + if err := st.CreateRun(ctx, "r1", g, false, time.Now()); err != nil { t.Fatal(err) } secret := "Bearer sk-verysecretkey1234567890abcdef" ts := time.Now().UnixMilli() if err := st.RecordAttempt(ctx, Attempt{ - ID: "w1/1", RunID: "r1", NodeID: "w1", No: 1, Status: "done", + ID: "r1/w1/1", RunID: "r1", NodeID: "w1", No: 1, Status: "done", Evidence: "verifier saw " + secret, FinishedAt: &ts, }); err != nil { t.Fatal(err) } if err := st.RecordArtifact(ctx, Artifact{ - RunID: "r1", AttemptID: "w1/1", NodeID: "w1", Name: "diff", + RunID: "r1", AttemptID: "r1/w1/1", NodeID: "w1", Name: "diff", Hash: "h", Content: "output with " + secret, }); err != nil { t.Fatal(err) } - if _, err := st.AppendEvent(ctx, "r1", "w1", EventVerdict, "", "", "w1/1", `{"note":"`+secret+`"}`, time.Now()); err != nil { + if _, err := st.AppendEvent(ctx, "r1", "w1", EventVerdict, "", "", "r1/w1/1", `{"note":"`+secret+`"}`, time.Now()); err != nil { t.Fatal(err) } @@ -64,7 +93,7 @@ func TestSecretsNeverPersisted(t *testing.T) { if strings.Contains(atts[0].Evidence, "sk-verysecretkey") { t.Fatalf("evidence leaked secret: %q", atts[0].Evidence) } - arts, _ := st.Artifacts(ctx, "r1", "w1/1") + arts, _ := st.Artifacts(ctx, "r1", "r1/w1/1") if strings.Contains(arts[0].Content, "sk-verysecretkey") { t.Fatalf("artifact leaked secret: %q", arts[0].Content) } diff --git a/internal/store/store.go b/internal/store/store.go index 824ed1f..f909238 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -9,6 +9,8 @@ import ( "encoding/json" "fmt" "regexp" + "strings" + "sync" "time" _ "modernc.org/sqlite" @@ -26,6 +28,7 @@ const ( EventRetry EventType = "retry" // retry scheduled EventRun EventType = "run" // run lifecycle (created/completed) EventGraph EventType = "graph" // graph version change + EventSteer EventType = "steer" // operator steering message to a running attempt ) type Event struct { @@ -58,10 +61,11 @@ type Attempt struct { } type Run struct { - ID string `json:"id"` - Graph *graph.Graph `json:"graph"` - Status string `json:"status"` // active|completed|canceled - CreatedAt int64 `json:"createdAt"` + ID string `json:"id"` + Graph *graph.Graph `json:"graph"` + Status string `json:"status"` // active|completed|canceled + AutoApproveGates bool `json:"autoApproveGates"` + CreatedAt int64 `json:"createdAt"` } // NodeRow is the materialized per-node state. @@ -77,6 +81,63 @@ type NodeRow struct { type Store struct { db *sql.DB + + eventMu sync.Mutex + eventSubscribers map[*eventSubscriber]struct{} + eventsClosed bool +} + +type eventSubscriber struct { + ch chan Event +} + +// eventSubscriberBuffer decouples durable writes from live observers. A +// subscriber that cannot keep up misses wakeups until it catches up, then +// reconciles the gap from the durable log using its last sequence number. +const eventSubscriberBuffer = 1024 + +// SubscribeEvents observes events after their transactions commit. Delivery +// is best effort and never blocks a store writer. The caller must unsubscribe; +// a slow subscriber stays attached so later wakeups resume after it catches up. +func (s *Store) SubscribeEvents() (<-chan Event, func()) { + s.eventMu.Lock() + defer s.eventMu.Unlock() + if s.eventsClosed { + ch := make(chan Event) + close(ch) + return ch, func() {} + } + sub := &eventSubscriber{ch: make(chan Event, eventSubscriberBuffer)} + s.eventSubscribers[sub] = struct{}{} + return sub.ch, func() { s.unsubscribeEvents(sub) } +} + +func (s *Store) unsubscribeEvents(sub *eventSubscriber) { + s.eventMu.Lock() + defer s.eventMu.Unlock() + if _, ok := s.eventSubscribers[sub]; !ok { + return + } + delete(s.eventSubscribers, sub) + close(sub.ch) +} + +// publish notifies every subscriber without waiting. Callers invoke it only +// after the transaction containing ev has committed successfully. +func (s *Store) publish(ev Event) { + s.eventMu.Lock() + defer s.eventMu.Unlock() + if s.eventsClosed { + return + } + for sub := range s.eventSubscribers { + select { + case sub.ch <- ev: + default: + // Notifications are wakeups, not the source of truth. Dropping one + // is safe; the next delivered wakeup triggers durable cursor replay. + } + } } func Open(path string) (*Store, error) { @@ -109,10 +170,22 @@ func Open(path string) (*Store, error) { db.Close() return nil, err } - return &Store{db: db}, nil + return &Store{db: db, eventSubscribers: map[*eventSubscriber]struct{}{}}, nil } -func (s *Store) Close() error { return s.db.Close() } +func (s *Store) Close() error { + err := s.db.Close() + s.eventMu.Lock() + if !s.eventsClosed { + s.eventsClosed = true + for sub := range s.eventSubscribers { + close(sub.ch) + } + s.eventSubscribers = nil + } + s.eventMu.Unlock() + return err +} func migrate(db *sql.DB) error { schema := ` @@ -120,6 +193,7 @@ CREATE TABLE IF NOT EXISTS runs ( id TEXT PRIMARY KEY, graph TEXT NOT NULL, status TEXT NOT NULL, + auto_approve_gates INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS events ( @@ -175,10 +249,18 @@ CREATE TABLE IF NOT EXISTS artifacts ( PRIMARY KEY(run_id, attempt_id, name) );` _, err := db.Exec(schema) - return err + if err != nil { + return err + } + // Migrate databases created before the auto-approve-gates column. + if _, err := db.Exec(`ALTER TABLE runs ADD COLUMN auto_approve_gates INTEGER NOT NULL DEFAULT 0`); err != nil && + !strings.Contains(err.Error(), "duplicate column") { + return err + } + return nil } -func (s *Store) CreateRun(ctx context.Context, runID string, g *graph.Graph, now time.Time) error { +func (s *Store) CreateRun(ctx context.Context, runID string, g *graph.Graph, autoApproveGates bool, now time.Time) error { gj, err := json.Marshal(g) if err != nil { return err @@ -189,8 +271,8 @@ func (s *Store) CreateRun(ctx context.Context, runID string, g *graph.Graph, now } defer tx.Rollback() if _, err := tx.ExecContext(ctx, - `INSERT INTO runs(id, graph, status, created_at) VALUES(?, ?, 'active', ?)`, - runID, string(gj), now.UnixMilli()); err != nil { + `INSERT INTO runs(id, graph, status, auto_approve_gates, created_at) VALUES(?, ?, 'active', ?, ?)`, + runID, string(gj), boolInt(autoApproveGates), now.UnixMilli()); err != nil { return err } for _, n := range g.Nodes { @@ -200,21 +282,26 @@ func (s *Store) CreateRun(ctx context.Context, runID string, g *graph.Graph, now return err } } - if _, err := tx.ExecContext(ctx, - `INSERT INTO events(run_id, seq, etype, payload, created_at) VALUES(?, 1, 'run', '{"status":"created"}', ?)`, - runID, now.UnixMilli()); err != nil { + ev, err := s.appendEventTx(ctx, tx, runID, 1, "", EventRun, "", "", "", "", `{"status":"created"}`, now) + if err != nil { return err } - return tx.Commit() + if err := tx.Commit(); err != nil { + return err + } + s.publish(*ev) + return nil } func (s *Store) Run(ctx context.Context, runID string) (*Run, error) { - row := s.db.QueryRowContext(ctx, `SELECT id, graph, status, created_at FROM runs WHERE id = ?`, runID) + row := s.db.QueryRowContext(ctx, `SELECT id, graph, status, auto_approve_gates, created_at FROM runs WHERE id = ?`, runID) var r Run var gj string - if err := row.Scan(&r.ID, &gj, &r.Status, &r.CreatedAt); err != nil { + var auto int + if err := row.Scan(&r.ID, &gj, &r.Status, &auto, &r.CreatedAt); err != nil { return nil, err } + r.AutoApproveGates = auto != 0 if err := json.Unmarshal([]byte(gj), &r.Graph); err != nil { return nil, fmt.Errorf("decode graph: %w", err) } @@ -223,7 +310,7 @@ func (s *Store) Run(ctx context.Context, runID string) (*Run, error) { // ListRuns returns all runs with their status. func (s *Store) ListRuns(ctx context.Context) ([]Run, error) { - rows, err := s.db.QueryContext(ctx, `SELECT id, graph, status, created_at FROM runs ORDER BY created_at`) + rows, err := s.db.QueryContext(ctx, `SELECT id, graph, status, auto_approve_gates, created_at FROM runs ORDER BY created_at`) if err != nil { return nil, err } @@ -232,9 +319,11 @@ func (s *Store) ListRuns(ctx context.Context) ([]Run, error) { for rows.Next() { var r Run var gj string - if err := rows.Scan(&r.ID, &gj, &r.Status, &r.CreatedAt); err != nil { + var auto int + if err := rows.Scan(&r.ID, &gj, &r.Status, &auto, &r.CreatedAt); err != nil { return nil, err } + r.AutoApproveGates = auto != 0 if err := json.Unmarshal([]byte(gj), &r.Graph); err != nil { return nil, fmt.Errorf("decode graph: %w", err) } @@ -243,6 +332,13 @@ func (s *Store) ListRuns(ctx context.Context) ([]Run, error) { return out, rows.Err() } +func boolInt(b bool) int { + if b { + return 1 + } + return 0 +} + func (s *Store) CompleteRun(ctx context.Context, runID string, status string, now time.Time) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { @@ -252,10 +348,15 @@ func (s *Store) CompleteRun(ctx context.Context, runID string, status string, no if _, err := tx.ExecContext(ctx, `UPDATE runs SET status = ? WHERE id = ?`, status, runID); err != nil { return err } - if _, err := appendEventTx(ctx, tx, runID, 0, "", EventRun, "", "", "", "", `{"status":"`+status+`"}`, now); err != nil { + ev, err := s.appendEventTx(ctx, tx, runID, 0, "", EventRun, "", "", "", "", `{"status":"`+status+`"}`, now) + if err != nil { + return err + } + if err := tx.Commit(); err != nil { return err } - return tx.Commit() + s.publish(*ev) + return nil } // AppendTransition records a state change and updates the materialized @@ -266,7 +367,7 @@ func (s *Store) AppendTransition(ctx context.Context, runID string, nodeID strin return 0, err } defer tx.Rollback() - seq, err := appendEventTx(ctx, tx, runID, 0, nodeID, EventTransition, string(from), string(to), "", "", payload, now) + ev, err := s.appendEventTx(ctx, tx, runID, 0, nodeID, EventTransition, string(from), string(to), "", "", payload, now) if err != nil { return 0, err } @@ -275,7 +376,11 @@ func (s *Store) AppendTransition(ctx context.Context, runID string, nodeID strin string(to), now.UnixMilli(), runID, nodeID); err != nil { return 0, err } - return seq, tx.Commit() + if err := tx.Commit(); err != nil { + return 0, err + } + s.publish(*ev) + return ev.Seq, nil } func (s *Store) AppendEvent(ctx context.Context, runID, nodeID string, typ EventType, from, to graph.State, attemptID, payload string, now time.Time) (int64, error) { @@ -284,38 +389,60 @@ func (s *Store) AppendEvent(ctx context.Context, runID, nodeID string, typ Event return 0, err } defer tx.Rollback() - seq, err := appendEventTx(ctx, tx, runID, 0, nodeID, typ, string(from), string(to), attemptID, "", payload, now) + ev, err := s.appendEventTx(ctx, tx, runID, 0, nodeID, typ, string(from), string(to), attemptID, "", payload, now) if err != nil { return 0, err } - return seq, tx.Commit() + if err := tx.Commit(); err != nil { + return 0, err + } + s.publish(*ev) + return ev.Seq, nil } -// appendEventTx computes the next per-run sequence and inserts the event. -func appendEventTx(ctx context.Context, tx *sql.Tx, runID string, forceSeq int64, nodeID string, typ EventType, from, to, attemptID, _, payload string, now time.Time) (int64, error) { +// appendEventTx computes the next per-run sequence, inserts the event and +// returns the fully populated event for the caller to publish after commit. +func (s *Store) appendEventTx(ctx context.Context, tx *sql.Tx, runID string, forceSeq int64, nodeID string, typ EventType, from, to, attemptID, _, payload string, now time.Time) (*Event, error) { var seq int64 if forceSeq > 0 { seq = forceSeq } else { if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(seq), 0) + 1 FROM events WHERE run_id = ?`, runID).Scan(&seq); err != nil { - return 0, err + return nil, err } } + redacted := Redact(payload) if _, err := tx.ExecContext(ctx, `INSERT INTO events(run_id, seq, node_id, etype, from_state, to_state, attempt_id, payload, created_at) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)`, - runID, seq, nodeID, string(typ), from, to, attemptID, Redact(payload), now.UnixMilli()); err != nil { - return 0, err + runID, seq, nodeID, string(typ), from, to, attemptID, redacted, now.UnixMilli()); err != nil { + return nil, err } - return seq, nil + return &Event{ + Seq: seq, + RunID: runID, + NodeID: nodeID, + Type: typ, + From: graph.State(from), + To: graph.State(to), + AttemptID: attemptID, + Payload: json.RawMessage(redacted), + CreatedAt: now.UnixMilli(), + }, nil } // Events returns the full event log of a run in sequence order. func (s *Store) Events(ctx context.Context, runID string) ([]Event, error) { + return s.EventsAfter(ctx, runID, 0) +} + +// EventsAfter returns the events of a run with seq greater than after, in +// sequence order. It backs both replay and the live-event SSE cursor. +func (s *Store) EventsAfter(ctx context.Context, runID string, after int64) ([]Event, error) { rows, err := s.db.QueryContext(ctx, `SELECT seq, node_id, etype, from_state, to_state, attempt_id, payload, created_at - FROM events WHERE run_id = ? ORDER BY seq`, runID) + FROM events WHERE run_id = ? AND seq > ? ORDER BY seq`, runID, after) if err != nil { return nil, err } @@ -524,6 +651,63 @@ func Redact(s string) string { if s == "" { return s } + if json.Valid([]byte(s)) { + var value any + decoder := json.NewDecoder(strings.NewReader(s)) + decoder.UseNumber() + if decoder.Decode(&value) == nil { + value = redactJSON(value) + if out, err := json.Marshal(value); err == nil { + return string(out) + } + } + } + return redactText(s) +} + +func redactJSON(value any) any { + switch value := value.(type) { + case map[string]any: + for key, item := range value { + if secretKey(key) { + value[key] = "[REDACTED]" + continue + } + value[key] = redactJSON(item) + } + case []any: + for i, item := range value { + value[i] = redactJSON(item) + } + case string: + return redactText(value) + } + return value +} + +func secretKey(key string) bool { + key = strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(key, "_", ""), "-", "")) + for _, marker := range []string{"apikey", "password", "passwd", "secret", "authorization", "privatekey", "accesskey"} { + if strings.Contains(key, marker) { + return true + } + } + // Token accounting fields are telemetry, not credentials. Other token + // keys (accessToken, refreshToken, tokenValue, etc.) are secrets. + accounting := map[string]bool{ + "tokens": true, "tokencount": true, "maxtokens": true, + "prompttokens": true, "completiontokens": true, "totaltokens": true, + "inputtokens": true, "outputtokens": true, "reasoningtokens": true, + "cachedtokens": true, "cachecreationtokens": true, + "cachereadtokens": true, "cachewritetokens": true, + } + if strings.Contains(key, "token") && !accounting[key] { + return true + } + return false +} + +func redactText(s string) string { repl := []struct{ re, to string }{ {`(?i)bearer\s+[A-Za-z0-9._~+/=-]+`, "bearer [REDACTED]"}, {`(?i)api[_-]?key["']?\s*[:=]\s*["']?[A-Za-z0-9._~+/=-]{6,}`, "apiKey [REDACTED]"}, diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 5601ee5..7d4941e 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -2,6 +2,7 @@ package store import ( "context" + "database/sql" "path/filepath" "testing" "time" @@ -32,7 +33,7 @@ func now() time.Time { return time.UnixMilli(1786000000000) } func TestCreateRunAndReplay(t *testing.T) { st := open(t) ctx := context.Background() - if err := st.CreateRun(ctx, "r1", testGraph(t), now()); err != nil { + if err := st.CreateRun(ctx, "r1", testGraph(t), false, now()); err != nil { t.Fatal(err) } if _, err := st.AppendTransition(ctx, "r1", "a", graph.StatePending, graph.StateReady, "", now()); err != nil { @@ -66,10 +67,89 @@ func TestCreateRunAndReplay(t *testing.T) { } } +func TestEventSubscriptionsPublishAfterCommitToMultipleObservers(t *testing.T) { + st := open(t) + ctx := context.Background() + one, unsubscribeOne := st.SubscribeEvents() + defer unsubscribeOne() + two, unsubscribeTwo := st.SubscribeEvents() + defer unsubscribeTwo() + + if err := st.CreateRun(ctx, "r1", testGraph(t), false, now()); err != nil { + t.Fatal(err) + } + for i, events := range []<-chan Event{one, two} { + select { + case event := <-events: + if event.RunID != "r1" || event.Seq != 1 || event.Type != EventRun { + t.Fatalf("observer %d got %+v", i, event) + } + persisted, err := st.EventsAfter(ctx, "r1", 0) + if err != nil || len(persisted) != 1 || persisted[0].Seq != event.Seq { + t.Fatalf("observer %d notified before durable read: events=%+v err=%v", i, persisted, err) + } + case <-time.After(time.Second): + t.Fatalf("observer %d received no event", i) + } + } +} + +func TestSlowEventSubscriberRecoversAfterDroppedWakeup(t *testing.T) { + st := open(t) + events, unsubscribe := st.SubscribeEvents() + defer unsubscribe() + + // Fill the best-effort wakeup queue, then overflow it once. Overflow may + // drop that wakeup, but must not permanently detach the subscriber: its + // consumer can reconcile from the durable log and keep listening. + for i := 1; i <= eventSubscriberBuffer+1; i++ { + st.publish(Event{Seq: int64(i), RunID: "r1", Type: EventGraph}) + } + for i := 0; i < eventSubscriberBuffer; i++ { + if _, ok := <-events; !ok { + t.Fatal("slow subscriber was permanently closed on overflow") + } + } + + const marker = int64(10_000) + st.publish(Event{Seq: marker, RunID: "r1", Type: EventGraph}) + select { + case event, ok := <-events: + if !ok { + t.Fatal("subscriber remained closed after catching up") + } + if event.Seq != marker { + t.Fatalf("event seq = %d, want %d", event.Seq, marker) + } + case <-time.After(time.Second): + t.Fatal("subscriber did not receive events after catching up") + } +} + +func TestEventsAfterUsesExclusiveCursor(t *testing.T) { + st := open(t) + ctx := context.Background() + if err := st.CreateRun(ctx, "r1", testGraph(t), false, now()); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if _, err := st.AppendEvent(ctx, "r1", "a", EventGraph, "", "", "", `{}`, now()); err != nil { + t.Fatal(err) + } + } + events, err := st.EventsAfter(ctx, "r1", 1) + if err != nil { + t.Fatal(err) + } + if len(events) != 2 || events[0].Seq != 2 || events[1].Seq != 3 { + t.Fatalf("events after 1 = %+v", events) + } +} + func TestLeaseAtomicity(t *testing.T) { st := open(t) ctx := context.Background() - if err := st.CreateRun(ctx, "r1", testGraph(t), now()); err != nil { + if err := st.CreateRun(ctx, "r1", testGraph(t), false, now()); err != nil { t.Fatal(err) } ok, err := st.AcquireLease(ctx, "r1", "a", "h1", time.Minute, now()) @@ -112,41 +192,119 @@ func TestLeaseAtomicity(t *testing.T) { func TestAttemptsUniquePerNode(t *testing.T) { st := open(t) ctx := context.Background() - if err := st.CreateRun(ctx, "r1", testGraph(t), now()); err != nil { + if err := st.CreateRun(ctx, "r1", testGraph(t), false, now()); err != nil { t.Fatal(err) } - a := Attempt{ID: "a/1", RunID: "r1", NodeID: "a", No: 1, Status: "running"} + a := Attempt{ID: "r1/a/1", RunID: "r1", NodeID: "a", No: 1, Status: "running"} if err := st.RecordAttempt(ctx, a); err != nil { t.Fatal(err) } - if err := st.RecordAttempt(ctx, Attempt{ID: "a/2", RunID: "r1", NodeID: "a", No: 2, Status: "running"}); err != nil { + if err := st.RecordAttempt(ctx, Attempt{ID: "r1/a/2", RunID: "r1", NodeID: "a", No: 2, Status: "running"}); err != nil { t.Fatal(err) } // Duplicate attempt number for the same node must fail. - if err := st.RecordAttempt(ctx, Attempt{ID: "a/2b", RunID: "r1", NodeID: "a", No: 2, Status: "running"}); err == nil { + if err := st.RecordAttempt(ctx, Attempt{ID: "r1/a/2b", RunID: "r1", NodeID: "a", No: 2, Status: "running"}); err == nil { t.Fatal("duplicate (node, no) accepted") } atts, err := st.Attempts(ctx, "r1", "a") if err != nil { t.Fatal(err) } - if len(atts) != 2 || atts[0].ID != "a/1" || atts[1].ID != "a/2" { + if len(atts) != 2 || atts[0].ID != "r1/a/1" || atts[1].ID != "r1/a/2" { t.Fatalf("attempts wrong: %+v", atts) } } +// TestMigrateAddsAutoApproveColumn opens a database created with the +// pre-auto-approve schema and verifies the column is added, so existing +// deployments keep working after an upgrade. +func TestMigrateAddsAutoApproveColumn(t *testing.T) { + path := filepath.Join(t.TempDir(), "old.db") + oldDB, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + if _, err := oldDB.Exec(`CREATE TABLE runs ( + id TEXT PRIMARY KEY, + graph TEXT NOT NULL, + status TEXT NOT NULL, + created_at INTEGER NOT NULL + );`); err != nil { + t.Fatal(err) + } + oldDB.Close() + + st, err := Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + ctx := context.Background() + if err := st.CreateRun(ctx, "r1", testGraph(t), false, now()); err != nil { + t.Fatal(err) + } + ru, err := st.Run(ctx, "r1") + if err != nil { + t.Fatal(err) + } + if ru.AutoApproveGates { + t.Fatal("migrated run should default autoApproveGates to false") + } +} + +func TestAutoApproveGatesPersisted(t *testing.T) { + st := open(t) + ctx := context.Background() + // Default run: flag off. + if err := st.CreateRun(ctx, "off", testGraph(t), false, now()); err != nil { + t.Fatal(err) + } + // Explicit run: flag on. + if err := st.CreateRun(ctx, "on", testGraph(t), true, now()); err != nil { + t.Fatal(err) + } + off, err := st.Run(ctx, "off") + if err != nil { + t.Fatal(err) + } + if off.AutoApproveGates { + t.Fatal("default run has autoApproveGates set") + } + on, err := st.Run(ctx, "on") + if err != nil { + t.Fatal(err) + } + if !on.AutoApproveGates { + t.Fatal("autoApproveGates not persisted on the run") + } + // ListRuns carries the flag too. + runs, err := st.ListRuns(ctx) + if err != nil { + t.Fatal(err) + } + if len(runs) != 2 { + t.Fatalf("runs = %d, want 2", len(runs)) + } + for _, r := range runs { + want := r.ID == "on" + if r.AutoApproveGates != want { + t.Fatalf("run %s autoApproveGates = %v, want %v", r.ID, r.AutoApproveGates, want) + } + } +} + func TestMarkInterrupted(t *testing.T) { st := open(t) ctx := context.Background() - if err := st.CreateRun(ctx, "r1", testGraph(t), now()); err != nil { + if err := st.CreateRun(ctx, "r1", testGraph(t), false, now()); err != nil { t.Fatal(err) } - for _, id := range []string{"a/1", "b/1"} { - if err := st.RecordAttempt(ctx, Attempt{ID: id, RunID: "r1", NodeID: string(id[0]), No: 1, Status: "running"}); err != nil { + for _, attempt := range []struct{ id, nodeID string }{{"r1/a/1", "a"}, {"r1/b/1", "b"}} { + if err := st.RecordAttempt(ctx, Attempt{ID: attempt.id, RunID: "r1", NodeID: attempt.nodeID, No: 1, Status: "running"}); err != nil { t.Fatal(err) } } - if err := st.RecordAttempt(ctx, Attempt{ID: "a/0", RunID: "r1", NodeID: "a", No: 0, Status: "done"}); err != nil { + if err := st.RecordAttempt(ctx, Attempt{ID: "r1/a/0", RunID: "r1", NodeID: "a", No: 0, Status: "done"}); err != nil { t.Fatal(err) } if err := st.MarkInterrupted(ctx, "r1", "a"); err != nil { @@ -154,12 +312,12 @@ func TestMarkInterrupted(t *testing.T) { } atts, _ := st.Attempts(ctx, "r1", "a") for _, at := range atts { - if at.ID == "a/0" && at.Status != "done" { + if at.ID == "r1/a/0" && at.Status != "done" { t.Fatalf("completed attempt flipped: %+v", at) } } for _, at := range atts { - if at.ID == "a/1" && at.Status != "interrupted" { + if at.ID == "r1/a/1" && at.Status != "interrupted" { t.Fatalf("running attempt not interrupted: %+v", at) } } diff --git a/internal/tui/api.go b/internal/tui/api.go index 632f5e7..c4b6472 100644 --- a/internal/tui/api.go +++ b/internal/tui/api.go @@ -4,12 +4,18 @@ package tui import ( + "bufio" "bytes" "context" "encoding/json" + "errors" "fmt" "io" + "mime" "net/http" + "net/url" + "strconv" + "strings" "time" ) @@ -38,7 +44,9 @@ type GraphNode struct { MaxRetries int `json:"maxRetries"` } `json:"retryPolicy"` Budget struct { - MaxDuration int64 `json:"maxDuration"` // nanoseconds (time.Duration) + MaxDuration int64 `json:"maxDuration"` // nanoseconds (time.Duration) + MaxTokens int `json:"maxTokens,omitempty"` + MaxCost float64 `json:"maxCost,omitempty"` } `json:"budget"` } @@ -58,13 +66,15 @@ type AttemptView struct { } type EventView struct { - Seq int64 `json:"seq"` - NodeID string `json:"nodeID,omitempty"` - Type string `json:"type"` - From string `json:"from,omitempty"` - To string `json:"to,omitempty"` - AttemptID string `json:"attemptID,omitempty"` - CreatedAt int64 `json:"createdAt"` + Seq int64 `json:"seq"` + RunID string `json:"runID,omitempty"` + NodeID string `json:"nodeID,omitempty"` + Type string `json:"type"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + AttemptID string `json:"attemptID,omitempty"` + Payload json.RawMessage `json:"payload,omitempty"` + CreatedAt int64 `json:"createdAt"` } type RunDetail struct { @@ -81,11 +91,14 @@ type RunDetail struct { type API interface { ListRuns(ctx context.Context) ([]RunSummary, error) GetRun(ctx context.Context, runID string) (*RunDetail, error) + StreamEvents(ctx context.Context, runID string, after int64, emit func(EventView) error) error + Tail(ctx context.Context, runID, nodeID string, lines int) ([]string, error) Approve(ctx context.Context, runID, nodeID string) error Reject(ctx context.Context, runID, nodeID string) error Cancel(ctx context.Context, runID, nodeID string) error Retry(ctx context.Context, runID, nodeID string) error Steer(ctx context.Context, runID, nodeID, message string) error + RespondPermission(ctx context.Context, runID, nodeID, permissionID string, allow bool) error } // Client talks to a corral daemon over HTTP. @@ -150,6 +163,134 @@ func (c *Client) GetRun(ctx context.Context, runID string) (*RunDetail, error) { return &out, err } +// StreamEvents consumes raw durable store.Event frames from the daemon's +// SSE endpoint. It blocks until the stream ends, context is canceled, or +// emit returns an error. Reconnect callers pass the last accepted Seq as +// after; duplicate frames are ignored defensively. +func (c *Client) StreamEvents(ctx context.Context, runID string, after int64, emit func(EventView) error) error { + if runID == "" { + return fmt.Errorf("runID required") + } + if after < 0 { + return fmt.Errorf("after must not be negative") + } + if emit == nil { + return errors.New("event emitter required") + } + path := fmt.Sprintf("/api/runs/%s/events?after=%d", url.PathEscape(runID), after) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.Base+path, nil) + if err != nil { + return err + } + req.Header.Set("Accept", "text/event-stream") + req.Header.Set("X-Corral-Role", c.Role) + if c.Key != "" { + req.Header.Set("Authorization", "Bearer "+c.Key) + } + client := &http.Client{} + if c.HTTP != nil { + // SSE is intentionally long-lived, so omit the normal request timeout + // while retaining caller-supplied transport/redirect/cookie behavior. + client.Transport = c.HTTP.Transport + client.CheckRedirect = c.HTTP.CheckRedirect + client.Jar = c.HTTP.Jar + } + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + data, _ := io.ReadAll(io.LimitReader(resp.Body, 201)) + return fmt.Errorf("GET %s: %d %s", path, resp.StatusCode, truncate(string(data), 200)) + } + mediaType, _, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) + if err != nil || mediaType != "text/event-stream" { + return fmt.Errorf("GET %s: unexpected content type %q", path, resp.Header.Get("Content-Type")) + } + + cursor := after + scanner := bufio.NewScanner(resp.Body) + scanner.Buffer(make([]byte, 4096), 1<<20) + var eventID string + var data []string + dispatch := func() error { + if len(data) == 0 { + eventID = "" + return nil + } + var event EventView + if err := json.Unmarshal([]byte(strings.Join(data, "\n")), &event); err != nil { + return fmt.Errorf("decode event: %w", err) + } + if event.Seq <= 0 { + return fmt.Errorf("event has invalid seq %d", event.Seq) + } + if eventID != "" { + id, err := strconv.ParseInt(eventID, 10, 64) + if err != nil || id != event.Seq { + return fmt.Errorf("event id %q does not match seq %d", eventID, event.Seq) + } + } + eventID, data = "", nil + if event.Seq <= cursor { + return nil + } + if err := emit(event); err != nil { + return err + } + cursor = event.Seq + return nil + } + for scanner.Scan() { + line := scanner.Text() + if line == "" { + if err := dispatch(); err != nil { + return err + } + continue + } + if strings.HasPrefix(line, ":") { + continue + } + field, value, ok := strings.Cut(line, ":") + if ok && strings.HasPrefix(value, " ") { + value = value[1:] + } + switch field { + case "id": + eventID = value + case "data": + data = append(data, value) + } + } + if err := scanner.Err(); err != nil { + return err + } + if err := dispatch(); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + return io.EOF +} + +func (c *Client) Tail(ctx context.Context, runID, nodeID string, lines int) ([]string, error) { + if runID == "" || nodeID == "" { + return nil, fmt.Errorf("runID and nodeID required") + } + if lines < 1 || lines > 500 { + return nil, fmt.Errorf("lines must be between 1 and 500") + } + var out struct { + Lines []string `json:"lines"` + } + path := fmt.Sprintf("/api/runs/%s/tail?node=%s&lines=%d", url.PathEscape(runID), url.QueryEscape(nodeID), lines) + err := c.do(ctx, http.MethodGet, path, nil, &out) + return out.Lines, err +} + func (c *Client) nodeAction(ctx context.Context, path, runID, nodeID string) error { return c.do(ctx, http.MethodPost, path, map[string]string{"nodeID": nodeID}, nil) } @@ -169,6 +310,10 @@ func (c *Client) Retry(ctx context.Context, runID, nodeID string) error { func (c *Client) Steer(ctx context.Context, runID, nodeID, message string) error { return c.do(ctx, http.MethodPost, "/api/runs/"+runID+"/steer", map[string]string{"nodeID": nodeID, "message": message}, nil) } +func (c *Client) RespondPermission(ctx context.Context, runID, nodeID, permissionID string, allow bool) error { + return c.do(ctx, http.MethodPost, "/api/runs/"+runID+"/permission", + map[string]any{"nodeID": nodeID, "permissionID": permissionID, "allow": allow}, nil) +} func truncate(s string, n int) string { if len(s) <= n { diff --git a/internal/tui/client_test.go b/internal/tui/client_test.go index 5ecbec8..6244c28 100644 --- a/internal/tui/client_test.go +++ b/internal/tui/client_test.go @@ -2,12 +2,16 @@ package tui import ( "context" + "encoding/json" + "fmt" + "net/http" "net/http/httptest" "path/filepath" "strings" "testing" "time" + "corral/internal/adapter" "corral/internal/clock" "corral/internal/daemon" "corral/internal/graph" @@ -31,6 +35,7 @@ func TestClientAgainstDaemon(t *testing.T) { eng := verify.New(workdir) s := sched.New(st, drv, &sched.EngineVerifier{Eng: eng}, clock.Real{}, sched.Options{Concurrency: 2}) d := daemon.New(st, s, nil, t.TempDir(), "") + t.Cleanup(d.Close) ctx, cancel := context.WithCancel(context.Background()) defer cancel() d.SetContext(ctx) @@ -155,3 +160,212 @@ func TestClientAgainstDaemon(t *testing.T) { t.Fatalf("events missing") } } + +// TestClientRespondPermission drives a permission-blocked node through the +// daemon: the client's RespondPermission allows it and the node completes. +func TestClientRespondPermission(t *testing.T) { + st, err := store.Open(filepath.Join(t.TempDir(), "t.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + + workdir := t.TempDir() + drv := sched.NewFakeDriver(clock.Real{}, nil) + eng := verify.New(workdir) + s := sched.New(st, drv, &sched.EngineVerifier{Eng: eng}, clock.Real{}, sched.Options{Concurrency: 2}) + d := daemon.New(st, s, nil, t.TempDir(), "") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + d.SetContext(ctx) + srv := httptest.NewServer(d.Handler()) + t.Cleanup(srv.Close) + + client := NewClient(srv.URL, "") + client.Role = "operator" + + g := &graph.Graph{Nodes: []*graph.Node{{ + ID: "w1", Type: graph.NodeAgent, Role: "worker", + Objective: "write a.txt", AcceptanceCriteria: []string{"a.txt"}, + Priority: graph.PriorityNormal, WriteScope: []string{"a.txt"}, + Verification: &graph.Verification{Kind: "command", Command: []string{"test", "-f", "a.txt"}}, + Meta: map[string]string{"cwd": workdir}, + }}} + drv.SetScript("w1", sched.Script{Delay: 5 * time.Second, Permission: "perm-9", Write: map[string]string{"a.txt": "A"}}) + var created struct{ RunID string } + if err := client.do(ctx, "POST", "/api/runs", map[string]any{"graph": g}, &created); err != nil { + t.Fatal(err) + } + + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + dd, err := client.GetRun(ctx, created.RunID) + if err == nil && dd.States["w1"] == "blocked" { + break + } + time.Sleep(100 * time.Millisecond) + } + + // The pending permission is visible in the detail payload. + dd, err := client.GetRun(ctx, created.RunID) + if err != nil { + t.Fatal(err) + } + pid := "" + for _, ev := range dd.Events { + if ev.NodeID == "w1" && ev.To == "blocked" { + var p struct { + Reason string `json:"reason"` + PermissionID string `json:"permissionID"` + } + if json.Unmarshal(ev.Payload, &p) == nil { + pid = p.PermissionID + } + } + } + if pid != "perm-9" { + t.Fatalf("blocked payload permissionID = %q, want perm-9", pid) + } + + if err := client.RespondPermission(ctx, created.RunID, "w1", "perm-9", true); err != nil { + t.Fatalf("respond permission: %v", err) + } + for time.Now().Before(deadline) { + dd, _ := client.GetRun(ctx, created.RunID) + if dd != nil && dd.States["w1"] == "done" { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("node never done after permission allowed") +} + +// TestTailAgainstDaemon exercises the tail endpoint against a live daemon +// with a running attempt: the transcript lines stream while the node runs. +func TestTailAgainstDaemon(t *testing.T) { + st, err := store.Open(filepath.Join(t.TempDir(), "t.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + + workdir := t.TempDir() + drv := sched.NewFakeDriver(clock.Real{}, nil) + eng := verify.New(workdir) + s := sched.New(st, drv, &sched.EngineVerifier{Eng: eng}, clock.Real{}, sched.Options{Concurrency: 2}) + d := daemon.New(st, s, nil, t.TempDir(), "") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + d.SetContext(ctx) + srv := httptest.NewServer(d.Handler()) + t.Cleanup(srv.Close) + + client := NewClient(srv.URL, "") + client.Role = "operator" + drv.SetScript("w1", sched.Script{ + Delay: 5 * time.Second, + Messages: []adapter.Message{ + {Role: "assistant", Text: "inspecting the workspace"}, + {Role: "assistant", Text: "found the bug\napplying the fix"}, + }, + }) + g := &graph.Graph{Nodes: []*graph.Node{{ + ID: "w1", Type: graph.NodeAgent, Role: "worker", + Objective: "write a.txt", AcceptanceCriteria: []string{"a.txt"}, + Priority: graph.PriorityNormal, WriteScope: []string{"a.txt"}, + Verification: &graph.Verification{Kind: "command", Command: []string{"test", "-f", "a.txt"}}, + Meta: map[string]string{"cwd": workdir}, + }}} + var created struct{ RunID string } + if err := client.do(ctx, "POST", "/api/runs", map[string]any{"graph": g}, &created); err != nil { + t.Fatal(err) + } + + // Wait until the node is running, then fetch the tail. + deadline := time.Now().Add(10 * time.Second) + var dd *RunDetail + for time.Now().Before(deadline) { + dd, _ = client.GetRun(ctx, created.RunID) + if dd != nil && dd.States["w1"] == "running" { + break + } + time.Sleep(50 * time.Millisecond) + } + if dd == nil || dd.States["w1"] != "running" { + t.Fatal("node never reached running") + } + lines, err := client.Tail(ctx, created.RunID, "w1", 40) + if err != nil { + t.Fatalf("tail: %v", err) + } + if len(lines) == 0 { + t.Fatal("tail returned no lines while running") + } + joined := strings.Join(lines, "\n") + for _, want := range []string{"inspecting", "applying"} { + if !strings.Contains(joined, want) { + t.Fatalf("tail missing %q: %q", want, joined) + } + } +} + +func TestTailRejectsInvalidLineCount(t *testing.T) { + client := NewClient("http://unused.invalid", "") + for _, lines := range []int{0, -1, 501} { + if _, err := client.Tail(context.Background(), "run", "node", lines); err == nil { + t.Fatalf("Tail accepted lines=%d", lines) + } + } +} + +func TestStreamEventsUsesCursorAndRawFrames(t *testing.T) { + var gotPath, gotAccept, gotAuth string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.RequestURI() + gotAccept = r.Header.Get("Accept") + gotAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + fmt.Fprint(w, ": ping\n\n") + fmt.Fprint(w, "id: 8\n") + fmt.Fprint(w, `data: {"seq":8,"runID":"run id","nodeID":"w1","type":"transition","from":"running","to":"blocked","payload":{"reason":"permission","permissionID":"p1"},"createdAt":123}`+"\n\n") + fmt.Fprint(w, "id: 8\n") // duplicate must be ignored + fmt.Fprint(w, `data: {"seq":8,"runID":"run id","nodeID":"w1","type":"transition"}`+"\n\n") + })) + t.Cleanup(srv.Close) + + client := NewClient(srv.URL, "secret") + var events []EventView + err := client.StreamEvents(context.Background(), "run id", 7, func(event EventView) error { + events = append(events, event) + return nil + }) + if err == nil || err.Error() != "EOF" { + t.Fatalf("closed stream err = %v, want EOF", err) + } + if gotPath != "/api/runs/run%20id/events?after=7" { + t.Fatalf("stream path = %q", gotPath) + } + if gotAccept != "text/event-stream" || gotAuth != "Bearer secret" { + t.Fatalf("stream headers accept=%q auth=%q", gotAccept, gotAuth) + } + if len(events) != 1 || events[0].Seq != 8 || events[0].To != "blocked" || string(events[0].Payload) == "" { + t.Fatalf("events = %+v", events) + } +} + +func TestStreamEventsRejectsNilEmitterBeforeRequest(t *testing.T) { + requests := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + w.Header().Set("Content-Type", "text/event-stream") + })) + t.Cleanup(srv.Close) + + client := NewClient(srv.URL, "") + if err := client.StreamEvents(context.Background(), "run", 0, nil); err == nil { + t.Fatal("StreamEvents accepted nil emitter") + } + if requests != 0 { + t.Fatalf("nil emitter opened %d HTTP requests", requests) + } +} diff --git a/internal/tui/model.go b/internal/tui/model.go index 323fc4c..3f9a187 100644 --- a/internal/tui/model.go +++ b/internal/tui/model.go @@ -2,6 +2,7 @@ package tui import ( "context" + "encoding/json" "fmt" "time" @@ -20,10 +21,35 @@ type fetchMsg struct { } type fetchRunMsg struct { + runID string detail *RunDetail err error } +type tailMsg struct { + runID string + node string + lines []string + err error +} + +type eventStreamItem struct { + event *EventView + err error +} + +type eventStreamReadyMsg struct { + runID string + items <-chan eventStreamItem + cancel context.CancelFunc +} + +type eventStreamItemMsg struct { + runID string + items <-chan eventStreamItem + item eventStreamItem +} + type actionMsg struct { label string err error @@ -56,6 +82,27 @@ type Model struct { steerNode string steerInput string + // tail holds the last-fetched live attempt tail for the inspected + // node. tailNode is set when the inspect view is active. + tailNode string + tail []string + + // Durable event stream state. eventCursor is the greatest sequence + // incorporated into detail. Full polling is used only while this stream + // is unavailable, plus the initial/action refreshes. + eventCursor int64 + streamRun string + streamItems <-chan eventStreamItem + streamCancel context.CancelFunc + streamConnecting bool + streamAttempted bool + + // prevStates records the last seen node states so attention only + // fires on transitions (gate awaiting approval, node failed). + prevStates map[string]string + // notify delivers terminal attention; overridden in tests. + notify func(title, body string) + status string err error @@ -67,6 +114,12 @@ func New(api API, ctx context.Context) *Model { return &Model{api: api, ctx: ctx, tick: time.Second} } +// EnableAttention arms terminal attention notifications (bell + desktop +// notification) for gates awaiting approval and failed nodes. The model +// ships with attention disabled so unit tests stay side-effect free; the +// real TUI calls this once at startup. +func (m *Model) EnableAttention() { m.notify = NotifyAttention } + func (m *Model) Init() tea.Cmd { return tea.Batch(fetchRunsCmd(m), tickCmd(m.tick)) } @@ -79,9 +132,53 @@ func fetchRunsCmd(m *Model) tea.Cmd { } func fetchRunCmd(m *Model) tea.Cmd { + runID := m.selectedID + return func() tea.Msg { + d, err := m.api.GetRun(m.ctx, runID) + return fetchRunMsg{runID: runID, detail: d, err: err} + } +} + +func fetchTailCmd(m *Model) tea.Cmd { + runID, node := m.selectedID, m.tailNode + return func() tea.Msg { + lines, err := m.api.Tail(m.ctx, runID, node, 40) + return tailMsg{runID: runID, node: node, lines: lines, err: err} + } +} + +func startEventStreamCmd(m *Model) tea.Cmd { + api, parent := m.api, m.ctx + runID, after := m.selectedID, m.eventCursor return func() tea.Msg { - d, err := m.api.GetRun(m.ctx, m.selectedID) - return fetchRunMsg{detail: d, err: err} + ctx, cancel := context.WithCancel(parent) + items := make(chan eventStreamItem, 64) + go func() { + err := api.StreamEvents(ctx, runID, after, func(event EventView) error { + select { + case items <- eventStreamItem{event: &event}: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) + select { + case items <- eventStreamItem{err: err}: + case <-ctx.Done(): + } + close(items) + }() + return eventStreamReadyMsg{runID: runID, items: items, cancel: cancel} + } +} + +func waitEventStreamCmd(runID string, items <-chan eventStreamItem) tea.Cmd { + return func() tea.Msg { + item, ok := <-items + if !ok { + item.err = fmt.Errorf("event stream closed") + } + return eventStreamItemMsg{runID: runID, items: items, item: item} } } @@ -102,14 +199,25 @@ func (m *Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { switch v := msg.(type) { case tickMsg: m.lastFetch = time.Time(v) - cmd := tickCmd(m.tick) + cmds := []tea.Cmd{tickCmd(m.tick)} switch m.mode { case modeList: - return m, tea.Batch(cmd, fetchRunsCmd(m)) + cmds = append(cmds, fetchRunsCmd(m)) case modeDetail, modeInspect: - return m, tea.Batch(cmd, fetchRunCmd(m)) + if m.detail == nil || (!m.runTerminal() && m.streamItems == nil && !m.streamConnecting) { + // Initial/full refresh and fallback polling while SSE is down. + cmds = append(cmds, fetchRunCmd(m)) + if m.detail != nil && !m.runTerminal() { + m.streamConnecting = true + m.streamAttempted = true + cmds = append(cmds, startEventStreamCmd(m)) + } + } + if m.mode == modeInspect && m.tailNode != "" { + cmds = append(cmds, fetchTailCmd(m)) + } } - return m, cmd + return m, tea.Batch(cmds...) case fetchMsg: if v.err != nil { @@ -124,19 +232,113 @@ func (m *Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil case fetchRunMsg: + if v.runID != "" && v.runID != m.selectedID { + return m, nil + } if v.err != nil || v.detail == nil { if v.err != nil { m.err = v.err } } else { + initial := m.detail == nil || m.detail.RunID != v.detail.RunID + snapshotCursor := int64(0) + for _, event := range v.detail.Events { + if event.Seq > snapshotCursor { + snapshotCursor = event.Seq + } + } + // An action/fallback GET can race a healthy stream. Never let an + // older snapshot overwrite state already materialized from SSE. + if !initial && snapshotCursor < m.eventCursor { + return m, nil + } m.detail = v.detail m.err = nil + if snapshotCursor > m.eventCursor { + m.eventCursor = snapshotCursor + } if m.nodeCursor >= len(m.detail.Graph.Nodes) { m.nodeCursor = 0 } + var cmds []tea.Cmd + if initial { + m.seedAttentionStates() + } else if cmd := m.checkAttention(); cmd != nil { + cmds = append(cmds, cmd) + } + if !m.runTerminal() && m.streamItems == nil && !m.streamConnecting && !m.streamAttempted { + m.streamConnecting = true + m.streamAttempted = true + cmds = append(cmds, startEventStreamCmd(m)) + } + return m, tea.Batch(cmds...) } return m, nil + case tailMsg: + if v.err == nil && v.runID == m.selectedID && v.node == m.tailNode { + m.tail = v.lines + } + return m, nil + + case eventStreamReadyMsg: + if v.runID != m.selectedID || (m.mode != modeDetail && m.mode != modeInspect) { + v.cancel() + return m, nil + } + if m.streamCancel != nil { + m.streamCancel() + } + m.streamRun = v.runID + m.streamItems = v.items + m.streamCancel = v.cancel + m.streamConnecting = false + return m, waitEventStreamCmd(v.runID, v.items) + + case eventStreamItemMsg: + if v.runID != m.selectedID || v.items != m.streamItems { + return m, nil + } + if v.item.err != nil { + m.stopEventStream() + m.status = "event stream unavailable; polling" + if m.detail != nil && !m.runTerminal() { + return m, fetchRunCmd(m) + } + return m, nil + } + if v.item.event == nil { + return m, waitEventStreamCmd(v.runID, v.items) + } + event := *v.item.event + if event.RunID != "" && event.RunID != m.selectedID { + m.stopEventStream() + m.status = "event stream mismatched run; polling" + return m, fetchRunCmd(m) + } + if event.Seq <= m.eventCursor { + return m, waitEventStreamCmd(v.runID, v.items) + } + if event.Seq != m.eventCursor+1 { + m.stopEventStream() + m.status = "event stream gap; polling" + return m, fetchRunCmd(m) + } + needsRefresh := m.applyEvent(event) + var cmds []tea.Cmd + // applyEvent stops the channel on a terminal run event. Only wait for + // another frame while this channel is still the active stream. + if m.streamItems == v.items { + cmds = append(cmds, waitEventStreamCmd(v.runID, v.items)) + } + if cmd := m.checkAttention(); cmd != nil { + cmds = append(cmds, cmd) + } + if needsRefresh { + cmds = append(cmds, fetchRunCmd(m)) + } + return m, tea.Batch(cmds...) + case actionMsg: m.status = v.label if v.err != nil { @@ -176,9 +378,13 @@ func (m *Model) handleKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { switch m.mode { case modeList: if m.cursor < len(m.runs) { + m.stopEventStream() m.selectedID = m.runs[m.cursor].ID m.mode = modeDetail m.nodeCursor = 0 + m.detail = nil + m.eventCursor = 0 + m.streamAttempted = false return m, fetchRunCmd(m) } case modeSteer: @@ -204,8 +410,10 @@ func (m *Model) handleKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { case "i": if id, ok := m.nodeAt(m.nodeCursor); ok { m.inspectNode = id + m.tailNode = id + m.tail = nil m.mode = modeInspect - return m, nil + return m, fetchTailCmd(m) } case "a": return m, m.nodeAction("approved", m.api.Approve) @@ -215,6 +423,10 @@ func (m *Model) handleKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { return m, m.nodeAction("canceled", m.api.Cancel) case "t": return m, m.nodeAction("retried", m.api.Retry) + case "p": + return m, m.permissionAction("allowed", true) + case "d": + return m, m.permissionAction("denied", false) case "s": if id, ok := m.nodeAt(m.nodeCursor); ok { m.steerNode = id @@ -249,15 +461,72 @@ func (m *Model) nodeAction(label string, fn func(context.Context, string, string }) } +// pendingPermissionDetails returns the permission request the node is +// currently blocked on, if its latest blocked transition carried one. +func (m *Model) pendingPermissionDetails(nodeID string) (string, string, string, bool) { + if m.detail == nil || m.detail.States[nodeID] != "blocked" { + return "", "", "", false + } + for i := len(m.detail.Events) - 1; i >= 0; i-- { + ev := m.detail.Events[i] + if ev.NodeID != nodeID || ev.Type != "transition" || ev.To != "blocked" { + continue + } + var p struct { + Reason string `json:"reason"` + PermissionID string `json:"permissionID"` + Tool string `json:"tool"` + Input string `json:"input"` + } + if json.Unmarshal(ev.Payload, &p) == nil && p.Reason == "permission" && p.PermissionID != "" { + return p.PermissionID, p.Tool, p.Input, true + } + // Only the latest transition into blocked describes why the + // current blocked state exists. Never fall back to an older request. + return "", "", "", false + } + return "", "", "", false +} + +// pendingPermission returns the permission id the node is currently blocked +// on, if its latest blocked transition carried a permission request. +func (m *Model) pendingPermission(nodeID string) (string, bool) { + id, _, _, ok := m.pendingPermissionDetails(nodeID) + return id, ok +} + +// permissionAction answers the pending permission of the node under the +// cursor; it is a no-op unless the node is blocked on a permission. +func (m *Model) permissionAction(label string, allow bool) tea.Cmd { + id, ok := m.nodeAt(m.nodeCursor) + if !ok { + return nil + } + pid, ok := m.pendingPermission(id) + if !ok { + return nil + } + runID := m.selectedID + m.status = fmt.Sprintf("%s permission %s/%s", label, runID, id) + return actionCmd(m, label, func(ctx context.Context) error { + return m.api.RespondPermission(ctx, runID, id, pid, allow) + }) +} + func (m *Model) back() { switch m.mode { case modeInspect: m.mode = modeDetail + m.tailNode = "" + m.tail = nil case modeSteer: m.mode = modeDetail case modeDetail: + m.stopEventStream() m.mode = modeList m.detail = nil + m.eventCursor = 0 + m.streamAttempted = false } } @@ -276,6 +545,8 @@ func (m *Model) move(delta int) { if m.mode == modeInspect { if id, ok := m.nodeAt(m.nodeCursor); ok { m.inspectNode = id + m.tailNode = id + m.tail = nil } } } @@ -290,6 +561,13 @@ func (m *Model) moveTo(idx int) { return } m.nodeCursor = clamp(idx, 0, len(m.detail.Graph.Nodes)-1) + if m.mode == modeInspect { + if id, ok := m.nodeAt(m.nodeCursor); ok { + m.inspectNode = id + m.tailNode = id + m.tail = nil + } + } } } @@ -308,3 +586,118 @@ func (m *Model) SelectedNode() (string, bool) { return m.nodeAt(m.nodeCursor) } // CurrentRun returns the selected run id (for tests). func (m *Model) CurrentRun() string { return m.selectedID } + +func (m *Model) stopEventStream() { + if m.streamCancel != nil { + m.streamCancel() + } + m.streamRun = "" + m.streamItems = nil + m.streamCancel = nil + m.streamConnecting = false +} + +func (m *Model) runTerminal() bool { + return m.detail != nil && terminalRunStatus(m.detail.Status) +} + +func terminalRunStatus(status string) bool { + return status == "completed" || status == "canceled" +} + +func (m *Model) seedAttentionStates() { + if m.detail == nil { + return + } + if m.prevStates == nil { + m.prevStates = map[string]string{} + } + for _, node := range m.detail.Graph.Nodes { + m.prevStates[m.detail.RunID+"/"+node.ID] = m.detail.States[node.ID] + } +} + +// applyEvent incrementally materializes the state carried by one raw +// durable store.Event. It returns true when a full detail refresh is +// useful for data not present in the event (attempt rows or graph edits). +func (m *Model) applyEvent(event EventView) bool { + if m.detail == nil { + return true + } + m.eventCursor = event.Seq + m.detail.Events = append(m.detail.Events, event) + switch event.Type { + case "transition", "recovery": + if event.NodeID != "" && event.To != "" { + if m.detail.States == nil { + m.detail.States = map[string]string{} + } + m.detail.States[event.NodeID] = event.To + } + return false + case "run": + var payload struct { + Status string `json:"status"` + } + if json.Unmarshal(event.Payload, &payload) == nil && payload.Status != "" { + m.detail.Status = payload.Status + m.detail.Done = payload.Status != "active" && payload.Status != "created" + } + // Attempt finalization is durable before the terminal/waiting run + // event. Refresh once here so transcript metadata and usage settle. + if m.runTerminal() { + m.stopEventStream() + } + return true + case "attempt", "verdict", "graph": + return true + default: + return false + } +} + +// checkAttention fires a terminal attention notification when a node's +// state demands it: a human gate awaiting approval (running) or any node +// that fails. Each condition is announced once per transition. +func (m *Model) checkAttention() tea.Cmd { + if m.detail == nil { + return nil + } + if m.prevStates == nil { + m.prevStates = map[string]string{} + } + var cmds []tea.Cmd + for _, n := range m.detail.Graph.Nodes { + cur := m.detail.States[n.ID] + key := m.detail.RunID + "/" + n.ID + prev := m.prevStates[key] + m.prevStates[key] = cur + if m.notify == nil { + continue + } + var title, body string + switch { + case cur == "running" && prev != "running" && n.Type == "human_gate": + title = "corral: gate awaits approval" + body = fmt.Sprintf("gate %s on run %s awaits approval", n.ID, m.detail.RunID) + case cur == "failed" && prev != "failed": + title = "corral: node failed" + body = fmt.Sprintf("node %s on run %s failed", n.ID, m.detail.RunID) + } + if title != "" { + notify := m.notify + titleCopy, bodyCopy := title, body + cmds = append(cmds, func() tea.Msg { + notify(titleCopy, bodyCopy) + return nil + }) + } + } + if len(cmds) == 1 { + return cmds[0] + } + if len(cmds) > 1 { + return tea.Batch(cmds...) + } + return nil +} diff --git a/internal/tui/notify.go b/internal/tui/notify.go new file mode 100644 index 0000000..859152b --- /dev/null +++ b/internal/tui/notify.go @@ -0,0 +1,78 @@ +package tui + +import ( + "context" + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "time" + "unicode/utf8" +) + +const notificationLimit = 512 + +// NotifyAttention raises terminal attention when the run needs a human: +// a terminal bell, plus a desktop notification on macOS (osascript) and +// Linux (notify-send). Best-effort and non-blocking; failures are ignored. +func NotifyAttention(title, body string) { + ringBell() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + cmd := desktopNotify(ctx, boundNotification(title), boundNotification(body)) + if cmd != nil { + if err := cmd.Start(); err != nil { + cancel() + return + } + // Reap the child and release the timeout asynchronously. NotifyAttention + // itself is already run as a Bubble Tea command, never from Update. + go func() { + _ = cmd.Wait() + cancel() + }() + return + } + cancel() +} + +// ringBell writes a terminal bell (BEL) to stdout. This is safe inside a +// bubbletea program: BEL is a control character that does not disturb the +// rendered frame. +func ringBell() { + _, _ = fmt.Fprint(os.Stdout, "\a") +} + +// desktopNotify returns the platform notification command, or nil when the +// platform has no supported notifier. +func desktopNotify(ctx context.Context, title, body string) *exec.Cmd { + switch runtime.GOOS { + case "darwin": + // Pass untrusted text as argv, outside the AppleScript source. No + // quoting or escaping can turn it into executable AppleScript. + return exec.CommandContext(ctx, "osascript", "-e", + `on run argv +display notification (item 2 of argv) with title (item 1 of argv) +end run`, title, body) + case "linux": + return exec.CommandContext(ctx, "notify-send", "--", title, body) + } + return nil +} + +func boundNotification(s string) string { + s = strings.Map(func(r rune) rune { + if r == 0 || (r < 32 && r != '\n' && r != '\t') { + return -1 + } + return r + }, s) + if len(s) > notificationLimit { + s = s[:notificationLimit] + // Avoid returning malformed UTF-8 after byte bounding. + for !utf8.ValidString(s) { + s = s[:len(s)-1] + } + } + return s +} diff --git a/internal/tui/notify_test.go b/internal/tui/notify_test.go new file mode 100644 index 0000000..b7a84d5 --- /dev/null +++ b/internal/tui/notify_test.go @@ -0,0 +1,46 @@ +package tui + +import ( + "context" + "runtime" + "strings" + "testing" + "unicode/utf8" +) + +func TestDesktopNotificationDoesNotEmbedInputInAppleScript(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + injection := `" & do shell script "touch /tmp/pwned" & "` + cmd := desktopNotify(ctx, injection, injection) + if cmd == nil { + // Unsupported host. Construction is platform-dependent. + return + } + if runtime.GOOS == "darwin" { + if len(cmd.Args) < 5 { + t.Fatalf("notification command args: %q", cmd.Args) + } + script := cmd.Args[2] + if strings.Contains(script, injection) || strings.Contains(script, "touch /tmp/pwned") { + t.Fatalf("untrusted input embedded in program source: %q", script) + } + } else if runtime.GOOS == "linux" { + if len(cmd.Args) != 4 || cmd.Args[1] != "--" { + t.Fatalf("unsafe notify-send argv: %q", cmd.Args) + } + } + if cmd.Args[len(cmd.Args)-2] != injection || cmd.Args[len(cmd.Args)-1] != injection { + t.Fatalf("notification text not passed as data argv: %q", cmd.Args) + } +} + +func TestBoundNotificationIsBoundedAndValidUTF8(t *testing.T) { + got := boundNotification(strings.Repeat("🙂", notificationLimit)) + if len(got) > notificationLimit { + t.Fatalf("notification = %d bytes, limit %d", len(got), notificationLimit) + } + if !utf8.ValidString(got) { + t.Fatal("notification truncation broke UTF-8") + } +} diff --git a/internal/tui/tui_test.go b/internal/tui/tui_test.go index a8cb9b0..27ac99e 100644 --- a/internal/tui/tui_test.go +++ b/internal/tui/tui_test.go @@ -2,8 +2,10 @@ package tui import ( "context" + "encoding/json" "fmt" "strings" + "sync" "testing" "time" @@ -11,11 +13,16 @@ import ( ) type fakeAPI struct { - runs []RunSummary - detail *RunDetail - actions []string - listErr error - detailErr error + mu sync.Mutex + runs []RunSummary + detail *RunDetail + actions []string + listErr error + detailErr error + tailErr error + tail []string + stream func(context.Context, string, int64, func(EventView) error) error + streamAfter []int64 } func (f *fakeAPI) ListRuns(ctx context.Context) ([]RunSummary, error) { @@ -32,6 +39,30 @@ func (f *fakeAPI) GetRun(ctx context.Context, runID string) (*RunDetail, error) return f.detail, nil } +func (f *fakeAPI) StreamEvents(ctx context.Context, runID string, after int64, emit func(EventView) error) error { + f.mu.Lock() + f.streamAfter = append(f.streamAfter, after) + stream := f.stream + f.mu.Unlock() + if stream != nil { + return stream(ctx, runID, after, emit) + } + return fmt.Errorf("stream unavailable") +} + +func (f *fakeAPI) streamCursors() []int64 { + f.mu.Lock() + defer f.mu.Unlock() + return append([]int64(nil), f.streamAfter...) +} + +func (f *fakeAPI) Tail(ctx context.Context, runID, nodeID string, lines int) ([]string, error) { + if f.tailErr != nil { + return nil, f.tailErr + } + return f.tail, nil +} + func (f *fakeAPI) act(label string) { f.actions = append(f.actions, label) } @@ -44,6 +75,10 @@ func (f *fakeAPI) Steer(ctx context.Context, r, n, m string) error { f.act("steer:" + n + ":" + m) return nil } +func (f *fakeAPI) RespondPermission(ctx context.Context, r, n, pid string, allow bool) error { + f.act(fmt.Sprintf("perm:%s:%s:%v", pid, n, allow)) + return nil +} func sampleDetail() *RunDetail { return &RunDetail{ @@ -65,8 +100,8 @@ func sampleDetail() *RunDetail { }}, States: map[string]string{"w1": "done", "gate": "running", "m": "pending"}, Attempts: map[string][]AttemptView{ - "w1": {{ID: "w1/1", No: 1, Status: "done", SessionID: "ses_x", Worktree: "/tmp/wt/w1", Evidence: `{"exit":0}`}}, - "gate": {{ID: "gate/1", No: 1, Status: "running"}}, + "w1": {{ID: "run_1/w1/1", No: 1, Status: "done", SessionID: "ses_x", Worktree: "/tmp/wt/w1", Evidence: `{"exit":0}`}}, + "gate": {{ID: "run_1/gate/1", No: 1, Status: "running"}}, }, Events: []EventView{{Seq: 1, Type: "transition", NodeID: "w1", From: "pending", To: "done"}}, } @@ -86,6 +121,19 @@ func send(t *testing.T, m *Model, msg tea.Msg) { } } +func runCmd(t *testing.T, cmd tea.Cmd) { + t.Helper() + if cmd == nil { + return + } + msg := cmd() + if batch, ok := msg.(tea.BatchMsg); ok { + for _, nested := range batch { + runCmd(t, nested) + } + } +} + func TestListAndNavigation(t *testing.T) { api := &fakeAPI{runs: []RunSummary{{ID: "run_1", Status: "active", States: map[string]string{"w1": "running"}}}, detail: sampleDetail()} m := New(api, context.Background()) @@ -176,6 +224,95 @@ func TestNodeActions(t *testing.T) { } } +func TestPermissionRespond(t *testing.T) { + d := sampleDetail() + // w1 is blocked on a permission request carried by its transition. + d.States["w1"] = "blocked" + d.Events = append(d.Events, EventView{ + Seq: 2, NodeID: "w1", Type: "transition", From: "running", To: "blocked", + Payload: json.RawMessage(`{"reason":"permission","permissionID":"perm-9","tool":"Bash","input":"{\"command\":\"git status\"}"}`), + }) + api := &fakeAPI{} + m := New(api, context.Background()) + m.runs = []RunSummary{{ID: "run_1"}} + m.selectedID = "run_1" + m.detail = d + m.mode = modeDetail + m.nodeCursor = 0 // w1 + + // The pending permission is surfaced in the detail view. + view := m.View() + for _, want := range []string{"perm:perm-9", "Bash", "p allow perm", "d deny perm"} { + if !strings.Contains(view, want) { + t.Fatalf("detail view missing %q:\n%s", want, view) + } + } + m.mode = modeInspect + m.inspectNode = "w1" + if view = m.View(); !strings.Contains(view, "git status") { + t.Fatalf("inspect view missing permission input:\n%s", view) + } + m.mode = modeDetail + + send(t, m, key("p")) + send(t, m, key("d")) + want := []string{"perm:perm-9:w1:true", "perm:perm-9:w1:false"} + if fmt.Sprint(api.actions) != fmt.Sprint(want) { + t.Fatalf("actions = %v, want %v", api.actions, want) + } + + // 'p'/'d' are no-ops on a node with no pending permission (the gate). + m.nodeCursor = 1 + send(t, m, key("p")) + send(t, m, key("d")) + if len(api.actions) != 2 { + t.Fatalf("permission keys acted on non-permission node: %v", api.actions) + } +} + +func TestResolvedPermissionIsNotExposedOrResent(t *testing.T) { + d := sampleDetail() + d.States["w1"] = "done" + d.Events = append(d.Events, EventView{ + Seq: 2, NodeID: "w1", Type: "transition", From: "running", To: "blocked", + Payload: json.RawMessage(`{"reason":"permission","permissionID":"perm-old"}`), + }) + d.Events = append(d.Events, EventView{ + Seq: 3, NodeID: "w1", Type: "transition", From: "blocked", To: "running", + }) + + api := &fakeAPI{} + m := New(api, context.Background()) + m.selectedID, m.detail, m.mode = "run_1", d, modeDetail + if pid, ok := m.pendingPermission("w1"); ok || pid != "" { + t.Fatalf("resolved permission exposed: %q, %v", pid, ok) + } + if strings.Contains(m.View(), "perm-old") { + t.Fatalf("resolved permission rendered:\n%s", m.View()) + } + send(t, m, key("p")) + send(t, m, key("d")) + if len(api.actions) != 0 { + t.Fatalf("resolved permission resent: %v", api.actions) + } +} + +func TestOnlyLatestBlockedTransitionCanExposePermission(t *testing.T) { + d := sampleDetail() + d.States["w1"] = "blocked" + d.Events = append(d.Events, + EventView{Seq: 2, NodeID: "w1", Type: "transition", From: "running", To: "blocked", + Payload: json.RawMessage(`{"reason":"permission","permissionID":"perm-old"}`)}, + EventView{Seq: 3, NodeID: "w1", Type: "transition", From: "ready", To: "blocked", + Payload: json.RawMessage(`{"reason":"dependency_failed"}`)}, + ) + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.detail, m.mode = "run_1", d, modeDetail + if pid, ok := m.pendingPermission("w1"); ok || pid != "" { + t.Fatalf("stale permission exposed after newer block: %q, %v", pid, ok) + } +} + func TestEmptyState(t *testing.T) { m := New(&fakeAPI{}, context.Background()) m.Update(fetchMsg{}) @@ -184,6 +321,411 @@ func TestEmptyState(t *testing.T) { } } +func TestLiveAttemptTail(t *testing.T) { + api := &fakeAPI{detail: sampleDetail(), tail: []string{"alpha", "beta", "gamma"}} + m := New(api, context.Background()) + m.selectedID = "run_1" + m.detail = sampleDetail() + m.mode = modeDetail + m.nodeCursor = 1 // gate + m.Update(key("i")) + if m.mode != modeInspect || m.tailNode != "gate" { + t.Fatalf("inspect mode = %d tailNode=%q", m.mode, m.tailNode) + } + // Simulate the tail fetch result. + m.Update(tailMsg{runID: "run_1", node: "gate", lines: api.tail}) + view := m.View() + for _, want := range []string{"live tail", "alpha", "beta", "gamma"} { + if !strings.Contains(view, want) { + t.Fatalf("inspect view missing %q:\n%s", want, view) + } + } + // Back clears the tail so it stops being fetched. + m.Update(key("esc")) + if m.tailNode != "" || len(m.tail) != 0 { + t.Fatalf("tail not cleared on back: node=%q tail=%v", m.tailNode, m.tail) + } +} + +func TestAttentionOnGateAndFailure(t *testing.T) { + var got []string + api := &fakeAPI{detail: sampleDetail()} + m := New(api, context.Background()) + m.notify = func(title, body string) { got = append(got, title+" | "+body) } + m.selectedID = "run_1" + m.detail = sampleDetail() + m.detail.States["gate"] = "pending" + m.seedAttentionStates() + m.streamAttempted = true + + // Gate transitions into running → gate attention fires. + next := sampleDetail() + _, cmd := m.Update(fetchRunMsg{detail: next}) + runCmd(t, cmd) + if len(got) != 1 || !strings.Contains(got[0], "gate") { + t.Fatalf("gate attention not fired: %v", got) + } + + // Same state again: no duplicate. + _, cmd = m.Update(fetchRunMsg{detail: next}) + runCmd(t, cmd) + if len(got) != 1 { + t.Fatalf("duplicate attention fired: %v", got) + } + + // A node fails → failure attention fires. + detail := sampleDetail() + detail.States["w1"] = "failed" + _, cmd = m.Update(fetchRunMsg{detail: detail}) + runCmd(t, cmd) + if len(got) != 2 || !strings.Contains(got[1], "failed") { + t.Fatalf("failure attention not fired: %v", got) + } +} + +func TestAttentionDisabledByDefault(t *testing.T) { + api := &fakeAPI{detail: sampleDetail()} + m := New(api, context.Background()) + m.selectedID = "run_1" + m.detail = sampleDetail() + m.detail.States["gate"] = "pending" + m.seedAttentionStates() + m.streamAttempted = true + _, cmd := m.Update(fetchRunMsg{detail: sampleDetail()}) + if cmd != nil { + t.Fatal("attention should be disabled unless EnableAttention is called") + } +} + +func TestBudgetBarInDetail(t *testing.T) { + d := sampleDetail() + d.Graph.Nodes[0].Budget.MaxDuration = int64(2 * time.Second) + d.Attempts["w1"] = []AttemptView{{ID: "w1/1", No: 1, Status: "done", StartedAt: int64Ptr(1000), FinishedAt: int64Ptr(2000)}} + m := New(&fakeAPI{detail: d}, context.Background()) + m.selectedID = "run_1" + m.detail = d + m.mode = modeDetail + view := m.View() + if !strings.Contains(view, "progress") { + t.Fatalf("detail view missing progress bar:\n%s", view) + } + if !strings.Contains(view, "1s/2s") { + t.Fatalf("detail view missing budget usage:\n%s", view) + } + if !strings.Contains(view, "wall elapsed") { + t.Fatalf("detail view must label timestamp-only duration as wall elapsed:\n%s", view) + } +} + +func TestBudgetBarExcludesPermissionWaitFromProviderRuntime(t *testing.T) { + d := sampleDetail() + n := &d.Graph.Nodes[0] + n.Budget.MaxDuration = int64(10 * time.Second) + d.Attempts["w1"] = []AttemptView{{ + ID: "run_1/w1/1", No: 1, Status: "done", + StartedAt: int64Ptr(1_000), FinishedAt: int64Ptr(26_000), + }} + d.Events = []EventView{ + {Seq: 1, NodeID: "w1", Type: "attempt", AttemptID: "run_1/w1/1", CreatedAt: 1_000}, + {Seq: 2, NodeID: "w1", Type: "transition", From: "running", To: "blocked", CreatedAt: 3_000}, + {Seq: 3, NodeID: "w1", Type: "transition", From: "blocked", To: "ready", CreatedAt: 23_000}, + {Seq: 4, NodeID: "w1", Type: "transition", From: "ready", To: "leased", CreatedAt: 23_000}, + {Seq: 5, NodeID: "w1", Type: "transition", From: "leased", To: "running", CreatedAt: 23_000}, + {Seq: 6, NodeID: "w1", Type: "transition", From: "running", To: "verifying", CreatedAt: 26_000}, + } + m := New(&fakeAPI{}, context.Background()) + m.detail = d + bar := m.nodeBudgetBar(*n, d.Attempts["w1"], "done") + if !strings.Contains(bar, "runtime 5s/10s") { + t.Fatalf("permission wait counted as provider runtime: %q", bar) + } +} + +func TestElapsedActiveAttemptUsesCurrentTime(t *testing.T) { + start := time.Now().Add(-2 * time.Second).UnixMilli() + got := elapsed(start, nil) + if got == "0ms" { + t.Fatal("active attempt elapsed time must advance") + } +} + +func TestInspectSanitizesProviderTerminalControls(t *testing.T) { + d := sampleDetail() + d.States["w1"] = "running" + d.Graph.Nodes[0].Objective = "safe\x1b]52;c;Y2xpcGJvYXJk\a\x1b[31mred\x1b[0m" + d.Attempts["w1"][0].Status = "run\x1b]52;c;c3RhdHVz\a\x1b[31mning\x1b[0m" + d.Attempts["w1"][0].Evidence = "proof\x1b]0;title\a" + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.inspectNode, m.detail, m.mode = "run_1", "w1", d, modeInspect + m.tail = []string{"tail\x1b]52;c;ZXZpbA==\a\x1b[2Jvisible"} + view := m.View() + for _, unsafe := range []string{"\x1b]52", "\x1b[31m", "\x1b[2J", "\a"} { + if strings.Contains(view, unsafe) { + t.Fatalf("view retained terminal control %q: %q", unsafe, view) + } + } + for _, want := range []string{"safe", "red", "proof", "tail", "visible"} { + if !strings.Contains(view, want) { + t.Fatalf("view lost safe text %q: %q", want, view) + } + } +} + +func TestCursorTargetsRenderedGraphOrder(t *testing.T) { + d := sampleDetail() + // Input order differs from the stable ID order rendered by viewDetail. + d.Graph.Nodes = []GraphNode{d.Graph.Nodes[2], d.Graph.Nodes[0], d.Graph.Nodes[1]} + api := &fakeAPI{} + m := New(api, context.Background()) + m.selectedID, m.detail, m.mode, m.nodeCursor = "run_1", d, modeDetail, 0 + if selected, ok := m.SelectedNode(); !ok || selected != "m" { + t.Fatalf("selected node = %q, %v; want first rendered node m", selected, ok) + } + _, cmd := m.Update(key("a")) + if cmd == nil { + t.Fatal("approve command missing") + } + cmd() + if len(api.actions) != 1 || api.actions[0] != "approve:m" { + t.Fatalf("cursor action = %v, want approve:m", api.actions) + } +} + +func TestBudgetBarUsesHighestUtilizationAndDoesNotFillDone(t *testing.T) { + d := sampleDetail() + n := &d.Graph.Nodes[0] + n.Budget.MaxDuration = int64(100 * time.Second) + n.Budget.MaxTokens = 100 + n.Budget.MaxCost = 10 + d.Attempts["w1"] = []AttemptView{{ + ID: "run_1/w1/1", No: 1, Status: "done", + StartedAt: int64Ptr(1_000), FinishedAt: int64Ptr(11_000), + Tokens: 75, Cost: 2, + }} + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.detail, m.mode = "run_1", d, modeDetail + bar := m.nodeBudgetBar(*n, d.Attempts["w1"], "done") + if !strings.Contains(bar, "tokens") { + t.Fatalf("dominant token budget not labeled: %q", bar) + } + if got := strings.Count(bar, "█"); got != 6 { + t.Fatalf("done node filled %d/8 cells, want actual 75%% (6/8): %q", got, bar) + } +} + +func TestDetailViewHandlesEmptyGraph(t *testing.T) { + d := &RunDetail{RunID: "empty", States: map[string]string{}, Attempts: map[string][]AttemptView{}} + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.detail, m.mode = "empty", d, modeDetail + if got := m.View(); !strings.Contains(got, "no nodes") { + t.Fatalf("empty graph view missing empty state:\n%s", got) + } +} + +func TestAttentionExecutesOutsideUpdate(t *testing.T) { + var got []string + m := New(&fakeAPI{}, context.Background()) + m.notify = func(title, body string) { got = append(got, title+" | "+body) } + m.selectedID = "run_1" + m.detail = sampleDetail() + m.detail.States["gate"] = "pending" + m.seedAttentionStates() + m.streamAttempted = true + _, cmd := m.Update(fetchRunMsg{detail: sampleDetail()}) + if len(got) != 0 { + t.Fatalf("notification blocked Update: %v", got) + } + if cmd == nil { + t.Fatal("transition did not return async attention command") + } + _ = cmd() + if len(got) != 1 || !strings.Contains(got[0], "gate") { + t.Fatalf("attention command did not notify: %v", got) + } +} + +func TestEventUpdatesStateIncrementallyAndAdvancesCursor(t *testing.T) { + d := sampleDetail() + d.States["w1"] = "running" + d.Events = []EventView{{Seq: 4, Type: "transition", NodeID: "w1", From: "leased", To: "running"}} + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.detail, m.mode, m.eventCursor = "run_1", d, modeDetail, 4 + items := make(chan eventStreamItem) + m.streamItems = items + + event := EventView{Seq: 5, RunID: "run_1", Type: "transition", NodeID: "w1", From: "running", To: "blocked", + Payload: json.RawMessage(`{"reason":"permission","permissionID":"perm-live"}`)} + _, cmd := m.Update(eventStreamItemMsg{runID: "run_1", items: items, item: eventStreamItem{event: &event}}) + if m.eventCursor != 5 || m.detail.States["w1"] != "blocked" { + t.Fatalf("incremental state cursor=%d state=%q", m.eventCursor, m.detail.States["w1"]) + } + if pid, ok := m.pendingPermission("w1"); !ok || pid != "perm-live" { + t.Fatalf("incremental permission = %q, %v", pid, ok) + } + if cmd == nil { + t.Fatal("stream listener was not continued") + } +} + +func TestFirstStreamEventGapFallsBackWithoutApplying(t *testing.T) { + d := sampleDetail() + d.Events = nil + d.States["w1"] = "pending" + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.detail, m.mode = "run_1", d, modeDetail + items := make(chan eventStreamItem) + m.streamItems = items + m.streamCancel = func() {} + event := EventView{Seq: 2, RunID: "run_1", Type: "transition", NodeID: "w1", From: "pending", To: "running"} + + _, cmd := m.Update(eventStreamItemMsg{runID: "run_1", items: items, item: eventStreamItem{event: &event}}) + if m.eventCursor != 0 || m.detail.States["w1"] != "pending" { + t.Fatalf("gapped event applied: cursor=%d state=%q", m.eventCursor, m.detail.States["w1"]) + } + if m.streamItems != nil || cmd == nil { + t.Fatal("gapped event did not stop stream and request full refresh") + } +} + +func TestStaleFullRefreshDoesNotOverwriteNewerEventState(t *testing.T) { + current := sampleDetail() + current.Events = []EventView{{Seq: 5, Type: "transition", NodeID: "w1", From: "leased", To: "running"}} + current.States["w1"] = "running" + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.detail, m.mode, m.eventCursor = "run_1", current, modeDetail, 5 + + stale := sampleDetail() + stale.Events = []EventView{{Seq: 4, Type: "transition", NodeID: "w1", From: "ready", To: "leased"}} + stale.States["w1"] = "leased" + m.Update(fetchRunMsg{runID: "run_1", detail: stale}) + if m.detail.States["w1"] != "running" || m.eventCursor != 5 { + t.Fatalf("stale refresh regressed state: cursor=%d state=%q", m.eventCursor, m.detail.States["w1"]) + } +} + +func TestDroppedStreamFallsBackToFullRefreshAndReconnectsFromCursor(t *testing.T) { + api := &fakeAPI{detail: sampleDetail()} + m := New(api, context.Background()) + m.selectedID, m.detail, m.mode, m.eventCursor = "run_1", sampleDetail(), modeDetail, 9 + items := make(chan eventStreamItem) + m.streamItems = items + canceled := false + m.streamCancel = func() { canceled = true } + + _, cmd := m.Update(eventStreamItemMsg{runID: "run_1", items: items, item: eventStreamItem{err: fmt.Errorf("dropped")}}) + if !canceled || m.streamItems != nil || cmd == nil { + t.Fatalf("drop did not stop stream/fallback: canceled=%v items=%v cmd=%v", canceled, m.streamItems, cmd) + } + msg := cmd() + refresh, ok := msg.(fetchRunMsg) + if !ok || refresh.detail == nil || refresh.runID != "run_1" { + t.Fatalf("fallback = %#v, want full run refresh", msg) + } + + _, cmd = m.Update(tickMsg(time.Now())) + batch, ok := cmd().(tea.BatchMsg) + if !ok { + t.Fatalf("reconnect tick = %T, want batch", cmd()) + } + var ready eventStreamReadyMsg + for _, nested := range batch { + if nested == nil { + continue + } + if msg := nested(); msg != nil { + if value, ok := msg.(eventStreamReadyMsg); ok { + ready = value + } + } + } + if ready.items == nil { + t.Fatal("fallback tick did not reconnect stream") + } + ready.cancel() + deadline := time.Now().Add(time.Second) + for len(api.streamCursors()) == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + cursors := api.streamCursors() + if len(cursors) == 0 || cursors[len(cursors)-1] != 9 { + t.Fatalf("reconnect cursors = %v, want last cursor 9", cursors) + } +} + +func TestHealthyStreamPreventsOneSecondFullPolling(t *testing.T) { + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.detail, m.mode = "run_1", sampleDetail(), modeDetail + m.streamItems = make(chan eventStreamItem) + _, cmd := m.Update(tickMsg(time.Now())) + if _, ok := cmd().(fetchRunMsg); ok { + t.Fatal("healthy event stream still triggered one-second full polling") + } +} + +func TestWaitingRunKeepsStreamForExternalResolution(t *testing.T) { + d := sampleDetail() + d.Status = "waiting" + d.Done = true + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.detail, m.mode, m.eventCursor = "run_1", d, modeDetail, 4 + items := make(chan eventStreamItem) + m.streamItems = items + canceled := false + m.streamCancel = func() { canceled = true } + event := EventView{Seq: 5, RunID: "run_1", Type: "run", Payload: json.RawMessage(`{"status":"waiting"}`)} + _, cmd := m.Update(eventStreamItemMsg{runID: "run_1", items: items, item: eventStreamItem{event: &event}}) + if canceled || m.streamItems == nil { + t.Fatal("waiting run stopped stream needed for later permission resolution") + } + if cmd == nil { + t.Fatal("waiting run did not keep stream listener") + } +} + +func TestTerminalRunEventStopsWithoutAnotherStreamWait(t *testing.T) { + d := sampleDetail() + d.States["w1"] = "done" + m := New(&fakeAPI{detail: d}, context.Background()) + m.selectedID, m.detail, m.mode, m.eventCursor = "run_1", d, modeDetail, 4 + items := make(chan eventStreamItem) + m.streamItems = items + m.streamCancel = func() {} + event := EventView{Seq: 5, RunID: "run_1", Type: "run", Payload: json.RawMessage(`{"status":"completed"}`)} + + _, cmd := m.Update(eventStreamItemMsg{runID: "run_1", items: items, item: eventStreamItem{event: &event}}) + if m.streamItems != nil || !m.runTerminal() { + t.Fatal("terminal run event did not stop stream") + } + if _, ok := cmd().(tea.BatchMsg); ok { + t.Fatal("terminal event enqueued another wait on stopped stream") + } +} + +func TestInitialWaitingRunStartsEventStream(t *testing.T) { + d := sampleDetail() + d.Status = "waiting" + d.Done = true + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.mode = "run_1", modeDetail + _, cmd := m.Update(fetchRunMsg{runID: "run_1", detail: d}) + if cmd == nil || !m.streamConnecting { + t.Fatal("initial waiting run did not start event stream") + } +} + +func TestWaitingRunDoesNotRenderDoneMarker(t *testing.T) { + d := sampleDetail() + d.Status = "waiting" + d.Done = true + m := New(&fakeAPI{}, context.Background()) + m.selectedID, m.detail, m.mode = "run_1", d, modeDetail + if got := m.View(); strings.Contains(got, "✓ done") { + t.Fatalf("waiting run rendered terminal marker:\n%s", got) + } +} + +func int64Ptr(v int64) *int64 { return &v } + func TestFetchErrorShown(t *testing.T) { api := &fakeAPI{listErr: fmt.Errorf("connection refused")} m := New(api, context.Background()) diff --git a/internal/tui/view.go b/internal/tui/view.go index 848be36..52ae545 100644 --- a/internal/tui/view.go +++ b/internal/tui/view.go @@ -5,8 +5,10 @@ import ( "sort" "strings" "time" + "unicode" "github.com/charmbracelet/lipgloss" + "github.com/charmbracelet/x/ansi" ) var ( @@ -19,6 +21,27 @@ var ( styleMuted = lipgloss.NewStyle().Foreground(lipgloss.Color("244")) ) +// progressBar renders a horizontal bar of the given width with the filled +// fraction (0..1). Empty fill uses a bright color, so near-full usage is +// easy to spot. +func progressBar(frac float64, width int) string { + if frac < 0 { + frac = 0 + } + if frac > 1 { + frac = 1 + } + filled := int(frac*float64(width) + 0.5) + body := strings.Repeat("█", filled) + strings.Repeat("░", width-filled) + if frac >= 1 { + return styleError.Render(body) + } + if frac >= 0.85 { + return lipgloss.NewStyle().Foreground(lipgloss.Color("214")).Render(body) + } + return styleOK.Render(body) +} + func stateColor(s string) lipgloss.Style { switch s { case "done": @@ -52,6 +75,23 @@ func (m *Model) View() string { return "" } +func stripUnsafeControls(s string) string { + return strings.Map(func(r rune) rune { + switch r { + case '\n', '\t': + return r + } + if unicode.IsControl(r) { + return -1 + } + return r + }, s) +} + +func safeText(s string) string { + return stripUnsafeControls(ansi.Strip(s)) +} + func (m *Model) viewList() string { var b strings.Builder b.WriteString(styleHeader.Render("corral — runs") + "\n\n") @@ -59,11 +99,11 @@ func (m *Model) viewList() string { b.WriteString(styleDim.Render("no runs yet; start one from OpenCode (corral_start) or the daemon API") + "\n") } else { for i, r := range m.runs { - line := fmt.Sprintf(" %-28s %-10s %s", r.ID, r.Status, r.StateSummary()) + line := fmt.Sprintf(" %-28s %-10s %s", safeText(r.ID), safeText(r.Status), r.StateSummary()) if i == m.cursor { line = styleSelected.Render(line) } - if r.Done { + if r.Status == "completed" { line += styleOK.Render(" ✓") } b.WriteString(line + "\n") @@ -78,47 +118,60 @@ func (m *Model) viewDetail() string { return "loading…" } var b strings.Builder - b.WriteString(styleTitle.Render(fmt.Sprintf("run %s [%s]", m.detail.RunID, m.detail.Status))) - if m.detail.Done { + b.WriteString(styleTitle.Render(fmt.Sprintf("run %s [%s]", safeText(m.detail.RunID), safeText(m.detail.Status)))) + if m.detail.Status == "completed" { b.WriteString(styleOK.Render(" ✓ done")) } + // Overall run progress: done nodes / total. + total := len(m.detail.Graph.Nodes) + if total > 0 { + done := 0 + for _, n := range m.detail.Graph.Nodes { + if m.detail.States[n.ID] == "done" { + done++ + } + } + b.WriteString("\n" + styleDim.Render("progress ") + progressBar(float64(done)/float64(total), 20) + + styleMuted.Render(fmt.Sprintf(" %d/%d done", done, total))) + } b.WriteString("\n\n") // DAG view: nodes in dependency order, arrows between them. b.WriteString(styleDim.Render("dag") + "\n") nodes := m.detail.Graph.Nodes + if len(nodes) == 0 { + b.WriteString(styleMuted.Render(" no nodes") + "\n") + } byID := map[string]GraphNode{} for _, n := range nodes { byID[n.ID] = n } - // Topological-ish order: by state then id for stable rendering. - order := append([]GraphNode(nil), nodes...) - sort.SliceStable(order, func(i, j int) bool { return order[i].ID < order[j].ID }) indeg := map[string]int{} - for _, n := range order { + for _, n := range nodes { indeg[n.ID] = len(n.DependsOn) } - for _, n := range order { + selected, _ := m.nodeAt(m.nodeCursor) + for _, n := range nodes { line := " " + m.nodeLine(n, indeg[n.ID]) - if n.ID == m.detail.Graph.Nodes[m.nodeCursor].ID { + if n.ID == selected { line = styleSelected.Render(line) } b.WriteString(line + "\n") for _, dep := range n.DependsOn { if dn, ok := byID[dep]; ok { - b.WriteString(styleDim.Render(fmt.Sprintf(" ← %s (%s)\n", dep, m.detail.States[dep]))) + b.WriteString(styleDim.Render(fmt.Sprintf(" ← %s (%s)\n", safeText(dep), safeText(m.detail.States[dep])))) _ = dn } } } - b.WriteString("\n" + m.footer("↑/↓ node · a approve · r reject · c cancel · t retry · s steer · i inspect · esc back · q quit")) + b.WriteString("\n" + m.footer("↑/↓ node · a approve · r reject · c cancel · t retry · s steer · p allow perm · d deny perm · i inspect · esc back · q quit")) return b.String() } func (m *Model) nodeLine(n GraphNode, deps int) string { state := m.detail.States[n.ID] - st := stateColor(state).Render(pad(state, 10)) - typ := styleDim.Render(n.Type) + st := stateColor(state).Render(pad(safeText(state), 10)) + typ := styleDim.Render(safeText(n.Type)) prio := styleDim.Render(fmt.Sprintf("p%d", n.Priority)) atts := m.detail.Attempts[n.ID] attempts := styleDim.Render(fmt.Sprintf("%d att", len(atts))) @@ -126,7 +179,149 @@ func (m *Model) nodeLine(n GraphNode, deps int) string { if deps > 0 { depsS = styleDim.Render(fmt.Sprintf("(%d deps)", deps)) } - return fmt.Sprintf("%-12s %s %-7s %s %s %s", n.ID, st, typ, prio, attempts, depsS) + permS := "" + if state == "blocked" { + if pid, tool, _, ok := m.pendingPermissionDetails(n.ID); ok { + permS = styleTitle.Render(fmt.Sprintf("perm:%s", safeText(pid))) + if tool != "" { + permS += styleMuted.Render(" " + safeText(tool)) + } + } + } + bar := m.nodeBudgetBar(n, atts, state) + return fmt.Sprintf("%-12s %s %-7s %s %s %s %s %s", safeText(n.ID), st, typ, prio, attempts, depsS, bar, permS) +} + +// nodeBudgetBar renders a compact budget-usage bar for a node. The +// dominant budget dimension (provider runtime, tokens, or cost) drives the +// bar, with the used/limit figures beside it. Nodes without a budget show "". +func (m *Model) nodeBudgetBar(n GraphNode, atts []AttemptView, _ string) string { + maxDur := n.Budget.MaxDuration + maxTok := n.Budget.MaxTokens + maxCost := n.Budget.MaxCost + if maxDur <= 0 && maxTok <= 0 && maxCost <= 0 { + return "" + } + // Used figures, from the most recent attempt for runtime and the sum for + // tokens/cost. + var usedDur time.Duration + timeLabel := "wall elapsed" + usedTok := 0 + usedCost := 0.0 + if len(atts) > 0 { + at := atts[len(atts)-1] + if runtime, exact := m.attemptRuntime(n.ID, at); exact { + usedDur = runtime + timeLabel = "runtime" + } else if at.StartedAt != nil { + start := time.UnixMilli(*at.StartedAt) + end := m.now() + if at.FinishedAt != nil { + end = time.UnixMilli(*at.FinishedAt) + } + if end.After(start) { + usedDur = end.Sub(start) + } + } + } + for _, at := range atts { + usedTok += at.Tokens + usedCost += at.Cost + } + // Choose the highest-utilization configured dimension. Completion is + // lifecycle progress, not budget consumption, so done nodes retain + // their actual utilization. + type usage struct { + fraction float64 + label string + } + var dominant usage + consider := func(candidate usage) { + if dominant.label == "" || candidate.fraction > dominant.fraction { + dominant = candidate + } + } + if maxDur > 0 { + consider(usage{ + fraction: float64(usedDur) / float64(time.Duration(maxDur)), + label: fmt.Sprintf("%s %s/%s", timeLabel, usedDur.Round(time.Second), time.Duration(maxDur).Round(time.Second)), + }) + } + if maxTok > 0 { + consider(usage{ + fraction: float64(usedTok) / float64(maxTok), + label: fmt.Sprintf("tokens %d/%d", usedTok, maxTok), + }) + } + if maxCost > 0 { + consider(usage{ + fraction: usedCost / maxCost, + label: fmt.Sprintf("cost $%.2f/$%.2f", usedCost, maxCost), + }) + } + return progressBar(dominant.fraction, 8) + " " + styleMuted.Render(dominant.label) +} + +// attemptRuntime sums intervals in which the provider session was running. +// Durable transition events make permission-blocked intervals visible, so +// those waits do not consume the displayed runtime budget. Older/incomplete +// event histories return exact=false and are shown explicitly as wall elapsed. +func (m *Model) attemptRuntime(nodeID string, at AttemptView) (time.Duration, bool) { + if m.detail == nil || at.StartedAt == nil { + return 0, false + } + var startSeq int64 + for _, event := range m.detail.Events { + if event.Type == "attempt" && event.AttemptID == at.ID { + startSeq = event.Seq + break + } + } + if startSeq == 0 { + return 0, false + } + + start := *at.StartedAt + end := m.now().UnixMilli() + if at.FinishedAt != nil { + end = *at.FinishedAt + } + if end < start { + end = start + } + + active := true + activeSince := start + var usedMillis int64 + for _, event := range m.detail.Events { + if event.Seq <= startSeq || event.NodeID != nodeID || event.Type != "transition" || event.CreatedAt < start || event.CreatedAt > end { + continue + } + if active && event.From == "running" && event.To != "running" { + if event.CreatedAt > activeSince { + usedMillis += event.CreatedAt - activeSince + } + active = false + continue + } + if !active && event.To == "running" { + active = true + activeSince = event.CreatedAt + } + } + if active && end > activeSince { + usedMillis += end - activeSince + } + return time.Duration(usedMillis) * time.Millisecond, true +} + +// now returns the model's wall-clock reference (last tick time, or real +// time before the first tick) for elapsed computations. +func (m *Model) now() time.Time { + if m.lastFetch.IsZero() { + return time.Now() + } + return m.lastFetch } func (m *Model) viewInspect() string { @@ -139,25 +334,43 @@ func (m *Model) viewInspect() string { return "node gone" } state := m.detail.States[n.ID] - b.WriteString(styleTitle.Render(fmt.Sprintf("node %s %s", n.ID, stateColor(state).Render(state))) + "\n\n") - b.WriteString(styleDim.Render("objective: ") + n.Objective + "\n") + b.WriteString(styleTitle.Render(fmt.Sprintf("node %s %s", safeText(n.ID), stateColor(state).Render(safeText(state)))) + "\n\n") + b.WriteString(styleDim.Render("objective: ") + safeText(n.Objective) + "\n") if n.Role != "" { - b.WriteString(styleDim.Render("role: ") + n.Role + "\n") + b.WriteString(styleDim.Render("role: ") + safeText(n.Role) + "\n") } if len(n.WriteScope) > 0 { - b.WriteString(styleDim.Render("write scope: ") + strings.Join(n.WriteScope, ", ") + "\n") + scopes := make([]string, len(n.WriteScope)) + for i, scope := range n.WriteScope { + scopes[i] = safeText(scope) + } + b.WriteString(styleDim.Render("write scope: ") + strings.Join(scopes, ", ") + "\n") } if n.Verification != nil { - b.WriteString(styleDim.Render("verification: ") + n.Verification.Kind + " " + strings.Join(n.Verification.Command, " ") + "\n") + command := make([]string, len(n.Verification.Command)) + for i, arg := range n.Verification.Command { + command[i] = safeText(arg) + } + b.WriteString(styleDim.Render("verification: ") + safeText(n.Verification.Kind) + " " + strings.Join(command, " ") + "\n") + } + if pid, tool, input, ok := m.pendingPermissionDetails(n.ID); ok { + b.WriteString(styleDim.Render("permission: ") + styleTitle.Render(safeText(pid))) + if tool != "" { + b.WriteString(styleMuted.Render(" tool=" + safeText(tool))) + } + if input != "" { + b.WriteString(styleMuted.Render(" input=" + shortLine(safeText(input), 100))) + } + b.WriteString(styleMuted.Render(" pending — p allow · d deny") + "\n") } b.WriteString("\n" + styleDim.Render("attempts") + "\n") for _, at := range m.detail.Attempts[n.ID] { - b.WriteString(fmt.Sprintf(" #%d %-10s", at.No, stateColor(at.Status).Render(at.Status))) + b.WriteString(fmt.Sprintf(" #%d %-10s", at.No, stateColor(at.Status).Render(safeText(at.Status)))) if at.SessionID != "" { - b.WriteString(styleMuted.Render(" session=" + at.SessionID)) + b.WriteString(styleMuted.Render(" session=" + safeText(at.SessionID))) } if at.Worktree != "" { - b.WriteString(styleMuted.Render(" worktree=" + shortPath(at.Worktree))) + b.WriteString(styleMuted.Render(" worktree=" + shortPath(safeText(at.Worktree)))) } if at.StartedAt != nil { b.WriteString(styleMuted.Render(" " + elapsed(*at.StartedAt, at.FinishedAt))) @@ -167,17 +380,28 @@ func (m *Model) viewInspect() string { b.WriteString(styleMuted.Render(fmt.Sprintf(" $%.4f %d tok\n", at.Cost, at.Tokens))) } if at.Evidence != "" { - b.WriteString(styleMuted.Render(" evidence: "+shortLine(at.Evidence, 90)) + "\n") + b.WriteString(styleMuted.Render(" evidence: "+shortLine(safeText(at.Evidence), 90)) + "\n") } } - b.WriteString("\n" + m.footer("esc back · ↑/↓ navigate · a/r/c/t/s act")) + // Live attempt tail for the current (running) attempt. + if state == "running" || state == "verifying" { + b.WriteString("\n" + styleDim.Render("live tail") + "\n") + if len(m.tail) == 0 { + b.WriteString(styleMuted.Render(" (no output yet)\n")) + } else { + for _, ln := range m.tail { + b.WriteString(styleMuted.Render(" "+shortLine(safeText(ln), 90)) + "\n") + } + } + } + b.WriteString("\n" + m.footer("esc back · ↑/↓ navigate · a/r/c/t/s act · p/d respond perm")) return b.String() } func (m *Model) viewSteer() string { var b strings.Builder - b.WriteString(styleTitle.Render(fmt.Sprintf("steer %s/%s", m.selectedID, m.steerNode)) + "\n\n") - b.WriteString("message: " + m.steerInput + "▌\n\n") + b.WriteString(styleTitle.Render(fmt.Sprintf("steer %s/%s", safeText(m.selectedID), safeText(m.steerNode))) + "\n\n") + b.WriteString("message: " + safeText(m.steerInput) + "▌\n\n") b.WriteString(m.footer("type message · enter send · esc cancel")) return b.String() } @@ -186,10 +410,10 @@ func (m *Model) footer(keys string) string { var b strings.Builder b.WriteString(styleDim.Render("── " + keys)) if m.status != "" { - b.WriteString(styleOK.Render(" · " + m.status)) + b.WriteString(styleOK.Render(" · " + safeText(m.status))) } if m.err != nil { - b.WriteString(styleError.Render(" · " + m.err.Error())) + b.WriteString(styleError.Render(" · " + safeText(m.err.Error()))) } return b.String() } @@ -209,7 +433,7 @@ func (m *Model) findNode(id string) *GraphNode { func (r RunSummary) StateSummary() string { var parts []string for _, id := range sortedKeys(r.States) { - parts = append(parts, fmt.Sprintf("%s:%s", id, stateColor(r.States[id]).Render(r.States[id]))) + parts = append(parts, fmt.Sprintf("%s:%s", safeText(id), stateColor(r.States[id]).Render(safeText(r.States[id])))) } return strings.Join(parts, " ") } @@ -246,11 +470,14 @@ func shortPath(p string) string { } func elapsed(start int64, end *int64) string { - e := start + e := time.Now().UnixMilli() if end != nil { e = *end } d := time.Duration(e-start) * time.Millisecond + if d < 0 { + d = 0 + } if d < time.Second { return fmt.Sprintf("%dms", d.Milliseconds()) } diff --git a/internal/worktree/worktree.go b/internal/worktree/worktree.go index ab5cbc6..bf91fe7 100644 --- a/internal/worktree/worktree.go +++ b/internal/worktree/worktree.go @@ -8,11 +8,13 @@ import ( "context" "crypto/sha256" "encoding/hex" + "errors" "fmt" "os" "os/exec" "path/filepath" "strings" + "time" ) // Manager owns worktrees for one repository. @@ -52,6 +54,23 @@ func (m *Manager) git(ctx context.Context, dir string, args ...string) (string, return string(out), nil } +// gitExit runs git like git but returns the exit code instead of wrapping +// non-zero exits as errors (for commands whose exit code is meaningful, +// e.g. merge-base --is-ancestor). +func (m *Manager) gitExit(ctx context.Context, dir string, args ...string) (string, int, error) { + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err == nil { + return string(out), 0, nil + } + var ee *exec.ExitError + if errors.As(err, &ee) { + return string(out), ee.ExitCode(), nil + } + return "", -1, err +} + // Add creates a worktree for branch at a fresh path, based on the main // checkout HEAD. func (m *Manager) Add(ctx context.Context, branch string) (string, error) { @@ -92,6 +111,15 @@ func (m *Manager) Files(ctx context.Context, worktree string) ([]string, error) return files, nil } +// ChangedFiles lists tracked and untracked (non-ignored) paths changed from +// HEAD. Intent-to-add makes untracked files visible without committing them. +func (m *Manager) ChangedFiles(ctx context.Context, worktree string) ([]string, error) { + if _, err := m.git(ctx, worktree, "add", "-A", "--intent-to-add"); err != nil { + return nil, err + } + return m.Files(ctx, worktree) +} + // HashContent returns the content address of a patch. func HashContent(content string) string { h := sha256.Sum256([]byte(content)) @@ -103,9 +131,23 @@ func (m *Manager) CommitWorktree(ctx context.Context, worktree string) error { if _, err := m.git(ctx, worktree, "add", "-A"); err != nil { return err } - if _, err := m.git(ctx, worktree, "-c", "user.name=corral", "-c", "user.email=corral@local", "commit", "-q", "-m", "corral: work"); err != nil { - // Nothing to commit is fine. + // Determine the no-op case explicitly. A non-zero commit can also mean + // a hook, signer, or filesystem failure; those errors must reach the + // caller so the staged work can be retried. + out, code, err := m.gitExit(ctx, worktree, "diff", "--cached", "--quiet", "--exit-code", "HEAD", "--") + if err != nil { + return err + } + switch code { + case 0: return nil + case 1: + // Staged changes exist; commit them below. + default: + return fmt.Errorf("inspect staged work: git diff exited %d: %s", code, tail([]byte(out), 400)) + } + if _, err := m.git(ctx, worktree, "-c", "user.name=corral", "-c", "user.email=corral@local", "commit", "-q", "-m", "corral: work"); err != nil { + return err } return nil } @@ -121,15 +163,37 @@ func (m *Manager) MergeBranch(ctx context.Context, branch string) error { if _, err := m.git(ctx, m.repo, "checkout", "-q", main); err != nil { return err } - out, err := m.git(ctx, m.repo, + _, err = m.git(ctx, m.repo, "-c", "user.name=corral", "-c", "user.email=corral@local", "merge", "--no-ff", "-m", "corral: merge "+branch, branch) if err != nil { - return fmt.Errorf("merge %s: %w: %s", branch, err, tail([]byte(out), 400)) + mergeErr := fmt.Errorf("merge %s: %w", branch, err) + if abortErr := m.abortMerge(); abortErr != nil { + return fmt.Errorf("%w; abort failed: %v", mergeErr, abortErr) + } + return mergeErr } return nil } +// abortMerge restores the main checkout after a merge that entered merge +// state and then failed (for example, a conflict or merge-commit hook). +// Cleanup uses its own bounded context so caller cancellation cannot strand +// the repository in an active merge. +func (m *Manager) abortMerge() error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, code, err := m.gitExit(ctx, m.repo, "rev-parse", "--verify", "--quiet", "MERGE_HEAD") + if err != nil { + return err + } + if code != 0 { + return nil + } + _, err = m.git(ctx, m.repo, "merge", "--abort") + return err +} + // MainBranch returns the currently checked-out branch of the main repo. func (m *Manager) MainBranch(ctx context.Context) (string, error) { out, err := m.git(ctx, m.repo, "branch", "--show-current") @@ -143,20 +207,221 @@ func (m *Manager) MainBranch(ctx context.Context) (string, error) { return b, nil } -// Remove deletes a worktree. Failed worktrees are kept for inspection by -// design; callers invoke Remove only after successful merge/cleanup. +// Remove deletes a clean worktree. Any tracked, untracked, or ignored content +// keeps the worktree for inspection, even after its branch was merged. func (m *Manager) Remove(ctx context.Context, branch string) error { path := filepath.Join(m.dir, branch) if _, err := os.Stat(path); err != nil { return nil // already gone } - if _, err := m.git(ctx, m.repo, "worktree", "remove", "--force", path); err != nil { + status, err := m.git(ctx, path, "status", "--porcelain=v1", "--untracked-files=normal", "--ignored=matching") + if err != nil { + return err + } + if strings.TrimSpace(status) != "" { + return fmt.Errorf("refuse to remove dirty worktree %s", path) + } + if _, err := m.git(ctx, m.repo, "worktree", "remove", path); err != nil { return err } _, _ = m.git(ctx, m.repo, "branch", "-D", branch) return nil } +// WorktreeInfo describes one attempt worktree owned by this manager. +type WorktreeInfo struct { + Path string + Branch string // branch name without the refs/heads/ prefix + Head string // full commit hash the worktree is on + Mtime time.Time + Locked bool + Detached bool + Orphaned bool // admin entry whose working directory is already gone + Dirty bool // tracked, staged, untracked, or ignored content not recorded in HEAD +} + +// List returns the attempt worktrees registered under the manager's +// sibling .corral-worktrees directory. Entries come from `git worktree +// list --porcelain`; the main checkout is never included. Mtime is the +// most recent change to the worktree directory or its gitdir HEAD/index. +func (m *Manager) List(ctx context.Context) ([]WorktreeInfo, error) { + out, err := m.git(ctx, m.repo, "worktree", "list", "--porcelain") + if err != nil { + return nil, err + } + var infos []WorktreeInfo + for _, block := range strings.Split(out, "\n\n") { + info, ok := parseWorktreeBlock(block) + if !ok || !insideDir(m.dir, info.Path) { + continue + } + info.Mtime = lastActivity(info.Path) + // Ignored files still carry user data. Treat them as dirty so automatic + // pruning never deletes content merely because .gitignore hides it. + status, err := m.git(ctx, info.Path, "status", "--porcelain=v1", "--untracked-files=normal", "--ignored=matching") + if err != nil { + return nil, err + } + info.Dirty = strings.TrimSpace(status) != "" + infos = append(infos, info) + } + return infos, nil +} + +func parseWorktreeBlock(block string) (WorktreeInfo, bool) { + var info WorktreeInfo + for _, line := range strings.Split(block, "\n") { + switch { + case strings.HasPrefix(line, "worktree "): + info.Path = strings.TrimSpace(strings.TrimPrefix(line, "worktree ")) + case strings.HasPrefix(line, "HEAD "): + info.Head = strings.TrimSpace(strings.TrimPrefix(line, "HEAD ")) + case strings.HasPrefix(line, "branch refs/heads/"): + info.Branch = strings.TrimPrefix(line, "branch refs/heads/") + case line == "detached": + info.Detached = true + case line == "locked" || strings.HasPrefix(line, "locked "): + info.Locked = true + case strings.HasPrefix(line, "prunable"): + info.Orphaned = true + } + } + return info, info.Path != "" +} + +// insideDir reports whether path lives strictly below dir. Both sides are +// resolved through symlinks because git reports canonical paths while the +// caller may hold a symlinked one (e.g. /var vs /private/var on macOS). +func insideDir(dir, path string) bool { + dir, err := filepath.EvalSymlinks(dir) + if err != nil { + return false + } + path, err = filepath.EvalSymlinks(path) + if err != nil { + return false + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return false + } + return rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// lastActivity returns the most recent modification time among the +// worktree directory and its gitdir HEAD/index files — a proxy for the +// last time anything changed in the worktree. +func lastActivity(path string) time.Time { + var latest time.Time + if fi, err := os.Stat(path); err == nil { + latest = fi.ModTime() + } + if gd := worktreeGitDir(path); gd != "" { + for _, f := range []string{"HEAD", "index"} { + if fi, err := os.Stat(filepath.Join(gd, f)); err == nil && fi.ModTime().After(latest) { + latest = fi.ModTime() + } + } + } + return latest +} + +// worktreeGitDir resolves the git directory backing a linked worktree by +// reading its .git file ("gitdir: "). Empty when unavailable. +func worktreeGitDir(path string) string { + data, err := os.ReadFile(filepath.Join(path, ".git")) + if err != nil { + return "" + } + for _, line := range strings.Split(string(data), "\n") { + if line, ok := strings.CutPrefix(line, "gitdir: "); ok { + return strings.TrimSpace(line) + } + } + return "" +} + +// BranchMerged reports whether branch has been folded into the main +// checkout branch with its own commits, or no longer exists. A branch +// still pointing at the main tip is not "merged": its worktree may hold +// uncommitted work. Neither check touches the main checkout. +func (m *Manager) BranchMerged(ctx context.Context, branch string) (bool, error) { + if _, code, err := m.gitExit(ctx, m.repo, "rev-parse", "--verify", "--quiet", "refs/heads/"+branch); err != nil { + return false, err + } else if code != 0 { + return true, nil // branch removed; its worktree is orphaned + } + main, err := m.MainBranch(ctx) + if err != nil { + return false, err + } + branchTip, err := m.git(ctx, m.repo, "rev-parse", "--verify", "refs/heads/"+branch) + if err != nil { + return false, err + } + mainTip, err := m.git(ctx, m.repo, "rev-parse", "--verify", "refs/heads/"+main) + if err != nil { + return false, err + } + if strings.TrimSpace(branchTip) == strings.TrimSpace(mainTip) { + return false, nil // no unique commits; uncommitted work may still live in the worktree + } + _, code, err := m.gitExit(ctx, m.repo, "merge-base", "--is-ancestor", "refs/heads/"+branch, "refs/heads/"+main) + if err != nil { + return false, err + } + return code == 0, nil +} + +// Prune removes clean worktrees that are safe to drop: their branch has +// been merged into the main checkout branch or no longer exists, or (when +// staleAfter > 0) their last activity is older than staleAfter. Dirty, +// locked, and detached worktrees are always skipped; orphaned git admin +// entries are pruned first via `git worktree prune`. The main checkout is +// never touched. Returns the paths removed. +func (m *Manager) Prune(ctx context.Context, staleAfter time.Duration, now time.Time) ([]string, error) { + if _, err := m.git(ctx, m.repo, "worktree", "prune"); err != nil { + return nil, err + } + infos, err := m.List(ctx) + if err != nil { + return nil, err + } + var pruned []string + for _, info := range infos { + if info.Dirty || info.Locked || info.Detached || info.Orphaned { + continue + } + merged, err := m.BranchMerged(ctx, info.Branch) + if err != nil { + return pruned, err + } + stale := staleAfter > 0 && now.Sub(info.Mtime) > staleAfter + if !merged && !stale { + continue + } + if err := m.removeInfo(ctx, info, merged); err != nil { + return pruned, fmt.Errorf("prune %s: %w", info.Path, err) + } + pruned = append(pruned, info.Path) + } + return pruned, nil +} + +// removeInfo removes a worktree by path. A merged branch is deleted too; +// an unmerged stale branch is kept so its commits stay recoverable. +func (m *Manager) removeInfo(ctx context.Context, info WorktreeInfo, merged bool) error { + // No --force here: if content appears after List's safety check, Git must + // refuse instead of deleting an agent's newly-written work. + if _, err := m.git(ctx, m.repo, "worktree", "remove", info.Path); err != nil { + return err + } + if merged && info.Branch != "" { + _, _ = m.git(ctx, m.repo, "branch", "-D", info.Branch) + } + return nil +} + // ScopesOverlap reports whether two declared write scopes can conflict. // Empty or "*" scopes mean the whole repository (a writing node with no // declared scope collides with everything). Paths overlap when one is a @@ -175,6 +440,29 @@ func ScopesOverlap(a, b []string) bool { return false } +// ScopeContains reports whether path is covered by at least one declared +// relative scope. Empty or "*" scopes intentionally mean the whole worktree. +func ScopeContains(scopes []string, path string) bool { + path = filepath.ToSlash(filepath.Clean(filepath.FromSlash(strings.TrimSpace(path)))) + if path == "." || path == ".." || strings.HasPrefix(path, "../") || filepath.IsAbs(filepath.FromSlash(path)) { + return false + } + if len(scopes) == 0 { + return true + } + for _, scope := range scopes { + scope = strings.TrimSpace(strings.ReplaceAll(scope, `\`, "/")) + if scope == "" || scope == "*" { + return true + } + scope = filepath.ToSlash(filepath.Clean(filepath.FromSlash(scope))) + if path == scope || strings.HasPrefix(path, scope+"/") { + return true + } + } + return false +} + func scopesTouch(x, y string) bool { x = strings.TrimSpace(x) y = strings.TrimSpace(y) diff --git a/internal/worktree/worktree_test.go b/internal/worktree/worktree_test.go index c203f4e..05a4297 100644 --- a/internal/worktree/worktree_test.go +++ b/internal/worktree/worktree_test.go @@ -5,7 +5,9 @@ import ( "os" "os/exec" "path/filepath" + "strings" "testing" + "time" ) func gitInit(t *testing.T, dir string) { @@ -22,6 +24,17 @@ func gitInit(t *testing.T, dir string) { } } +// resolved returns the canonical (symlink-resolved) form of p, the form +// git reports on macOS (/var vs /private/var). +func resolved(t *testing.T, p string) string { + t.Helper() + r, err := filepath.EvalSymlinks(p) + if err != nil { + t.Fatal(err) + } + return r +} + func TestManagerLifecycle(t *testing.T) { repo := t.TempDir() gitInit(t, repo) @@ -88,6 +101,524 @@ func TestManagerLifecycle(t *testing.T) { } } +func TestRemovePreservesIgnoredContent(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + if err := os.WriteFile(filepath.Join(repo, ".gitignore"), []byte("secret.env\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := m.git(ctx, repo, "add", ".gitignore"); err != nil { + t.Fatal(err) + } + if _, err := m.git(ctx, repo, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "ignore"); err != nil { + t.Fatal(err) + } + branch := "corral/r1/w1/1" + path, err := m.Add(ctx, branch) + if err != nil { + t.Fatal(err) + } + secret := filepath.Join(path, "secret.env") + if err := os.WriteFile(secret, []byte("keep me"), 0o600); err != nil { + t.Fatal(err) + } + if err := m.Remove(ctx, branch); err == nil || !strings.Contains(err.Error(), "dirty worktree") { + t.Fatalf("Remove ignored-content error = %v", err) + } + if got, err := os.ReadFile(secret); err != nil || string(got) != "keep me" { + t.Fatalf("ignored content lost: %q, %v", got, err) + } +} + +func TestCommitWorktreePropagatesCommitFailureAndCanRetry(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + hook := filepath.Join(repo, ".git", "hooks", "pre-commit") + if err := os.WriteFile(hook, []byte("#!/bin/sh\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + + if err := m.CommitWorktree(ctx, path); err == nil { + t.Fatal("CommitWorktree hid a real git commit failure") + } + status, err := m.git(ctx, path, "status", "--porcelain=v1") + if err != nil { + t.Fatal(err) + } + if status != "A a.txt\n" { + t.Fatalf("failed commit did not preserve staged work: %q", status) + } + mainStatus, err := m.git(ctx, repo, "status", "--porcelain=v1") + if err != nil { + t.Fatal(err) + } + if mainStatus != "" { + t.Fatalf("failed worktree commit dirtied main: %q", mainStatus) + } + + if err := os.Remove(hook); err != nil { + t.Fatal(err) + } + if err := m.CommitWorktree(ctx, path); err != nil { + t.Fatalf("retry commit: %v", err) + } + if err := m.MergeBranch(ctx, "corral/r1/w1/1"); err != nil { + t.Fatalf("merge after retry: %v", err) + } +} + +func TestCommitWorktreeAllowsNothingToCommit(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := m.CommitWorktree(ctx, path); err != nil { + t.Fatalf("empty commit: %v", err) + } +} + +func TestMergeBranchAbortsConflictAndCanRetry(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + if err := os.WriteFile(filepath.Join(repo, "conflict.txt"), []byte("base\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := m.git(ctx, repo, "add", "conflict.txt"); err != nil { + t.Fatal(err) + } + if _, err := m.git(ctx, repo, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "base"); err != nil { + t.Fatal(err) + } + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "conflict.txt"), []byte("branch\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := m.CommitWorktree(ctx, path); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, "conflict.txt"), []byte("main\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := m.git(ctx, repo, "add", "conflict.txt"); err != nil { + t.Fatal(err) + } + if _, err := m.git(ctx, repo, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-q", "-m", "main change"); err != nil { + t.Fatal(err) + } + mainHead, err := m.git(ctx, repo, "rev-parse", "HEAD") + if err != nil { + t.Fatal(err) + } + + if err := m.MergeBranch(ctx, "corral/r1/w1/1"); err == nil { + t.Fatal("conflicting merge unexpectedly succeeded") + } + assertCleanMainWithoutMerge(t, m, ctx, mainHead) + + // Fold main into the worktree branch, making a retry conflict-free. + if _, err := m.git(ctx, path, + "-c", "user.name=t", "-c", "user.email=t@t", + "merge", "-q", "-s", "ours", "-m", "resolve main", "main"); err != nil { + t.Fatal(err) + } + if err := m.MergeBranch(ctx, "corral/r1/w1/1"); err != nil { + t.Fatalf("merge retry: %v", err) + } +} + +func TestMergeBranchAbortsFailedMergeCommitAndCanRetry(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + if err := m.CommitWorktree(ctx, path); err != nil { + t.Fatal(err) + } + hook := filepath.Join(repo, ".git", "hooks", "pre-merge-commit") + if err := os.WriteFile(hook, []byte("#!/bin/sh\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + mainHead, err := m.git(ctx, repo, "rev-parse", "HEAD") + if err != nil { + t.Fatal(err) + } + + if err := m.MergeBranch(ctx, "corral/r1/w1/1"); err == nil { + t.Fatal("merge commit hook failure was hidden") + } + assertCleanMainWithoutMerge(t, m, ctx, mainHead) + + if err := os.Remove(hook); err != nil { + t.Fatal(err) + } + if err := m.MergeBranch(ctx, "corral/r1/w1/1"); err != nil { + t.Fatalf("merge retry: %v", err) + } +} + +func assertCleanMainWithoutMerge(t *testing.T, m *Manager, ctx context.Context, wantHead string) { + t.Helper() + if _, code, err := m.gitExit(ctx, m.repo, "rev-parse", "--verify", "--quiet", "MERGE_HEAD"); err != nil { + t.Fatal(err) + } else if code == 0 { + t.Fatal("failed merge left MERGE_HEAD active") + } + status, err := m.git(ctx, m.repo, "status", "--porcelain=v1") + if err != nil { + t.Fatal(err) + } + if status != "" { + t.Fatalf("failed merge left main dirty: %q", status) + } + head, err := m.git(ctx, m.repo, "rev-parse", "HEAD") + if err != nil { + t.Fatal(err) + } + if head != wantHead { + t.Fatalf("failed merge moved main HEAD: got %q want %q", head, wantHead) + } +} + +func TestManagerList(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + if infos, err := m.List(ctx); err != nil { + t.Fatalf("list: %v", err) + } else if len(infos) != 0 { + t.Fatalf("List = %v, want none", infos) + } + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + + infos, err := m.List(ctx) + if err != nil { + t.Fatal(err) + } + if len(infos) != 1 { + t.Fatalf("List = %v, want 1 worktree", infos) + } + info := infos[0] + if info.Branch != "corral/r1/w1/1" { + t.Fatalf("branch = %q", info.Branch) + } + if info.Path != resolved(t, path) { + t.Fatalf("path = %q, want %q", info.Path, resolved(t, path)) + } + if info.Head == "" { + t.Fatal("head empty") + } + if info.Mtime.IsZero() { + t.Fatal("mtime zero") + } + if !info.Dirty { + t.Fatal("dirty worktree reported clean") + } + if info.Detached || info.Locked || info.Orphaned { + t.Fatalf("unexpected flags: %+v", info) + } + // The main checkout is never listed. + for _, i := range infos { + if i.Path == repo { + t.Fatalf("main checkout listed: %+v", i) + } + } +} + +func TestManagerPruneMerged(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + if err := m.CommitWorktree(ctx, path); err != nil { + t.Fatal(err) + } + if err := m.MergeBranch(ctx, "corral/r1/w1/1"); err != nil { + t.Fatal(err) + } + merged, err := m.BranchMerged(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if !merged { + t.Fatal("merged branch should report merged") + } + want := resolved(t, path) + + pruned, err := m.Prune(ctx, 0, time.Now()) + if err != nil { + t.Fatal(err) + } + if len(pruned) != 1 || pruned[0] != want { + t.Fatalf("pruned = %v, want [%s]", pruned, want) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("worktree dir still exists") + } + if infos, err := m.List(ctx); err != nil || len(infos) != 0 { + t.Fatalf("List after prune = %v (err %v)", infos, err) + } + // Main checkout keeps the merged content and is untouched. + data, err := os.ReadFile(filepath.Join(repo, "a.txt")) + if err != nil || string(data) != "hello" { + t.Fatalf("main checkout wrong: %v %q", err, data) + } +} + +func TestManagerPruneSkipsDirtyMerged(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "merged.txt"), []byte("merged"), 0o644); err != nil { + t.Fatal(err) + } + if err := m.CommitWorktree(ctx, path); err != nil { + t.Fatal(err) + } + if err := m.MergeBranch(ctx, "corral/r1/w1/1"); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "unfinished.txt"), []byte("keep me"), 0o644); err != nil { + t.Fatal(err) + } + + pruned, err := m.Prune(ctx, 0, time.Now()) + if err != nil { + t.Fatal(err) + } + if len(pruned) != 0 { + t.Fatalf("dirty merged worktree pruned: %v", pruned) + } + if data, err := os.ReadFile(filepath.Join(path, "unfinished.txt")); err != nil || string(data) != "keep me" { + t.Fatalf("uncommitted work lost: %q (%v)", data, err) + } +} + +func TestManagerPruneStale(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + if err := m.CommitWorktree(ctx, path); err != nil { + t.Fatal(err) + } + now := time.Now() + + // An unmerged worktree survives --prune alone (age criterion off). + if pruned, err := m.Prune(ctx, 0, now); err != nil || len(pruned) != 0 { + t.Fatalf("prune without stale removed %v (err %v)", pruned, err) + } + // It survives --prune --stale while fresh. + if pruned, err := m.Prune(ctx, 24*time.Hour, now); err != nil || len(pruned) != 0 { + t.Fatalf("fresh prune removed %v (err %v)", pruned, err) + } + // Idle past the threshold: removed, but its branch is kept. + want := resolved(t, path) + pruned, err := m.Prune(ctx, 24*time.Hour, now.Add(30*24*time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(pruned) != 1 || pruned[0] != want { + t.Fatalf("stale pruned = %v, want [%s]", pruned, want) + } + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("stale worktree dir still exists") + } + if _, code, _ := m.gitExit(ctx, m.repo, "rev-parse", "--verify", "--quiet", "refs/heads/corral/r1/w1/1"); code != 0 { + t.Fatal("unmerged stale branch should be kept") + } +} + +func TestManagerPruneSkipsDirtyStale(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "unfinished.txt"), []byte("keep me"), 0o644); err != nil { + t.Fatal(err) + } + + pruned, err := m.Prune(ctx, time.Hour, time.Now().Add(30*24*time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(pruned) != 0 { + t.Fatalf("dirty stale worktree pruned: %v", pruned) + } + if _, err := os.Stat(filepath.Join(path, "unfinished.txt")); err != nil { + t.Fatalf("uncommitted work lost: %v", err) + } +} + +func TestManagerPruneSkipsIgnoredContent(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + if err := os.WriteFile(filepath.Join(repo, ".gitignore"), []byte("secret.env\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := m.git(ctx, repo, "add", ".gitignore"); err != nil { + t.Fatal(err) + } + if _, err := m.git(ctx, repo, "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "ignore secret"); err != nil { + t.Fatal(err) + } + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + secret := filepath.Join(path, "secret.env") + if err := os.WriteFile(secret, []byte("keep me"), 0o600); err != nil { + t.Fatal(err) + } + pruned, err := m.Prune(ctx, time.Hour, time.Now().Add(30*24*time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(pruned) != 0 { + t.Fatalf("worktree with ignored content pruned: %v", pruned) + } + if got, err := os.ReadFile(secret); err != nil || string(got) != "keep me" { + t.Fatalf("ignored content lost: %q, %v", got, err) + } +} + +func TestManagerPruneSkipsLocked(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := m.git(ctx, m.repo, "worktree", "lock", "--reason", "active run", path); err != nil { + t.Fatal(err) + } + if err := m.CommitWorktree(ctx, path); err != nil { + t.Fatal(err) + } + if err := m.MergeBranch(ctx, "corral/r1/w1/1"); err != nil { + t.Fatal(err) + } + // Merged, but locked: never pruned. + if pruned, err := m.Prune(ctx, 0, time.Now()); err != nil || len(pruned) != 0 { + t.Fatalf("locked worktree pruned: %v (err %v)", pruned, err) + } + if _, err := os.Stat(path); err != nil { + t.Fatalf("locked worktree dir removed: %v", err) + } +} + +func TestManagerPruneOrphaned(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + path, err := m.Add(ctx, "corral/r1/w1/1") + if err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(path); err != nil { + t.Fatal(err) + } + // The admin entry still exists until a prune sweeps it. + if _, err := m.Prune(ctx, 0, time.Now()); err != nil { + t.Fatal(err) + } + if infos, err := m.List(ctx); err != nil || len(infos) != 0 { + t.Fatalf("orphaned worktree not cleaned: %v (err %v)", infos, err) + } +} + +func TestManagerBranchMergedMissing(t *testing.T) { + repo := t.TempDir() + gitInit(t, repo) + ctx := context.Background() + m := NewManager(repo) + + merged, err := m.BranchMerged(ctx, "corral/r1/never/1") + if err != nil { + t.Fatal(err) + } + if !merged { + t.Fatal("missing branch should report merged (safe to prune)") + } +} + func TestScopesOverlap(t *testing.T) { cases := []struct { a, b []string