diff --git a/Cargo.lock b/Cargo.lock index 7f0ed207b..3f3a22cad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1290,7 +1290,9 @@ version = "0.1.0" dependencies = [ "futures-util", "libc", + "serde", "serde_json", + "tempfile", "tokio", "tokio-tungstenite 0.24.0", "tracing", diff --git a/crates/freshell-codex/Cargo.toml b/crates/freshell-codex/Cargo.toml index d71da507b..d35758714 100644 --- a/crates/freshell-codex/Cargo.toml +++ b/crates/freshell-codex/Cargo.toml @@ -34,6 +34,10 @@ default = [] real-transport = ["dep:tokio-tungstenite", "dep:futures-util", "dep:libc", "tokio/net", "tokio/io-util", "tokio/process"] [dependencies] +# Derive for the durable sidecar record rows (`sidecar_store`) — the versioned +# camelCase JSON schema a restarted server reads back (BindingRow precedent, +# pane_ledger.rs:93-130). +serde = { workspace = true } # Dynamic JSON parsing that mirrors the TS `Record` params/result model # (JSON.parse parity, corruption-tolerant). preserve_order matches the wire object model. serde_json = { workspace = true } @@ -57,3 +61,8 @@ futures-util = { version = "0.3", default-features = false, features = ["sink", # FRESHELL_CODEX_SIDECAR_ID tag (runtime.ts:494) so no orphan survives (the oracle # `ownership.cleanup` invariant). Only needed by the real transport. libc = { version = "0.2", optional = true } + +[dev-dependencies] +# Tempdirs for the sidecar-store unit tests — no global state, nothing outside +# each test's own temp dir (never `~/.freshell/`). +tempfile = "3" diff --git a/crates/freshell-codex/src/app_server.rs b/crates/freshell-codex/src/app_server.rs index 5af6c3524..7174d3769 100644 --- a/crates/freshell-codex/src/app_server.rs +++ b/crates/freshell-codex/src/app_server.rs @@ -413,6 +413,31 @@ impl CodexAppServerClient { .await } + /// `thread/loaded/list` — the ids of threads this app-server currently + /// has loaded in memory (result shape `{ data: string[], nextCursor? }`, + /// contract-foundation plan §thread/loaded/list; the committed fixture + /// returns `{ data: behavior.loadedThreadIds }`). NOTE: `loaded` alone + /// does NOT mean mid-turn — idle threads stay loaded forever + /// (reports/V1.md); pair with [`Self::read_thread`]'s status. Added for + /// Task 9's sweep probe. + pub async fn list_loaded_threads(&self) -> Result, CodexAppServerError> { + let result = self.request("thread/loaded/list", json!({})).await?; + // Absent/non-array `data` is NOT an empty list: the sweep probe reads + // Ok(vec![]) as proof of "reachable, no loaded threads" and may REAP + // on it — a malformed payload must fail loudly instead, sending the + // caller down the conservative writer-evidence path (final review F2). + let Some(ids) = result.get("data").and_then(Value::as_array) else { + return Err(CodexAppServerError::InvalidResponse { + method: "thread/loaded/list".to_string(), + detail: format!("expected result.data to be an array, got: {result}"), + }); + }; + Ok(ids + .iter() + .filter_map(|id| id.as_str().map(str::to_string)) + .collect()) + } + /// Send a notification frame (no response awaited) — `notify`, `client.ts:805-808`. pub async fn notify( &self, diff --git a/crates/freshell-codex/src/launch_lifecycle.rs b/crates/freshell-codex/src/launch_lifecycle.rs index 156bb88d2..ad98f57af 100644 --- a/crates/freshell-codex/src/launch_lifecycle.rs +++ b/crates/freshell-codex/src/launch_lifecycle.rs @@ -35,6 +35,8 @@ //! explicitly. use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; @@ -42,13 +44,19 @@ use std::time::Duration; use tokio::sync::mpsc; use crate::app_server::BoxFuture; -use crate::durability::mint_ownership_id; +use crate::durability::{default_server_instance_id, mint_ownership_id}; use crate::launch_plan::{ codex_sidecar_spawn_spec, plan_codex_launch, plan_codex_launch_retry, CodexLaunchConfigError, CodexLaunchPlan, CodexLaunchPlanInput, CodexLaunchRetryDecision, CODEX_INITIAL_LAUNCH_RETRY_DELAY_MS, }; use crate::remote_proxy::{CodexRemoteProxy, CodexRemoteProxyOptions, RemoteProxyEvent}; +use crate::runtime_select::select_codex_runtime; +use crate::sidecar_reconcile::{codex_sidecar_reconciler, write_record_loudly}; +use crate::sidecar_store::{ + codex_sidecar_store, proc_cmdline, proc_starttime, CodexSidecarRecord, CodexSidecarStore, + SidecarRecordState, SIDECAR_RECORD_VERSION, +}; use crate::transport::reap_owned_codex_sidecars; /// `assertAcceptingPlans` (`launch-planner.ts:199`), byte-identical. @@ -89,13 +97,50 @@ pub trait CodexLaunchRuntime: Send + Sync { generation: u64, ) -> BoxFuture<'_, Result<(), String>>; + /// Task 4: note the codex session/thread id once it is known — resume + /// launches at plan time, fresh launches when the proxy captures the + /// thread candidate. Default no-op: only runtimes with a durable sidecar + /// record have anything to enrich. + fn note_session_id(&self, session_id: String) -> BoxFuture<'_, Result<(), String>> { + let _ = session_id; + Box::pin(async { Ok(()) }) + } + + /// Task 10: server-shutdown retention — the runtime is asked to KEEP its + /// sidecar alive across the restart (record flipped to + /// `Retained{reason}`, process never signalled) instead of tearing it + /// down. Default no-op `Ok(())` (the `note_session_id` pattern): only + /// runtimes with a durable sidecar record have anything to retain. The + /// retention gate lives in the real impls — a runtime whose sidecar has + /// NO persisted record (disabled store / non-Linux) MUST tear down + /// exactly as `shutdown` would; "retaining" a record-less sidecar would + /// orphan it silently (the ynfn hole). + fn prepare_retention(&self, reason: String) -> BoxFuture<'_, Result<(), String>> { + let _ = reason; + Box::pin(async { Ok(()) }) + } + /// Tear the app-server down (`runtime.shutdown()`, `launch-planner.ts:302`). fn shutdown(&self) -> BoxFuture<'_, Result<(), String>>; } /// The planner's runtime factory (`CodexLaunchPlanner` ctor `runtimeOrFactory`, -/// `launch-planner.ts:115-121`): one fresh runtime per plan. -pub type CodexRuntimeFactory = Box Arc + Send + Sync>; +/// `launch-planner.ts:115-121`): one fresh runtime per plan. Plan-aware and +/// async (Task 7): the factory receives the S3 pure plan and returns a boxed +/// future that async [`CodexLaunchPlanner::plan_create`] AWAITS — the +/// production selection ([`crate::runtime_select::select_codex_runtime`]) +/// must await the reconciler's claim +/// ([`crate::sidecar_reconcile::SidecarReconciler::claim_for_session`], +/// whose duplicate arm runs a bounded ws writer probe) before deciding +/// reattach-vs-spawn. +pub type CodexRuntimeFactory = Box< + dyn for<'a> Fn( + &'a CodexLaunchPlan, + ) + -> Pin> + Send + 'a>> + + Send + + Sync, +>; // ─── errors ────────────────────────────────────────────────────────────────────────────── @@ -216,6 +261,34 @@ impl CodexLaunchSidecar { Ok(()) } + /// Task 10 server-shutdown retention: close the proxy (its listener dies + /// with this process anyway) and ask the runtime to + /// `prepare_retention(reason)` INSTEAD of tearing it down. Marks the + /// sidecar shutdown-complete so any late teardown path (a double-fired + /// PTY exit hook, `manager.shutdown()`'s drain) no-ops via the + /// idempotence flag instead of re-killing the retained survivor. The + /// retention GATE lives in the runtime: a record-less runtime tears its + /// sidecar down exactly as today. + pub async fn retain(&self, reason: &str) -> Result<(), String> { + let mut inner = self.inner.lock().await; + if inner.shutdown_succeeded { + return Ok(()); + } + inner.shutdown_started = true; + if let Some(proxy) = inner.proxy.take() { + proxy.close().await; + } + let result = self.runtime.prepare_retention(reason.to_string()).await; + // Final-review H3c: the retention DECISION stands even when the + // record rewrite fails (prepare_retention already logs loudly): mark + // shutdown-complete either way, so a later shutdown() (a double-fired + // PTY exit hook, `manager.shutdown()`'s drain) can never kill a + // sidecar we chose to retain. + inner.shutdown_succeeded = true; + self.planner_active.lock().unwrap().remove(&self.id); + result + } + /// `sidecar.shutdown()` (`launch-planner.ts:281-316`): idempotent, single-flight /// (concurrent callers serialize on the inner lock and observe the succeeded flag). /// Tears down the proxy (listener + socket pairs) and the runtime (spawned child). @@ -305,7 +378,7 @@ impl CodexLaunchPlanner { self.assert_accepting_plans()?; let plan = plan_codex_launch(input).map_err(CodexLaunchError::Config)?; - let runtime = (self.runtime_factory)(); + let runtime = (self.runtime_factory)(&plan).await; let id = self.next_id.fetch_add(1, Ordering::SeqCst); let sidecar = Arc::new(CodexLaunchSidecar { id, @@ -323,6 +396,13 @@ impl CodexLaunchPlanner { let started: Result<(CodexRemoteProxy, mpsc::UnboundedReceiver), String> = async { let ready = runtime.ensure_ready(plan.runtime_cwd.clone()).await?; + // Task 4: resume launches know their session id at plan time — + // note it so the runtime's durable record carries the + // restore-time reattach key. Best-effort (the record write + // path logs its own failures); never fails the plan. + if let Some(sid) = plan.session_id.clone() { + let _ = runtime.note_session_id(sid).await; + } CodexRemoteProxy::start(CodexRemoteProxyOptions::new( ready.ws_url, plan.require_candidate_persistence, @@ -537,19 +617,31 @@ pub fn set_global_codex_launch_manager_for_tests(manager: CodexTerminalLaunchMan pub struct CodexTerminalLaunchManager { planner: CodexLaunchPlanner, adopted: Mutex>, - teardown_tx: OnceLock>, + /// Teardown/retention worker feed: the bool is the RETAIN decision, + /// made by the sender at hand-off time (Task 10). + teardown_tx: OnceLock>, + /// Task 10: server-shutdown retention mode — set once by + /// [`Self::begin_shutdown_retention`], never cleared (the process is + /// exiting). + shutdown_retention: AtomicBool, plan_budget: Arc, plan_budget_wait: Duration, plan_queue_cap: usize, plan_waiting: std::sync::Arc, } +/// The durable `Retained{reason}` every server-shutdown retention records +/// (both [`CodexTerminalLaunchManager::shutdown`]'s drain and the retention +/// arm of the PTY exit hook). +const SERVER_SHUTDOWN_RETENTION_REASON: &str = "server-shutdown"; + impl CodexTerminalLaunchManager { pub fn new(runtime_factory: CodexRuntimeFactory) -> Self { Self { planner: CodexLaunchPlanner::new(runtime_factory), adopted: Mutex::new(HashMap::new()), teardown_tx: OnceLock::new(), + shutdown_retention: AtomicBool::new(false), plan_budget: Arc::new(tokio::sync::Semaphore::new(CODEX_SIDECAR_PLAN_CONCURRENCY)), plan_budget_wait: CODEX_SIDECAR_PLAN_WAIT, // The env read MUST live here — `global()` calls `new()`, so @@ -579,12 +671,24 @@ impl CodexTerminalLaunchManager { self.plan_waiting.load(std::sync::atomic::Ordering::SeqCst) } - /// The process-wide manager over the REAL spawn runtime — legacy has exactly one - /// `CodexLaunchPlanner` per server (`server/index.ts:359`). + /// The process-wide manager over the REAL selection — legacy has exactly one + /// `CodexLaunchPlanner` per server (`server/index.ts:359`). Task 7: the + /// factory dispatches through [`select_codex_runtime`] — a claimable + /// verified survivor for a resume plan reattaches + /// ([`crate::sidecar_reconcile::ReattachedCodexAppServerRuntime`]); every + /// other plan (and a `None` reconciler/store — nothing installed at boot) + /// gets the spawn runtime, exactly the pre-Task-7 behavior. pub fn global() -> &'static CodexTerminalLaunchManager { GLOBAL_MANAGER.get_or_init(|| { - CodexTerminalLaunchManager::new(Box::new(|| { - Arc::new(SpawnedCodexAppServerRuntime::new()) as Arc + CodexTerminalLaunchManager::new(Box::new(|plan| { + Box::pin(async move { + select_codex_runtime( + codex_sidecar_reconciler().as_ref(), + codex_sidecar_store().as_ref(), + plan, + ) + .await + }) })) }) } @@ -779,6 +883,25 @@ impl CodexTerminalLaunchManager { } } + /// Task 4: forward a captured codex session/thread id to an adopted + /// terminal's runtime so its durable sidecar record carries the + /// restore-time reattach key (katas ynfn/da92). Called by the freshell-ws + /// proxy-event router beside [`Self::mark_candidate_persisted`]; resume + /// launches get theirs at plan time instead. Unknown terminal ids are a + /// silent no-op (the mark_candidate_persisted discipline). + pub async fn note_session_id(&self, terminal_id: &str, session_id: &str) { + let runtime = { + self.adopted + .lock() + .unwrap() + .get(terminal_id) + .map(|entry| entry.sidecar.runtime.clone()) + }; + if let Some(runtime) = runtime { + let _ = runtime.note_session_id(session_id.to_string()).await; + } + } + /// S5.c: fail the gate for an adopted terminal (candidate refused). pub async fn fail_candidate_capture(&self, terminal_id: &str, message: &str) { let sidecar = { @@ -793,42 +916,69 @@ impl CodexTerminalLaunchManager { } } + /// Server-shutdown mode: adopted (terminal-owned) sidecars are RETAINED + /// across the restart — proxies close, runtimes are asked to + /// prepare_retention(reason) instead of shutdown. Unadopted planner + /// sidecars (mid-plan) are still torn down. Call BEFORE registry.kill_all() + /// so PTY-exit hooks (notify_terminal_exit) also retain instead of reap. + pub fn begin_shutdown_retention(&self) { + self.shutdown_retention.store(true, Ordering::SeqCst); + } + /// Sync-safe (callable from the PTY exit hook's non-async thread): detach the /// terminal's launch and hand it to the teardown worker. No-op for terminals without - /// a managed launch. + /// a managed launch. Task 10: under server-shutdown retention + /// ([`Self::begin_shutdown_retention`] runs BEFORE `registry.kill_all()`, + /// so every shutdown-driven exit sees the flag set) the entry is routed + /// through retention instead of teardown. pub fn notify_terminal_exit(&self, terminal_id: &str) { let Some(entry) = self.adopted.lock().unwrap().remove(terminal_id) else { return; }; + let retain = self.shutdown_retention.load(Ordering::SeqCst); if let Some(tx) = self.teardown_tx.get() { - let _ = tx.send(entry); + let _ = tx.send((entry, retain)); } } /// Server-exit teardown (main.rs graceful shutdown): mirrors legacy's close-time /// `codexLaunchPlanner.shutdown()` (`server/index.ts:981-1049` shutdown owners) — - /// the planner stops accepting plans and tears down its unadopted sidecars — PLUS - /// the adopted (terminal-owned) launches this manager keys, since server exit ends - /// those terminals too (their exit hooks may also queue teardown; sidecar shutdown - /// is idempotent, so both paths are safe). + /// the planner stops accepting plans and tears down its unadopted sidecars + /// unconditionally (they have no pane to reattach to, and a fresh-plan proxy may + /// hold the candidate timer) — PLUS the adopted (terminal-owned) launches this + /// manager keys. Task 10: with [`Self::begin_shutdown_retention`] set, adopted + /// entries get proxy-close + `prepare_retention("server-shutdown")` instead of + /// teardown (kata ynfn: surviving restarts is a feature); the runtime-level + /// retention gate still tears down record-less sidecars exactly as today. Exit + /// hooks may also route the same entries; retain/shutdown share the sidecar's + /// idempotence flag, so both paths stay safe. pub async fn shutdown(&self) { self.planner.shutdown().await; + let retain = self.shutdown_retention.load(Ordering::SeqCst); let adopted: Vec = { let mut map = self.adopted.lock().unwrap(); map.drain().map(|(_, entry)| entry).collect() }; for entry in adopted { - let _ = entry.sidecar.shutdown().await; + if retain { + let _ = entry.sidecar.retain(SERVER_SHUTDOWN_RETENTION_REASON).await; + } else { + let _ = entry.sidecar.shutdown().await; + } entry.drain.abort(); } } fn ensure_teardown_worker(&self) { self.teardown_tx.get_or_init(|| { - let (tx, mut rx) = mpsc::unbounded_channel::(); + let (tx, mut rx) = mpsc::unbounded_channel::<(AdoptedTerminalLaunch, bool)>(); tokio::spawn(async move { - while let Some(entry) = rx.recv().await { - let _ = entry.sidecar.shutdown().await; + while let Some((entry, retain)) = rx.recv().await { + if retain { + let _ = entry.sidecar.retain(SERVER_SHUTDOWN_RETENTION_REASON).await; + } else { + let _ = entry.sidecar.shutdown().await; + } entry.drain.abort(); } }); @@ -843,6 +993,9 @@ struct SpawnedSidecar { ws_url: String, ownership_id: String, child: tokio::process::Child, + /// The durable record written at spawn (tracked spawns only) — kept so + /// `update_ownership_metadata` can enrich + rewrite it without a re-read. + record: Option, } /// The real [`CodexLaunchRuntime`]: spawns `codex -c features.apps=false app-server @@ -855,6 +1008,12 @@ struct SpawnedSidecar { pub struct SpawnedCodexAppServerRuntime { codex_command: Option, start_budget: Duration, + /// Durable sidecar record store (Task 3). Production resolves the + /// process-global handle ([`crate::sidecar_store::set_codex_sidecar_store`], + /// wired at boot in Task 10); absent global ⇒ disabled store ⇒ behavior + /// identical to the pre-store world (attached `kill_on_drop(true)` spawn, + /// no record). + store: Arc, state: tokio::sync::Mutex>, adopted_metadata: Mutex>, } @@ -867,11 +1026,13 @@ impl Default for SpawnedCodexAppServerRuntime { impl SpawnedCodexAppServerRuntime { /// Command from `CODEX_CMD` (whitespace-split, matching `codex.rs::spawn_sidecar`'s - /// interpreter-plus-script support) falling back to `codex`. + /// interpreter-plus-script support) falling back to `codex`. The record + /// store resolves from the process-global handle; absent ⇒ disabled. pub fn new() -> Self { Self { codex_command: None, start_budget: SIDECAR_START_BUDGET, + store: codex_sidecar_store().unwrap_or_else(|| Arc::new(CodexSidecarStore::disabled())), state: tokio::sync::Mutex::new(None), adopted_metadata: Mutex::new(None), } @@ -886,6 +1047,19 @@ impl SpawnedCodexAppServerRuntime { } } + /// Explicit command AND store injection (tests: a lock-free store over a + /// tempdir) — per-instance, never the process-global handle. + pub fn with_command_and_store( + command: impl Into, + store: Arc, + ) -> Self { + Self { + codex_command: Some(command.into()), + store, + ..Self::new() + } + } + /// The spawned app-server's pid, if running (test observability). pub async fn child_pid(&self) -> Option { self.state.lock().await.as_ref().and_then(|s| s.child.id()) @@ -904,6 +1078,29 @@ impl SpawnedCodexAppServerRuntime { .filter(|value| !value.trim().is_empty()) .unwrap_or_else(|| "codex".to_string()) } + + /// Remove the durable record after teardown/failure reaping. Idempotent + /// (missing rows are `Ok`); failures are logged loudly, never propagated — + /// the reap already happened, so the worst case is a stale row the boot + /// reconciler re-verifies (and finds Dead) later. + fn scrub_record(&self, ownership_id: &str) { + if let Err(error) = self.store.remove(ownership_id) { + tracing::error!( + target: "freshell_codex::launch", + ownership_id = %ownership_id, + error = %error, + "sidecar_record_remove_failed: stale row left for boot reconcile" + ); + } + } +} + +/// Wall-clock unix millis for record `created_at`/`updated_at` stamps. +fn unix_millis() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0) } /// Allocate a loopback ephemeral port (`allocateLocalhostPort`-shaped: bind @@ -971,7 +1168,28 @@ impl CodexLaunchRuntime for SpawnedCodexAppServerRuntime { cmd.stdin(std::process::Stdio::null()) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()); - cmd.kill_on_drop(true); + + // Detach CONDITIONALLY — only when the sidecar will actually be + // TRACKED (kata ynfn: "surviving restarts is a feature"; Node + // parity: `detached: true`, runtime.ts:1828-1843). The store + // record now plays the safety-net role kill_on_drop played: an + // unclean server death leaves a tracked record for boot + // reconciliation (Tasks 5/9). A sidecar with NO record must keep + // the kill_on_drop backstop — detaching it would be the + // silently-orphaned ynfn hole with no reconcile path — and + // non-Linux identity can never be /proc-verified, so a detached + // sidecar there would be untracked AND unreapable. + let detach = cfg!(target_os = "linux") && self.store.is_enabled(); + if detach { + cmd.kill_on_drop(false); + // `tokio::process::Command::process_group` is Unix-only, so + // the call must stay cfg-gated even though detach is + // Linux-only today — keeps any non-Unix build compiling. + #[cfg(unix)] + cmd.process_group(0); + } else { + cmd.kill_on_drop(true); + } let mut child = cmd .spawn() @@ -1009,6 +1227,7 @@ impl CodexLaunchRuntime for SpawnedCodexAppServerRuntime { }; if let Ok(Some(status)) = child.try_wait() { reap_owned_codex_sidecars(&ownership_id); + self.scrub_record(&ownership_id); return Err(format!( "codex app-server exited before listening: {status}" )); @@ -1016,15 +1235,72 @@ impl CodexLaunchRuntime for SpawnedCodexAppServerRuntime { if tokio::time::Instant::now() >= deadline { let _ = child.start_kill(); reap_owned_codex_sidecars(&ownership_id); + self.scrub_record(&ownership_id); return Err(format!("codex app-server WS never came up: {probe_error}")); } tokio::time::sleep(Duration::from_millis(100)).await; } + // Persist the durable record for TRACKED spawns (Task 3): the + // listener is up, so capture the child's /proc identity evidence + // and write the row a restarted server reconciles against. + // Untracked spawns (detach == false) skip the write — they keep + // the kill_on_drop backstop and need no reconcile row. + let mut record = None; + if detach { + match child.id() { + Some(pid) => { + // Fall back to the constructed argv if /proc is + // momentarily unreadable; a starttime of 0 can never + // match a live process, so the worst outcome is the + // conservative Mismatch (never signalled), not a + // wrong kill. + let constructed_cmdline: Vec = std::iter::once(program.clone()) + .chain(leading_args.iter().cloned()) + .chain(spec.args.iter().cloned()) + .collect(); + let now = unix_millis(); + let row = CodexSidecarRecord { + record_version: SIDECAR_RECORD_VERSION, + ownership_id: ownership_id.clone(), + pid, + starttime: proc_starttime(pid as i32).unwrap_or(0), + cmdline: proc_cmdline(pid as i32).unwrap_or(constructed_cmdline), + ws_url: ws_url.clone(), + session_id: None, + terminal_id: None, + server_instance_id: default_server_instance_id(), + created_at: now, + updated_at: now, + state: SidecarRecordState::Active, + }; + // Write failures are logged LOUDLY, never abort the + // launch (the pane-ledger write-failure policy). + if let Err(error) = self.store.write(&row) { + tracing::error!( + target: "freshell_codex::launch", + ownership_id = %row.ownership_id, + pid = row.pid, + error = %error, + "sidecar_record_write_failed: spawn proceeds UNTRACKED \ + (detached; boot reconcile cannot see this sidecar)" + ); + } + record = Some(row); + } + None => tracing::error!( + target: "freshell_codex::launch", + ownership_id = %ownership_id, + "sidecar_record_skipped: child pid unavailable after probe success" + ), + } + } + *state = Some(SpawnedSidecar { ws_url: ws_url.clone(), ownership_id, child, + record, }); Ok(CodexRuntimeReady { ws_url }) }) @@ -1036,11 +1312,96 @@ impl CodexLaunchRuntime for SpawnedCodexAppServerRuntime { generation: u64, ) -> BoxFuture<'_, Result<(), String>> { Box::pin(async move { - *self.adopted_metadata.lock().unwrap() = Some((terminal_id, generation)); + *self.adopted_metadata.lock().unwrap() = Some((terminal_id.clone(), generation)); + // Enrich the durable record at adopt (Task 3): the terminal id is + // what boot reconcile reports a surviving sidecar under. + let mut state = self.state.lock().await; + if let Some(record) = state.as_mut().and_then(|s| s.record.as_mut()) { + record.terminal_id = Some(terminal_id); + record.updated_at = unix_millis(); + if let Err(error) = self.store.write(record) { + tracing::error!( + target: "freshell_codex::launch", + ownership_id = %record.ownership_id, + error = %error, + "sidecar_record_enrich_failed: adopt proceeds; record keeps \ + its spawn-time shape (pane-ledger write-failure policy)" + ); + } + } + Ok(()) + }) + } + + fn note_session_id(&self, session_id: String) -> BoxFuture<'_, Result<(), String>> { + Box::pin(async move { + // Enrich the durable record with the codex session/thread id + // (Task 4): the restore-time reattach key boot reconcile matches + // records against. Untracked spawns (no record) are a no-op. + let mut state = self.state.lock().await; + if let Some(record) = state.as_mut().and_then(|s| s.record.as_mut()) { + record.session_id = Some(session_id); + record.updated_at = unix_millis(); + if let Err(error) = self.store.write(record) { + tracing::error!( + target: "freshell_codex::launch", + ownership_id = %record.ownership_id, + error = %error, + "sidecar_record_enrich_failed: session id kept in memory only \ + (pane-ledger write-failure policy)" + ); + } + } Ok(()) }) } + /// Task 10: server-shutdown retention. Tracked spawns (persisted record; + /// `kill_on_drop(false)`, Task 3) flip their record to `Retained{reason}` + /// and DROP the `Child` handle without a signal — the sidecar outlives + /// this process and the record is what the next generation reconciles + /// against. The retention GATE: a record-less spawn (disabled store / + /// non-Linux ⇒ `kill_on_drop(true)`, NO record) is torn down exactly as + /// [`Self::shutdown`] would — "retaining" it would orphan it silently + /// with no reconcile path (the ynfn hole). + fn prepare_retention(&self, reason: String) -> BoxFuture<'_, Result<(), String>> { + Box::pin(async move { + let mut state = self.state.lock().await; + let Some(mut spawned) = state.take() else { + return Ok(()); // never spawned / already torn down + }; + match spawned.record.as_mut() { + Some(record) => { + record.state = SidecarRecordState::Retained { reason }; + record.updated_at = unix_millis(); + // Write failures log loudly, never propagate (pane-ledger + // policy): the row stays Active on disk and boot + // reconcile still finds + re-verifies the survivor. + write_record_loudly(&self.store, record); + tracing::info!( + target: "freshell_codex::launch", + ownership_id = %record.ownership_id, + pid = record.pid, + "sidecar_retained: tracked sidecar left running across \ + server shutdown (kata ynfn); record state = Retained" + ); + // `spawned` drops at scope end: kill_on_drop is false for + // tracked spawns, so the child is released untouched. + Ok(()) + } + None => { + // Record-less: the ynfn gate — teardown exactly as today. + let _ = spawned.child.start_kill(); + let _ = + tokio::time::timeout(Duration::from_secs(5), spawned.child.wait()).await; + reap_owned_codex_sidecars(&spawned.ownership_id); + self.scrub_record(&spawned.ownership_id); + Ok(()) + } + } + }) + } + fn shutdown(&self) -> BoxFuture<'_, Result<(), String>> { Box::pin(async move { let mut state = self.state.lock().await; @@ -1048,6 +1409,9 @@ impl CodexLaunchRuntime for SpawnedCodexAppServerRuntime { let _ = spawned.child.start_kill(); let _ = tokio::time::timeout(Duration::from_secs(5), spawned.child.wait()).await; reap_owned_codex_sidecars(&spawned.ownership_id); + // Explicit teardown scrubs the record (Task 3): a cleanly + // shut-down sidecar must leave nothing for boot reconcile. + self.scrub_record(&spawned.ownership_id); } Ok(()) }) diff --git a/crates/freshell-codex/src/lib.rs b/crates/freshell-codex/src/lib.rs index d2738c4da..56bae4344 100644 --- a/crates/freshell-codex/src/lib.rs +++ b/crates/freshell-codex/src/lib.rs @@ -52,6 +52,18 @@ pub mod launch_lifecycle; #[cfg(feature = "real-transport")] pub mod remote_proxy; #[cfg(feature = "real-transport")] +pub mod runtime_select; +#[cfg(feature = "real-transport")] +pub mod sidecar_reconcile; +#[cfg(feature = "real-transport")] +pub mod sidecar_store; +#[cfg(feature = "real-transport")] +pub mod sidecar_sweep; +// Shared helpers for the sidecar lifecycle test suites (never shipped; +// Linux-only like the suites themselves — they read `/proc` evidence). +#[cfg(all(test, target_os = "linux", feature = "real-transport"))] +pub(crate) mod sidecar_test_support; +#[cfg(feature = "real-transport")] pub mod transport; pub use app_server::{ @@ -72,6 +84,22 @@ pub use model::{ CodexEffortError, CHEAPEST_T2_MODEL, FRESHCODEX_DEFAULT_EFFORT, FRESHCODEX_DEFAULT_MODEL, FRESHCODEX_EFFORTS_VERBATIM, }; +#[cfg(feature = "real-transport")] +pub use runtime_select::select_codex_runtime; +#[cfg(feature = "real-transport")] +pub use sidecar_reconcile::{ + codex_sidecar_reconciler, set_codex_sidecar_reconciler, BootReconcileReport, + ReattachedCodexAppServerRuntime, SidecarReconciler, +}; +#[cfg(feature = "real-transport")] +pub use sidecar_store::{ + proc_cmdline, proc_starttime, set_codex_sidecar_store, verify_sidecar_identity, + CodexSidecarRecord, CodexSidecarStore, IdentityVerdict, SidecarRecordState, + SIDECAR_RECORD_VERSION, +}; +#[cfg(feature = "real-transport")] +pub use sidecar_sweep::{kill_verified_sidecar_tree, KillOutcome, KillTreeOutcome}; + pub use protocol::{ build_notification_frame, build_request_frame, classify_notification, extract_turn_notification_event, parse_client_frame, parse_incoming_frame, turn_status, diff --git a/crates/freshell-codex/src/runtime_select.rs b/crates/freshell-codex/src/runtime_select.rs new file mode 100644 index 000000000..683b3270c --- /dev/null +++ b/crates/freshell-codex/src/runtime_select.rs @@ -0,0 +1,40 @@ +//! Plan-aware runtime selection (Task 7) — the reattach-vs-spawn seam the +//! production [`CodexRuntimeFactory`] dispatches through. +//! +//! Sibling of [`crate::sidecar_reconcile`] (the pre-authorized split: the +//! reconcile module sits at its 1,000-line ceiling): the reconciler owns the +//! CLAIM; this module owns the SELECTION the claim's outcome drives. +//! +//! [`CodexRuntimeFactory`]: crate::launch_lifecycle::CodexRuntimeFactory + +use std::sync::Arc; + +use crate::launch_lifecycle::{CodexLaunchRuntime, SpawnedCodexAppServerRuntime}; +use crate::launch_plan::CodexLaunchPlan; +use crate::sidecar_reconcile::{ReattachedCodexAppServerRuntime, SidecarReconciler}; +use crate::sidecar_store::CodexSidecarStore; + +/// The production selection: a claimable verified survivor for the plan's +/// resume session ⇒ reattach; otherwise the spawn runtime. Reattach applies +/// only to resume plans (`plan.session_id` is `Some` ⇔ resume, +/// [`CodexLaunchPlan::session_id`]), so the A4 fresh-restore exclusion and +/// the 45s candidate-capture timer are untouched. `None` reconciler/store +/// (nothing installed at boot) ⇒ spawn — behavior identical to the +/// pre-reconciler world. +pub async fn select_codex_runtime( + reconciler: Option<&Arc>, + store: Option<&Arc>, + plan: &CodexLaunchPlan, +) -> Arc { + if let (Some(reconciler), Some(store), Some(session_id)) = + (reconciler, store, plan.session_id.as_deref()) + { + if let Some(record) = reconciler.claim_for_session(session_id).await { + return Arc::new(ReattachedCodexAppServerRuntime::new( + record, + Arc::clone(store), + )); + } + } + Arc::new(SpawnedCodexAppServerRuntime::new()) +} diff --git a/crates/freshell-codex/src/sidecar_reconcile.rs b/crates/freshell-codex/src/sidecar_reconcile.rs new file mode 100644 index 000000000..7be2941bf --- /dev/null +++ b/crates/freshell-codex/src/sidecar_reconcile.rs @@ -0,0 +1,701 @@ +//! Boot-time **sidecar reconciler** — loads the durable +//! `rust-codex-sidecars` records a previous server generation left behind +//! ([`crate::sidecar_store`]), prunes rows whose identity evidence no longer +//! matches live `/proc` (Dead / Mismatch — remove only, NEVER signal), and +//! holds the survivors as one-shot claimable by codex session id for +//! restore-time reattach (katas ynfn/da92; the adopt/sweep sides land in +//! Tasks 6–9). +//! +//! Every prune/claim decision emits structured tracing with the ownership id +//! and identity verdict — auditability is half the invariant. +//! +//! The plan-aware reattach-vs-spawn selection OVER a claim lives in the +//! sibling [`crate::runtime_select`] (Task 7). + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::Duration; + +use crate::app_server::{BoxFuture, CodexAppServerClient}; +use crate::launch_lifecycle::{CodexLaunchRuntime, CodexRuntimeReady}; +use crate::sidecar_store::{ + verify_sidecar_identity, CodexSidecarRecord, CodexSidecarStore, IdentityVerdict, + SidecarRecordState, +}; +use crate::sidecar_sweep::kill_verified_sidecar_tree; +use crate::transport::TungsteniteTransport; + +/// Per-candidate budget for the duplicate-arm writer probe (connect + the +/// `initialize`/`initialized` handshake + one `thread/loaded/list` round +/// trip). Bounded so a wedged survivor cannot stall a restore; on timeout +/// the candidate is simply NOT the writer. +const WRITER_PROBE_BUDGET: Duration = Duration::from_millis(1000); + +/// Reattach `ensure_ready` probe budget: ONE bounded connect against the +/// survivor's recorded `ws_url` (the spawn path's A6-fixed probe shape, +/// `launch_lifecycle.rs:1088-1131`, but a single short attempt) — reattach +/// must fail FAST into the structural fresh-spawn fallback, never sit out +/// the 45s spawn budget. +const REATTACH_PROBE_BUDGET: Duration = Duration::from_secs(3); + +/// Boot-log summary returned by [`SidecarReconciler::boot_reconcile`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BootReconcileReport { + /// Healthy rows loaded from the store (corrupt rows were quarantined by + /// `load_all`, not counted here). + pub loaded: usize, + /// Dead-verdict rows removed (stale records of exited sidecars). + pub pruned_dead: usize, + /// Mismatch-verdict rows removed (pid reuse — the pid is NOT ours; the + /// row is dropped and the process is NEVER signalled). + pub pruned_mismatch: usize, + /// Rows held for claim/sweep (Verified + Unverifiable). + pub held: usize, +} + +/// The boot reconciler: holds every surviving record until a restore claims +/// it (by session id) or the sweep (Task 9) disposes of it. +pub struct SidecarReconciler { + /// pub(crate): shared with the [`crate::sidecar_sweep`] sibling (the + /// pre-authorized 1,000-line split) — one logical brick, two files. + pub(crate) store: Arc, + /// ALL held records, keyed by ownership_id — NOT by session id. Two live + /// records can legitimately share a session id (a mid-turn survivor + /// retained at sweep + a later fresh spawn enriched with the same session; + /// validated reachable — reports/V3.md), and Verified-without-session / + /// Unverifiable records must also be held for the sweep. Keying by + /// session_id would silently drop records (a fifth-fate ynfn violation). + pub(crate) held: Mutex>, + /// Secondary index for restore-time claims. + pub(crate) by_session: Mutex>>, +} + +/// Outcome of the sync (lock-holding) phase of a claim. The `Claimed` record +/// is boxed to keep the enum small (clippy `large_enum_variant`). +enum FastClaim { + /// No claimable candidate for this session. + Empty, + /// Exactly one verified candidate — claimed under the locks, no probe. + Claimed(Box), + /// Two or more verified candidates, snapshotted OUT of the locks for the + /// async writer probe. + Duplicates(Vec), +} + +impl SidecarReconciler { + /// Boot: load_all(); prune records whose identity verdict is Dead + /// (remove) or Mismatch (remove — the pid is NOT ours, never signal); + /// hold every remaining record by ownership_id (Verified with session = + /// claimable via the index; Verified without session and Unverifiable = + /// held for the sweep only). Returns a summary for boot logs. + pub fn boot_reconcile(store: Arc) -> (Self, BootReconcileReport) { + let records = store.load_all(); + let loaded = records.len(); + let mut pruned_dead = 0; + let mut pruned_mismatch = 0; + let mut held: HashMap = HashMap::new(); + let mut by_session: HashMap> = HashMap::new(); + + for record in records { + let verdict = verify_sidecar_identity(&record); + match verdict { + IdentityVerdict::Dead => { + pruned_dead += 1; + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + verdict = ?verdict, + "sidecar_record_pruned: recorded sidecar exited; stale row removed" + ); + remove_pruned(&store, &record.ownership_id); + } + IdentityVerdict::Mismatch => { + pruned_mismatch += 1; + tracing::warn!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + verdict = ?verdict, + "sidecar_record_pruned: pid reuse — the pid is NOT ours; \ + row removed, process NEVER signalled" + ); + remove_pruned(&store, &record.ownership_id); + } + IdentityVerdict::Verified | IdentityVerdict::Unverifiable => { + // Verified with a session id is claimable via the index; + // Verified without one and Unverifiable are held for the + // sweep only. + let claimable = + verdict == IdentityVerdict::Verified && record.session_id.is_some(); + if claimable { + by_session + .entry(record.session_id.clone().expect("claimable has a session")) + .or_default() + .push(record.ownership_id.clone()); + } + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + verdict = ?verdict, + session_id = record.session_id.as_deref().unwrap_or(""), + claimable, + "sidecar_record_held: survivor held for claim/sweep" + ); + held.insert(record.ownership_id.clone(), record); + } + } + } + + let report = BootReconcileReport { + loaded, + pruned_dead, + pruned_mismatch, + held: held.len(), + }; + ( + Self { + store, + held: Mutex::new(held), + by_session: Mutex::new(by_session), + }, + report, + ) + } + + /// Restore-time claim: re-verify identity at claim time and return ONE + /// record for this session. With duplicates, pick the WRITER: prefer the + /// candidate whose live sidecar reports this session in + /// thread/loaded/list (a bounded ws probe — duplicate arm only, ~1s per + /// candidate), else newest updated_at. Losers + /// STAY held (they keep their sweep fate — never silently dropped). + /// Retained-state records ARE claimable (re-verified; adopt flips them + /// back to Active) — a late restore after the sweep must still reattach + /// a mid-turn survivor instead of reproducing the -32600 (reports/V3.md). + /// Only the returned record leaves `held`; each record is claimable ONCE. + /// ASYNC because of the writer probe (Task 7's factory is async-aware and + /// awaits this): the 0/1-candidate fast path opens no connection; the + /// duplicate arm snapshots candidates OUT of the `held`/`by_session` + /// locks before any await (std Mutex guards must never be held across an + /// await point — clippy `await_holding_lock`). + /// After the probe await, the winner is claimed by re-acquiring the + /// locks and removing it from `held`/`by_session` ONLY if still present; + /// a candidate the sweep consumed during the await is skipped (fall + /// through to the remaining candidates, else None). Membership in + /// `held` is the single source of truth for claim-vs-sweep ownership — + /// every exit from `held` happens under its lock (Task 9's sweep + /// TOCTOU guard is the mirror of this rule). + pub async fn claim_for_session(&self, session_id: &str) -> Option { + let candidates = match self.verify_and_fast_claim(session_id) { + FastClaim::Empty => return None, + FastClaim::Claimed(record) => return Some(*record), + FastClaim::Duplicates(candidates) => candidates, + }; + + // Duplicate arm — NO locks held across these awaits. + let mut ranked: Vec<(bool, CodexSidecarRecord)> = Vec::with_capacity(candidates.len()); + for record in candidates { + let is_writer = writer_probe(&record.ws_url, session_id).await; + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + session_id, + is_writer, + "sidecar_claim_probe: duplicate-arm writer probe result" + ); + ranked.push((is_writer, record)); + } + // Writers first, then newest updated_at; ownership_id is a + // deterministic final tiebreak. + ranked.sort_by(|a, b| { + b.0.cmp(&a.0) + .then(b.1.updated_at.cmp(&a.1.updated_at)) + .then(a.1.ownership_id.cmp(&b.1.ownership_id)) + }); + + self.claim_first_still_held(session_id, &ranked) + } + + /// Records still held (unclaimed) — the sweep's future workload. + pub fn unclaimed_len(&self) -> usize { + self.held.lock().unwrap().len() + } + + /// Sync phase of a claim (all lock work, no awaits): re-verify every + /// indexed candidate, prune Dead/Mismatch rows (store + held), skip + /// Unverifiable ones (held for the sweep), and either claim a single + /// verified candidate outright or snapshot the duplicates for the probe. + fn verify_and_fast_claim(&self, session_id: &str) -> FastClaim { + let mut held = self.held.lock().unwrap(); + let mut by_session = self.by_session.lock().unwrap(); + let Some(indexed_ids) = by_session.get(session_id).cloned() else { + return FastClaim::Empty; + }; + + let mut retained_ids: Vec = Vec::new(); + let mut candidates: Vec = Vec::new(); + for ownership_id in indexed_ids { + let Some(record) = held.get(&ownership_id) else { + // Consumed by an earlier claim or the sweep — membership in + // `held` is the single source of truth; drop the stale index + // entry. + continue; + }; + let verdict = verify_sidecar_identity(record); + match verdict { + IdentityVerdict::Verified => { + candidates.push(record.clone()); + retained_ids.push(ownership_id); + } + IdentityVerdict::Dead => { + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %ownership_id, + verdict = ?verdict, + session_id, + "sidecar_claim_pruned: candidate died since boot; row removed" + ); + remove_pruned(&self.store, &ownership_id); + held.remove(&ownership_id); + } + IdentityVerdict::Mismatch => { + tracing::warn!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %ownership_id, + verdict = ?verdict, + session_id, + "sidecar_claim_pruned: pid reuse since boot — the pid is NOT \ + ours; row removed, process NEVER signalled" + ); + remove_pruned(&self.store, &ownership_id); + held.remove(&ownership_id); + } + IdentityVerdict::Unverifiable => { + // Not provably ours ⇒ not claimable; not provably stale + // ⇒ stays held for the sweep (never silently dropped). + tracing::warn!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %ownership_id, + verdict = ?verdict, + session_id, + "sidecar_claim_skipped: identity unverifiable at claim time; \ + record stays held for the sweep" + ); + retained_ids.push(ownership_id); + } + } + } + + if candidates.len() == 1 { + let claimed = candidates.remove(0); + held.remove(&claimed.ownership_id); + retained_ids.retain(|id| id != &claimed.ownership_id); + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %claimed.ownership_id, + verdict = ?IdentityVerdict::Verified, + session_id, + decided_by = "single_candidate", + "sidecar_record_claimed: sole verified candidate claimed (no probe)" + ); + rewrite_index(&mut by_session, session_id, retained_ids); + return FastClaim::Claimed(Box::new(claimed)); + } + + rewrite_index(&mut by_session, session_id, retained_ids); + if candidates.is_empty() { + FastClaim::Empty + } else { + FastClaim::Duplicates(candidates) + } + } + + /// Post-probe phase (locks re-acquired, no awaits): claim the first + /// ranked candidate still present in `held`; skip candidates consumed + /// during the probe await. + fn claim_first_still_held( + &self, + session_id: &str, + ranked: &[(bool, CodexSidecarRecord)], + ) -> Option { + let mut held = self.held.lock().unwrap(); + let mut by_session = self.by_session.lock().unwrap(); + for (is_writer, candidate) in ranked { + let Some(record) = held.remove(&candidate.ownership_id) else { + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %candidate.ownership_id, + session_id, + "sidecar_claim_candidate_consumed: candidate left `held` during \ + the probe await; skipped" + ); + continue; + }; + if let Some(ids) = by_session.get_mut(session_id) { + ids.retain(|id| id != &record.ownership_id); + if ids.is_empty() { + by_session.remove(session_id); + } + } + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + verdict = ?IdentityVerdict::Verified, + session_id, + decided_by = if *is_writer { "writer_probe" } else { "updated_at_fallback" }, + "sidecar_record_claimed: duplicate-arm winner claimed; losers stay held" + ); + return Some(record); + } + None + } +} + +/// Remove a pruned row from the store; a removal failure is logged loudly +/// (the row will be re-pruned next boot) and never fails the reconcile. +pub(crate) fn remove_pruned(store: &CodexSidecarStore, ownership_id: &str) { + if let Err(error) = store.remove(ownership_id) { + tracing::error!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %ownership_id, + error = %error, + "sidecar_record_prune_remove_failed: row removal failed; retried next boot" + ); + } +} + +/// Rewrite (or drop, when empty) a session's index entry. +fn rewrite_index( + by_session: &mut HashMap>, + session_id: &str, + retained_ids: Vec, +) { + if retained_ids.is_empty() { + by_session.remove(session_id); + } else { + by_session.insert(session_id.to_string(), retained_ids); + } +} + +/// Bounded writer probe (duplicate arm only): does the candidate's live +/// sidecar report `session_id` in `thread/loaded/list`? Reuses the crate's +/// own client ([`CodexAppServerClient`] over [`TungsteniteTransport`], the +/// sweep-probe shape, `sidecar_sweep.rs::probe_mid_turn`) so the +/// `initialize`/`initialized` handshake ALWAYS precedes the list RPC — real +/// codex gates pre-initialize RPCs, and a hand-rolled first-frame list would +/// silently degrade every probe to the newest-`updated_at` fallback (final +/// review F1). All under a single [`WRITER_PROBE_BUDGET`]; any error/timeout +/// ⇒ NOT the writer (the fallback decides). The positive arm is pinned by +/// `duplicate_claim_prefers_the_live_writer_over_newer_updated_at` (this +/// module's tests, against an initialize-gated fixture); the other +/// duplicate-claim tests use `sleep` children that speak no ws, so their +/// probes fail fast. +async fn writer_probe(ws_url: &str, session_id: &str) -> bool { + tokio::time::timeout(WRITER_PROBE_BUDGET, writer_probe_inner(ws_url, session_id)) + .await + .unwrap_or(false) +} + +async fn writer_probe_inner(ws_url: &str, session_id: &str) -> bool { + let Ok(transport) = TungsteniteTransport::connect(ws_url).await else { + return false; + }; + // Keep the notification receiver alive for the probe's lifetime; the + // client Drop aborts the background consumer (even on the outer timeout). + let (client, _notifications) = CodexAppServerClient::connect(Arc::new(transport)); + // `list_loaded_threads` runs the initialize/initialized handshake first + // (every non-initialize request gates on it, app_server.rs). + let is_writer = match client.list_loaded_threads().await { + Ok(loaded) => loaded.iter().any(|id| id == session_id), + Err(_) => false, + }; + client.close().await; + is_writer +} + +// --------------------------------------------------------------------------- +// The reattach runtime (Task 6): a second `CodexLaunchRuntime` impl over a +// CLAIMED record — adopt the surviving app-server instead of spawning. +// `plan_create`'s existing cleanup-on-plan-failure (`launch_lifecycle.rs:373-380` +// calls `sidecar.shutdown()` on `ensure_ready` error) composes with the +// failure arms here: a failed reattach tears down via the SAME conservative +// path, and the retry loop (`plan_create_with_retry`, +// `launch_lifecycle.rs:389-416`) re-invokes the factory, which — the claim +// being consumed — mints a fresh `SpawnedCodexAppServerRuntime`: fallback is +// structural, not special-cased. +// --------------------------------------------------------------------------- + +/// A second [`CodexLaunchRuntime`]: wraps a record claimed from the +/// [`SidecarReconciler`] and reattaches to the surviving app-server instead +/// of spawning a fresh one (kata da92). +pub struct ReattachedCodexAppServerRuntime { + /// The claimed record. Interior mutability (std `Mutex`) because the + /// trait's enrich hooks (`update_ownership_metadata`/`note_session_id`) + /// rewrite it through `&self`; the guard is NEVER held across an await — + /// every async path clones the record out first (the + /// [`kill_verified_sidecar_tree`] caller contract). + record: Mutex, + store: Arc, + /// Set by a successful `ensure_ready`; gates `shutdown`'s kill. + verified_usable: AtomicBool, +} + +impl ReattachedCodexAppServerRuntime { + /// Wrap a record claimed via [`SidecarReconciler::claim_for_session`] + /// (Task 7's factory constructs this when a claim succeeds). + pub fn new(record: CodexSidecarRecord, store: Arc) -> Self { + Self { + record: Mutex::new(record), + store, + verified_usable: AtomicBool::new(false), + } + } +} + +impl CodexLaunchRuntime for ReattachedCodexAppServerRuntime { + /// Reattach readiness: `cwd` is IGNORED — the survivor already has one + /// (it was spawned with the original create cwd). Re-verify identity, + /// then probe-dial `record.ws_url` with ONE bounded connect + /// ([`REATTACH_PROBE_BUDGET`]). On success: mark `verified_usable` and + /// return the survivor's ws url. On failure: + /// - `Mismatch`/`Unverifiable` → `store.remove`, `Err` — **no signal is + /// ever sent** (this pid is not provably ours). + /// - `Dead` → `store.remove`, `Err`. + /// - `Verified` but probe failed (dead port / handshake failure) → the + /// survivor is unusable: [`kill_verified_sidecar_tree`], + /// `store.remove`, `Err`. An unusable tracked sidecar must not leak; + /// killing it releases codex's per-thread writer-lock files on exit, + /// so the retry's fresh spawn can resume the thread (reports/V1.md). + fn ensure_ready( + &self, + _cwd: Option, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + let record = self.record.lock().unwrap().clone(); + let verdict = verify_sidecar_identity(&record); + match verdict { + IdentityVerdict::Mismatch | IdentityVerdict::Unverifiable => { + tracing::warn!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + pid = record.pid, + verdict = ?verdict, + "sidecar_reattach_refused: identity not provably ours; \ + record removed, process NEVER signalled" + ); + remove_pruned(&self.store, &record.ownership_id); + Err(format!( + "codex sidecar reattach refused: identity {verdict:?} for pid {}; \ + record removed, process never signalled", + record.pid + )) + } + IdentityVerdict::Dead => { + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + pid = record.pid, + verdict = ?verdict, + "sidecar_reattach_failed: recorded sidecar is dead; stale row removed" + ); + remove_pruned(&self.store, &record.ownership_id); + Err( + "codex sidecar reattach failed: recorded sidecar is dead; record removed" + .to_string(), + ) + } + IdentityVerdict::Verified => { + let probe_error = match tokio::time::timeout( + REATTACH_PROBE_BUDGET, + tokio_tungstenite::connect_async(&record.ws_url), + ) + .await + { + Ok(Ok((probe, _response))) => { + drop(probe); + self.verified_usable.store(true, Ordering::SeqCst); + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + pid = record.pid, + ws_url = %record.ws_url, + "sidecar_reattached: surviving app-server adopted; no spawn" + ); + return Ok(CodexRuntimeReady { + ws_url: record.ws_url.clone(), + }); + } + Ok(Err(error)) => error.to_string(), + Err(_elapsed) => "probe timed out awaiting the WS handshake".to_string(), + }; + // Verified but unusable (dead port / handshake failure): + // the survivor must not leak — reap the whole tree. + let outcome = kill_verified_sidecar_tree(&record).await; + tracing::warn!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + pid = record.pid, + probe_error = %probe_error, + outcome = ?outcome.outcomes, + "sidecar_reattach_reaped: verified survivor unusable; \ + tree reaped, record removed" + ); + remove_pruned(&self.store, &record.ownership_id); + Err(format!( + "codex sidecar reattach failed: verified survivor unusable \ + ({probe_error}); tree reaped, record removed" + )) + } + } + }) + } + + /// Adopt-time enrich: rewrite the record (new terminal id, updated_at) + /// and flip a claimed `Retained{..}` row back to `Active` — the sidecar + /// is pane-owned again; a stale retention reason would lie to auditors + /// (final review H3a). + fn update_ownership_metadata( + &self, + terminal_id: String, + _generation: u64, + ) -> BoxFuture<'_, Result<(), String>> { + Box::pin(async move { + let snapshot = { + let mut record = self.record.lock().unwrap(); + record.terminal_id = Some(terminal_id); + record.state = SidecarRecordState::Active; + record.updated_at = unix_millis(); + record.clone() + }; + write_record_loudly(&self.store, &snapshot); + Ok(()) + }) + } + + /// Session enrich: rewrite the record (new session id, updated_at); + /// `Active` for the same H3a reason as `update_ownership_metadata`. + fn note_session_id(&self, session_id: String) -> BoxFuture<'_, Result<(), String>> { + Box::pin(async move { + let snapshot = { + let mut record = self.record.lock().unwrap(); + record.session_id = Some(session_id); + record.state = SidecarRecordState::Active; + record.updated_at = unix_millis(); + record.clone() + }; + write_record_loudly(&self.store, &snapshot); + Ok(()) + }) + } + + /// Task 10: server-shutdown retention — the reattached survivor stays + /// alive across ANOTHER restart. A reattached runtime always wraps a + /// persisted record (claims only exist over an enabled store), so + /// retention applies unconditionally: NO signal is ever sent, the record + /// flips to `Retained{reason}`, and the `verified_usable` gate drops so + /// no later teardown path can kill the retained survivor. + fn prepare_retention(&self, reason: String) -> BoxFuture<'_, Result<(), String>> { + Box::pin(async move { + self.verified_usable.store(false, Ordering::SeqCst); + let snapshot = { + let mut record = self.record.lock().unwrap(); + record.state = SidecarRecordState::Retained { reason }; + record.updated_at = unix_millis(); + record.clone() + }; + write_record_loudly(&self.store, &snapshot); + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %snapshot.ownership_id, + pid = snapshot.pid, + "sidecar_retained: reattached sidecar left running across \ + server shutdown (kata ynfn); record state = Retained" + ); + Ok(()) + }) + } + + /// Teardown (pane closed, or the plan raced the planner's shutdown): + /// [`kill_verified_sidecar_tree`] + `store.remove`. Gated on + /// `verified_usable` — if `ensure_ready` never positively adopted this + /// survivor (or its failure arm already disposed of it), shutdown + /// removes the record ONLY and never signals. The kill helper re-verifies + /// identity immediately before each signal, so `Mismatch`/`Dead`/ + /// `Unverifiable` at kill time ⇒ remove record only. + fn shutdown(&self) -> BoxFuture<'_, Result<(), String>> { + Box::pin(async move { + let record = self.record.lock().unwrap().clone(); + if !self.verified_usable.swap(false, Ordering::SeqCst) { + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + pid = record.pid, + "sidecar_reattach_shutdown_skipped_kill: survivor never verified \ + usable by this runtime; record removed, nothing signalled" + ); + remove_pruned(&self.store, &record.ownership_id); + return Ok(()); + } + let outcome = kill_verified_sidecar_tree(&record).await; + tracing::info!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + pid = record.pid, + outcome = ?outcome.outcomes, + "sidecar_reattach_shutdown: reattached sidecar torn down; record removed" + ); + remove_pruned(&self.store, &record.ownership_id); + Ok(()) + }) + } +} + +/// Rewrite the runtime's record durably; write failures are logged LOUDLY, +/// never propagated (the pane-ledger write-failure policy, +/// `launch_lifecycle.rs:1211-1219` precedent) — the in-memory record stays +/// authoritative for teardown. +pub(crate) fn write_record_loudly(store: &CodexSidecarStore, record: &CodexSidecarRecord) { + if let Err(error) = store.write(record) { + tracing::error!( + target: "freshell_codex::sidecar_reconcile", + ownership_id = %record.ownership_id, + error = %error, + "sidecar_record_rewrite_failed: reattached record kept in memory only \ + (pane-ledger write-failure policy)" + ); + } +} + +/// Wall-clock unix millis for record `updated_at` stamps. Deliberate +/// duplicate of the private `launch_lifecycle::unix_millis` +/// (`launch_lifecycle.rs:988-993`) — a one-liner not worth a shared-helper +/// dependency; keep the two bodies in sync. +pub(crate) fn unix_millis() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0) +} + +// --------------------------------------------------------------------------- +// Process-global reconciler handle (wired at server boot in Task 10). A +// re-settable RwLock seam, mirroring `sidecar_store`'s store global. +// --------------------------------------------------------------------------- + +static GLOBAL_SIDECAR_RECONCILER: RwLock>> = RwLock::new(None); + +/// Install the process-wide reconciler (server boot, after +/// [`SidecarReconciler::boot_reconcile`]). Later calls replace the handle. +pub fn set_codex_sidecar_reconciler(r: Arc) { + *GLOBAL_SIDECAR_RECONCILER.write().unwrap() = Some(r); +} + +/// The installed process-wide reconciler, if any. `None` (nothing installed) +/// means restore-time callers have nothing to claim from — behavior identical +/// to the pre-reconciler world. +pub fn codex_sidecar_reconciler() -> Option> { + GLOBAL_SIDECAR_RECONCILER.read().unwrap().clone() +} + +#[cfg(test)] +#[path = "sidecar_reconcile_tests.rs"] +mod tests; diff --git a/crates/freshell-codex/src/sidecar_reconcile_tests.rs b/crates/freshell-codex/src/sidecar_reconcile_tests.rs new file mode 100644 index 000000000..ba824d858 --- /dev/null +++ b/crates/freshell-codex/src/sidecar_reconcile_tests.rs @@ -0,0 +1,714 @@ +//! Unit tests for the boot-time sidecar reconciler ([`super`]). +//! +//! Tempfile tempdirs ONLY — no global state, nothing outside each test's own +//! temp dir. In particular these tests must NEVER touch +//! `~/.freshell/codex-sidecars/` (Node's store), the production +//! `~/.freshell/rust-codex-sidecars/` root (wired in Task 10), or any live +//! process the test did not itself spawn. +//! +//! PROCESS SAFETY: each test spawns and reaps ONLY its own children +//! (`sleep 300`), killed in a [`ChildGuard`] drop guard — nothing else on the +//! machine is ever signalled. Reconciliation itself NEVER signals any pid +//! (prune is `store.remove` only), and the tests assert exactly that by +//! checking their children are still alive afterwards. +//! +//! Writer-probe note: `sleep` children speak no ws, and every record's +//! `ws_url` points at a loopback port nothing listens on, so the duplicate +//! arm's probe fails fast (connection refused, bounded by the ~1s budget) and +//! the newest-`updated_at` fallback decides — deterministic. The probe's +//! POSITIVE arm is pinned here by +//! `duplicate_claim_prefers_the_live_writer_over_newer_updated_at` (an +//! initialize-gated fixture — the real-codex shape). Tests bind loopback +//! ephemeral ports only; never port 3001. +//! +//! /proc semantics are Linux-only, so these tests are +//! `#[cfg(target_os = "linux")]` (the sidecar_store_tests precedent). + +#![cfg(target_os = "linux")] + +use std::sync::Arc; + +use super::*; +use crate::sidecar_store::CodexSidecarRecord; +use crate::sidecar_test_support::{ + record_for_child, spawn_own_fake_app_server, spawn_own_fake_app_server_with_behavior, + spawn_own_sleep_child, store_in, NEVER_SIGNALLED_GRACE, SESSION, +}; + +#[test] +fn boot_reconcile_prunes_dead_and_mismatched_records() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + + // Dead: real evidence captured live, then OUR OWN child is killed+reaped. + let mut dead_child = spawn_own_sleep_child(); + let dead = record_for_child( + "codex-sidecar-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + dead_child.0.id(), + Some(SESSION), + ); + dead_child.0.kill().expect("kill own child"); + dead_child.0.wait().expect("reap own child"); + + // Mismatch: a live child's pid+starttime but a DIFFERENT cmdline — + // the pid-reuse shape. This pid is NOT ours; it must never be signalled. + let mut mismatch_child = spawn_own_sleep_child(); + let mismatch = CodexSidecarRecord { + cmdline: vec!["codex".to_string(), "app-server".to_string()], + ..record_for_child( + "codex-sidecar-bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + mismatch_child.0.id(), + Some(SESSION), + ) + }; + + // Verified: a live child's real evidence. + let mut verified_child = spawn_own_sleep_child(); + let verified = record_for_child( + "codex-sidecar-cccccccc-cccc-4ccc-8ccc-cccccccccccc", + verified_child.0.id(), + Some(SESSION), + ); + + store.write(&dead).expect("write dead"); + store.write(&mismatch).expect("write mismatch"); + store.write(&verified).expect("write verified"); + + let (reconciler, report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + + assert_eq!( + report, + BootReconcileReport { + loaded: 3, + pruned_dead: 1, + pruned_mismatch: 1, + held: 1, + } + ); + assert_eq!(reconciler.unclaimed_len(), 1, "only the verified row held"); + assert_eq!( + store.load_all(), + vec![verified], + "the store holds ONLY the verified row after pruning" + ); + + // Prune NEVER signals: both live children are still alive afterwards. + assert_eq!( + mismatch_child + .0 + .try_wait() + .expect("try_wait mismatch child"), + None, + "the mismatching pid must never be signalled" + ); + assert_eq!( + verified_child + .0 + .try_wait() + .expect("try_wait verified child"), + None, + "the verified child must not be signalled by boot" + ); +} + +#[tokio::test] +async fn boot_reconcile_holds_sessionless_records_for_the_sweep() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + + let child = spawn_own_sleep_child(); + let sessionless = record_for_child( + "codex-sidecar-dddddddd-dddd-4ddd-8ddd-dddddddddddd", + child.0.id(), + None, + ); + store.write(&sessionless).expect("write sessionless"); + + let (reconciler, report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + assert_eq!(report.held, 1); + assert_eq!( + reconciler.unclaimed_len(), + 1, + "a verified record WITHOUT a session is held for the sweep" + ); + + // Not claimable by any session — and NOT dropped by the attempt. + assert_eq!(reconciler.claim_for_session(SESSION).await, None); + assert_eq!( + reconciler.unclaimed_len(), + 1, + "the sessionless record stays held after a foreign claim attempt" + ); + assert_eq!( + store.load_all(), + vec![sessionless], + "the sessionless row survives in the store" + ); +} + +#[tokio::test] +async fn claim_for_session_returns_each_record_once() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + + let child = spawn_own_sleep_child(); + let record = record_for_child( + "codex-sidecar-eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee", + child.0.id(), + Some(SESSION), + ); + store.write(&record).expect("write record"); + + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + assert_eq!(reconciler.unclaimed_len(), 1); + + let first = reconciler.claim_for_session(SESSION).await; + assert_eq!(first, Some(record), "first claim returns the record"); + assert_eq!(reconciler.unclaimed_len(), 0, "the claim left held"); + + let second = reconciler.claim_for_session(SESSION).await; + assert_eq!(second, None, "each record is claimable ONCE"); +} + +#[tokio::test] +async fn claim_reverifies_identity_at_claim_time() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + + let mut child = spawn_own_sleep_child(); + let record = record_for_child( + "codex-sidecar-ffffffff-ffff-4fff-8fff-ffffffffffff", + child.0.id(), + Some(SESSION), + ); + store.write(&record).expect("write record"); + + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + assert_eq!(reconciler.unclaimed_len(), 1, "held while the child lives"); + + // The sidecar dies BETWEEN boot and claim (kill+reap OUR OWN child). + child.0.kill().expect("kill own child"); + child.0.wait().expect("reap own child"); + + assert_eq!( + reconciler.claim_for_session(SESSION).await, + None, + "claim re-verifies identity and refuses a dead sidecar" + ); + assert_eq!(reconciler.unclaimed_len(), 0, "the dead record left held"); + assert!( + store.load_all().is_empty(), + "the dead record was removed from the store" + ); +} + +#[tokio::test] +async fn duplicate_session_records_claim_one_keep_the_loser_held() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + + // Two VERIFIED records sharing one session id (two live test children) — + // the mid-turn-survivor + fresh-spawn shape (reports/V3.md). + let mut older_child = spawn_own_sleep_child(); + let older = CodexSidecarRecord { + updated_at: 1_700_000_000_001, + ..record_for_child( + "codex-sidecar-11111111-2222-4333-8444-555555555555", + older_child.0.id(), + Some(SESSION), + ) + }; + let mut newer_child = spawn_own_sleep_child(); + let newer = CodexSidecarRecord { + updated_at: 1_700_000_000_002, + ..record_for_child( + "codex-sidecar-66666666-7777-4888-8999-aaaaaaaaaaaa", + newer_child.0.id(), + Some(SESSION), + ) + }; + store.write(&older).expect("write older"); + store.write(&newer).expect("write newer"); + + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + assert_eq!(reconciler.unclaimed_len(), 2, "both duplicates held"); + + // `sleep` children speak no ws (and the ws_urls point at closed ports), + // so the writer probe fails fast on both and the newest-`updated_at` + // fallback decides. + let claimed = reconciler.claim_for_session(SESSION).await; + assert_eq!( + claimed, + Some(newer), + "the newest-updated_at candidate wins the fallback" + ); + assert_eq!( + reconciler.unclaimed_len(), + 1, + "the loser STAYS held for the sweep — never silently dropped" + ); + + // Claiming NEVER signals: both children are still alive. + assert_eq!( + older_child.0.try_wait().expect("try_wait older child"), + None, + "the losing candidate's sidecar must not be signalled" + ); + assert_eq!( + newer_child.0.try_wait().expect("try_wait newer child"), + None, + "the winning candidate's sidecar must not be signalled" + ); +} + +#[tokio::test] +async fn duplicate_claim_prefers_the_live_writer_over_newer_updated_at() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + + // The WRITER candidate: a live fixture reporting this session in + // thread/loaded/list, with the OLDER updated_at — writer preference must + // beat the newest-updated_at fallback (final review F1: the fallback + // tends to pick the NON-writer in exactly this scenario). The fixture + // GATES pre-initialize RPCs (requireInitializeBeforeOtherMethods + + // requireInitializedNotification, the real-codex shape): a probe that + // skipped the initialize/initialized handshake would get -32000, read as + // not-writer, and this test would fail on the fallback picking the + // newer non-writer. + let ownership_writer = "codex-sidecar-f1000001-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + let behavior = format!( + r#"{{"loadedThreadIds": ["{SESSION}"], "requireInitializeBeforeOtherMethods": true, "requireInitializedNotification": true}}"# + ); + let (mut fixture, ws_url) = + spawn_own_fake_app_server_with_behavior(ownership_writer, Some(&behavior)).await; + let writer = CodexSidecarRecord { + ws_url: ws_url.clone(), + updated_at: 1_700_000_000_001, + ..record_for_child( + ownership_writer, + fixture.id().expect("live fixture pid"), + Some(SESSION), + ) + }; + + // The NON-writer duplicate: NEWER updated_at, no ws listener (sleep + // child + closed loopback port) — the fallback's pick. + let mut non_writer_child = spawn_own_sleep_child(); + let non_writer = CodexSidecarRecord { + updated_at: 1_700_000_000_002, + ..record_for_child( + "codex-sidecar-f1000002-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + non_writer_child.0.id(), + Some(SESSION), + ) + }; + store.write(&writer).expect("write writer"); + store.write(&non_writer).expect("write non-writer"); + + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + assert_eq!(reconciler.unclaimed_len(), 2, "both duplicates held"); + + let claimed = reconciler.claim_for_session(SESSION).await; + assert_eq!( + claimed, + Some(writer), + "the live WRITER wins the duplicate claim despite the older updated_at" + ); + assert_eq!( + reconciler.unclaimed_len(), + 1, + "the non-writer loser STAYS held for the sweep" + ); + + // Claiming NEVER signals: both candidates are still alive. + assert_eq!( + non_writer_child + .0 + .try_wait() + .expect("try_wait non-writer child"), + None, + "the losing candidate's sidecar must not be signalled" + ); + assert_eq!( + fixture.try_wait().expect("try_wait fixture"), + None, + "the winning writer's sidecar must not be signalled" + ); + + fixture + .kill() + .await + .expect("cleanup: kill this test's own fixture"); +} + +// --------------------------------------------------------------------------- +// Task 6: ReattachedCodexAppServerRuntime + kill_verified_sidecar_tree. +// +// Each reattach test spawns ITS OWN fake app-server fixture +// (`node test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs +// --listen ws://127.0.0.1:`) as a direct tokio::process child tagged +// `FRESHELL_CODEX_SIDECAR_ID=`, records that pid, and +// kills ONLY that pid in cleanup (`kill_on_drop(true)` plus explicit kills) +// — nothing else on the machine is ever signalled. Loopback ephemeral ports +// only; never 3001/3002. +// --------------------------------------------------------------------------- + +use crate::launch_lifecycle::CodexLaunchRuntime; + +/// Count live processes whose `/proc//environ` carries OUR unique +/// ownership tag — a read-only `/proc` scan keyed on this test's own id, +/// used to prove a reattach spawned NO new sidecar process. +fn count_own_tagged_processes(ownership_id: &str) -> usize { + let needle = crate::durability::ownership_needle(ownership_id); + let Ok(entries) = std::fs::read_dir("/proc") else { + return 0; + }; + entries + .flatten() + .filter(|entry| { + let name = entry.file_name(); + let Some(name) = name.to_str() else { + return false; + }; + let Ok(pid) = name.parse::() else { + return false; + }; + let Ok(environ) = std::fs::read(format!("/proc/{pid}/environ")) else { + return false; + }; + environ + .split(|&b| b == 0) + .any(|var| var == needle.as_bytes()) + }) + .count() +} + +#[tokio::test] +async fn reattach_ensure_ready_returns_the_existing_listener() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + let ownership_id = "codex-sidecar-a6000001-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + let (mut child, ws_url) = spawn_own_fake_app_server(ownership_id).await; + // Record built from the live fixture's REAL /proc evidence + real ws_url. + let record = CodexSidecarRecord { + ws_url: ws_url.clone(), + ..record_for_child( + ownership_id, + child.id().expect("live fixture pid"), + Some(SESSION), + ) + }; + store.write(&record).expect("write record"); + + let runtime = ReattachedCodexAppServerRuntime::new(record.clone(), Arc::clone(&store)); + let ready = runtime + .ensure_ready(Some("/tmp/ignored-reattach-cwd".to_string())) + .await + .expect("reattach ensure_ready adopts the surviving listener"); + assert_eq!( + ready.ws_url, ws_url, + "reattach returns the SURVIVOR's ws url" + ); + + // The survivor is still alive and NO new process was spawned: exactly + // one live process carries this test's unique ownership tag. + assert_eq!( + child.try_wait().expect("try_wait fixture"), + None, + "the adopted survivor must still be alive" + ); + assert_eq!( + count_own_tagged_processes(ownership_id), + 1, + "reattach must spawn NO new sidecar process" + ); + assert_eq!( + store.load_all(), + vec![record], + "a usable survivor's record stays in the store" + ); + + child + .kill() + .await + .expect("cleanup: kill this test's own fixture"); +} + +#[tokio::test] +async fn reattach_refuses_on_identity_mismatch_without_signalling() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + let ownership_id = "codex-sidecar-a6000002-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + let (mut child, ws_url) = spawn_own_fake_app_server(ownership_id).await; + // The fixture's pid + starttime but a WRONG cmdline — the pid-reuse + // shape. This pid is NOT ours; it must never be signalled. + let record = CodexSidecarRecord { + ws_url: ws_url.clone(), + cmdline: vec!["codex".to_string(), "app-server".to_string()], + ..record_for_child( + ownership_id, + child.id().expect("live fixture pid"), + Some(SESSION), + ) + }; + store.write(&record).expect("write record"); + + let runtime = ReattachedCodexAppServerRuntime::new(record, Arc::clone(&store)); + runtime + .ensure_ready(None) + .await + .expect_err("a mismatched identity must refuse the reattach"); + + assert!( + store.load_all().is_empty(), + "the mismatched record is removed" + ); + tokio::time::sleep(NEVER_SIGNALLED_GRACE).await; + assert_eq!( + child.try_wait().expect("try_wait fixture"), + None, + "the mismatching pid must NEVER be signalled" + ); + + child + .kill() + .await + .expect("cleanup: kill this test's own fixture"); +} + +#[tokio::test] +async fn reattach_reaps_verified_but_unusable_survivor() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + let ownership_id = "codex-sidecar-a6000003-cccc-4ccc-8ccc-cccccccccccc"; + // Fixture listens on port A; the record's ws_url points at port B where + // NOTHING listens (record_for_child's default) — pid evidence stays + // valid, so identity is Verified but the probe fails fast. + let (mut child, _fixture_ws_url) = spawn_own_fake_app_server(ownership_id).await; + let record = record_for_child( + ownership_id, + child.id().expect("live fixture pid"), + Some(SESSION), + ); + store.write(&record).expect("write record"); + + let runtime = ReattachedCodexAppServerRuntime::new(record, Arc::clone(&store)); + runtime + .ensure_ready(None) + .await + .expect_err("a verified-but-unusable survivor must fail into fallback"); + + // The unusable survivor was REAPED — an unusable tracked sidecar must + // not leak (killing it releases codex's writer-lock files on exit). + tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("the unusable survivor must be reaped within the drain budget") + .expect("wait fixture"); + assert!( + store.load_all().is_empty(), + "the unusable survivor's record is removed" + ); +} + +#[tokio::test] +async fn reattach_shutdown_kills_only_after_reverification() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + + // Positive arm: successful ensure_ready, then shutdown() → fixture gone, + // record removed. + let ownership_a = "codex-sidecar-a6000004-dddd-4ddd-8ddd-dddddddddddd"; + let (mut child_a, ws_a) = spawn_own_fake_app_server(ownership_a).await; + let record_a = CodexSidecarRecord { + ws_url: ws_a.clone(), + ..record_for_child( + ownership_a, + child_a.id().expect("live fixture pid"), + Some(SESSION), + ) + }; + store.write(&record_a).expect("write record a"); + let runtime_a = ReattachedCodexAppServerRuntime::new(record_a, Arc::clone(&store)); + runtime_a + .ensure_ready(None) + .await + .expect("ensure_ready adopts survivor A"); + runtime_a.shutdown().await.expect("shutdown A"); + tokio::time::timeout(Duration::from_secs(10), child_a.wait()) + .await + .expect("shutdown must reap the adopted survivor within the drain budget") + .expect("wait fixture a"); + assert!( + store.load_all().is_empty(), + "shutdown removes the adopted survivor's record" + ); + + // Negative arm: successful ensure_ready, THEN the record's starttime is + // replaced — the kill-time re-verification sees Mismatch and NEVER + // signals; the record is still removed. + let ownership_b = "codex-sidecar-a6000005-eeee-4eee-8eee-eeeeeeeeeeee"; + let (mut child_b, ws_b) = spawn_own_fake_app_server(ownership_b).await; + let record_b = CodexSidecarRecord { + ws_url: ws_b.clone(), + ..record_for_child( + ownership_b, + child_b.id().expect("live fixture pid"), + Some(SESSION), + ) + }; + store.write(&record_b).expect("write record b"); + let runtime_b = ReattachedCodexAppServerRuntime::new(record_b, Arc::clone(&store)); + runtime_b + .ensure_ready(None) + .await + .expect("ensure_ready adopts survivor B"); + // Tamper the held record's starttime (tests are a child module of the + // runtime, so private field access is available): the pid-reuse shape + // appearing AFTER a successful adopt. + runtime_b.record.lock().unwrap().starttime += 1; + runtime_b + .shutdown() + .await + .expect("shutdown returns Ok even when re-verification refuses the kill"); + tokio::time::sleep(NEVER_SIGNALLED_GRACE).await; + assert_eq!( + child_b.try_wait().expect("try_wait fixture b"), + None, + "a kill-time identity mismatch must NEVER be signalled" + ); + assert!( + store.load_all().is_empty(), + "shutdown removes the record even when the kill is refused" + ); + + child_b + .kill() + .await + .expect("cleanup: kill this test's own fixture"); +} + +// --------------------------------------------------------------------------- +// Task 7: the plan-aware selection seam ([`crate::runtime_select`]). +// +// The spawn arm is asserted BEHAVIORALLY without ever spawning: the returned +// runtime must not have consumed the claim, and its `shutdown` (a no-op for +// an un-started spawn runtime) must leave the survivor's record untouched — +// a reattach runtime's shutdown would scrub it. +// --------------------------------------------------------------------------- + +use crate::launch_plan::{plan_codex_launch, CodexLaunchPlanInput}; +use crate::runtime_select::select_codex_runtime; + +#[tokio::test] +async fn select_codex_runtime_prefers_a_claimable_survivor() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + let ownership_id = "codex-sidecar-a7000001-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + let (mut child, ws_url) = spawn_own_fake_app_server(ownership_id).await; + let record = CodexSidecarRecord { + ws_url: ws_url.clone(), + ..record_for_child( + ownership_id, + child.id().expect("live fixture pid"), + Some(SESSION), + ) + }; + store.write(&record).expect("write record"); + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + let reconciler = Arc::new(reconciler); + + let resume_plan = plan_codex_launch(&CodexLaunchPlanInput { + resume_session_id: Some(SESSION), + ..Default::default() + }) + .expect("resume plan"); + let fresh_plan = plan_codex_launch(&CodexLaunchPlanInput::default()).expect("fresh plan"); + let unknown_plan = plan_codex_launch(&CodexLaunchPlanInput { + resume_session_id: Some("s-unknown"), + ..Default::default() + }) + .expect("unknown-session resume plan"); + + // A fresh plan NEVER claims (the A4 fresh-restore exclusion). + let runtime = select_codex_runtime(Some(&reconciler), Some(&store), &fresh_plan).await; + assert_eq!( + reconciler.unclaimed_len(), + 1, + "a fresh plan must not claim the survivor" + ); + runtime.shutdown().await.expect("spawn-type shutdown"); + assert_eq!( + store.load_all(), + vec![record.clone()], + "fresh plan: the survivor's record must stay untouched" + ); + + // An unknown resume session has nothing to claim: spawn, survivor held. + let runtime = select_codex_runtime(Some(&reconciler), Some(&store), &unknown_plan).await; + assert_eq!( + reconciler.unclaimed_len(), + 1, + "an unknown session must not claim the survivor" + ); + runtime.shutdown().await.expect("spawn-type shutdown"); + assert_eq!( + store.load_all(), + vec![record.clone()], + "unknown session: the survivor's record must stay untouched" + ); + + // No reconciler installed: spawn, even for the claimable resume session. + let runtime = select_codex_runtime(None, Some(&store), &resume_plan).await; + assert_eq!( + reconciler.unclaimed_len(), + 1, + "a None reconciler must claim nothing" + ); + runtime.shutdown().await.expect("spawn-type shutdown"); + assert_eq!( + store.load_all(), + vec![record.clone()], + "no reconciler: the survivor's record must stay untouched" + ); + + // No store installed: spawn — a reattach runtime cannot be minted + // without the store Arc, so the claim must stay unconsumed even with a + // live reconciler and a claimable resume session. + let runtime = select_codex_runtime(Some(&reconciler), None, &resume_plan).await; + assert_eq!( + reconciler.unclaimed_len(), + 1, + "a None store must claim nothing" + ); + runtime.shutdown().await.expect("spawn-type shutdown"); + assert_eq!( + store.load_all(), + vec![record.clone()], + "no store: the survivor's record must stay untouched" + ); + + // The resume plan for the held session claims the survivor and mints the + // reattach runtime: the claim leaves `held`, and `ensure_ready` adopts + // the record's live listener (no spawn). + let runtime = select_codex_runtime(Some(&reconciler), Some(&store), &resume_plan).await; + assert_eq!( + reconciler.unclaimed_len(), + 0, + "the resume plan must consume the claim" + ); + let ready = runtime + .ensure_ready(None) + .await + .expect("reattach adopts the surviving listener"); + assert_eq!(ready.ws_url, ws_url, "reattach returns the RECORD's ws url"); + assert_eq!( + child.try_wait().expect("try_wait fixture"), + None, + "the adopted survivor must still be alive" + ); + + child + .kill() + .await + .expect("cleanup: kill this test's own fixture"); +} diff --git a/crates/freshell-codex/src/sidecar_store.rs b/crates/freshell-codex/src/sidecar_store.rs new file mode 100644 index 000000000..439451a0f --- /dev/null +++ b/crates/freshell-codex/src/sidecar_store.rs @@ -0,0 +1,449 @@ +//! Durable codex **sidecar record store** — one JSON file per owned +//! `codex app-server` sidecar, so a restarted server can reattach to (or +//! conservatively reap) processes a previous generation spawned (kata +//! ynfn/da92 groundwork; the reconciler lands in later tasks). Identity +//! verification against a record's `/proc` evidence lives below +//! ([`verify_sidecar_identity`]). +//! +//! Production root (wired in Task 10): `/.freshell/rust-codex-sidecars/`. +//! The `rust-` prefix is the anti-collision convention with Node's +//! `~/.freshell/codex-sidecars/` store (precedent: `rust-session-cache.json`) +//! — the two servers must never share a writer on one directory. +//! +//! Layout: `/.json` (ownership ids are +//! `codex-sidecar-`, filesystem-safe as-is — `durability.rs:36`), +//! `/lock` (single-writer flock), corrupt rows renamed aside to +//! `.quarantined-`. +//! +//! Policies, all inherited from `freshell_ws::pane_ledger` (the store this is +//! modelled on): +//! - **Single writer:** [`CodexSidecarStore::new_locked`] holds an exclusive +//! advisory `flock(2)` on `/lock` for the process lifetime; on +//! contention the store comes up DISABLED (every write an `Ok(())` no-op) — +//! never two writers on one store (`pane_ledger.rs:236-274`). +//! - **Atomic, durable writes:** sibling tmp → write → `sync_all` → rename → +//! fsync parent dir (`tabs_persist.rs:682-708`). +//! - **Corruption:** fail loud PER-ROW — quarantine (rename aside + ERROR +//! log), never silently drop, never fail the whole store (`pane_ledger.rs` +//! module header). + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock}; + +/// Schema version stamped into every record. Rows with a different version +/// are quarantined loudly at load, never silently reinterpreted (the +/// `LEDGER_VERSION` policy, `pane_ledger.rs:57`). +pub const SIDECAR_RECORD_VERSION: u32 = 1; + +/// One durable sidecar record — everything a restarted server needs to +/// re-verify (pid + starttime + cmdline), reattach to (ws_url), or attribute +/// (ownership/session/terminal ids) a codex app-server it spawned. +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CodexSidecarRecord { + pub record_version: u32, + /// `"codex-sidecar-"` (`durability.rs:36`) — also the file name stem. + pub ownership_id: String, + pub pid: u32, + /// `/proc//stat` field 22 — the pid-reuse guard: `(pid, starttime)` + /// uniquely identifies a process incarnation. + pub starttime: u64, + /// `/proc//cmdline` argv, NUL-split. + pub cmdline: Vec, + /// The app-server's `--listen` URL. + pub ws_url: String, + /// Codex thread id, enriched when known. + #[serde(skip_serializing_if = "Option::is_none")] + pub session_id: Option, + /// Enriched at adopt. + #[serde(skip_serializing_if = "Option::is_none")] + pub terminal_id: Option, + /// `durability.rs::default_server_instance_id()`. + pub server_instance_id: String, + pub created_at: i64, + pub updated_at: i64, + pub state: SidecarRecordState, +} + +/// Lifecycle state of a recorded sidecar: `Active` (owned by a live server +/// generation) or `Retained { reason }` (deliberately left running across a +/// server death, awaiting reconciliation). +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case", tag = "kind")] +pub enum SidecarRecordState { + Active, + Retained { reason: String }, +} + +/// The durable record store. `root: None` ⇒ DISABLED: every write/remove is +/// an `Ok(())` no-op and `load_all` is empty (the PaneLedger +/// disabled-fallback shape, `pane_ledger.rs:200-212`). +pub struct CodexSidecarStore { + root: Option, + /// Held for the process lifetime by `new_locked` (single-writer guard); + /// the kernel releases the flock on process death. + #[allow(dead_code)] // read only by the kernel (flock lifetime) + lock_file: Option, +} + +impl CodexSidecarStore { + /// Production construction: exclusive advisory `flock(2)` on + /// `/lock`; on contention log a loud structured ERROR and come up + /// DISABLED (`pane_ledger.rs:236-274` pattern). + pub fn new_locked(root: Option) -> Self { + let Some(r) = root else { + return Self::disabled(); + }; + match Self::acquire_store_lock(&r) { + Ok(lock_file) => Self { + root: Some(r), + lock_file, + }, + Err(err) => { + tracing::error!( + target: "freshell_codex::sidecar_store", + root = %r.display(), + error = %err, + "sidecar_store_lock_unavailable: another writer holds /lock; \ + store DISABLED for this process (never two writers on one store)" + ); + Self::disabled() + } + } + } + + #[cfg(unix)] + fn acquire_store_lock(root: &Path) -> std::io::Result> { + use std::os::unix::io::AsRawFd; + std::fs::create_dir_all(root)?; + // Content irrelevant (only existence + flock state matter); + // truncate(false) avoids clippy's suspicious_open_options. + // + // O_CLOEXEC: `std::fs::File` opens close-on-exec by DEFAULT — KEEP it + // that way (no custom_flags stripping it). flock state rides the open + // file description, so a leaked lock fd inherited by a detached, + // retained sidecar (Task 3 removes kill_on_drop) would keep holding + // the flock after the server dies and silently disable the store for + // every future server generation (reports/V6.md NA-3). + let file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(root.join("lock"))?; + // SAFETY: `fd` is a valid open descriptor owned by `file` for the + // duration of the call; flock only mutates kernel lock state. + let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if rc == 0 { + Ok(Some(file)) + } else { + Err(std::io::Error::last_os_error()) + } + } + + #[cfg(not(unix))] + fn acquire_store_lock(root: &Path) -> std::io::Result> { + // No advisory-lock primitive on this platform — construct normally + // (PaneLedger / ConfigLock non-unix parity). + std::fs::create_dir_all(root)?; + Ok(None) + } + + /// Lock-free construction — tests and verification handles over a live + /// server's dir must not fight the server's flock (`pane_ledger.rs:214-229`). + /// Production uses [`CodexSidecarStore::new_locked`]. + pub fn new(root: PathBuf) -> Self { + Self { + root: Some(root), + lock_file: None, + } + } + + /// A store that stores nothing. + pub fn disabled() -> Self { + Self { + root: None, + lock_file: None, + } + } + + /// Whether this store actually stores anything (`root: Some`). + pub fn is_enabled(&self) -> bool { + self.root.is_some() + } + + /// Write (create or replace) one record atomically, fsync'd: sibling tmp + /// (PID+millis unique) + write + `sync_all` + rename + fsync parent dir + /// (the `write_row_atomic` idiom, `pane_ledger.rs:983-1000`). + pub fn write(&self, record: &CodexSidecarRecord) -> std::io::Result<()> { + let Some(root) = self.root.as_ref() else { + return Ok(()); + }; + let bytes = serde_json::to_vec_pretty(record) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + let dest = record_path(root, &record.ownership_id); + let tmp = root.join(format!( + "{}.json.tmp-{}-{}", + record.ownership_id, + std::process::id(), + now_millis() + )); + atomic_write_durable(&dest, &tmp, &bytes) + } + + /// Remove one record; idempotent (a missing row is `Ok(())`). + pub fn remove(&self, ownership_id: &str) -> std::io::Result<()> { + let Some(root) = self.root.as_ref() else { + return Ok(()); + }; + match std::fs::remove_file(record_path(root, ownership_id)) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } + } + + /// Load every healthy record. Corrupt rows are quarantined loudly — + /// renamed aside to `.quarantined-` + ERROR log — never + /// silently dropped, and never fail the healthy rows (the + /// fail-loud-PER-ROW policy, `pane_ledger.rs` module header). + pub fn load_all(&self) -> Vec { + let Some(root) = self.root.as_ref() else { + return Vec::new(); + }; + let Ok(entries) = std::fs::read_dir(root) else { + return Vec::new(); // no root dir yet ⇒ nothing recorded + }; + let mut records = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; // `lock`, *.tmp-* and *.quarantined-* residue + } + let bytes = match std::fs::read(&path) { + Ok(b) => b, + Err(e) => { + tracing::error!( + target: "freshell_codex::sidecar_store", + path = %path.display(), + error = %e, + "sidecar_record_unreadable: row skipped (io error, not corruption)" + ); + continue; + } + }; + match serde_json::from_slice::(&bytes) { + Ok(record) if record.record_version == SIDECAR_RECORD_VERSION => { + records.push(record); + } + Ok(record) => quarantine_row( + &path, + &format!("unsupported recordVersion {}", record.record_version), + ), + Err(e) => quarantine_row(&path, &format!("parse: {e}")), + } + } + records + } +} + +// --------------------------------------------------------------------------- +// Pid identity evidence + verification. A durable record is only ever +// TRUSTED after its `(pid, starttime, cmdline)` evidence is re-verified +// against live `/proc` — only [`IdentityVerdict::Verified`] may ever be +// signalled. Environ tags are deliberately NOT required here: YAMA can hide +// `/proc//environ` for reparented orphans, while `stat` and `cmdline` +// are world-readable. +// --------------------------------------------------------------------------- + +/// /proc//stat field 22; None for gone/zombie. (pid, starttime) is the +/// pid-reuse guard. Deliberate duplicate of the private +/// freshell-freshagent/src/session_lease.rs:144-160 helper (dependency +/// direction forbids importing it) — keep the parsing identical: split at the +/// LAST ')' then index 19, rejecting Z/X states. +#[cfg(target_os = "linux")] +pub fn proc_starttime(pid: i32) -> Option { + let stat = std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + // comm (field 2) may contain spaces/parens: split at the LAST ')' — the + // remainder starts at field 3 (state), so starttime (field 22) is index + // 19 there (session_lease.rs:144-160 parsing, kept identical). + let rest = stat.rsplit(')').next()?; + let fields: Vec<&str> = rest.split_whitespace().collect(); + match fields.first() { + Some(&"Z") | Some(&"X") | None => return None, + Some(_) => {} + } + fields.get(19)?.parse().ok() +} + +/// Non-Linux stub: no `/proc`, no evidence — `None` (never verified ⇒ never +/// killed). +#[cfg(not(target_os = "linux"))] +pub fn proc_starttime(_pid: i32) -> Option { + None +} + +/// /proc//cmdline, NUL-split into argv. World-readable (no ptrace/YAMA +/// constraint, unlike /proc//environ). +#[cfg(target_os = "linux")] +pub fn proc_cmdline(pid: i32) -> Option> { + let bytes = std::fs::read(format!("/proc/{pid}/cmdline")).ok()?; + Some( + bytes + .split(|b| *b == 0) + .filter(|arg| !arg.is_empty()) + .map(|arg| String::from_utf8_lossy(arg).into_owned()) + .collect(), + ) +} + +/// Non-Linux stub: no `/proc`, no evidence — `None` (never verified ⇒ never +/// killed). +#[cfg(not(target_os = "linux"))] +pub fn proc_cmdline(_pid: i32) -> Option> { + None +} + +/// The answer to "is `/proc/` still the process this record describes?" +/// — the gate every reattach/reap decision goes through. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IdentityVerdict { + /// (pid, starttime, cmdline) all match the record — this IS our sidecar. + Verified, + /// pid gone or zombie — the sidecar is dead; the record is stale. + Dead, + /// pid alive but starttime or cmdline differ — pid reuse; NEVER signal. + Mismatch, + /// non-Linux / evidence unreadable — NEVER signal. + Unverifiable, +} + +/// Re-verify a record's `(pid, starttime, cmdline)` evidence against live +/// `/proc`. Read-only — never signals anything. A pid that vanishes between +/// the two reads yields [`IdentityVerdict::Unverifiable`] (conservative: +/// never signalled), not a guess. +#[cfg(target_os = "linux")] +pub fn verify_sidecar_identity(record: &CodexSidecarRecord) -> IdentityVerdict { + // pid > i32::MAX cannot exist on Linux (PID_MAX_LIMIT = 2^22); the `as` + // wrap would produce a negative pid whose /proc entry never exists, so + // the verdict is still the safe `Dead`. + let pid = record.pid as i32; + let Some(starttime) = proc_starttime(pid) else { + return IdentityVerdict::Dead; + }; + if starttime != record.starttime { + return IdentityVerdict::Mismatch; + } + let Some(cmdline) = proc_cmdline(pid) else { + return IdentityVerdict::Unverifiable; + }; + if cmdline != record.cmdline { + return IdentityVerdict::Mismatch; + } + IdentityVerdict::Verified +} + +/// Non-Linux stub: no `/proc` evidence — [`IdentityVerdict::Unverifiable`] +/// (never verified ⇒ never killed). +#[cfg(not(target_os = "linux"))] +pub fn verify_sidecar_identity(_record: &CodexSidecarRecord) -> IdentityVerdict { + IdentityVerdict::Unverifiable +} + +// --------------------------------------------------------------------------- +// Process-global store handle (Task 3 seam; wired at server boot in Task 10). +// A re-settable RwLock rather than the manager's OnceLock: tests inject a +// per-instance store (`SpawnedCodexAppServerRuntime::with_command_and_store`) +// and never touch this global, so no set-once ratchet is needed here. +// --------------------------------------------------------------------------- + +static GLOBAL_SIDECAR_STORE: RwLock>> = RwLock::new(None); + +/// Install the process-wide sidecar store (server boot, before any codex +/// terminal can spawn). Later calls replace the handle. +pub fn set_codex_sidecar_store(store: Arc) { + *GLOBAL_SIDECAR_STORE.write().unwrap() = Some(store); +} + +/// The installed process-wide store, if any. `None` (nothing installed) means +/// callers fall back to [`CodexSidecarStore::disabled`] — behavior identical +/// to the pre-store world. +pub(crate) fn codex_sidecar_store() -> Option> { + GLOBAL_SIDECAR_STORE.read().unwrap().clone() +} + +fn record_path(root: &Path, ownership_id: &str) -> PathBuf { + root.join(format!("{ownership_id}.json")) +} + +fn now_millis() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis()) + .unwrap_or(0) +} + +/// Rename a corrupt row aside to `.quarantined-` and log a +/// loud structured ERROR — the row is preserved for forensics, never +/// silently deleted (`pane_ledger.rs` quarantine policy). +fn quarantine_row(path: &Path, why: &str) { + let file_name = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("record"); + let quarantined = path.with_file_name(format!("{file_name}.quarantined-{}", now_millis())); + match std::fs::rename(path, &quarantined) { + Ok(()) => tracing::error!( + target: "freshell_codex::sidecar_store", + path = %path.display(), + quarantined = %quarantined.display(), + why, + "sidecar_record_quarantined: corrupt row renamed aside (fail loud per-row)" + ), + Err(e) => tracing::error!( + target: "freshell_codex::sidecar_store", + path = %path.display(), + why, + error = %e, + "sidecar_record_quarantine_failed: corrupt row could not be renamed aside" + ), + } +} + +/// Durably replace `destination` with `bytes`: write + `sync_all` the +/// temporary file, rename it atomically, then `sync_all` the parent directory +/// so the new name survives a power/kernel failure. +/// +/// PROVENANCE: a deliberate, verbatim duplicate of +/// `freshell_ws::tabs_persist::atomic_write_durable` (`tabs_persist.rs:682-708`). +/// It cannot be imported: `freshell-ws` depends on `freshell-codex`, so the +/// dependency direction is forbidden — the same reason the repo already +/// duplicates `CODEX_MANAGED_REMOTE_CONFIG_ARGS` (`launch_plan.rs:33` vs +/// `cli_launch.rs:177`). Keep the two bodies in sync. +fn atomic_write_durable(destination: &Path, temporary: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + let parent = destination.parent().ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("destination has no parent: {}", destination.display()), + ) + })?; + if temporary.parent() != Some(parent) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "atomic-write temporary file must be a sibling of the destination", + )); + } + std::fs::create_dir_all(parent)?; + let mut file = std::fs::File::create(temporary)?; + file.write_all(bytes)?; + file.sync_all()?; + drop(file); + std::fs::rename(temporary, destination)?; + #[cfg(unix)] + std::fs::File::open(parent)?.sync_all()?; + Ok(()) +} + +#[cfg(test)] +#[path = "sidecar_store_tests.rs"] +mod tests; diff --git a/crates/freshell-codex/src/sidecar_store_tests.rs b/crates/freshell-codex/src/sidecar_store_tests.rs new file mode 100644 index 000000000..ed6fe7ac2 --- /dev/null +++ b/crates/freshell-codex/src/sidecar_store_tests.rs @@ -0,0 +1,282 @@ +//! Unit tests for the durable codex sidecar record store ([`super`]). +//! +//! Tempfile tempdirs ONLY — no global state, nothing outside each test's own +//! temp dir. In particular these tests must NEVER touch +//! `~/.freshell/codex-sidecars/` (Node's store), the production +//! `~/.freshell/rust-codex-sidecars/` root (wired in Task 10), or any live +//! process the test did not itself spawn. +//! +//! PROCESS SAFETY (identity tests): each identity test spawns and reaps ONLY +//! its own child (`sleep 300`), killed in a [`ChildGuard`] drop guard — +//! nothing else on the machine is ever signalled. + +use super::*; + +fn sample_record(ownership_id: &str) -> CodexSidecarRecord { + CodexSidecarRecord { + record_version: SIDECAR_RECORD_VERSION, + ownership_id: ownership_id.to_string(), + pid: 4242, + starttime: 123_456_789, + cmdline: vec![ + "codex".to_string(), + "-c".to_string(), + "features.apps=false".to_string(), + "app-server".to_string(), + "--listen".to_string(), + "ws://127.0.0.1:7777".to_string(), + ], + ws_url: "ws://127.0.0.1:7777".to_string(), + session_id: Some("019810de-1e5f-7db3-9c47-1c2a3b4c5d6e".to_string()), + terminal_id: None, + server_instance_id: "srv-1".to_string(), + created_at: 1_700_000_000_000, + updated_at: 1_700_000_000_001, + state: SidecarRecordState::Active, + } +} + +fn dir_names(root: &Path) -> Vec { + let mut names: Vec = std::fs::read_dir(root) + .expect("read_dir root") + .map(|e| { + e.expect("dir entry") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + names.sort(); + names +} + +#[test] +fn record_roundtrips_through_disk() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = CodexSidecarStore::new(dir.path().to_path_buf()); + let active = sample_record("codex-sidecar-11111111-1111-4111-8111-111111111111"); + // A second record pins the Retained tagged-enum shape and the + // Option-field round-trip (session_id absent, terminal_id present). + let retained = CodexSidecarRecord { + ownership_id: "codex-sidecar-22222222-2222-4222-8222-222222222222".to_string(), + session_id: None, + terminal_id: Some("term-9".to_string()), + state: SidecarRecordState::Retained { + reason: "server_death_with_live_sidecar".to_string(), + }, + ..sample_record("") + }; + store.write(&active).expect("write active"); + store.write(&retained).expect("write retained"); + + let mut loaded = store.load_all(); + loaded.sort_by(|a, b| a.ownership_id.cmp(&b.ownership_id)); + assert_eq!(loaded, vec![active, retained]); +} + +#[test] +fn write_is_atomic_sibling_tmp_then_rename() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = CodexSidecarStore::new(dir.path().to_path_buf()); + let record = sample_record("codex-sidecar-33333333-3333-4333-8333-333333333333"); + store.write(&record).expect("write"); + + // No *.tmp-* residue after a successful write (sibling tmp was renamed + // into place, tabs_persist.rs:682-708 discipline). + let names = dir_names(dir.path()); + assert!( + names.iter().all(|n| !n.contains(".tmp-")), + "no tmp residue may remain: {names:?}" + ); + + // The destination is `/.json` and parses back. + let dest = dir.path().join(format!("{}.json", record.ownership_id)); + let bytes = std::fs::read(&dest).expect("destination file exists"); + let parsed: CodexSidecarRecord = + serde_json::from_slice(&bytes).expect("destination parses as a record"); + assert_eq!(parsed, record); +} + +#[test] +fn disabled_store_is_a_silent_noop() { + let store = CodexSidecarStore::disabled(); + assert!(!store.is_enabled()); + let record = sample_record("codex-sidecar-44444444-4444-4444-8444-444444444444"); + store.write(&record).expect("disabled write is Ok(())"); + store + .remove(&record.ownership_id) + .expect("disabled remove is Ok(())"); + assert!(store.load_all().is_empty()); +} + +#[test] +fn corrupt_record_is_quarantined_not_fatal() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = CodexSidecarStore::new(dir.path().to_path_buf()); + let healthy = sample_record("codex-sidecar-55555555-5555-4555-8555-555555555555"); + store.write(&healthy).expect("write healthy"); + // Hand-written garbage beside it (fail-loud-per-row policy, + // pane_ledger.rs module header). + let garbage = dir.path().join("codex-sidecar-garbage.json"); + std::fs::write(&garbage, b"{ this is not json").expect("write garbage"); + + let loaded = store.load_all(); + assert_eq!(loaded, vec![healthy], "the healthy row survives"); + + assert!(!garbage.exists(), "the garbage row must be renamed aside"); + let names = dir_names(dir.path()); + assert!( + names + .iter() + .any(|n| n.starts_with("codex-sidecar-garbage.json.quarantined-")), + "quarantine residue must exist: {names:?}" + ); +} + +#[cfg(unix)] // flock is the unix-only single-writer primitive (pane_ledger parity) +#[test] +fn second_locked_open_comes_up_disabled() { + // Single-writer flock (pane_ledger.rs:236-274): never two writers on one + // store. flock state rides the open file description, so a second open + // in the SAME process still contends — no child process needed. + let dir = tempfile::tempdir().expect("tempdir"); + let root = dir.path().to_path_buf(); + let holder = CodexSidecarStore::new_locked(Some(root.clone())); + assert!(holder.is_enabled(), "first locked open owns the store"); + let record = sample_record("codex-sidecar-66666666-6666-4666-8666-666666666666"); + holder.write(&record).expect("holder write"); + + let loser = CodexSidecarStore::new_locked(Some(root.clone())); + assert!(!loser.is_enabled(), "second locked open must be DISABLED"); + let loser_record = sample_record("codex-sidecar-77777777-7777-4777-8777-777777777777"); + loser + .write(&loser_record) + .expect("disabled write is an Ok(()) no-op"); + assert!(loser.load_all().is_empty(), "disabled loser reads nothing"); + assert!( + !root + .join(format!("{}.json", loser_record.ownership_id)) + .exists(), + "the disabled loser's no-op write left no file behind" + ); + drop(holder); +} + +// --------------------------------------------------------------------------- +// Pid identity evidence + verification (Task 2). /proc semantics are +// Linux-only, so these tests are #[cfg(target_os = "linux")]; the non-Linux +// stubs (None / Unverifiable) are covered by the type system, not spawned +// processes. +// --------------------------------------------------------------------------- + +/// Kills and reaps ONLY the guarded child on drop (defer-style guard) — the +/// test's own `sleep 300`, nothing else on the machine. `kill` on an +/// already-reaped `Child` is a no-op error inside std (no signal is sent to +/// a possibly-recycled pid), so double-cleanup is safe. +#[cfg(target_os = "linux")] +struct ChildGuard(std::process::Child); + +#[cfg(target_os = "linux")] +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +#[cfg(target_os = "linux")] +fn spawn_own_sleep_child() -> ChildGuard { + ChildGuard( + std::process::Command::new("sleep") + .arg("300") + .spawn() + .expect("spawn this test's own sleep child"), + ) +} + +/// A record carrying the spawned child's REAL `/proc` evidence. +#[cfg(target_os = "linux")] +fn record_for_child(pid: u32) -> CodexSidecarRecord { + CodexSidecarRecord { + pid, + starttime: proc_starttime(pid as i32).expect("live child has a starttime"), + cmdline: proc_cmdline(pid as i32).expect("live child has a cmdline"), + ..sample_record("codex-sidecar-88888888-8888-4888-8888-888888888888") + } +} + +#[cfg(target_os = "linux")] +#[test] +fn proc_starttime_identifies_a_live_child_and_none_after_exit() { + let mut child = spawn_own_sleep_child(); + let pid = child.0.id() as i32; + assert!( + proc_starttime(pid).is_some(), + "a live child must have a readable starttime" + ); + // Kill + reap OUR OWN child; after the reap the pid is gone from /proc. + child.0.kill().expect("kill own child"); + child.0.wait().expect("reap own child"); + assert_eq!(proc_starttime(pid), None, "a reaped pid must read as gone"); +} + +#[cfg(target_os = "linux")] +#[test] +fn verify_identity_confirms_own_spawned_child() { + let child = spawn_own_sleep_child(); + let record = record_for_child(child.0.id()); + assert_eq!(verify_sidecar_identity(&record), IdentityVerdict::Verified); +} + +#[cfg(target_os = "linux")] +#[test] +fn verify_identity_rejects_cmdline_mismatch_without_signalling() { + let mut child = spawn_own_sleep_child(); + // The live child's pid+starttime but a DIFFERENT cmdline: pid-reuse shape. + let record = CodexSidecarRecord { + cmdline: vec!["codex".to_string(), "app-server".to_string()], + ..record_for_child(child.0.id()) + }; + assert_eq!(verify_sidecar_identity(&record), IdentityVerdict::Mismatch); + // Verification is read-only: the mismatching child must still be alive + // (NEVER signalled) afterwards. + assert_eq!( + child.0.try_wait().expect("try_wait own child"), + None, + "verification must never signal a mismatching pid" + ); + assert!( + proc_starttime(child.0.id() as i32).is_some(), + "the mismatching child is still visible in /proc" + ); +} + +#[cfg(target_os = "linux")] +#[test] +fn verify_identity_reports_dead_for_missing_pid() { + // A reaped child: real evidence captured live, then the pid goes away. + let mut child = spawn_own_sleep_child(); + let record = record_for_child(child.0.id()); + child.0.kill().expect("kill own child"); + child.0.wait().expect("reap own child"); + assert_eq!(verify_sidecar_identity(&record), IdentityVerdict::Dead); + + // And a pid that cannot exist: far beyond /proc/sys/kernel/pid_max + // (kernel ceiling PID_MAX_LIMIT = 4_194_304), verified against the + // machine's actual setting so the "impossible" claim is real evidence. + let pid_max: u64 = std::fs::read_to_string("/proc/sys/kernel/pid_max") + .expect("read pid_max") + .trim() + .parse() + .expect("parse pid_max"); + let impossible = CodexSidecarRecord { + pid: 999_999_999, + ..sample_record("codex-sidecar-99999999-9999-4999-8999-999999999999") + }; + assert!( + u64::from(impossible.pid) > pid_max, + "test pid {} must exceed this machine's pid_max {pid_max}", + impossible.pid + ); + assert_eq!(verify_sidecar_identity(&impossible), IdentityVerdict::Dead); +} diff --git a/crates/freshell-codex/src/sidecar_sweep.rs b/crates/freshell-codex/src/sidecar_sweep.rs new file mode 100644 index 000000000..cea576c04 --- /dev/null +++ b/crates/freshell-codex/src/sidecar_sweep.rs @@ -0,0 +1,769 @@ +//! Conservative **disposal** side of the sidecar lifecycle (katas ynfn/da92): +//! the tree-aware verified kill helper (Task 6) and the boot-time reap sweep +//! over the [`SidecarReconciler`]'s unclaimed survivors (Task 9). +//! +//! Sibling of [`crate::sidecar_reconcile`] (the pre-authorized split: the +//! reconcile module sits at its 1,000-line ceiling, the `runtime_select.rs` +//! precedent from Task 7): the reconciler owns boot/claim; this module owns +//! everything that may ever SIGNAL a recorded sidecar — and therefore also +//! the never-signal refusals. +//! +//! PROCESS SAFETY (binding, plan-wide): nothing here ever kills by +//! process-name pattern. Only pids recorded AND re-verified via +//! `(pid, starttime, cmdline)` immediately before each signal are ever +//! signalled; `Mismatch`/`Dead`/`Unverifiable` are NEVER signalled. + +use std::sync::Arc; +use std::time::Duration; + +use crate::app_server::CodexAppServerClient; +use crate::events::{normalize_codex_thread_status, CodexStatus}; +use crate::sidecar_reconcile::{ + remove_pruned, unix_millis, write_record_loudly, SidecarReconciler, +}; +#[cfg(target_os = "linux")] +use crate::sidecar_store::{proc_cmdline, proc_starttime}; +use crate::sidecar_store::{ + verify_sidecar_identity, CodexSidecarRecord, IdentityVerdict, SidecarRecordState, +}; +use crate::transport::TungsteniteTransport; + +/// Env override for the boot-time reap grace window (milliseconds): how long +/// a freshly booted server waits before sweeping unclaimed survivors, so +/// restores get to claim them first. Consumed by the boot wiring (Task 10); +/// [`SidecarReconciler::sweep_unclaimed`] itself takes no age into account — +/// the grace decides WHEN the sweep runs, not what it sees. +pub const FRESHELL_CODEX_SIDECAR_REAP_GRACE_MS_ENV: &str = "FRESHELL_CODEX_SIDECAR_REAP_GRACE_MS"; + +/// Default reap grace: 30 minutes. +pub const CODEX_SIDECAR_REAP_GRACE_MS_DEFAULT: u64 = 30 * 60 * 1000; // incident gap was 18 min + +/// The boot wiring's grace read (Task 10): +/// [`FRESHELL_CODEX_SIDECAR_REAP_GRACE_MS_ENV`] parsed as u64 millis; unset +/// or non-numeric falls back to [`CODEX_SIDECAR_REAP_GRACE_MS_DEFAULT`]. +/// `0` IS honored — an operator/test asking for an immediate sweep (unlike +/// `FRESHELL_CODEX_PLAN_QUEUE_CAP`, where 0 is meaningless). +pub fn reap_grace_from_env() -> Duration { + reap_grace_from_value( + std::env::var(FRESHELL_CODEX_SIDECAR_REAP_GRACE_MS_ENV) + .ok() + .as_deref(), + ) +} + +/// Pure parse half of [`reap_grace_from_env`] — unit-testable without +/// process-global env mutation (parallel test runs share the env). +fn reap_grace_from_value(value: Option<&str>) -> Duration { + value + .and_then(|v| v.parse::().ok()) + .map(Duration::from_millis) + .unwrap_or(Duration::from_millis(CODEX_SIDECAR_REAP_GRACE_MS_DEFAULT)) +} + +/// Whole-probe budget per sweep candidate (connect → initialize → +/// thread/loaded/list → thread/read per loaded thread). Bounded so a wedged +/// survivor cannot stall the sweep; on timeout the candidate is treated as +/// ws-unreachable and falls to the conservative writer-evidence check (a +/// wedged mid-turn writer still holds its rollout handle ⇒ retained). +const SWEEP_PROBE_BUDGET: Duration = Duration::from_secs(10); + +/// Poll-gone drain budget per signalled pid: 5s, not 500ms — codex's SIGTERM +/// handler is a graceful drain (reports/V2.md), so give it time to exit +/// before escalating to SIGKILL. +#[cfg(target_os = "linux")] +const KILL_DRAIN_BUDGET: Duration = Duration::from_secs(5); + +/// Poll interval while waiting for a signalled pid to go away. +#[cfg(target_os = "linux")] +const KILL_POLL_INTERVAL: Duration = Duration::from_millis(50); + +/// What the sweep did with one still-held record. +#[derive(Debug, PartialEq)] +pub enum SweepOutcome { + /// Verified, reap-eligible, and killed tree-wide; record removed. + Reaped, + /// A thread/read reported status ACTIVE — retained, never signalled; + /// state recorded as `Retained{reason:"mid-turn-active-thread"}`. + RetainedMidTurn, + /// ws unreachable but writer evidence held (`/proc//fd`) — retained, + /// never signalled; `Retained{reason:"ws-unreachable-writer-held"}`. + RetainedWriterHeld, + /// Dead/Mismatch verdict at sweep time — record removed, NEVER signalled. + RecordRemovedStale, + /// Identity unverifiable — retained, never signalled; + /// `Retained{reason:"identity-unverifiable"}`. + RetainedUnverifiable, + /// The record left `held` (claimed by a restore) during the probe + /// window — skipped, NO signal sent. + SkippedClaimedDuringSweep, +} + +/// The decide-phase verdict the commit arm executes. Fieldless (Copy) so the +/// TOCTOU test can drive [`SidecarReconciler::commit_sweep_decision`] +/// directly with a pre-claim snapshot. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum SweepDecision { + /// Dead/Mismatch — remove the record, NEVER signal. + RemoveStale, + /// Retain with reason `"mid-turn-active-thread"`. + RetainMidTurn, + /// Retain with reason `"ws-unreachable-writer-held"`. + RetainWriterHeld, + /// Retain with reason `"identity-unverifiable"`. + RetainUnverifiable, + /// Verified + reap-eligible — kill the tree, remove the record. + Kill, +} + +impl SweepDecision { + /// The durable `Retained{reason}` string for retain decisions. + fn retain_reason(self) -> Option<&'static str> { + match self { + SweepDecision::RetainMidTurn => Some("mid-turn-active-thread"), + SweepDecision::RetainWriterHeld => Some("ws-unreachable-writer-held"), + SweepDecision::RetainUnverifiable => Some("identity-unverifiable"), + SweepDecision::RemoveStale | SweepDecision::Kill => None, + } + } +} + +impl SidecarReconciler { + /// For every still-held, unclaimed record: re-verify identity, then + /// Dead / Mismatch -> remove record, NEVER signal (RecordRemovedStale) + /// Unverifiable -> retain, state = Retained{reason:"identity-unverifiable"} + /// Verified + ws probe (initialize -> thread/loaded/list -> thread/read + /// per loaded thread; `loaded` alone does NOT mean mid-turn — idle + /// threads stay loaded forever, reports/V1.md): + /// any thread/read status ACTIVE -> retain, Retained{reason:"mid-turn-active-thread"} + /// reachable, no active thread -> kill_verified_sidecar_tree, remove record (Reaped) + /// ws UNREACHABLE -> /proc//fd writer-evidence check + /// (open rollout .jsonl write handle or + /// thread-writer-locks/ file — readable + /// same-uid on this host, reports/V2.md): + /// evidence held -> retain, Retained{reason:"ws-unreachable-writer-held"} + /// no evidence -> kill_verified_sidecar_tree, remove record (Reaped) + /// TOCTOU guard (binding): the probe phase runs on a SNAPSHOT of `held` + /// (no locks across awaits), and a late claim_for_session may adopt a + /// snapshotted record while a probe awaits. Per-pid identity + /// re-verification CANNOT detect that (same live process), so it never + /// authorizes a kill alone. Structure the sweep decide → commit: for + /// each kill decision, re-acquire the `held` lock, confirm the record + /// is STILL held (unclaimed), REMOVE it from `held`/`by_session` under + /// that lock, release, and only then `kill_verified_sidecar_tree(...) + /// .await` + `store.remove`. If the record already left `held` + /// (claimed mid-sweep), skip with outcome SkippedClaimedDuringSweep and + /// send NO signal — a restore just reattached to that sidecar; killing + /// it is the exact da92 harm. Membership in `held` is the single source + /// of truth for claim-vs-sweep ownership (Task 5's claim removes + /// winners under the same lock). + /// Sweep CONSUMES only Reaped/RecordRemovedStale entries from `held`; + /// every Retained row STAYS held and claimable (a late restore must still + /// reattach a mid-turn survivor — reports/V3.md), and is re-evaluated at + /// next boot. Every decision logged with ownership id + verdict + outcome. + pub async fn sweep_unclaimed(&self) -> Vec<(String, SweepOutcome)> { + // Probe-phase SNAPSHOT: no `held` lock is held across any await. + // Sorted for deterministic result/log ordering. + let snapshot: Vec = { + let held = self.held.lock().unwrap(); + let mut records: Vec = held.values().cloned().collect(); + records.sort_by(|a, b| a.ownership_id.cmp(&b.ownership_id)); + records + }; + + let mut results = Vec::with_capacity(snapshot.len()); + for record in snapshot { + let verdict = verify_sidecar_identity(&record); + let decision = decide_sweep_action(&record, &verdict).await; + let outcome = self.commit_sweep_decision(&record, decision).await; + tracing::info!( + target: "freshell_codex::sidecar_sweep", + ownership_id = %record.ownership_id, + pid = record.pid, + verdict = ?verdict, + outcome = ?outcome, + "sidecar_sweep_decision: unclaimed survivor swept to an explicit fate" + ); + results.push((record.ownership_id, outcome)); + } + results + } + + /// The commit arm of one sweep decision (the decide → commit structure + /// from [`SidecarReconciler::sweep_unclaimed`]'s TOCTOU guard). + /// pub(crate) so the TOCTOU test can drive it directly with a pre-claim + /// snapshot — deterministic, no timing dependence. + pub(crate) async fn commit_sweep_decision( + &self, + record: &CodexSidecarRecord, + decision: SweepDecision, + ) -> SweepOutcome { + match decision { + SweepDecision::RemoveStale => { + if !self.take_if_still_held(record) { + return SweepOutcome::SkippedClaimedDuringSweep; + } + // Dead/Mismatch: remove the row, NEVER signal (the pid is + // either gone or NOT ours). + remove_pruned(&self.store, &record.ownership_id); + SweepOutcome::RecordRemovedStale + } + SweepDecision::Kill => { + if !self.take_if_still_held(record) { + return SweepOutcome::SkippedClaimedDuringSweep; + } + // Removed from `held` under its lock — the claim can no + // longer adopt this record. Kill AFTER the lock is released; + // the helper re-verifies (pid, starttime, cmdline) + // immediately before every signal. + let outcome = kill_verified_sidecar_tree(record).await; + tracing::info!( + target: "freshell_codex::sidecar_sweep", + ownership_id = %record.ownership_id, + pid = record.pid, + outcomes = ?outcome.outcomes, + "sidecar_sweep_reaped: unclaimed sidecar tree reaped" + ); + remove_pruned(&self.store, &record.ownership_id); + SweepOutcome::Reaped + } + SweepDecision::RetainMidTurn + | SweepDecision::RetainWriterHeld + | SweepDecision::RetainUnverifiable => { + let reason = decision + .retain_reason() + .expect("retain decisions carry a reason"); + // Retained rows STAY held (and claimable): update the held + // record in place — under the lock, no await — then persist + // the recorded reason. + let snapshot = { + let mut held = self.held.lock().unwrap(); + let Some(held_record) = held.get_mut(&record.ownership_id) else { + return SweepOutcome::SkippedClaimedDuringSweep; + }; + held_record.state = SidecarRecordState::Retained { + reason: reason.to_string(), + }; + held_record.updated_at = unix_millis(); + held_record.clone() + }; + write_record_loudly(&self.store, &snapshot); + match decision { + SweepDecision::RetainMidTurn => SweepOutcome::RetainedMidTurn, + SweepDecision::RetainWriterHeld => SweepOutcome::RetainedWriterHeld, + _ => SweepOutcome::RetainedUnverifiable, + } + } + } + } + + /// The TOCTOU commit gate: re-acquire `held`, confirm the record is + /// STILL held (unclaimed), and remove it from `held`/`by_session` under + /// that lock. `false` ⇒ a claim consumed the record during the probe + /// await — the caller must send NO signal (a restore just reattached to + /// that sidecar). Lock order (held, then by_session) matches every + /// claim-path acquisition. + fn take_if_still_held(&self, record: &CodexSidecarRecord) -> bool { + let mut held = self.held.lock().unwrap(); + if held.remove(&record.ownership_id).is_none() { + tracing::info!( + target: "freshell_codex::sidecar_sweep", + ownership_id = %record.ownership_id, + pid = record.pid, + "sidecar_sweep_skipped_claimed: record left `held` during the \ + probe window (claimed by a restore); NO signal sent" + ); + return false; + } + let mut by_session = self.by_session.lock().unwrap(); + if let Some(session_id) = &record.session_id { + if let Some(ids) = by_session.get_mut(session_id) { + ids.retain(|id| id != &record.ownership_id); + if ids.is_empty() { + by_session.remove(session_id); + } + } + } + true + } +} + +/// The decide phase for one record (no locks held; may await the ws probe). +/// Pure decision — the commit arm re-checks `held` membership before acting. +/// pub(crate) so the Unverifiable mapping (unsynthesizable end-to-end for a +/// test-owned child) is directly testable. +pub(crate) async fn decide_sweep_action( + record: &CodexSidecarRecord, + verdict: &IdentityVerdict, +) -> SweepDecision { + match verdict { + // Dead or pid-reuse: the row is stale; the pid is never signalled. + IdentityVerdict::Dead | IdentityVerdict::Mismatch => SweepDecision::RemoveStale, + // Not provably ours ⇒ never signalled; not provably stale ⇒ kept on + // the books with a recorded reason. + IdentityVerdict::Unverifiable => SweepDecision::RetainUnverifiable, + IdentityVerdict::Verified => match probe_mid_turn(&record.ws_url).await { + MidTurnProbe::ActiveThread => SweepDecision::RetainMidTurn, + MidTurnProbe::ReachableIdle => SweepDecision::Kill, + MidTurnProbe::Unreachable => { + if writer_evidence_held(record.pid as i32) { + SweepDecision::RetainWriterHeld + } else { + SweepDecision::Kill + } + } + }, + } +} + +/// What the bounded mid-turn ws probe concluded about one verified survivor. +enum MidTurnProbe { + /// Some loaded thread's `thread/read` status is `active` — mid-turn. + ActiveThread, + /// Reachable and every loaded thread reads idle (`loaded` alone does NOT + /// mean mid-turn — idle threads stay loaded forever, reports/V1.md). + ReachableIdle, + /// Connect/handshake/read failure or budget exhausted — fall to the + /// writer-evidence check. + Unreachable, +} + +/// The mid-turn probe over the crate's own client ([`CodexAppServerClient`] +/// on [`TungsteniteTransport`]): connect → `initialize`/`initialized` → +/// `thread/loaded/list` → `thread/read` per loaded thread, discriminating on +/// status `active`. Whole-candidate budget: [`SWEEP_PROBE_BUDGET`]; any +/// failure ⇒ [`MidTurnProbe::Unreachable`] (conservative — a wedged mid-turn +/// writer still holds its rollout handle, so the fd check retains it). +async fn probe_mid_turn(ws_url: &str) -> MidTurnProbe { + let inner = async { + let Ok(transport) = TungsteniteTransport::connect(ws_url).await else { + return MidTurnProbe::Unreachable; + }; + // Keep the notification receiver alive for the probe's lifetime; the + // client Drop aborts the background consumer. + let (client, _notifications) = CodexAppServerClient::connect(Arc::new(transport)); + let result = probe_loaded_threads(&client).await; + client.close().await; + result + }; + tokio::time::timeout(SWEEP_PROBE_BUDGET, inner) + .await + .unwrap_or(MidTurnProbe::Unreachable) +} + +/// `initialize` → `thread/loaded/list` → `thread/read` per loaded thread. +async fn probe_loaded_threads(client: &CodexAppServerClient) -> MidTurnProbe { + if client.initialize().await.is_err() { + return MidTurnProbe::Unreachable; + } + let Ok(loaded) = client.list_loaded_threads().await else { + return MidTurnProbe::Unreachable; + }; + for thread_id in loaded { + let Ok(result) = client.read_thread(&thread_id, false).await else { + // Reachable but the read failed: cannot prove idle — treat as + // unreachable and let the writer-evidence check decide. + return MidTurnProbe::Unreachable; + }; + let status = result + .get("thread") + .and_then(|thread| thread.get("status")) + .cloned() + .unwrap_or(serde_json::Value::Null); + if normalize_codex_thread_status(&status) == CodexStatus::Running { + return MidTurnProbe::ActiveThread; + } + } + MidTurnProbe::ReachableIdle +} + +/// Writer-evidence check for a ws-unreachable Verified survivor: does +/// `/proc//fd` hold an open rollout `.jsonl` WRITE handle or any +/// `thread-writer-locks/` file handle? Same-uid readable on this host +/// (reports/V2.md). Unreadable fd table / fdinfo ⇒ `true` (evidence-held — +/// conservative: retain, never kill on missing evidence). +#[cfg(target_os = "linux")] +pub(crate) fn writer_evidence_held(pid: i32) -> bool { + let Ok(entries) = std::fs::read_dir(format!("/proc/{pid}/fd")) else { + return true; // fd table unreadable ⇒ treat as evidence-held + }; + for entry in entries.flatten() { + let Ok(target) = std::fs::read_link(entry.path()) else { + continue; // this fd vanished mid-scan — not evidence + }; + let target = target.to_string_lossy(); + if target.contains("thread-writer-locks/") { + return true; + } + let file_name = target.rsplit('/').next().unwrap_or(&target); + if file_name.starts_with("rollout-") + && file_name.ends_with(".jsonl") + && fd_opened_for_write(pid, &entry.file_name().to_string_lossy()) + { + return true; + } + } + false +} + +/// Non-Linux: no `/proc` — evidence can never be ruled out (conservative). +/// Structurally unreachable today (non-Linux identity is never Verified). +#[cfg(not(target_os = "linux"))] +pub(crate) fn writer_evidence_held(_pid: i32) -> bool { + true +} + +/// Is `/proc//fdinfo/`'s `flags:` octal opened for write +/// (O_WRONLY/O_RDWR)? Unreadable/unparsable ⇒ `true` (conservative). +#[cfg(target_os = "linux")] +fn fd_opened_for_write(pid: i32, fd: &str) -> bool { + let Ok(info) = std::fs::read_to_string(format!("/proc/{pid}/fdinfo/{fd}")) else { + return true; + }; + for line in info.lines() { + if let Some(rest) = line.strip_prefix("flags:") { + let Ok(flags) = u32::from_str_radix(rest.trim(), 8) else { + return true; + }; + return flags & (libc::O_ACCMODE as u32) != libc::O_RDONLY as u32; + } + } + true +} + +// --------------------------------------------------------------------------- +// The shared tree-aware kill helper (Task 6; reused by Task 9's sweep). +// A3 was FALSIFIED (reports/V2.md): sidecars are process TREES (children +// like `codex-code-mode-host` live in their OWN pgids/sessions, so neither +// single-pid signalling nor a pgid group-kill covers them), codex's SIGTERM +// handler is a graceful drain, and SIGKILL provably orphans its children +// (no PDEATHSIG; cleanup is userspace-only). "Reaped" must mean the whole +// tree is gone. +// --------------------------------------------------------------------------- + +/// What happened to one pid during [`kill_verified_sidecar_tree`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KillOutcome { + /// Gone (exited, reaped, or zombie) within the drain budget after SIGTERM. + ExitedAfterSigterm, + /// Survived the SIGTERM drain budget; SIGKILL was sent once. + SigkilledAfterBudget, + /// Already gone before any signal was needed. + AlreadyDead, + /// Live pid whose evidence no longer matches its snapshot (pid reuse) — + /// NEVER signalled. + SkippedIdentityMismatch, + /// Evidence unreadable / non-Linux — NEVER signalled. + SkippedUnverifiable, + /// SIGTERM WAS sent to the verified pid, the drain budget expired, and + /// the pre-SIGKILL re-verify no longer matched (Mismatch/Unverifiable — + /// e.g. a mid-drain re-exec rewrote the argv on the same incarnation). + /// The escalation was REFUSED: no SIGKILL was sent. Distinct from the + /// pre-signal `Skipped*` outcomes, which would under-report the SIGTERM + /// that was actually delivered (Task 6 review carry-forward). + SigtermSentEscalationRefused, +} + +/// Per-pid outcomes of one [`kill_verified_sidecar_tree`] call: the root +/// first, then each captured descendant in capture order. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct KillTreeOutcome { + pub outcomes: Vec<(u32, KillOutcome)>, +} + +/// Re-verify (pid, starttime, cmdline); capture the pid's live descendant +/// set from `/proc` (children recursively via `/proc//task/*/children`, +/// each snapshotted with its own (pid, starttime, cmdline) so nothing is +/// ever signalled on a stale pid); SIGTERM the root; poll-gone with a +/// drain-tolerant budget (`KILL_DRAIN_BUDGET` = 5s, not 500ms — codex +/// drains gracefully); SIGKILL the root once if needed; then SIGTERM → +/// poll → SIGKILL each captured descendant that survived, re-verified by its +/// snapshot immediately before each signal. Returns what happened per pid. +/// Never signals anything whose snapshot no longer matches. +/// +/// ASYNC (binding): every call site is async on the tokio runtime — +/// [`ReattachedCodexAppServerRuntime`]'s `ensure_ready` (inside the +/// user-facing restore path, holding one of the manager's two plan permits), +/// `shutdown`, and Task 9's `sweep_unclaimed` — and the poll-gone budgets +/// wait for multi-second intervals. All waits are `tokio::time::sleep` +/// awaits, never `std::thread::sleep`: a sync fn here would block executor +/// workers for the whole SIGTERM→poll→SIGKILL sequence (the same +/// sync/async impedance class the claim path already fixed). Callers must +/// not hold any `held`/store lock across this await. +/// +/// [`ReattachedCodexAppServerRuntime`]: crate::sidecar_reconcile::ReattachedCodexAppServerRuntime +pub async fn kill_verified_sidecar_tree(record: &CodexSidecarRecord) -> KillTreeOutcome { + let verdict = verify_sidecar_identity(record); + match verdict { + IdentityVerdict::Dead => KillTreeOutcome { + outcomes: vec![(record.pid, KillOutcome::AlreadyDead)], + }, + IdentityVerdict::Mismatch | IdentityVerdict::Unverifiable => { + tracing::warn!( + target: "freshell_codex::sidecar_sweep", + ownership_id = %record.ownership_id, + pid = record.pid, + verdict = ?verdict, + "sidecar_tree_kill_skipped: identity not provably ours; NEVER signalled" + ); + let outcome = if verdict == IdentityVerdict::Mismatch { + KillOutcome::SkippedIdentityMismatch + } else { + KillOutcome::SkippedUnverifiable + }; + KillTreeOutcome { + outcomes: vec![(record.pid, outcome)], + } + } + IdentityVerdict::Verified => { + #[cfg(target_os = "linux")] + { + kill_verified_tree_linux(record).await + } + #[cfg(not(target_os = "linux"))] + { + // Structurally unreachable: non-Linux identity is always + // Unverifiable (never Verified). Kept as the conservative + // never-signal posture should that ever change. + KillTreeOutcome { + outcomes: vec![(record.pid, KillOutcome::SkippedUnverifiable)], + } + } + } + } +} + +/// One captured descendant's OWN identity evidence, snapshotted at capture +/// time — the only thing a descendant is ever verified (and signalled) +/// against. +#[cfg(target_os = "linux")] +struct PidSnapshot { + pid: i32, + starttime: u64, + cmdline: Vec, +} + +/// Does `/proc/` still hold the process this snapshot captured? +#[cfg(target_os = "linux")] +enum SnapshotVerdict { + Matches, + Gone, + /// Live pid, different incarnation or changed cmdline — NEVER signal. + Mismatch, + /// Live pid, unreadable cmdline — not provably the captured process; + /// NEVER signal (kept distinct from Mismatch so outcomes stay truthful, + /// Task 6 review carry-forward). + Unverifiable, +} + +#[cfg(target_os = "linux")] +fn verify_snapshot(snapshot: &PidSnapshot) -> SnapshotVerdict { + let Some(starttime) = proc_starttime(snapshot.pid) else { + return SnapshotVerdict::Gone; + }; + if starttime != snapshot.starttime { + return SnapshotVerdict::Mismatch; + } + match proc_cmdline(snapshot.pid) { + Some(cmdline) if cmdline == snapshot.cmdline => SnapshotVerdict::Matches, + Some(_) => SnapshotVerdict::Mismatch, + None => SnapshotVerdict::Unverifiable, + } +} + +/// The Verified arm of [`kill_verified_sidecar_tree`]: capture, then the +/// root and per-descendant SIGTERM→poll→SIGKILL sequences, each signal +/// preceded by its own fresh verification. +#[cfg(target_os = "linux")] +async fn kill_verified_tree_linux(record: &CodexSidecarRecord) -> KillTreeOutcome { + let root_pid = record.pid as i32; + // Capture the descendants BEFORE the root is signalled: codex's SIGTERM + // drain tears down (some of) its children in userspace and a SIGKILLed + // root orphans them — either way the parent links that let /proc find + // them are gone once the root dies (reports/V2.md). + let descendants = capture_descendants(root_pid); + let mut outcomes = Vec::with_capacity(1 + descendants.len()); + + // The caller's verify dispatched here, but re-verify immediately before + // the signal — nothing is ever signalled on a stale pid. + let root_outcome = match verify_sidecar_identity(record) { + IdentityVerdict::Verified => { + signal_pid(root_pid, libc::SIGTERM); + if poll_incarnation_gone(root_pid, record.starttime, KILL_DRAIN_BUDGET).await { + KillOutcome::ExitedAfterSigterm + } else { + // Re-verify immediately before the escalation too. + match verify_sidecar_identity(record) { + IdentityVerdict::Verified => { + signal_pid(root_pid, libc::SIGKILL); + KillOutcome::SigkilledAfterBudget + } + IdentityVerdict::Dead => KillOutcome::ExitedAfterSigterm, + verdict @ (IdentityVerdict::Mismatch | IdentityVerdict::Unverifiable) => { + tracing::warn!( + target: "freshell_codex::sidecar_sweep", + ownership_id = %record.ownership_id, + pid = record.pid, + verdict = ?verdict, + "sidecar_tree_kill_escalation_refused: identity decayed \ + after SIGTERM; SIGKILL NOT sent" + ); + KillOutcome::SigtermSentEscalationRefused + } + } + } + } + IdentityVerdict::Dead => KillOutcome::AlreadyDead, + IdentityVerdict::Mismatch => KillOutcome::SkippedIdentityMismatch, + IdentityVerdict::Unverifiable => KillOutcome::SkippedUnverifiable, + }; + outcomes.push((record.pid, root_outcome)); + + // Final-review H3b: a PRE-signal root refusal (Mismatch/Unverifiable) + // throws the whole capture's provenance into doubt — the descendants were + // snapshotted as children of a root we can no longer prove is ours, so + // NONE of them are processed (nothing is signalled). Post-SIGTERM + // outcomes (incl. SigtermSentEscalationRefused) keep the per-descendant + // sweep: the tree was captured while the root still verified as ours. + if matches!( + root_outcome, + KillOutcome::SkippedIdentityMismatch | KillOutcome::SkippedUnverifiable + ) { + if !descendants.is_empty() { + tracing::warn!( + target: "freshell_codex::sidecar_sweep", + ownership_id = %record.ownership_id, + pid = record.pid, + skipped_descendants = descendants.len(), + "sidecar_tree_kill_descendants_skipped: root identity refused \ + pre-signal; captured descendants NOT processed" + ); + } + return KillTreeOutcome { outcomes }; + } + + for snapshot in &descendants { + let outcome = kill_captured_descendant(snapshot).await; + outcomes.push((snapshot.pid as u32, outcome)); + } + + let result = KillTreeOutcome { outcomes }; + tracing::info!( + target: "freshell_codex::sidecar_sweep", + ownership_id = %record.ownership_id, + outcomes = ?result.outcomes, + "sidecar_tree_killed: verified sidecar tree torn down" + ); + result +} + +/// SIGTERM → poll-gone → SIGKILL one captured descendant, re-verified by +/// its OWN snapshot immediately before EACH signal. +#[cfg(target_os = "linux")] +async fn kill_captured_descendant(snapshot: &PidSnapshot) -> KillOutcome { + match verify_snapshot(snapshot) { + SnapshotVerdict::Gone => return KillOutcome::AlreadyDead, + SnapshotVerdict::Mismatch => return KillOutcome::SkippedIdentityMismatch, + SnapshotVerdict::Unverifiable => return KillOutcome::SkippedUnverifiable, + SnapshotVerdict::Matches => {} + } + signal_pid(snapshot.pid, libc::SIGTERM); + if poll_incarnation_gone(snapshot.pid, snapshot.starttime, KILL_DRAIN_BUDGET).await { + return KillOutcome::ExitedAfterSigterm; + } + match verify_snapshot(snapshot) { + SnapshotVerdict::Gone => KillOutcome::ExitedAfterSigterm, + // SIGTERM WAS sent; the escalation is refused on decayed identity — + // reported truthfully, never as a pre-signal "Skipped*". + SnapshotVerdict::Mismatch | SnapshotVerdict::Unverifiable => { + KillOutcome::SigtermSentEscalationRefused + } + SnapshotVerdict::Matches => { + signal_pid(snapshot.pid, libc::SIGKILL); + KillOutcome::SigkilledAfterBudget + } + } +} + +/// Walk `/proc//task/*/children` recursively (a visited set guards +/// against reparenting races) and snapshot each live descendant's own +/// evidence. Descendants that are gone — or whose evidence is unreadable — +/// at capture time are NOT captured: no snapshot ⇒ never signalled. +#[cfg(target_os = "linux")] +fn capture_descendants(root_pid: i32) -> Vec { + let mut snapshots = Vec::new(); + let mut visited = std::collections::HashSet::new(); + visited.insert(root_pid); + let mut frontier = vec![root_pid]; + while let Some(pid) = frontier.pop() { + for child in proc_children(pid) { + if !visited.insert(child) { + continue; + } + frontier.push(child); + let Some(starttime) = proc_starttime(child) else { + continue; // gone/zombie — nothing to signal + }; + let Some(cmdline) = proc_cmdline(child) else { + continue; // unreadable — never signalled + }; + snapshots.push(PidSnapshot { + pid: child, + starttime, + cmdline, + }); + } + } + snapshots +} + +/// One pid's direct children, from every thread's +/// `/proc//task//children` row (space-separated child pids). +#[cfg(target_os = "linux")] +fn proc_children(pid: i32) -> Vec { + let mut children = Vec::new(); + let Ok(tasks) = std::fs::read_dir(format!("/proc/{pid}/task")) else { + return children; + }; + for task in tasks.flatten() { + let tid = task.file_name(); + let Some(tid) = tid.to_str() else { continue }; + let Ok(row) = std::fs::read_to_string(format!("/proc/{pid}/task/{tid}/children")) else { + continue; + }; + children.extend(row.split_whitespace().filter_map(|p| p.parse::().ok())); + } + children +} + +/// Send one signal to one just-verified pid. +#[cfg(target_os = "linux")] +fn signal_pid(pid: i32, signal: libc::c_int) { + // SAFETY: kill(2) only dispatches a signal — no memory is touched. The + // caller verified the pid's identity evidence immediately before this + // call (transport.rs:110 precedent). + unsafe { + libc::kill(pid, signal); + } +} + +/// Poll until `(pid, starttime)` no longer names a live incarnation +/// (exited, reaped, zombie, or pid reused) or the budget expires. All waits +/// are `tokio::time::sleep` — never `std::thread::sleep` (the +/// [`kill_verified_sidecar_tree`] ASYNC contract). +#[cfg(target_os = "linux")] +async fn poll_incarnation_gone(pid: i32, starttime: u64, budget: Duration) -> bool { + let deadline = tokio::time::Instant::now() + budget; + loop { + if proc_starttime(pid) != Some(starttime) { + return true; + } + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(KILL_POLL_INTERVAL).await; + } +} + +#[cfg(test)] +#[path = "sidecar_sweep_tests.rs"] +mod tests; diff --git a/crates/freshell-codex/src/sidecar_sweep_tests.rs b/crates/freshell-codex/src/sidecar_sweep_tests.rs new file mode 100644 index 000000000..ffd8caa2c --- /dev/null +++ b/crates/freshell-codex/src/sidecar_sweep_tests.rs @@ -0,0 +1,826 @@ +//! Unit tests for the conservative reap sweep + the tree-aware kill helper +//! ([`super`]) — Task 9's never-silently-orphaned invariant (ynfn). +//! +//! Tempfile tempdirs ONLY — no global state, nothing outside each test's own +//! temp dir. These tests must NEVER touch `~/.freshell/codex-sidecars/` +//! (Node's store), the production `~/.freshell/rust-codex-sidecars/` root +//! (wired in Task 10), or any live process the test did not itself spawn. +//! +//! PROCESS SAFETY: each test spawns and signals ONLY its own children +//! (`sleep 300` trees / fake app-server fixtures), cleaned up by drop guards +//! (`ChildGuard` / `kill_on_drop(true)` / snapshot-verified orphan guards) — +//! nothing else on the machine is ever signalled. The machine's live orphaned +//! codex app-servers stay structurally unreachable: every record here names a +//! test-spawned pid, and the sweep only ever signals recorded AND re-verified +//! pids. Loopback ephemeral ports only; never 3001/3002. +//! +//! /proc semantics are Linux-only, so these tests are +//! `#[cfg(target_os = "linux")]` (the sidecar_reconcile_tests precedent). + +#![cfg(target_os = "linux")] + +use std::sync::Arc; + +use super::*; +use crate::launch_lifecycle::CodexLaunchRuntime; +use crate::sidecar_reconcile::{ReattachedCodexAppServerRuntime, SidecarReconciler}; +use crate::sidecar_store::SidecarRecordState; +use crate::sidecar_test_support::{ + record_for_child, spawn_own_fake_app_server_with_behavior, spawn_own_shell_child, + spawn_own_sleep_child, store_in, NEVER_SIGNALLED_GRACE, SESSION, +}; + +/// Poll this test's OWN spawned-tree root until `want` direct children exist +/// whose cmdline reads `sleep 300` (post-exec — a pre-exec fork window would +/// snapshot the parent's argv and flake the kill as a Mismatch skip). +fn wait_for_sleep_children(root: i32, want: usize) -> Vec<(i32, u64)> { + let want_cmdline = vec!["sleep".to_string(), "300".to_string()]; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let settled: Vec<(i32, u64)> = proc_children(root) + .into_iter() + .filter(|&pid| proc_cmdline(pid).as_ref() == Some(&want_cmdline)) + .filter_map(|pid| proc_starttime(pid).map(|starttime| (pid, starttime))) + .collect(); + if settled.len() >= want { + return settled; + } + assert!( + std::time::Instant::now() < deadline, + "the test tree's sleep children never appeared" + ); + std::thread::sleep(std::time::Duration::from_millis(5)); + } +} + +/// Cleanup-on-failure guard for indirectly spawned pids (a bash tree's +/// sleeps): on drop, SIGKILL any pid whose `(pid, starttime)` incarnation +/// still matches the snapshot — a recycled pid is never signalled. The happy +/// path leaves every snapshot dead, making drop a no-op. +struct OrphanSnapshotGuard(Vec<(i32, u64)>); + +impl Drop for OrphanSnapshotGuard { + fn drop(&mut self) { + for &(pid, starttime) in &self.0 { + if proc_starttime(pid) == Some(starttime) { + signal_pid(pid, libc::SIGKILL); + } + } + } +} + +/// Poll a std child until it is observed exited+reaped (SIGTERM'd by the +/// sweep) or the deadline passes. +fn wait_child_gone(child: &mut std::process::Child, why: &str) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + if child.try_wait().expect("try_wait own child").is_some() { + return; + } + assert!(std::time::Instant::now() < deadline, "{why}"); + std::thread::sleep(std::time::Duration::from_millis(20)); + } +} + +// --------------------------------------------------------------------------- +// The seven Task 9 tests (names verbatim from the plan). +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn sweep_reaps_verified_idle_unclaimed_sidecar() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + let ownership_id = "codex-sidecar-a9000001-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + // Deliberately loaded-but-idle: pins the `loaded ≠ mid-turn` + // discriminator (idle threads stay loaded forever, reports/V1.md). + let (mut child, ws_url) = spawn_own_fake_app_server_with_behavior( + ownership_id, + Some(r#"{"loadedThreadIds": ["t-1"], "threadStatuses": {"t-1": "idle"}}"#), + ) + .await; + let record = CodexSidecarRecord { + ws_url: ws_url.clone(), + ..record_for_child( + ownership_id, + child.id().expect("live fixture pid"), + Some(SESSION), + ) + }; + store.write(&record).expect("write record"); + + let (reconciler, report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + assert_eq!(report.held, 1, "the survivor is held, unclaimed"); + + let outcomes = reconciler.sweep_unclaimed().await; + assert_eq!( + outcomes, + vec![(ownership_id.to_string(), SweepOutcome::Reaped)], + "a verified, reachable, idle, unclaimed sidecar is reaped" + ); + + // The whole tree is gone within the drain budget... + tokio::time::timeout(Duration::from_secs(10), child.wait()) + .await + .expect("the reaped sidecar must exit within the drain budget") + .expect("wait fixture"); + // ...and the record left both the store and `held`. + assert!(store.load_all().is_empty(), "the reaped record is removed"); + assert_eq!(reconciler.unclaimed_len(), 0, "nothing left to sweep"); +} + +#[tokio::test] +async fn sweep_retains_mid_turn_sidecar_with_recorded_reason() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + let ownership_id = "codex-sidecar-a9000002-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + let (mut child, ws_url) = spawn_own_fake_app_server_with_behavior( + ownership_id, + Some(r#"{"loadedThreadIds": ["t-1"], "threadStatuses": {"t-1": "active"}}"#), + ) + .await; + let record = CodexSidecarRecord { + ws_url: ws_url.clone(), + ..record_for_child( + ownership_id, + child.id().expect("live fixture pid"), + Some(SESSION), + ) + }; + store.write(&record).expect("write record"); + + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + let outcomes = reconciler.sweep_unclaimed().await; + assert_eq!( + outcomes, + vec![(ownership_id.to_string(), SweepOutcome::RetainedMidTurn)], + "a mid-turn survivor must end up retained, not killed and not leaked" + ); + + // NEVER signalled: still alive after the grace window. + tokio::time::sleep(NEVER_SIGNALLED_GRACE).await; + assert_eq!( + child.try_wait().expect("try_wait fixture"), + None, + "a mid-turn sidecar must never be signalled by the sweep" + ); + // The reason is durably recorded... + let rows = store.load_all(); + assert_eq!(rows.len(), 1, "the retained record stays in the store"); + assert_eq!(rows[0].ownership_id, ownership_id); + assert_eq!( + rows[0].state, + SidecarRecordState::Retained { + reason: "mid-turn-active-thread".to_string() + } + ); + // ...and the record STAYS held (claimable; re-evaluated at next boot). + assert_eq!(reconciler.unclaimed_len(), 1, "retained rows stay held"); + + child + .kill() + .await + .expect("cleanup: kill this test's own fixture"); +} + +#[tokio::test] +async fn late_restore_after_sweep_reattaches_mid_turn_survivor() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + let ownership_id = "codex-sidecar-a9000003-cccc-4ccc-8ccc-cccccccccccc"; + let (mut child, ws_url) = spawn_own_fake_app_server_with_behavior( + ownership_id, + Some(r#"{"loadedThreadIds": ["t-1"], "threadStatuses": {"t-1": "active"}}"#), + ) + .await; + let record = CodexSidecarRecord { + ws_url: ws_url.clone(), + ..record_for_child( + ownership_id, + child.id().expect("live fixture pid"), + Some(SESSION), + ) + }; + store.write(&record).expect("write record"); + + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + let outcomes = reconciler.sweep_unclaimed().await; + assert_eq!( + outcomes, + vec![(ownership_id.to_string(), SweepOutcome::RetainedMidTurn)] + ); + + // A LATE restore still claims the retained survivor (re-verified) — it + // reattaches instead of fresh-spawning into the -32600 (A5 fix, + // reports/V3.md). + let claimed = reconciler + .claim_for_session(SESSION) + .await + .expect("the retained mid-turn survivor is still claimable"); + assert_eq!(claimed.ownership_id, ownership_id); + assert_eq!( + claimed.state, + SidecarRecordState::Retained { + reason: "mid-turn-active-thread".to_string() + }, + "the claim returns the retained record as recorded" + ); + assert_eq!(reconciler.unclaimed_len(), 0, "the claim left held"); + assert_eq!( + child.try_wait().expect("try_wait fixture"), + None, + "the survivor is alive for the reattach" + ); + + // Final-review H3a: the adopt-time enrich flips the claimed record back + // to Active — a reattached pane's row must not keep reading + // Retained{mid-turn} (that would lie to auditors). + let runtime = ReattachedCodexAppServerRuntime::new(claimed, Arc::clone(&store)); + runtime + .update_ownership_metadata("term-late-restore".to_string(), 1) + .await + .expect("adopt-time enrich"); + let rows = store.load_all(); + assert_eq!(rows.len(), 1, "one durable row for the reattached survivor"); + assert_eq!( + rows[0].state, + SidecarRecordState::Active, + "adopt flips a Retained record back to Active" + ); + assert_eq!(rows[0].terminal_id.as_deref(), Some("term-late-restore")); + + child + .kill() + .await + .expect("cleanup: kill this test's own fixture"); +} + +#[tokio::test] +async fn sweep_never_touches_unverified_pids() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + let child = spawn_own_sleep_child(); + let ownership_id = "codex-sidecar-a9000004-dddd-4ddd-8ddd-dddddddddddd"; + let record = record_for_child(ownership_id, child.0.id(), Some(SESSION)); + store.write(&record).expect("write record"); + + let (reconciler, report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + assert_eq!(report.held, 1); + // The pid-reuse shape appearing AFTER boot: the held record's cmdline no + // longer matches the live process — this pid is NOT ours. (A record + // mismatched at boot time is pruned by boot_reconcile before the sweep + // ever sees it; the sweep's own Mismatch arm exists for post-boot decay.) + reconciler + .held + .lock() + .unwrap() + .get_mut(ownership_id) + .expect("held record") + .cmdline = vec!["codex".to_string(), "app-server".to_string()]; + + let outcomes = reconciler.sweep_unclaimed().await; + assert_eq!( + outcomes, + vec![(ownership_id.to_string(), SweepOutcome::RecordRemovedStale)], + "a mismatched record is removed, NEVER signalled" + ); + + tokio::time::sleep(NEVER_SIGNALLED_GRACE).await; + let mut child = child; + assert_eq!( + child.0.try_wait().expect("try_wait own sleep child"), + None, + "the mismatching pid must never be signalled" + ); + assert!(store.load_all().is_empty(), "the stale record is removed"); + assert_eq!(reconciler.unclaimed_len(), 0); +} + +#[tokio::test] +async fn sweep_never_kills_a_record_claimed_during_the_probe_window() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + let ownership_id = "codex-sidecar-a9000005-eeee-4eee-8eee-eeeeeeeeeeee"; + // Default fixture behavior: reachable, no loaded threads — exactly the + // shape the decide phase turns into a Kill decision. + let (mut child, ws_url) = spawn_own_fake_app_server_with_behavior(ownership_id, None).await; + let record = CodexSidecarRecord { + ws_url: ws_url.clone(), + ..record_for_child( + ownership_id, + child.id().expect("live fixture pid"), + Some(SESSION), + ) + }; + store.write(&record).expect("write record"); + + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + + // The TOCTOU shape, deterministic (no timing dependence): the sweep's + // probe phase snapshotted the record... + let pre_claim_snapshot = record.clone(); + // ...then a restore claimed it while the probe awaited. Per-pid identity + // re-verification CANNOT detect this (same live process) — membership in + // `held` is the single source of truth. + let claimed = reconciler + .claim_for_session(SESSION) + .await + .expect("the restore's claim wins the record"); + assert_eq!(claimed.ownership_id, ownership_id); + + // Drive the sweep's commit arm with the PRE-claim snapshot. + let outcome = reconciler + .commit_sweep_decision(&pre_claim_snapshot, SweepDecision::Kill) + .await; + assert_eq!( + outcome, + SweepOutcome::SkippedClaimedDuringSweep, + "a record claimed mid-sweep is skipped — killing it is the exact da92 harm" + ); + + // NO signal was sent: the claimant's sidecar is still alive... + tokio::time::sleep(NEVER_SIGNALLED_GRACE).await; + assert_eq!( + child.try_wait().expect("try_wait fixture"), + None, + "the claimed sidecar must never be signalled by the sweep" + ); + // ...and the claimant's record is untouched. + assert_eq!( + store.load_all(), + vec![record], + "the claimant's record must stay untouched" + ); + + child + .kill() + .await + .expect("cleanup: kill this test's own fixture"); +} + +#[tokio::test] +async fn kill_verified_sidecar_tree_reaps_descendants() { + // A tree THIS TEST owns: bash root + two background sleeps. bash dies on + // SIGTERM without propagating it, orphaning the sleeps — the exact shape + // that makes single-pid signalling insufficient (A3 falsified). + let script = "sleep 300 & sleep 300 & wait"; + let root = spawn_own_shell_child("bash", &["-c", script], &["bash", "-c", script]); + let root_pid = root.0.id() as i32; + let children = wait_for_sleep_children(root_pid, 2); + let _orphan_guard = OrphanSnapshotGuard(children.clone()); + let record = record_for_child( + "codex-sidecar-a9000006-ffff-4fff-8fff-ffffffffffff", + root.0.id(), + Some(SESSION), + ); + + let outcome = kill_verified_sidecar_tree(&record).await; + assert_eq!( + outcome.outcomes.len(), + 3, + "root + both captured descendants are accounted for" + ); + assert_eq!( + outcome.outcomes[0], + (record.pid, KillOutcome::ExitedAfterSigterm), + "the root drains on SIGTERM" + ); + for &(pid, starttime) in &children { + assert!( + outcome + .outcomes + .iter() + .any(|&(p, o)| p == pid as u32 && o == KillOutcome::ExitedAfterSigterm), + "each captured descendant is individually reaped" + ); + assert_ne!( + proc_starttime(pid), + Some(starttime), + "the descendant incarnation is gone" + ); + } + + // The negative: a snapshot-mismatched descendant is NEVER signalled. + let bystander = spawn_own_sleep_child(); + let bystander_pid = bystander.0.id() as i32; + let mismatched = PidSnapshot { + pid: bystander_pid, + starttime: proc_starttime(bystander_pid).expect("live bystander") + 1, + cmdline: proc_cmdline(bystander_pid).expect("live bystander"), + }; + assert_eq!( + kill_captured_descendant(&mismatched).await, + KillOutcome::SkippedIdentityMismatch, + "a stale snapshot refuses the kill" + ); + tokio::time::sleep(NEVER_SIGNALLED_GRACE).await; + let mut bystander = bystander; + assert_eq!( + bystander.0.try_wait().expect("try_wait bystander"), + None, + "the snapshot-mismatched pid must never be signalled" + ); +} + +#[tokio::test] +async fn restart_reconciliation_leaves_no_sidecar_silently_orphaned() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + + // (a) The claimable verified survivor (newest updated_at of the + // session-id duplicates — the claim's fallback winner). + let child_a = spawn_own_sleep_child(); + let id_a = "codex-sidecar-a9000011-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + let record_a = CodexSidecarRecord { + updated_at: 1_700_000_000_002, + ..record_for_child(id_a, child_a.0.id(), Some(SESSION)) + }; + + // (b) A dead pid: real evidence captured live, then OUR OWN child is + // killed+reaped — pruned at boot. + let mut child_b = spawn_own_sleep_child(); + let id_b = "codex-sidecar-a9000012-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + let record_b = record_for_child(id_b, child_b.0.id(), Some(SESSION)); + child_b.0.kill().expect("kill own child"); + child_b.0.wait().expect("reap own child"); + + // (c) A verified idle sidecar (loaded-but-idle fixture) — reaped. + let id_c = "codex-sidecar-a9000013-cccc-4ccc-8ccc-cccccccccccc"; + let (mut child_c, ws_c) = spawn_own_fake_app_server_with_behavior( + id_c, + Some(r#"{"loadedThreadIds": ["t-c"], "threadStatuses": {"t-c": "idle"}}"#), + ) + .await; + let record_c = CodexSidecarRecord { + ws_url: ws_c, + ..record_for_child(id_c, child_c.id().expect("live fixture pid"), None) + }; + + // (d) A verified mid-turn sidecar — retained with a recorded reason. + let id_d = "codex-sidecar-a9000014-dddd-4ddd-8ddd-dddddddddddd"; + let (mut child_d, ws_d) = spawn_own_fake_app_server_with_behavior( + id_d, + Some(r#"{"loadedThreadIds": ["t-d"], "threadStatuses": {"t-d": "active"}}"#), + ) + .await; + let record_d = CodexSidecarRecord { + ws_url: ws_d, + ..record_for_child( + id_d, + child_d.id().expect("live fixture pid"), + Some("0198f00d-0d0d-7db3-9c47-1c2a3b4c5d6e"), + ) + }; + + // (e) A DUPLICATE verified record sharing (a)'s session id (the A4 + // shape, reports/V3.md) — the claim LOSER, swept to its own fate + // (reaped here — idle). + let child_e = spawn_own_sleep_child(); + let id_e = "codex-sidecar-a9000015-eeee-4eee-8eee-eeeeeeeeeeee"; + let record_e = CodexSidecarRecord { + updated_at: 1_700_000_000_001, + ..record_for_child(id_e, child_e.0.id(), Some(SESSION)) + }; + + for record in [&record_a, &record_b, &record_c, &record_d, &record_e] { + store.write(record).expect("write record"); + } + + // Boot → claim (a restore for (a)'s session) → sweep. + let (reconciler, report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + assert_eq!(report.loaded, 5); + assert_eq!(report.pruned_dead, 1, "(b) is removed at boot"); + assert_eq!(report.held, 4); + + let claimed = reconciler + .claim_for_session(SESSION) + .await + .expect("the restore claims one of the session duplicates"); + assert_eq!( + claimed, record_a, + "the newest-updated_at duplicate wins the claim (reattached-by-construction)" + ); + assert_eq!( + reconciler.unclaimed_len(), + 3, + "(c), (d), (e) remain unclaimed" + ); + + let mut outcomes = reconciler.sweep_unclaimed().await; + outcomes.sort_by(|x, y| x.0.cmp(&y.0)); + assert_eq!( + outcomes, + vec![ + (id_c.to_string(), SweepOutcome::Reaped), + (id_d.to_string(), SweepOutcome::RetainedMidTurn), + (id_e.to_string(), SweepOutcome::Reaped), + ], + "every unclaimed record is swept to an explicit fate" + ); + + // The exhaustive end-state: every sidecar is accounted for — reattached, + // reaped, or intentionally retained with a recorded reason. Never + // silently dropped from the books. + tokio::time::timeout(Duration::from_secs(10), child_c.wait()) + .await + .expect("(c) must be reaped within the drain budget") + .expect("wait fixture c"); + let mut child_e = child_e; + wait_child_gone(&mut child_e.0, "(e) — the claim loser — must be reaped"); + let mut child_a = child_a; + assert_eq!( + child_a.0.try_wait().expect("try_wait child a"), + None, + "(a) — the claimed survivor — must stay alive" + ); + assert_eq!( + child_d.try_wait().expect("try_wait fixture d"), + None, + "(d) — mid-turn — must stay alive" + ); + + let mut rows = store.load_all(); + rows.sort_by(|x, y| x.ownership_id.cmp(&y.ownership_id)); + assert_eq!( + rows.len(), + 2, + "the store holds ONLY the claimed Active record and the retained record" + ); + assert_eq!(rows[0], record_a, "(a)'s claimed record stays Active"); + assert_eq!(rows[1].ownership_id, id_d); + assert_eq!( + rows[1].state, + SidecarRecordState::Retained { + reason: "mid-turn-active-thread".to_string() + }, + "(d)'s retention reason is durably recorded" + ); + assert_eq!( + reconciler.unclaimed_len(), + 1, + "(d) stays held — claimable by a late restore, re-evaluated at next boot" + ); + + child_d + .kill() + .await + .expect("cleanup: kill this test's own fixture"); +} + +// --------------------------------------------------------------------------- +// Supplementary coverage: the remaining SweepOutcome paths + the Task 6 +// carry-forward (truthful KillOutcome fidelity). +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn sweep_retains_ws_unreachable_writer_holding_survivor() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + // A verified survivor whose ws is UNREACHABLE but which holds an open + // rollout .jsonl WRITE handle — the writer-evidence shape (reports/V2.md). + let rollout = dir.path().join("rollout-t-writer.jsonl"); + let script = format!("exec sleep 300 >> '{}'", rollout.display()); + let child = spawn_own_shell_child("bash", &["-c", &script], &["sleep", "300"]); + let ownership_id = "codex-sidecar-a9000021-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + // record_for_child's default ws_url points at a closed loopback port. + let record = record_for_child(ownership_id, child.0.id(), Some(SESSION)); + store.write(&record).expect("write record"); + + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + let outcomes = reconciler.sweep_unclaimed().await; + assert_eq!( + outcomes, + vec![(ownership_id.to_string(), SweepOutcome::RetainedWriterHeld)], + "an unreachable-but-writer-holding survivor is retained, not killed" + ); + + tokio::time::sleep(NEVER_SIGNALLED_GRACE).await; + let mut child = child; + assert_eq!( + child.0.try_wait().expect("try_wait own writer child"), + None, + "the writer-holding survivor must never be signalled" + ); + let rows = store.load_all(); + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0].state, + SidecarRecordState::Retained { + reason: "ws-unreachable-writer-held".to_string() + } + ); + assert_eq!(reconciler.unclaimed_len(), 1, "retained rows stay held"); +} + +#[tokio::test] +async fn unverifiable_verdict_decides_retain_and_commit_records_reason() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + let child = spawn_own_sleep_child(); + let ownership_id = "codex-sidecar-a9000022-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + let record = record_for_child(ownership_id, child.0.id(), Some(SESSION)); + store.write(&record).expect("write record"); + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + + // An Unverifiable verdict cannot be synthesized end-to-end for a + // test-owned child on Linux (its cmdline is always readable), so the + // decide-phase mapping and the commit arm are pinned directly. + let decision = decide_sweep_action(&record, &IdentityVerdict::Unverifiable).await; + assert!( + matches!(decision, SweepDecision::RetainUnverifiable), + "not provably ours + not provably stale ⇒ retained, never signalled" + ); + let outcome = reconciler.commit_sweep_decision(&record, decision).await; + assert_eq!(outcome, SweepOutcome::RetainedUnverifiable); + + tokio::time::sleep(NEVER_SIGNALLED_GRACE).await; + let mut child = child; + assert_eq!( + child.0.try_wait().expect("try_wait own sleep child"), + None, + "an unverifiable pid must never be signalled" + ); + let rows = store.load_all(); + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0].state, + SidecarRecordState::Retained { + reason: "identity-unverifiable".to_string() + } + ); + // Still held AND still claimable (evidence re-verifies fine at claim). + assert_eq!(reconciler.unclaimed_len(), 1); + let claimed = reconciler + .claim_for_session(SESSION) + .await + .expect("the retained record is still claimable"); + assert_eq!(claimed.ownership_id, ownership_id); +} + +#[tokio::test] +async fn kill_tree_reports_sigterm_sent_when_escalation_is_refused() { + // The Task 6 carry-forward: after SIGTERM is sent and the drain budget + // expires, a pre-SIGKILL re-verify that no longer matches must report + // that a SIGTERM WAS sent and the escalation was refused — not a + // "Skipped*" outcome that under-reports the signal. The tree here execs + // a NEW argv on TERM (same pid, same starttime, new cmdline): the + // pid-identity decays mid-drain. + let script = r#"trap 'exec sleep 301' TERM; sleep 300 & wait"#; + let root = spawn_own_shell_child("bash", &["-c", script], &["bash", "-c", script]); + let root_pid = root.0.id() as i32; + let children = wait_for_sleep_children(root_pid, 1); + let _orphan_guard = OrphanSnapshotGuard(children.clone()); + let record = record_for_child( + "codex-sidecar-a9000023-cccc-4ccc-8ccc-cccccccccccc", + root.0.id(), + Some(SESSION), + ); + + let outcome = kill_verified_sidecar_tree(&record).await; + assert_eq!( + outcome.outcomes[0], + (record.pid, KillOutcome::SigtermSentEscalationRefused), + "the outcome must report the sent SIGTERM and the refused SIGKILL truthfully" + ); + // No SIGKILL was sent: the same incarnation is still alive (as its new + // argv). + assert_eq!( + proc_starttime(root_pid), + Some(record.starttime), + "the re-exec'd root incarnation survives — escalation was refused" + ); + assert_eq!( + proc_cmdline(root_pid).as_deref(), + Some(&["sleep".to_string(), "301".to_string()][..]), + "the root re-exec'd mid-drain (the identity-decay shape)" + ); + // Its captured descendant was still individually reaped. + let (child_pid, child_starttime) = children[0]; + assert!( + outcome + .outcomes + .iter() + .any(|&(p, o)| p == child_pid as u32 && o == KillOutcome::ExitedAfterSigterm), + "the captured descendant is reaped independently of the root refusal" + ); + assert_ne!(proc_starttime(child_pid), Some(child_starttime)); + // ChildGuard drop reaps the re-exec'd root (still this test's child). +} + +/// Task 10: the boot wiring's grace parse — env value in millis, default on +/// unset/non-numeric, `0` honored (immediate sweep). Pure half only: env +/// mutation races parallel test binaries. +#[test] +fn reap_grace_parse_honors_value_zero_and_default() { + let default = std::time::Duration::from_millis(CODEX_SIDECAR_REAP_GRACE_MS_DEFAULT); + assert_eq!( + reap_grace_from_value(Some("1500")), + std::time::Duration::from_millis(1500) + ); + assert_eq!( + reap_grace_from_value(Some("0")), + std::time::Duration::ZERO, + "0 is a legitimate immediate-sweep knob" + ); + assert_eq!(reap_grace_from_value(None), default); + assert_eq!(reap_grace_from_value(Some("not-a-number")), default); +} + +#[tokio::test] +async fn sweep_treats_malformed_loaded_list_as_unreachable_not_idle() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = store_in(&dir); + // A reachable fixture whose thread/loaded/list result is MALFORMED (no + // `data` array) via the per-method overrides knob (final review F2): + // Ok(vec![]) parsing would read this as "reachable, no loaded threads" + // and REAP — the client must surface an error instead, sending the sweep + // down the conservative Unreachable → writer-evidence path. + let fixture_ownership = "codex-sidecar-f2000001-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + let (mut fixture, ws_url) = spawn_own_fake_app_server_with_behavior( + fixture_ownership, + Some(r#"{"overrides": {"thread/loaded/list": {"result": {"unexpected": true}}}}"#), + ) + .await; + // The RECORD's verified pid is a SEPARATE test-owned child holding real + // writer evidence (an open rollout .jsonl write fd), while its ws_url + // points at the malformed fixture. The outcome discriminates the two + // paths deterministically: ReachableIdle would Kill/Reap this record; + // the Unreachable arm consults /proc//fd and RETAINS it. + let rollout = dir.path().join("rollout-t-malformed.jsonl"); + let script = format!("exec sleep 300 >> '{}'", rollout.display()); + let writer_child = spawn_own_shell_child("bash", &["-c", &script], &["sleep", "300"]); + let ownership_id = "codex-sidecar-f2000002-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + let record = CodexSidecarRecord { + ws_url, + ..record_for_child(ownership_id, writer_child.0.id(), Some(SESSION)) + }; + store.write(&record).expect("write record"); + + let (reconciler, _report) = SidecarReconciler::boot_reconcile(Arc::clone(&store)); + let outcomes = reconciler.sweep_unclaimed().await; + assert_eq!( + outcomes, + vec![(ownership_id.to_string(), SweepOutcome::RetainedWriterHeld)], + "a malformed loaded-list must take the writer-evidence path, not ReachableIdle" + ); + + tokio::time::sleep(NEVER_SIGNALLED_GRACE).await; + let mut writer_child = writer_child; + assert_eq!( + writer_child.0.try_wait().expect("try_wait writer child"), + None, + "the writer-holding survivor must never be signalled on malformed data" + ); + + fixture + .kill() + .await + .expect("cleanup: kill this test's own fixture"); +} + +#[tokio::test] +async fn kill_tree_root_refusal_skips_captured_descendants() { + // Final-review H3b: when the root's PRE-signal re-verify refuses + // (Mismatch/Unverifiable), the captured descendants' provenance is in + // doubt (they were snapshotted as children of an unproven root) — the + // descendant loop must be skipped entirely, nothing signalled. Driven + // through the private linux arm directly: the public entry refuses a + // Mismatch before ever capturing, so the inner re-verify refusal is the + // only reachable shape for this guard. + let script = "sleep 300 & sleep 300 & wait"; + let root = spawn_own_shell_child("bash", &["-c", script], &["bash", "-c", script]); + let root_pid = root.0.id() as i32; + let children = wait_for_sleep_children(root_pid, 2); + let _orphan_guard = OrphanSnapshotGuard(children.clone()); + // The pid-reuse shape: live pid + starttime, WRONG cmdline. + let record = CodexSidecarRecord { + cmdline: vec!["codex".to_string(), "app-server".to_string()], + ..record_for_child( + "codex-sidecar-b3000001-cccc-4ccc-8ccc-cccccccccccc", + root.0.id(), + Some(SESSION), + ) + }; + + let outcome = kill_verified_tree_linux(&record).await; + assert_eq!( + outcome.outcomes, + vec![(record.pid, KillOutcome::SkippedIdentityMismatch)], + "a pre-signal root refusal reports ONLY the root; no descendant is processed" + ); + + // NOTHING was signalled: the root and both captured descendants live on. + tokio::time::sleep(NEVER_SIGNALLED_GRACE).await; + let mut root = root; + assert_eq!( + root.0.try_wait().expect("try_wait root"), + None, + "the mismatched root must never be signalled" + ); + for &(pid, starttime) in &children { + assert_eq!( + proc_starttime(pid), + Some(starttime), + "descendants of an unproven root must never be signalled" + ); + } +} diff --git a/crates/freshell-codex/src/sidecar_test_support.rs b/crates/freshell-codex/src/sidecar_test_support.rs new file mode 100644 index 000000000..dff0dd16d --- /dev/null +++ b/crates/freshell-codex/src/sidecar_test_support.rs @@ -0,0 +1,171 @@ +//! Shared test helpers for the sidecar lifecycle suites +//! ([`crate::sidecar_reconcile`] + [`crate::sidecar_sweep`] tests) — extracted +//! from `sidecar_reconcile_tests.rs` when Task 9's sweep suite landed in its +//! own file (the pre-authorized 1,000-line split). Compiled only for +//! `cfg(test)` on Linux (the `/proc` evidence helpers), never shipped. +//! +//! PROCESS SAFETY: every helper spawns and signals ONLY the calling test's +//! own children; temp stores only; loopback ephemeral ports only, never +//! 3001/3002. + +use std::sync::Arc; +use std::time::Duration; + +use crate::sidecar_store::{ + proc_cmdline, proc_starttime, CodexSidecarRecord, CodexSidecarStore, SidecarRecordState, + SIDECAR_RECORD_VERSION, +}; + +/// The codex session (thread) id the suites share. +pub(crate) const SESSION: &str = "019810de-1e5f-7db3-9c47-1c2a3b4c5d6e"; + +/// Grace window before a "never signalled" assertion: long enough for the +/// fixture's graceful SIGTERM exit to become observable if a signal HAD +/// (wrongly) been sent. +pub(crate) const NEVER_SIGNALLED_GRACE: Duration = Duration::from_millis(300); + +/// Kills and reaps ONLY the guarded child on drop (defer-style guard) — the +/// test's own `sleep 300`, nothing else on the machine. `kill` on an +/// already-reaped `Child` is a no-op error inside std (no signal is sent to +/// a possibly-recycled pid), so double-cleanup is safe. +pub(crate) struct ChildGuard(pub(crate) std::process::Child); + +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +/// Spawn this test's own `sleep 300` child and wait for exec to complete. +pub(crate) fn spawn_own_sleep_child() -> ChildGuard { + spawn_own_shell_child("sleep", &["300"], &["sleep", "300"]) +} + +/// Spawn this test's own child process and poll until `/proc//cmdline` +/// reads as `want_cmdline`: immediately after spawn the child may still be +/// post-fork/pre-exec, so evidence captured in that window verifies as a +/// cmdline Mismatch at boot/claim time (observed flake). +pub(crate) fn spawn_own_shell_child( + program: &str, + args: &[&str], + want_cmdline: &[&str], +) -> ChildGuard { + let guard = ChildGuard( + std::process::Command::new(program) + .args(args) + .spawn() + .expect("spawn this test's own child"), + ); + let pid = guard.0.id() as i32; + let want: Vec = want_cmdline.iter().map(|s| s.to_string()).collect(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while proc_cmdline(pid).as_ref() != Some(&want) { + assert!( + std::time::Instant::now() < deadline, + "test child failed to exec within 5s" + ); + std::thread::sleep(Duration::from_millis(2)); + } + guard +} + +/// A loopback `ws://` URL on an ephemeral port NOTHING listens on (bound, +/// read, dropped) — probe dials fail fast with connection-refused. Never +/// port 3001. +pub(crate) fn unused_loopback_ws_url() -> String { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port"); + let port = listener.local_addr().expect("local_addr").port(); + drop(listener); + format!("ws://127.0.0.1:{port}") +} + +/// A record carrying a spawned child's REAL `/proc` evidence. +pub(crate) fn record_for_child( + ownership_id: &str, + pid: u32, + session_id: Option<&str>, +) -> CodexSidecarRecord { + CodexSidecarRecord { + record_version: SIDECAR_RECORD_VERSION, + ownership_id: ownership_id.to_string(), + pid, + starttime: proc_starttime(pid as i32).expect("live child has a starttime"), + cmdline: proc_cmdline(pid as i32).expect("live child has a cmdline"), + ws_url: unused_loopback_ws_url(), + session_id: session_id.map(str::to_string), + terminal_id: None, + server_instance_id: "srv-prev".to_string(), + created_at: 1_700_000_000_000, + updated_at: 1_700_000_000_001, + state: SidecarRecordState::Active, + } +} + +/// A tempdir-backed store (lock-free test construction). +pub(crate) fn store_in(dir: &tempfile::TempDir) -> Arc { + Arc::new(CodexSidecarStore::new(dir.path().to_path_buf())) +} + +/// The committed fake app-server fixture (repo-owned test harness). +pub(crate) fn fake_app_server_fixture() -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs") +} + +/// Spawn THIS TEST'S OWN fake app-server on a loopback ephemeral port and +/// wait for its WS listener to accept. `kill_on_drop(true)` guarantees +/// cleanup kills ONLY this recorded child, even on panic. +pub(crate) async fn spawn_own_fake_app_server( + ownership_id: &str, +) -> (tokio::process::Child, String) { + spawn_own_fake_app_server_with_behavior(ownership_id, None).await +} + +/// [`spawn_own_fake_app_server`] with a scripted +/// `FAKE_CODEX_APP_SERVER_BEHAVIOR` JSON (e.g. the Task 9 `threadStatuses` +/// knob). +pub(crate) async fn spawn_own_fake_app_server_with_behavior( + ownership_id: &str, + behavior_json: Option<&str>, +) -> (tokio::process::Child, String) { + // Allocate a free loopback ephemeral port for the fixture to listen on. + let ws_url = unused_loopback_ws_url(); + let mut command = tokio::process::Command::new("node"); + command + .arg(fake_app_server_fixture()) + .arg("--listen") + .arg(&ws_url) + .env(crate::durability::CODEX_SIDECAR_OWNERSHIP_ENV, ownership_id) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true); + if let Some(behavior) = behavior_json { + command.env("FAKE_CODEX_APP_SERVER_BEHAVIOR", behavior); + } + let mut child = command + .spawn() + .expect("spawn this test's own fake app-server"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + if let Ok(Ok((probe, _response))) = tokio::time::timeout( + Duration::from_secs(1), + tokio_tungstenite::connect_async(&ws_url), + ) + .await + { + drop(probe); + break; + } + if let Ok(Some(status)) = child.try_wait() { + panic!("fake app-server exited before listening: {status}"); + } + assert!( + tokio::time::Instant::now() < deadline, + "fake app-server WS never came up" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + (child, ws_url) +} diff --git a/crates/freshell-codex/tests/global_manager_install.rs b/crates/freshell-codex/tests/global_manager_install.rs index a79627ff5..390c559e3 100644 --- a/crates/freshell-codex/tests/global_manager_install.rs +++ b/crates/freshell-codex/tests/global_manager_install.rs @@ -112,7 +112,10 @@ async fn installed_manager_is_returned_by_global_and_set_twice_fails() { let runtime = FakeRuntime::start().await; let factory_runtime = runtime.clone(); let manager = CodexTerminalLaunchManager::with_plan_budget( - Box::new(move || factory_runtime.clone() as std::sync::Arc), + Box::new(move |_plan| { + let rt = factory_runtime.clone() as std::sync::Arc; + Box::pin(async move { rt }) + }), 2, std::time::Duration::from_secs(30), 64, @@ -140,7 +143,10 @@ async fn installed_manager_is_returned_by_global_and_set_twice_fails() { let runtime2 = FakeRuntime::start().await; let second = CodexTerminalLaunchManager::with_plan_budget( - Box::new(move || runtime2.clone() as std::sync::Arc), + Box::new(move |_plan| { + let rt = runtime2.clone() as std::sync::Arc; + Box::pin(async move { rt }) + }), 2, std::time::Duration::from_secs(30), 64, diff --git a/crates/freshell-codex/tests/launch_lifecycle.rs b/crates/freshell-codex/tests/launch_lifecycle.rs index f52851008..9cabbe17b 100644 --- a/crates/freshell-codex/tests/launch_lifecycle.rs +++ b/crates/freshell-codex/tests/launch_lifecycle.rs @@ -29,7 +29,11 @@ use freshell_codex::launch_lifecycle::{ CODEX_LAUNCH_PLANNER_SHUTDOWN_MESSAGE, CODEX_SIDECAR_NOT_ADOPTABLE_MESSAGE, }; use freshell_codex::launch_plan::{codex_remote_args, CodexLaunchPlanInput}; -use freshell_codex::BoxFuture; +use freshell_codex::{ + proc_cmdline, proc_starttime, verify_sidecar_identity, BoxFuture, CodexSidecarRecord, + CodexSidecarStore, IdentityVerdict, ReattachedCodexAppServerRuntime, SidecarReconciler, + SidecarRecordState, CODEX_SIDECAR_OWNERSHIP_ENV, SIDECAR_RECORD_VERSION, +}; const RECV_TIMEOUT: Duration = Duration::from_secs(10); @@ -39,8 +43,10 @@ struct FakeRuntime { ws_url: String, ensure_ready_calls: Mutex>>, fail_ensure_ready: AtomicBool, + fail_prepare_retention: AtomicBool, shutdown_calls: AtomicU32, ownership_updates: Mutex>, + noted_session_ids: Mutex>, } impl FakeRuntime { @@ -74,8 +80,10 @@ impl FakeRuntime { ws_url, ensure_ready_calls: Mutex::new(Vec::new()), fail_ensure_ready: AtomicBool::new(false), + fail_prepare_retention: AtomicBool::new(false), shutdown_calls: AtomicU32::new(0), ownership_updates: Mutex::new(Vec::new()), + noted_session_ids: Mutex::new(Vec::new()), }) } } @@ -110,6 +118,22 @@ impl CodexLaunchRuntime for FakeRuntime { }) } + fn note_session_id(&self, session_id: String) -> BoxFuture<'_, Result<(), String>> { + Box::pin(async move { + self.noted_session_ids.lock().unwrap().push(session_id); + Ok(()) + }) + } + + fn prepare_retention(&self, _reason: String) -> BoxFuture<'_, Result<(), String>> { + Box::pin(async move { + if self.fail_prepare_retention.load(Ordering::SeqCst) { + return Err("fake runtime: prepare_retention failed".to_string()); + } + Ok(()) + }) + } + fn shutdown(&self) -> BoxFuture<'_, Result<(), String>> { Box::pin(async move { self.shutdown_calls.fetch_add(1, Ordering::SeqCst); @@ -119,8 +143,9 @@ impl CodexLaunchRuntime for FakeRuntime { } fn planner_for(runtime: Arc) -> CodexLaunchPlanner { - CodexLaunchPlanner::new(Box::new(move || { - runtime.clone() as Arc + CodexLaunchPlanner::new(Box::new(move |_plan| { + let rt = runtime.clone() as Arc; + Box::pin(async move { rt }) })) } @@ -183,6 +208,40 @@ async fn resume_plan_sets_session_id_and_disables_candidate_persistence() { launch.sidecar.shutdown().await.unwrap(); } +/// Task 4: resume launches know their session id at plan time, so +/// `plan_create` notes it on the runtime; fresh launches have no id yet +/// (theirs arrives via the proxy's thread candidate), so no call is made. +#[tokio::test] +async fn plan_create_notes_the_resume_session_id_on_the_runtime() { + let runtime = FakeRuntime::start().await; + let planner = planner_for(runtime.clone()); + + let resume = planner + .plan_create(&CodexLaunchPlanInput { + resume_session_id: Some("s-1"), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!( + runtime.noted_session_ids.lock().unwrap().as_slice(), + &["s-1".to_string()], + "the resume plan must note its session id on the runtime" + ); + resume.sidecar.shutdown().await.unwrap(); + + let fresh = planner + .plan_create(&CodexLaunchPlanInput::default()) + .await + .unwrap(); + assert_eq!( + runtime.noted_session_ids.lock().unwrap().len(), + 1, + "a fresh plan has no session id at plan time; no note call" + ); + fresh.sidecar.shutdown().await.unwrap(); +} + #[tokio::test] async fn relay_works_through_the_planned_proxy() { // The plan's remote_ws_url accepts a TUI connection and relays to the upstream: @@ -357,8 +416,9 @@ async fn retry_never_retries_configuration_errors() { async fn manager_adopts_by_terminal_id_and_tears_down_on_exit() { let runtime = FakeRuntime::start().await; let factory_runtime = runtime.clone(); - let manager = CodexTerminalLaunchManager::new(Box::new(move || { - factory_runtime.clone() as Arc + let manager = CodexTerminalLaunchManager::new(Box::new(move |_plan| { + let rt = factory_runtime.clone() as Arc; + Box::pin(async move { rt }) })); let launch = manager @@ -398,8 +458,9 @@ async fn manager_adopts_by_terminal_id_and_tears_down_on_exit() { async fn manager_discard_tears_down_an_unadopted_plan() { let runtime = FakeRuntime::start().await; let factory_runtime = runtime.clone(); - let manager = CodexTerminalLaunchManager::new(Box::new(move || { - factory_runtime.clone() as Arc + let manager = CodexTerminalLaunchManager::new(Box::new(move |_plan| { + let rt = factory_runtime.clone() as Arc; + Box::pin(async move { rt }) })); let launch = manager .plan_create_with_retry_uncancellable( @@ -420,7 +481,10 @@ async fn discard_sync_tears_down_an_unadopted_plan() { let runtime = FakeRuntime::start().await; let factory_runtime = runtime.clone(); let manager = CodexTerminalLaunchManager::with_plan_budget( - Box::new(move || factory_runtime.clone() as std::sync::Arc), + Box::new(move |_plan| { + let rt = factory_runtime.clone() as std::sync::Arc; + Box::pin(async move { rt }) + }), 2, std::time::Duration::from_secs(30), 64, @@ -470,7 +534,10 @@ fn discard_sync_outside_runtime_context_does_not_panic() { let runtime = FakeRuntime::start().await; let factory_runtime = runtime.clone(); let manager = CodexTerminalLaunchManager::with_plan_budget( - Box::new(move || factory_runtime.clone() as std::sync::Arc), + Box::new(move |_plan| { + let rt = factory_runtime.clone() as std::sync::Arc; + Box::pin(async move { rt }) + }), 2, std::time::Duration::from_secs(30), 64, @@ -498,8 +565,9 @@ async fn manager_shutdown_tears_down_adopted_and_unadopted_and_rejects_new_plans // launches the Rust manager keys, since server exit ends those terminals too. let runtime = FakeRuntime::start().await; let factory_runtime = runtime.clone(); - let manager = CodexTerminalLaunchManager::new(Box::new(move || { - factory_runtime.clone() as Arc + let manager = CodexTerminalLaunchManager::new(Box::new(move |_plan| { + let rt = factory_runtime.clone() as Arc; + Box::pin(async move { rt }) })); // One adopted launch + one unadopted plan. @@ -547,8 +615,9 @@ async fn manager_shutdown_tears_down_adopted_and_unadopted_and_rejects_new_plans async fn manager_exit_for_unknown_terminal_is_a_noop() { let runtime = FakeRuntime::start().await; let factory_runtime = runtime.clone(); - let manager = CodexTerminalLaunchManager::new(Box::new(move || { - factory_runtime.clone() as Arc + let manager = CodexTerminalLaunchManager::new(Box::new(move |_plan| { + let rt = factory_runtime.clone() as Arc; + Box::pin(async move { rt }) })); manager.notify_terminal_exit("never-created"); } @@ -595,10 +664,11 @@ fn blocking_test_runtime_factory() -> ( ) { let release = Arc::new(tokio::sync::Notify::new()); let factory_release = release.clone(); - let factory: freshell_codex::launch_lifecycle::CodexRuntimeFactory = Box::new(move || { - Arc::new(BlockingRuntime { + let factory: freshell_codex::launch_lifecycle::CodexRuntimeFactory = Box::new(move |_plan| { + let rt = Arc::new(BlockingRuntime { release: factory_release.clone(), - }) as Arc + }) as Arc; + Box::pin(async move { rt }) }); (factory, release) } @@ -698,12 +768,13 @@ async fn eight_restore_class_plans_queue_and_drain_without_error() { let in_flight = std::sync::Arc::new(AtomicUsize::new(0)); let peak = std::sync::Arc::new(AtomicUsize::new(0)); let (rt_in, rt_peak) = (in_flight.clone(), peak.clone()); - let factory: freshell_codex::launch_lifecycle::CodexRuntimeFactory = Box::new(move || { - std::sync::Arc::new(CountingRuntime { + let factory: freshell_codex::launch_lifecycle::CodexRuntimeFactory = Box::new(move |_plan| { + let rt = std::sync::Arc::new(CountingRuntime { in_flight: rt_in.clone(), peak: rt_peak.clone(), plan_delay: std::time::Duration::from_millis(200), - }) as std::sync::Arc + }) as std::sync::Arc; + Box::pin(async move { rt }) }); // wait = 200ms: 8 plans / 2 permits * 200ms = ~800ms of queueing. // Interactive would die; Restore must drain. @@ -887,8 +958,9 @@ async fn spawned_runtime_launches_the_app_server_and_relays_through_the_proxy() fake_app_server_command(), )); let spawn_runtime = runtime.clone(); - let planner = CodexLaunchPlanner::new(Box::new(move || { - spawn_runtime.clone() as Arc + let planner = CodexLaunchPlanner::new(Box::new(move |_plan| { + let rt = spawn_runtime.clone() as Arc; + Box::pin(async move { rt }) })); let launch = planner @@ -944,14 +1016,335 @@ async fn spawned_runtime_launches_the_app_server_and_relays_through_the_proxy() } } +// ── Task 7: reattach failure falls back to a fresh spawn through the plan retry ──── + +/// Task 7 (kata da92): the plan-aware factory claims a survivor for a resume +/// plan. A survivor that DIED between boot reconcile and the restore is +/// pruned by the claim-time re-verification (record removed, NOTHING +/// signalled — the pid is dead), the claim returns `None`, and the SAME +/// factory invocation falls through to the fresh spawn: the launch succeeds +/// within the retry budget, served by a fresh fixture. +#[tokio::test] +async fn plan_retry_falls_back_to_fresh_spawn_after_reattach_failure() { + let (_dir, store) = temp_sidecar_store(); + + // A survivor record for s-1 whose process is ALIVE at boot reconcile: + // this test's own `sleep 300` child, with its REAL /proc evidence. + let mut sleep_child = std::process::Command::new("sleep") + .arg("300") + .spawn() + .expect("spawn this test's own sleep child"); + let dead_pid = sleep_child.id(); + // Wait for exec to complete so the captured cmdline is really `sleep 300` + // (the sidecar_reconcile_tests post-fork/pre-exec flake guard). + let want = vec!["sleep".to_string(), "300".to_string()]; + let exec_deadline = std::time::Instant::now() + Duration::from_secs(5); + while proc_cmdline(dead_pid as i32).as_ref() != Some(&want) { + assert!( + std::time::Instant::now() < exec_deadline, + "sleep child failed to exec within 5s" + ); + std::thread::sleep(Duration::from_millis(2)); + } + // The record's ws_url points at a loopback ephemeral port NOTHING + // listens on (bound, read, dropped) — never dialed here (the claim + // refuses the dead pid first), and fail-fast if it ever were. + let unused_ws_url = { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port"); + let port = listener.local_addr().expect("local_addr").port(); + drop(listener); + format!("ws://127.0.0.1:{port}") + }; + let dead_record = CodexSidecarRecord { + record_version: SIDECAR_RECORD_VERSION, + ownership_id: "codex-sidecar-a7000002-bbbb-4bbb-8bbb-bbbbbbbbbbbb".to_string(), + pid: dead_pid, + starttime: proc_starttime(dead_pid as i32).expect("live child has a starttime"), + cmdline: want, + ws_url: unused_ws_url, + session_id: Some("s-1".to_string()), + terminal_id: None, + server_instance_id: "srv-prev".to_string(), + created_at: 1_700_000_000_000, + updated_at: 1_700_000_000_001, + state: SidecarRecordState::Active, + }; + store.write(&dead_record).expect("write survivor record"); + let (reconciler, report) = SidecarReconciler::boot_reconcile(store.clone()); + assert_eq!(report.held, 1, "the survivor is held at boot"); + let reconciler = Arc::new(reconciler); + + // …then the sidecar DIES before the restore claims it (kill + reap OUR + // OWN child): the claim-time re-verification prunes the record and the + // reattach arm never mints — no signal is ever sent. + sleep_child.kill().expect("kill own sleep child"); + sleep_child.wait().expect("reap own sleep child"); + + // The plan-aware factory: the production selection shape, with the spawn + // fallback pinned to the committed fixture (never the real `codex`). + let factory_reconciler = reconciler.clone(); + let factory_store = store.clone(); + let planner = CodexLaunchPlanner::new(Box::new(move |plan| { + let reconciler = factory_reconciler.clone(); + let store = factory_store.clone(); + Box::pin(async move { + if let Some(session_id) = plan.session_id.as_deref() { + if let Some(record) = reconciler.claim_for_session(session_id).await { + return Arc::new(ReattachedCodexAppServerRuntime::new(record, store)) + as Arc; + } + } + Arc::new(SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + store, + )) as Arc + }) + })); + + let tmp = std::env::temp_dir().join(format!("freshell-codex-t7-{}", std::process::id())); + std::fs::create_dir_all(&tmp).unwrap(); + let launch = planner + .plan_create_with_retry( + &CodexLaunchPlanInput { + cwd: Some(tmp.to_str().unwrap()), + resume_session_id: Some("s-1"), + ..Default::default() + }, + 2, + /* retry_delay_ms */ 1, + ) + .await + .expect("the resume plan must fall back to a fresh spawn"); + + // The dead survivor's record is GONE (pruned at claim) and the store's + // one record is the FRESH spawn's — a different sidecar entirely. + assert_eq!(reconciler.unclaimed_len(), 0, "the dead record left held"); + let records = store.load_all(); + assert_eq!( + records.len(), + 1, + "exactly the fresh spawn's record remains: {records:?}" + ); + assert_ne!( + records[0].ownership_id, dead_record.ownership_id, + "the dead survivor's record was removed" + ); + assert_ne!( + records[0].pid, dead_pid, + "the launch is served by a FRESH sidecar" + ); + + // The fresh fixture actually serves the launch: an initialize round trip + // relays through the planned proxy. + let (mut tui, _) = connect_async(&launch.remote_ws_url).await.unwrap(); + tui.send(Message::Text( + json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}).to_string(), + )) + .await + .unwrap(); + let reply = loop { + let msg = timeout(RECV_TIMEOUT, tui.next()) + .await + .expect("timed out waiting for the initialize reply through the proxy") + .expect("proxy closed before replying") + .unwrap(); + if let Message::Text(text) = msg { + let value: serde_json::Value = serde_json::from_str(&text).unwrap(); + if value.get("id") == Some(&json!(1)) { + break value; + } + } + }; + assert!(reply.get("result").is_some(), "initialize failed: {reply}"); + + launch + .sidecar + .shutdown() + .await + .expect("teardown the fresh spawn"); + assert!( + store.load_all().is_empty(), + "teardown scrubs the fresh spawn's record" + ); +} + +/// Task 7 review follow-up: the TRUE retry path. The claim SUCCEEDS (the +/// survivor's /proc evidence verifies) but the minted +/// [`ReattachedCodexAppServerRuntime`]'s `ensure_ready` FAILS (the record's +/// ws_url points at a port where nothing listens), which reaps the +/// verified-but-unusable survivor's tree (the test's OWN fixture child) and +/// removes its record; `plan_create`'s cleanup-on-plan-failure tears the +/// sidecar down and the retry loop's SECOND factory invocation finds +/// nothing left to claim (claims are one-shot) and spawns fresh. +#[tokio::test] +async fn plan_retry_spawns_fresh_after_claimed_reattach_ensure_ready_fails() { + let (_dir, store) = temp_sidecar_store(); + + // The survivor: this test's OWN fake app-server fixture on loopback + // ephemeral port A (spawn shape copied from + // `sidecar_reconcile_tests::spawn_own_fake_app_server`; test binaries + // cannot share code — the repo's copy-with-attribution convention). + let survivor_ownership = "codex-sidecar-a7000003-cccc-4ccc-8ccc-cccccccccccc"; + let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs"); + let bind_unused_ws_url = || { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port"); + let port = listener.local_addr().expect("local_addr").port(); + drop(listener); + format!("ws://127.0.0.1:{port}") + }; + let listen_ws_url = bind_unused_ws_url(); + let mut survivor = tokio::process::Command::new("node") + .arg(&fixture) + .arg("--listen") + .arg(&listen_ws_url) + .env(CODEX_SIDECAR_OWNERSHIP_ENV, survivor_ownership) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .expect("spawn this test's own fake app-server"); + let survivor_pid = survivor.id().expect("live fixture pid"); + // Wait for the WS listener: by then exec has long completed, so the + // /proc evidence captured below is really the fixture's (no + // post-fork/pre-exec cmdline flake). + let listen_deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + if let Ok(Ok((probe, _response))) = + timeout(Duration::from_secs(1), connect_async(&listen_ws_url)).await + { + drop(probe); + break; + } + if let Ok(Some(status)) = survivor.try_wait() { + panic!("fake app-server exited before listening: {status}"); + } + assert!( + tokio::time::Instant::now() < listen_deadline, + "fake app-server WS never came up" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + + // The record: the fixture's REAL /proc evidence (claim-time + // re-verification: Verified) BUT a ws_url on ephemeral port B where + // NOTHING listens (bind-then-drop) — the reattach probe fails fast. + let survivor_record = CodexSidecarRecord { + record_version: SIDECAR_RECORD_VERSION, + ownership_id: survivor_ownership.to_string(), + pid: survivor_pid, + starttime: proc_starttime(survivor_pid as i32).expect("live fixture has a starttime"), + cmdline: proc_cmdline(survivor_pid as i32).expect("live fixture has a cmdline"), + ws_url: bind_unused_ws_url(), + session_id: Some("s-1".to_string()), + terminal_id: None, + server_instance_id: "srv-prev".to_string(), + created_at: 1_700_000_000_000, + updated_at: 1_700_000_000_001, + state: SidecarRecordState::Active, + }; + store + .write(&survivor_record) + .expect("write survivor record"); + let (reconciler, report) = SidecarReconciler::boot_reconcile(store.clone()); + assert_eq!(report.held, 1, "the survivor is held at boot"); + let reconciler = Arc::new(reconciler); + + // The plan-aware factory (production selection shape + an invocation + // counter): attempt 1 claims and mints the reattach runtime; attempt 2 + // finds the one-shot claim consumed and spawns the fixture fresh. + let factory_invocations = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let invocations = factory_invocations.clone(); + let factory_reconciler = reconciler.clone(); + let factory_store = store.clone(); + let planner = CodexLaunchPlanner::new(Box::new(move |plan| { + factory_invocations.fetch_add(1, Ordering::SeqCst); + let reconciler = factory_reconciler.clone(); + let store = factory_store.clone(); + Box::pin(async move { + if let Some(session_id) = plan.session_id.as_deref() { + if let Some(record) = reconciler.claim_for_session(session_id).await { + return Arc::new(ReattachedCodexAppServerRuntime::new(record, store)) + as Arc; + } + } + Arc::new(SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + store, + )) as Arc + }) + })); + + let tmp = std::env::temp_dir().join(format!("freshell-codex-t7r-{}", std::process::id())); + std::fs::create_dir_all(&tmp).unwrap(); + let launch = planner + .plan_create_with_retry( + &CodexLaunchPlanInput { + cwd: Some(tmp.to_str().unwrap()), + resume_session_id: Some("s-1"), + ..Default::default() + }, + 2, + /* retry_delay_ms */ 1, + ) + .await + .expect("attempt 2 must spawn fresh after the claimed reattach fails"); + + assert_eq!( + invocations.load(Ordering::SeqCst), + 2, + "the failed reattach consumes attempt 1; the fresh spawn is attempt 2" + ); + assert_eq!( + reconciler.unclaimed_len(), + 0, + "the one-shot claim was consumed by attempt 1" + ); + + // The verified-but-unusable survivor was REAPED by the reattach failure + // arm (an unusable tracked sidecar must not leak) — the fixture is this + // test's own child, and nothing else was ever signalled. + wait_pid_gone(survivor_pid).await; + let _ = survivor.wait().await; // reap our own child + + // The store holds exactly the FRESH spawn's record — the survivor's is + // gone, and the launch is served by a different sidecar entirely. + let records = store.load_all(); + assert_eq!( + records.len(), + 1, + "exactly the fresh spawn's record remains: {records:?}" + ); + assert_ne!( + records[0].ownership_id, survivor_record.ownership_id, + "the unusable survivor's record was removed" + ); + assert_ne!( + records[0].pid, survivor_pid, + "the launch is served by a FRESH sidecar" + ); + + launch + .sidecar + .shutdown() + .await + .expect("teardown the fresh spawn"); + assert!( + store.load_all().is_empty(), + "teardown scrubs the fresh spawn's record" + ); +} + // ────── S5.c persistence plumbing (mark_candidate_persisted, fail_candidate_capture) ────── #[tokio::test] async fn mark_candidate_persisted_is_a_noop_for_unknown_terminals() { let runtime = FakeRuntime::start().await; let factory_runtime = runtime.clone(); - let manager = CodexTerminalLaunchManager::new(Box::new(move || { - factory_runtime.clone() as Arc + let manager = CodexTerminalLaunchManager::new(Box::new(move |_plan| { + let rt = factory_runtime.clone() as Arc; + Box::pin(async move { rt }) })); // Must not panic, hang, or error for a terminal that was never adopted. manager.mark_candidate_persisted("no-such-terminal").await; @@ -962,8 +1355,9 @@ async fn mark_candidate_persisted_is_a_noop_for_unknown_terminals() { // no-op methods on unknown terminals does not affect it (observable: the // adopted launch can still be shut down cleanly). let planner_runtime = runtime.clone(); - let planner = CodexLaunchPlanner::new(Box::new(move || { - planner_runtime.clone() as Arc + let planner = CodexLaunchPlanner::new(Box::new(move |_plan| { + let rt = planner_runtime.clone() as Arc; + Box::pin(async move { rt }) })); let launch = planner .plan_create(&CodexLaunchPlanInput::default()) @@ -981,3 +1375,528 @@ async fn mark_candidate_persisted_is_a_noop_for_unknown_terminals() { // The adopted terminal is unaffected (observable: manager can shut down cleanly). manager.shutdown().await; } + +/// Task 4: the manager seam forwards a captured session/thread id to the +/// ADOPTED terminal's runtime; unknown terminal ids are a silent no-op +/// (mirrors `mark_candidate_persisted`). +#[tokio::test] +async fn manager_note_session_id_reaches_adopted_runtime() { + let runtime = FakeRuntime::start().await; + let factory_runtime = runtime.clone(); + let manager = CodexTerminalLaunchManager::new(Box::new(move |_plan| { + let rt = factory_runtime.clone() as Arc; + Box::pin(async move { rt }) + })); + let launch = manager + .plan_create_with_retry_uncancellable( + &CodexLaunchPlanInput::default(), + 5, + LaunchClass::Interactive, + ) + .await + .unwrap(); + manager.adopt("term-sid", launch, 0).await.unwrap(); + + // Unknown terminal id: silent no-op — nothing reaches the runtime. + manager + .note_session_id("no-such-terminal", "s-ignored") + .await; + assert!( + runtime.noted_session_ids.lock().unwrap().is_empty(), + "an unknown terminal id must not forward to any runtime" + ); + + manager.note_session_id("term-sid", "s-1").await; + assert_eq!( + runtime.noted_session_ids.lock().unwrap().as_slice(), + &["s-1".to_string()], + "the adopted terminal's runtime must see the noted session id" + ); + + manager.shutdown().await; +} + +// ────── Task 3: durable sidecar records — persist on spawn, scrub on teardown, +// survive server death (kata ynfn "surviving restarts is a feature") ────────────── + +/// A lock-free store over its own tempdir (the tests-and-verification +/// construction, `sidecar_store.rs` docs). The `TempDir` guard is returned so +/// the record files outlive every runtime the test builds over the store. +fn temp_sidecar_store() -> (tempfile::TempDir, Arc) { + let dir = tempfile::tempdir().expect("tempdir for the sidecar store"); + let store = Arc::new(CodexSidecarStore::new(dir.path().to_path_buf())); + (dir, store) +} + +/// Poll until `pid` reads gone from `/proc`. `proc_starttime` returns `None` +/// for reaped AND zombie (Z) states, so this stays robust to tokio's +/// background orphan-reaping timing. +async fn wait_pid_gone(pid: u32) { + let deadline = tokio::time::Instant::now() + RECV_TIMEOUT; + loop { + if proc_starttime(pid as i32).is_none() { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "pid {pid} still alive past the {RECV_TIMEOUT:?} deadline" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +#[tokio::test] +async fn ensure_ready_persists_a_verified_sidecar_record() { + let (_dir, store) = temp_sidecar_store(); + let runtime = SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + store.clone(), + ); + + let ready = runtime.ensure_ready(None).await.expect("ensure_ready"); + let pid = runtime.child_pid().await.expect("child pid"); + + let records = store.load_all(); + assert_eq!(records.len(), 1, "exactly one record: {records:?}"); + let record = &records[0]; + assert_eq!(record.state, SidecarRecordState::Active); + assert_eq!(record.pid, pid, "record pid is the live child's pid"); + assert_eq!( + record.ws_url, ready.ws_url, + "record ws_url is the returned one" + ); + assert_eq!( + verify_sidecar_identity(record), + IdentityVerdict::Verified, + "(starttime, cmdline) must verify against the live child" + ); + + runtime + .shutdown() + .await + .expect("shutdown cleans up the child"); +} + +#[tokio::test] +async fn runtime_shutdown_removes_the_sidecar_record() { + let (_dir, store) = temp_sidecar_store(); + let runtime = SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + store.clone(), + ); + runtime.ensure_ready(None).await.expect("ensure_ready"); + let pid = runtime.child_pid().await.expect("child pid"); + assert_eq!(store.load_all().len(), 1, "record present before shutdown"); + + runtime.shutdown().await.expect("shutdown"); + + assert!( + store.load_all().is_empty(), + "explicit shutdown must scrub the record" + ); + wait_pid_gone(pid).await; +} + +#[tokio::test] +async fn update_ownership_metadata_enriches_the_record() { + let (_dir, store) = temp_sidecar_store(); + let runtime = SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + store.clone(), + ); + runtime.ensure_ready(None).await.expect("ensure_ready"); + + runtime + .update_ownership_metadata("term-42".to_string(), 7) + .await + .expect("update_ownership_metadata"); + + let records = store.load_all(); + assert_eq!(records.len(), 1, "still exactly one record: {records:?}"); + assert_eq!( + records[0].terminal_id.as_deref(), + Some("term-42"), + "adopt must enrich the record with the terminal id" + ); + + runtime + .shutdown() + .await + .expect("shutdown cleans up the child"); +} + +/// Task 4: `note_session_id` rewrites the durable record with the codex +/// session/thread id — the restore-time reattach key (katas ynfn/da92). +#[tokio::test] +async fn spawned_runtime_note_session_id_enriches_the_record() { + let (_dir, store) = temp_sidecar_store(); + let runtime = SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + store.clone(), + ); + runtime.ensure_ready(None).await.expect("ensure_ready"); + + runtime + .note_session_id("s-1".to_string()) + .await + .expect("note_session_id"); + + let records = store.load_all(); + assert_eq!(records.len(), 1, "still exactly one record: {records:?}"); + assert_eq!( + records[0].session_id.as_deref(), + Some("s-1"), + "note_session_id must enrich the record with the session id" + ); + + runtime + .shutdown() + .await + .expect("shutdown cleans up the child"); +} + +#[tokio::test] +async fn spawned_sidecar_survives_runtime_drop_without_shutdown() { + let (_dir, store) = temp_sidecar_store(); + let runtime = SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + store.clone(), + ); + runtime.ensure_ready(None).await.expect("ensure_ready"); + let pid = runtime.child_pid().await.expect("child pid"); + + // Server death without shutdown(): kill_on_drop is OFF for tracked spawns. + drop(runtime); + + // Give any (wrong) kill-on-drop a chance to land before asserting liveness. + tokio::time::sleep(Duration::from_millis(300)).await; + assert!( + proc_starttime(pid as i32).is_some(), + "detached sidecar (pid {pid}) must survive the runtime drop" + ); + + // The record still exists: an uncleanly-dying server leaves a TRACKED, + // reconcilable sidecar — not an invisible orphan (the whole point). + let records = store.load_all(); + assert_eq!( + records.len(), + 1, + "record must survive the drop: {records:?}" + ); + let record = &records[0]; + assert_eq!(record.pid, pid); + + // PROCESS SAFETY: kill ONLY the child this test spawned, and only after + // re-verifying (pid, starttime, cmdline) identity via the record. + assert_eq!(verify_sidecar_identity(record), IdentityVerdict::Verified); + // SAFETY: plain FFI signal send to our own verified child pid. + unsafe { + assert_eq!( + libc::kill(pid as i32, libc::SIGTERM), + 0, + "SIGTERM this test's own child" + ); + } + wait_pid_gone(pid).await; +} + +#[tokio::test] +async fn drop_without_shutdown_with_disabled_store_keeps_the_kill_on_drop_backstop() { + let runtime = SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + Arc::new(CodexSidecarStore::disabled()), + ); + runtime.ensure_ready(None).await.expect("ensure_ready"); + let pid = runtime.child_pid().await.expect("child pid"); + + // A record-less sidecar must NEVER outlive the server: detaching it would + // be the silently-orphaned ynfn hole with no reconcile path, so untracked + // spawns keep today's kill_on_drop backstop. + drop(runtime); + + wait_pid_gone(pid).await; +} + +// ────── Task 10: server-shutdown retention (katas ynfn/da92) ────── +// +// `begin_shutdown_retention()` flips the manager into server-shutdown mode: +// adopted (terminal-owned) TRACKED sidecars are retained across the restart +// (record → `Retained{reason:"server-shutdown"}`, process never signalled); +// unadopted planner sidecars and record-less (disabled-store) sidecars are +// torn down exactly as today — retaining a record-less sidecar would orphan +// it silently with no reconcile path (the ynfn hole). + +/// Manager over a single pre-built spawned runtime (the Task 3 test shape: +/// per-instance store injection, never the process-global handle). +fn manager_over(runtime: Arc) -> CodexTerminalLaunchManager { + CodexTerminalLaunchManager::new(Box::new(move |_plan| { + let rt = runtime.clone() as Arc; + Box::pin(async move { rt }) + })) +} + +#[tokio::test] +async fn shutdown_retention_retains_adopted_sidecars_and_records_reason() { + let (_dir, store) = temp_sidecar_store(); + let runtime = Arc::new(SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + store.clone(), + )); + let manager = manager_over(runtime.clone()); + + let launch = manager + .plan_create_with_retry_uncancellable( + &CodexLaunchPlanInput::default(), + 1, + LaunchClass::Interactive, + ) + .await + .expect("plan"); + manager + .adopt("term-retained", launch, 0) + .await + .expect("adopt"); + let pid = runtime.child_pid().await.expect("child pid"); + + manager.begin_shutdown_retention(); + manager.shutdown().await; + + // The fixture pid is STILL ALIVE: retention never signals. (Give any + // wrong kill a moment to land before asserting liveness — the + // spawned_sidecar_survives_runtime_drop_without_shutdown pattern.) + tokio::time::sleep(Duration::from_millis(300)).await; + assert!( + proc_starttime(pid as i32).is_some(), + "retained sidecar (pid {pid}) must survive the server shutdown" + ); + + // The record carries the reason a restarted server reconciles against. + let records = store.load_all(); + assert_eq!(records.len(), 1, "exactly one record: {records:?}"); + assert_eq!(records[0].pid, pid); + assert_eq!( + records[0].state, + SidecarRecordState::Retained { + reason: "server-shutdown".to_string() + }, + "retention must record its reason" + ); + + // PROCESS SAFETY cleanup: reap this test's OWN fixture pid, identity + // re-verified via the record immediately before the signal. + assert_eq!( + verify_sidecar_identity(&records[0]), + IdentityVerdict::Verified + ); + // SAFETY: plain FFI signal send to our own verified child pid. + unsafe { + assert_eq!( + libc::kill(pid as i32, libc::SIGTERM), + 0, + "SIGTERM this test's own child" + ); + } + wait_pid_gone(pid).await; +} + +/// Final-review H3c: even when `prepare_retention` fails (record rewrite +/// error — retention already logs loudly), the DECISION to retain stands: a +/// later `shutdown()` (a double-fired PTY exit hook, `manager.shutdown()`'s +/// drain) must NOT kill the sidecar we chose to retain. +#[tokio::test] +async fn retain_failure_still_blocks_a_later_shutdown_kill() { + let runtime = FakeRuntime::start().await; + runtime.fail_prepare_retention.store(true, Ordering::SeqCst); + let planner = planner_for(runtime.clone()); + let launch = planner + .plan_create(&CodexLaunchPlanInput::default()) + .await + .expect("plan"); + + launch + .sidecar + .retain("server-shutdown") + .await + .expect_err("the prepare_retention failure propagates to the caller"); + + // The retention decision stands: shutdown() must no-op via the + // idempotence flag, never reaching the runtime's kill path. + launch + .sidecar + .shutdown() + .await + .expect("post-retain shutdown is an idempotent no-op"); + assert_eq!( + runtime.shutdown_calls.load(Ordering::SeqCst), + 0, + "a sidecar we decided to retain must never be killed by a later shutdown()" + ); +} + +#[tokio::test] +async fn shutdown_still_tears_down_unadopted_planner_sidecars() { + let (_dir, store) = temp_sidecar_store(); + let runtime = Arc::new(SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + store.clone(), + )); + let manager = manager_over(runtime.clone()); + + // Plan WITHOUT adopt: a mid-plan sidecar has no pane to reattach to (and + // a fresh-plan proxy may hold the candidate timer) — still torn down. + let _unadopted = manager + .plan_create_with_retry_uncancellable( + &CodexLaunchPlanInput::default(), + 1, + LaunchClass::Interactive, + ) + .await + .expect("plan"); + let pid = runtime.child_pid().await.expect("child pid"); + + manager.begin_shutdown_retention(); + manager.shutdown().await; + + wait_pid_gone(pid).await; + assert!( + store.load_all().is_empty(), + "unadopted teardown must scrub the record" + ); +} + +#[tokio::test] +async fn retention_with_disabled_store_tears_down_as_today() { + // Task 3's conditional detach: disabled store ⇒ kill_on_drop(true), NO + // record. The retention gate says record-less sidecars are NEVER + // retained — "retaining" one would orphan it silently (the ynfn hole). + let runtime = Arc::new(SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + Arc::new(CodexSidecarStore::disabled()), + )); + let manager = manager_over(runtime.clone()); + + let launch = manager + .plan_create_with_retry_uncancellable( + &CodexLaunchPlanInput::default(), + 1, + LaunchClass::Interactive, + ) + .await + .expect("plan"); + manager + .adopt("term-untracked", launch, 0) + .await + .expect("adopt"); + let pid = runtime.child_pid().await.expect("child pid"); + + manager.begin_shutdown_retention(); + manager.shutdown().await; + + wait_pid_gone(pid).await; +} + +#[tokio::test] +async fn notify_terminal_exit_retains_under_retention_flag() { + let (_dir, store) = temp_sidecar_store(); + // One runtime per plan (the exit hook consumes its adopted entry, so the + // two arms need independent sidecars), each recorded for pid access. + let spawned: Arc>>> = + Arc::new(Mutex::new(Vec::new())); + let factory_store = store.clone(); + let factory_spawned = spawned.clone(); + let manager = CodexTerminalLaunchManager::new(Box::new(move |_plan| { + let runtime = Arc::new(SpawnedCodexAppServerRuntime::with_command_and_store( + fake_app_server_command(), + factory_store.clone(), + )); + factory_spawned.lock().unwrap().push(runtime.clone()); + let rt = runtime as Arc; + Box::pin(async move { rt }) + })); + + // Arm 1 — flag OFF (existing behavior, asserted so the contrast is + // pinned): the exit hook hands the entry to the teardown worker, which + // reaps the sidecar and scrubs its record. + let launch_a = manager + .plan_create_with_retry_uncancellable( + &CodexLaunchPlanInput::default(), + 1, + LaunchClass::Interactive, + ) + .await + .expect("plan a"); + manager + .adopt("term-exit-a", launch_a, 0) + .await + .expect("adopt a"); + let runtime_a = spawned.lock().unwrap()[0].clone(); + let pid_a = runtime_a.child_pid().await.expect("pid a"); + + manager.notify_terminal_exit("term-exit-a"); + wait_pid_gone(pid_a).await; + // The record scrub is a separate write after the reap — poll for it. + let deadline = tokio::time::Instant::now() + RECV_TIMEOUT; + while !store.load_all().is_empty() { + assert!( + tokio::time::Instant::now() < deadline, + "flag-off teardown must scrub the record" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + + // Arm 2 — flag ON: the SAME exit hook retains instead of reaping. + let launch_b = manager + .plan_create_with_retry_uncancellable( + &CodexLaunchPlanInput::default(), + 1, + LaunchClass::Interactive, + ) + .await + .expect("plan b"); + manager + .adopt("term-exit-b", launch_b, 0) + .await + .expect("adopt b"); + let runtime_b = spawned.lock().unwrap()[1].clone(); + let pid_b = runtime_b.child_pid().await.expect("pid b"); + + manager.begin_shutdown_retention(); + manager.notify_terminal_exit("term-exit-b"); + + // Retention flows through the async worker: poll for the Retained write. + let deadline = tokio::time::Instant::now() + RECV_TIMEOUT; + let record = loop { + let mut records = store.load_all(); + if records.len() == 1 && matches!(records[0].state, SidecarRecordState::Retained { .. }) { + break records.remove(0); + } + assert!( + tokio::time::Instant::now() < deadline, + "retained record never appeared: {records:?}" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + }; + assert_eq!(record.pid, pid_b); + assert_eq!( + record.state, + SidecarRecordState::Retained { + reason: "server-shutdown".to_string() + } + ); + assert!( + proc_starttime(pid_b as i32).is_some(), + "retained sidecar (pid {pid_b}) must be alive after the exit hook" + ); + + // PROCESS SAFETY cleanup: verified reap of this test's own fixture pid. + assert_eq!(verify_sidecar_identity(&record), IdentityVerdict::Verified); + // SAFETY: plain FFI signal send to our own verified child pid. + unsafe { + assert_eq!( + libc::kill(pid_b as i32, libc::SIGTERM), + 0, + "SIGTERM this test's own child" + ); + } + wait_pid_gone(pid_b).await; +} diff --git a/crates/freshell-server/src/main.rs b/crates/freshell-server/src/main.rs index 69a20e78b..346e7d964 100644 --- a/crates/freshell-server/src/main.rs +++ b/crates/freshell-server/src/main.rs @@ -407,6 +407,20 @@ async fn main() -> ExitCode { home.as_ref() .map(|h| h.join(".freshell").join("pane-ledger")), )); + // Codex sidecar record store (katas ynfn/da92, Task 10 wiring): the + // flock'd single-writer store of the `codex app-server` sidecars that + // terminal panes spawn, so a restarted server can reattach to (or + // conservatively reap) survivors. Same root policy as the pane ledger + // above (no home => disabled no-op store); the `rust-` prefix keeps it + // disjoint from Node's `~/.freshell/codex-sidecars/` store. The global + // handle is the Task 3 seam every tracked spawn records through. + let codex_sidecar_store = std::sync::Arc::new( + freshell_codex::sidecar_store::CodexSidecarStore::new_locked( + home.as_ref() + .map(|h| h.join(".freshell").join("rust-codex-sidecars")), + ), + ); + freshell_codex::sidecar_store::set_codex_sidecar_store(codex_sidecar_store.clone()); // OpenCode terminal-pane restore fix // (`docs/plans/2026-07-18-opencode-terminal-restore-spec.md`): the // opencode locator, resolved against the SAME `default_opencode_data_home()` @@ -1029,6 +1043,53 @@ async fn main() -> ExitCode { } } + // Codex sidecar boot reconcile (Task 10, katas ynfn/da92): load the + // previous generation's sidecar records, prune stale rows (Dead/Mismatch + // — remove only, NEVER signal), hold verified survivors claimable for + // restore-time reattach, then arm the grace-delayed conservative sweep. + { + // Disablement must be LOUD (A10 validation, reports/V6.md): the + // restart script provably waits for old-process exit, so lock + // contention is not expected on the normal path — but a same-HOME + // scratch server on another port (e.g. the evidenced `--port 3499` + // runs) silently loses the flock, disabling sidecar tracking for + // this whole generation with no timing race at all. + if !codex_sidecar_store.is_enabled() { + tracing::error!( + "codex_sidecar_store_disabled: sidecar records are NOT being written or \ + reconciled this generation (lock contention or no resolvable home) — codex \ + terminal-pane sidecars spawned now will NOT survive a restart of this process" + ); + } + let (reconciler, report) = + freshell_codex::sidecar_reconcile::SidecarReconciler::boot_reconcile( + codex_sidecar_store.clone(), + ); + tracing::info!( + codex_sidecar_store_enabled = codex_sidecar_store.is_enabled(), + loaded = report.loaded, + pruned_dead = report.pruned_dead, + pruned_mismatch = report.pruned_mismatch, + held = report.held, + "codex_sidecar_boot_reconcile: previous generation's sidecar records reconciled" + ); + let reconciler = std::sync::Arc::new(reconciler); + freshell_codex::sidecar_reconcile::set_codex_sidecar_reconciler(reconciler.clone()); + // The grace-delayed conservative sweep (Task 9): restores get the + // whole grace window (default 30m — the incident's restores arrived + // 18m post-boot) to claim survivors before any unclaimed one is + // probed and, only when verified AND reap-eligible, reaped. + tokio::spawn(async move { + tokio::time::sleep(freshell_codex::sidecar_sweep::reap_grace_from_env()).await; + let outcomes = reconciler.sweep_unclaimed().await; + tracing::info!( + swept = outcomes.len(), + "codex_sidecar_sweep_done: unclaimed survivors swept \ + (per-record decisions logged above)" + ); + }); + } + // P1.8 periodic GC (boot-time + periodic, spec §4.2 lifecycle). { let ledger = std::sync::Arc::clone(&pane_ledger); @@ -1755,6 +1816,28 @@ async fn main() -> ExitCode { // port matches that (already implemented before this fix). // * `fresh_codex_state.shutdown()` / `fresh_claude_state.shutdown()` — // the Codex app-server and claude Node sidecars (already implemented). + // + // SAFE-11 deliberate deviation (Task 10, kata ynfn): TRACKED codex + // terminal-pane sidecars (the launch manager's adopted, record-bearing + // spawns) are now RETAINED across this shutdown instead of reaped — + // "killing sidecars at shutdown is NOT acceptable; surviving restarts is + // a feature." Everything else above still reaps exactly as before, and + // record-less codex sidecars (disabled store / non-Linux) are still torn + // down (retaining them would orphan silently). + // + // Codex sidecar retention flag: flipped BEFORE the PTY kill below so the + // exit hooks (`notify_terminal_exit`) retain adopted terminal-pane codex + // sidecars — proxies close, tracked runtimes get + // `prepare_retention("server-shutdown")` and their records flip to + // Retained — instead of handing them to the teardown worker. + // + // Supervisor caveat (recorded, no code — reports/V6.md NA-1): the in-repo + // systemd unit is NOT installed today (restarts are script-driven). If it + // is ever adopted, its KillMode must not be `control-group` — a cgroup + // kill would slaughter the retained sidecars this deliberately keeps + // alive (`process_group(0)` detaches the pgid, not the cgroup). + freshell_codex::launch_lifecycle::CodexTerminalLaunchManager::global() + .begin_shutdown_retention(); registry.kill_all(); // A10 re-sweep (V3): kill_all() snapshots the id set ONCE // (registry.rs:889-892); a detached gated create settling during the @@ -1774,13 +1857,16 @@ async fn main() -> ExitCode { // SDK abort → SIGKILL straggler + `/proc` ownership sweep) so a freshclaude T2 run // leaves no orphaned sidecar or claude CLI grandchild. fresh_claude_state.shutdown().await; - // DEV-0006 S4: stop accepting codex managed-launch plans and tear down every - // launch sidecar + remote proxy the terminal-launch manager still owns (mirrors - // legacy's close-time `codexLaunchPlanner.shutdown()` among the shutdown owners, - // `server/index.ts:981-1049`). Runs AFTER `registry.kill_all()` above, so adopted - // launches whose exit hooks already queued teardown are simply re-shut-down - // (idempotent) and unadopted in-flight plans are reaped here. No-op when the - // managed-launch flag never planned anything. + // DEV-0006 S4 + Task 10 retention: stop accepting codex managed-launch plans + // (mirrors legacy's close-time `codexLaunchPlanner.shutdown()` among the shutdown + // owners, `server/index.ts:981-1049`). With the retention flag set above, this now + // RETAINS adopted terminal-pane sidecars (proxy-close + + // `prepare_retention("server-shutdown")`; records flip to Retained) and still + // tears down unadopted in-flight plans (no pane to reattach to). The runtime-level + // retention gate still reaps record-less sidecars exactly as before. Runs AFTER + // `registry.kill_all()` above, so adopted launches whose exit hooks already queued + // retention are simply re-retained (idempotent). No-op when the managed-launch + // flag never planned anything. freshell_codex::launch_lifecycle::CodexTerminalLaunchManager::global() .shutdown() .await; diff --git a/crates/freshell-server/tests/safe11_term22_shutdown_reaping.rs b/crates/freshell-server/tests/safe11_term22_shutdown_reaping.rs index aca48ad9a..ac4c4c994 100644 --- a/crates/freshell-server/tests/safe11_term22_shutdown_reaping.rs +++ b/crates/freshell-server/tests/safe11_term22_shutdown_reaping.rs @@ -17,6 +17,20 @@ //! target, and reaping is only meaningful at OS-process granularity anyway //! -- there is no in-process way to observe "did the child process actually //! die" other than asking the OS. +//! +//! DELIBERATE DEVIATION (Task 10, kata ynfn): the parity checklist's +//! acceptance text "terminate exact terminal/provider/extension trees" +//! (`docs/plans/2026-07-14-rust-tauri-parity-completion-checklist.md:615`) +//! is deliberately INVERTED for TRACKED codex terminal-pane sidecars — the +//! launch manager's adopted, record-bearing spawns are now RETAINED across a +//! graceful shutdown ("killing sidecars at shutdown is NOT acceptable — +//! surviving restarts is a feature"). This suite's coverage is untouched by +//! that: its codex leg is a freshagent-lane sidecar (`freshAgent.create +//! {sessionType:"freshcodex"}`) plus a shell PTY, both OUTSIDE the retention +//! scope, so it doubles as the tripwire that retention did not leak into the +//! freshagent lane or shell-PTY reaping. The retention behavior itself is +//! pinned by `crates/freshell-codex/tests/launch_lifecycle.rs`'s Task 10 +//! section. use std::io::Read; use std::path::{Path, PathBuf}; diff --git a/crates/freshell-ws/src/codex_proxy_route.rs b/crates/freshell-ws/src/codex_proxy_route.rs index c6ae66e1d..37ef14ed7 100644 --- a/crates/freshell-ws/src/codex_proxy_route.rs +++ b/crates/freshell-ws/src/codex_proxy_route.rs @@ -178,6 +178,12 @@ async fn route_candidate( CodexTerminalLaunchManager::global() .mark_candidate_persisted(terminal_id) .await; + // Task 4: the captured thread id is the durable sidecar record's + // restore-time reattach key — note it beside the persistence release. + // No-op without an adopted spawned runtime + enabled store. + CodexTerminalLaunchManager::global() + .note_session_id(terminal_id, &candidate.thread.id) + .await; // D-FORK: give managed panes the disk fork watch resume panes get. // `watch_fork` snapshots the sessions tree (bounded fs walk), so it // runs on the blocking pool like the association sweep's lane -- a diff --git a/crates/freshell-ws/tests/codex_sidecar_reattach_e2e.rs b/crates/freshell-ws/tests/codex_sidecar_reattach_e2e.rs new file mode 100644 index 000000000..a8c77b2e1 --- /dev/null +++ b/crates/freshell-ws/tests/codex_sidecar_reattach_e2e.rs @@ -0,0 +1,976 @@ +//! da92 end-to-end — codex pane restore reattaches to a surviving sidecar +//! through the WS door (codex sidecar lifecycle, Task 8). +//! +//! Three scenarios over the REAL Rust WS server, each sending the frozen +//! restore create (`terminal.create {mode:'codex', shell:'system', cwd, +//! restore:true, sessionRef:{provider:'codex', sessionId}}`) and playing the +//! TUI against the pane's `--remote` proxy URL: +//! +//! 1. **Reattach**: a tracked, verified SURVIVOR (this test's own fake +//! app-server child, mid-turn shape via `loadedThreadIds`) is claimed for +//! the restore — the TUI's `thread/resume` lands on the SURVIVOR (its op +//! log records it), its pid is untouched, and its durable record gains the +//! new terminal id at adopt. +//! 2. **Fresh fallback**: no tracked survivor ⇒ today's spawn path is +//! byte-compatible (the managed `--remote` 4-tuple + bel pair + resume +//! pair last) and a NEW fixture instance serves the plan (its op log +//! records the traffic). +//! 3. **da92 control**: the scripted `-32600` "active writer" rejection is +//! confined to the fresh path — the same `thread/resume` that SUCCEEDS +//! against a claimed survivor comes back as the incident-shaped error. +//! +//! This binary OWNS process env (`CODEX_CMD`, `FAKE_CODEX_APP_SERVER_BEHAVIOR`, +//! `CODEX_ARGV_CAPTURE_PATH`) — the `resume_validation_gate.rs` convention: +//! env mutation is process-global, and nothing else in this binary reads +//! these vars. The codex launch manager global is SET-ONCE per process, so +//! the scenarios are serialized on ONE never-dropped runtime (the +//! `restore_storm.rs` ground rule — the manager's lazily-armed teardown +//! worker must outlive every test fn) and swap the test-owned +//! reconciler/store through statics the one installed factory closes over. +//! +//! PROCESS SAFETY: kills ONLY pids this test spawned — the survivor fixture +//! child (reaped by the reattach teardown path, with an explicit own-child +//! kill fallback) and panes via `registry.kill`. Loopback ephemeral ports +//! only — never 3001/3002. Temp stores only — the machine's live sidecars +//! stay structurally unreachable. +//! +//! Linux-only: reattach identity evidence is `/proc`-based +//! (`verify_sidecar_identity` is Unverifiable elsewhere, so no claim path +//! exists off-Linux) and the tracked-spawn detach arm is Linux-gated. +#![cfg(target_os = "linux")] + +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::Duration; + +use futures_util::{SinkExt, StreamExt}; +use serde_json::json; +use tokio::net::TcpListener; +use tokio_tungstenite::tungstenite::Message as WsMessage; + +use freshell_codex::launch_lifecycle::{ + set_global_codex_launch_manager_for_tests, CodexTerminalLaunchManager, +}; +use freshell_codex::{ + proc_cmdline, proc_starttime, select_codex_runtime, CodexSidecarRecord, CodexSidecarStore, + SidecarReconciler, SidecarRecordState, CODEX_SIDECAR_OWNERSHIP_ENV, SIDECAR_RECORD_VERSION, +}; +use freshell_ws::WsState; + +const AUTH_TOKEN: &str = "e2e-codex-sidecar-reattach-token"; +const RECV_TIMEOUT: Duration = Duration::from_secs(20); + +// ─── the set-once global manager over a swappable test reconciler/store ────── + +static TEST_RECONCILER: Mutex>> = Mutex::new(None); +static TEST_STORE: Mutex>> = Mutex::new(None); + +/// Install the ONE process-wide launch manager (set-once): its factory +/// re-reads the statics per plan and dispatches through the REAL production +/// selection ([`select_codex_runtime`] — claim a verified survivor for resume +/// plans, else spawn), so each serialized scenario swaps in its own +/// reconciler/store. +fn install_global_manager() { + static INSTALLED: OnceLock<()> = OnceLock::new(); + INSTALLED.get_or_init(|| { + let manager = CodexTerminalLaunchManager::new(Box::new(|plan| { + // Clone the handles OUT of the statics before the await — std + // MutexGuards must never cross an await point. + let reconciler = TEST_RECONCILER.lock().unwrap().clone(); + let store = TEST_STORE.lock().unwrap().clone(); + Box::pin(async move { + select_codex_runtime(reconciler.as_ref(), store.as_ref(), plan).await + }) + })); + assert!( + set_global_codex_launch_manager_for_tests(manager), + "this binary must be the first global() toucher in its process" + ); + }); +} + +fn swap_test_reconciler( + reconciler: Option>, + store: Option>, +) { + *TEST_RECONCILER.lock().unwrap() = reconciler; + *TEST_STORE.lock().unwrap() = store; +} + +/// One tokio runtime for the WHOLE binary, never dropped (the +/// `restore_storm.rs` ground rule): the global manager's lazily-armed +/// teardown worker must outlive every test fn, so tests are +/// `#[test] fn .. { reattach_rt().block_on(async { .. }) }`. +fn reattach_rt() -> &'static tokio::runtime::Runtime { + static RT: OnceLock = OnceLock::new(); + RT.get_or_init(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("reattach runtime") + }) +} + +/// Serialize the scenarios: process env + the reconciler/store statics are +/// process-global, so exactly one scenario runs at a time. +fn test_lock() -> &'static tokio::sync::Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| tokio::sync::Mutex::new(())) +} + +/// The codex create door injects the freshell MCP server into the TUI argv +/// (`mcp_inject.rs::server_command_args`), resolving `tsx` from the freshell +/// repo root nearest the process CWD. A `.worktrees/` checkout carries no +/// `node_modules` of its own — the committed fixture already resolves `ws` +/// from the parent checkout via node's upward walk, and this shim gives the +/// MCP resolver the same reach: when the cwd-nearest freshell root lacks +/// `node_modules/tsx`, chdir to the nearest ANCESTOR freshell root that has +/// it (the parent checkout). Process-global exactly like the env this binary +/// already owns; a no-op on a dependency-bearing checkout. The injected args +/// only ever reach the fake dispatcher, which ignores them. +fn ensure_mcp_deps_resolvable() { + static DONE: OnceLock<()> = OnceLock::new(); + DONE.get_or_init(|| { + let has_tsx = + |root: &std::path::Path| root.join("node_modules/tsx/dist/loader.mjs").is_file(); + let is_freshell_root = |dir: &std::path::Path| { + std::fs::read_to_string(dir.join("package.json")) + .ok() + .and_then(|text| serde_json::from_str::(&text).ok()) + .is_some_and(|pkg| pkg.get("name").and_then(|n| n.as_str()) == Some("freshell")) + }; + // The root `mcp_inject::find_repo_root` would resolve from this cwd + // (walk up, max 5, first `package.json` named "freshell"). + let cwd = std::env::current_dir().expect("cwd"); + let mut nearest = cwd.clone(); + let mut dir = cwd.as_path(); + for _ in 0..5 { + if is_freshell_root(dir) { + nearest = dir.to_path_buf(); + break; + } + match dir.parent() { + Some(parent) => dir = parent, + None => break, + } + } + if has_tsx(&nearest) { + return; // normal checkout: nothing to do + } + let mut dir = nearest.as_path(); + while let Some(parent) = dir.parent() { + if is_freshell_root(parent) && has_tsx(parent) { + std::env::set_current_dir(parent) + .expect("chdir to the dependency-bearing parent checkout"); + return; + } + dir = parent; + } + panic!( + "no freshell checkout with node_modules/tsx found at or above {} — \ + the codex create door cannot resolve its MCP injection here", + nearest.display() + ); + }); +} + +// ─── harness (copied with attribution from codex_managed_launch_e2e.rs) ────── + +fn test_settings_value() -> serde_json::Value { + serde_json::json!({ + "ai": {}, + "codingCli": { "enabledProviders": [], "mcpServer": true, "providers": {} }, + "editor": { "externalEditor": "auto" }, + "extensions": { "disabled": [] }, + "freshAgent": { "defaultPlugins": [], "enabled": false, "providers": {} }, + "logging": { "debug": false }, + "network": { "configured": true, "host": "127.0.0.1" }, + "panes": { "defaultNewPane": "ask" }, + "safety": { "autoKillIdleMinutes": 15 }, + "sidebar": { + "autoGenerateTitles": true, + "excludeFirstChatMustStart": false, + "excludeFirstChatSubstrings": [] + }, + "terminal": { "scrollback": 10000 } + }) +} + +/// The shipped codex CLI spec shape (`server/index.ts:231-255`, mirrored from +/// `codex_managed_launch_e2e.rs::codex_cli_spec`), so the resolver takes the +/// REAL codex branch (notification pair, `--remote` when a proxy URL is +/// present, `resume {{sessionId}}`, `CODEX_CMD` override). +fn codex_cli_spec() -> freshell_platform::CliCommandSpec { + fn s(items: &[&str]) -> Vec { + items.iter().map(|i| i.to_string()).collect() + } + freshell_platform::CliCommandSpec { + name: "codex".into(), + label: "Codex CLI".into(), + env_var: Some("CODEX_CMD".into()), + default_cmd: "codex".into(), + resume_args: Some(s(&["resume", "{{sessionId}}"])), + model_args: Some(s(&["--model", "{{model}}"])), + sandbox_args: Some(s(&["--sandbox", "{{sandbox}}"])), + ..Default::default() + } +} + +fn fixture_path() -> std::path::PathBuf { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs") + .canonicalize() + .expect("fake-app-server fixture exists") +} + +/// Write the node dispatcher that plays BOTH codex roles (copied with +/// attribution from `codex_managed_launch_e2e.rs::write_codex_dispatcher`): +/// - argv contains `app-server` → run the committed fake app-server fixture +/// (the manager-spawned sidecar; reads `FAKE_CODEX_APP_SERVER_BEHAVIOR`). +/// - otherwise (the TUI launch) → dump argv JSON to +/// `$CODEX_ARGV_CAPTURE_PATH` and stay alive until the test kills the pane. +fn codex_dispatcher() -> &'static std::path::PathBuf { + static DISPATCHER: OnceLock = OnceLock::new(); + DISPATCHER.get_or_init(|| { + let fixture = fixture_path(); + let dispatcher = std::env::temp_dir().join(format!( + "freshell-codex-reattach-e2e-dispatcher-{}.mjs", + std::process::id() + )); + let script = format!( + "#!/usr/bin/env node\n\ + import fs from 'node:fs'\n\ + const args = process.argv.slice(2)\n\ + if (args.includes('app-server')) {{\n\ + await import('file://{fixture}')\n\ + }} else {{\n\ + fs.writeFileSync(process.env.CODEX_ARGV_CAPTURE_PATH, JSON.stringify(args))\n\ + setInterval(() => undefined, 1000)\n\ + }}\n", + fixture = fixture.display() + ); + std::fs::write(&dispatcher, script).expect("write dispatcher"); + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&dispatcher).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&dispatcher, perms).unwrap(); + dispatcher + }) +} + +async fn spawn_server() -> (String, freshell_terminal::TerminalRegistry) { + let auth_token = Arc::new(AUTH_TOKEN.to_string()); + let broadcast_tx = Arc::new(tokio::sync::broadcast::channel::(64).0); + let settings = + Arc::new(serde_json::from_value(test_settings_value()).expect("valid settings fixture")); + let registry = freshell_terminal::TerminalRegistry::new(); + + let state = WsState { + pane_ledger: std::sync::Arc::new(freshell_ws::pane_ledger::PaneLedger::disabled()), + layout: Default::default(), + identity: freshell_ws::identity::TerminalIdentityRegistry::new(), + terminal_meta: Default::default(), + auth_token: Arc::clone(&auth_token), + server_instance_id: Arc::new("srv-reattach-e2e".to_string()), + boot_id: Arc::new("boot-reattach-e2e".to_string()), + settings, + handshake_settings: Arc::new(tokio::sync::RwLock::new( + serde_json::from_value(test_settings_value()).expect("valid settings fixture"), + )), + broadcast_tx: Arc::clone(&broadcast_tx), + auto_resume_tx: tokio::sync::mpsc::unbounded_channel().0, + auto_resume_cancels: Default::default(), + fresh_codex: freshell_freshagent::FreshCodexState::new( + Arc::clone(&auth_token), + Arc::clone(&broadcast_tx), + serde_json::json!({ "freshAgent": { "enabled": false } }), + ), + fresh_claude: freshell_freshagent::FreshClaudeState::new(Arc::clone(&broadcast_tx)), + fresh_opencode: freshell_freshagent::FreshOpencodeState::new( + freshell_freshagent::FreshAgentState::new( + Arc::clone(&auth_token), + Arc::clone(&broadcast_tx), + ), + ), + registry: registry.clone(), + tabs: freshell_ws::tabs::TabsRegistry::new(), + screenshots: freshell_ws::screenshot::ScreenshotBroker::new(Arc::clone(&broadcast_tx)), + terminals_revision: Arc::new(std::sync::atomic::AtomicI64::new(0)), + sessions_revision: Arc::new(std::sync::atomic::AtomicI64::new(0)), + cli_commands: Arc::new(vec![codex_cli_spec()]), + shutdown: Arc::new(tokio::sync::Notify::new()), + ping_interval_ms: 30_000, + hello_timeout_ms: 5_000, + allowed_origins: Arc::new(freshell_ws::origin::default_allowed_origins()), + ws_max_payload_bytes: 16 * 1024 * 1024, + term09: freshell_ws::backpressure::Term09Config::default(), + create_protect: freshell_ws::create_limit::CreateProtectConfig::default(), + spawn_gate: std::sync::Arc::new(freshell_ws::spawn_gate::SpawnGate::new(4, 64)), + shutdown_started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + create_dedupe: std::sync::Arc::new(freshell_ws::create_dedupe::CreateDedupe::default()), + config_fallback: None, + opencode_locator: None, + codex_locator: None, + activity: None, + // `NoIndexProbe` answers Unknown ⇒ the resume-validation gate + // proceeds (fail-open), the codex_session_ref_resume.rs handshake + // convention. + session_existence: std::sync::Arc::new(freshell_ws::existence::NoIndexProbe::default()), + reconcile_deferral_budget_ms: freshell_ws::reconcile::RECONCILE_DEFERRAL_BUDGET_MS_DEFAULT, + fresh_agent_respawn_counts: Default::default(), + }; + + let router = freshell_ws::router(state); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral loopback port"); + let addr = listener.local_addr().expect("local addr"); + tokio::spawn(async move { + let _ = axum::serve(listener, router).await; + }); + + (format!("ws://{addr}/ws"), registry) +} + +type TestWs = + tokio_tungstenite::WebSocketStream>; + +async fn connect_and_handshake(url: &str) -> TestWs { + let (mut ws, _resp) = tokio_tungstenite::connect_async(url) + .await + .expect("ws connect"); + ws.send(WsMessage::Text( + json!({ + "type": "hello", + "token": AUTH_TOKEN, + "protocolVersion": freshell_protocol::WS_PROTOCOL_VERSION, + }) + .to_string(), + )) + .await + .expect("send hello"); + // Drain the 4-frame handshake (ready → settings.updated → perf.logging → + // terminal.inventory; config_fallback is None in this harness). + for _ in 0..4u8 { + let _ = tokio::time::timeout(RECV_TIMEOUT, ws.next()) + .await + .expect("handshake frame within timeout") + .expect("stream open") + .expect("no ws error"); + } + ws +} + +/// Send the frozen restore create (§17 field-for-field: +/// `{"type":"terminal.create","requestId":…,"mode":"codex","shell":"system", +/// "cwd":…,"restore":true,"sessionRef":{"provider":"codex","sessionId":…}}`) +/// and return the `terminal.created` frame (panicking on an `error` frame +/// for diagnosis). +async fn create_codex_restore_terminal( + ws: &mut TestWs, + request_id: &str, + cwd: &str, + session_id: &str, +) -> serde_json::Value { + ws.send(WsMessage::Text( + json!({ + "type": "terminal.create", + "requestId": request_id, + "mode": "codex", + "shell": "system", + "cwd": cwd, + "restore": true, + "sessionRef": { "provider": "codex", "sessionId": session_id }, + }) + .to_string(), + )) + .await + .expect("send terminal.create"); + loop { + let msg = tokio::time::timeout(RECV_TIMEOUT, ws.next()) + .await + .expect("terminal.created within timeout") + .expect("stream open") + .expect("no ws error"); + if let WsMessage::Text(text) = msg { + let value: serde_json::Value = serde_json::from_str(&text).expect("json frame"); + match value["type"].as_str() { + Some("terminal.created") if value["requestId"] == json!(request_id) => { + return value; + } + Some("error") => panic!("terminal.create failed: {value}"), + _ => {} + } + } + } +} + +/// Poll the capture file the dispatcher writes until it appears, then parse +/// the argv (JSON array — the dispatcher shape). +fn wait_for_captured_argv(path: &std::path::Path) -> Vec { + let deadline = std::time::Instant::now() + RECV_TIMEOUT; + loop { + if let Ok(raw) = std::fs::read_to_string(path) { + if !raw.is_empty() { + return serde_json::from_str(&raw).expect("captured argv is a JSON array"); + } + } + assert!( + std::time::Instant::now() < deadline, + "spawned codex child never wrote its argv capture at {}", + path.display() + ); + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn resume_pair_position(argv: &[String], session_id: &str) -> Option { + argv.windows(2) + .position(|w| w[0] == "resume" && w[1] == session_id) +} + +// ─── the survivor: this test's OWN fake app-server child ───────────────────── + +/// Allocate a loopback ephemeral ws URL (bind 127.0.0.1:0, read, release). +fn bind_loopback_ws_url() -> String { + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind ephemeral port"); + let port = listener.local_addr().expect("local_addr").port(); + drop(listener); + format!("ws://127.0.0.1:{port}") +} + +/// Spawn the SURVIVOR fixture on an ephemeral loopback port with a +/// per-instance behavior JSON (Command env overrides the inherited process +/// env, so the survivor's knobs never touch the dispatcher-spawned sidecars'). +/// `kill_on_drop(true)` is the leak backstop — an assertion failure anywhere +/// still reaps this test's own child. Returns (child, pid, listen ws url). +async fn spawn_survivor( + ownership_id: &str, + behavior: &serde_json::Value, +) -> (tokio::process::Child, u32, String) { + let listen_ws_url = bind_loopback_ws_url(); + let mut child = tokio::process::Command::new("node") + .arg(fixture_path()) + .arg("--listen") + .arg(&listen_ws_url) + .env(CODEX_SIDECAR_OWNERSHIP_ENV, ownership_id) + .env("FAKE_CODEX_APP_SERVER_BEHAVIOR", behavior.to_string()) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .expect("spawn this test's own survivor fixture"); + let pid = child.id().expect("live survivor pid"); + // Wait for the WS listener: by then exec has long completed, so the + // /proc evidence captured afterwards is really the fixture's (the Task 7 + // no-post-fork-flake convention). + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + loop { + if let Ok(Ok((probe, _response))) = tokio::time::timeout( + Duration::from_secs(1), + tokio_tungstenite::connect_async(&listen_ws_url), + ) + .await + { + drop(probe); + break; + } + if let Ok(Some(status)) = child.try_wait() { + panic!("survivor fixture exited before listening: {status}"); + } + assert!( + tokio::time::Instant::now() < deadline, + "survivor fixture WS never came up" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + (child, pid, listen_ws_url) +} + +// ─── the TUI: dial the proxy, initialize, thread/resume ────────────────────── + +/// Read frames until the JSON-RPC response for `id` arrives. +async fn wait_for_rpc_id(tui: &mut TestWs, id: i64) -> serde_json::Value { + loop { + let msg = tokio::time::timeout(RECV_TIMEOUT, tui.next()) + .await + .expect("rpc reply through the proxy within timeout") + .expect("proxy stream open") + .expect("no ws error"); + if let WsMessage::Text(text) = msg { + let value: serde_json::Value = serde_json::from_str(&text).expect("json frame"); + if value.get("id") == Some(&json!(id)) { + return value; + } + } + } +} + +/// Play the TUI: dial the pane's `--remote` proxy URL, complete +/// `initialize`/`initialized`, then send `thread/resume {threadId}` and +/// return its response frame (result OR error — the caller asserts which). +async fn tui_resume_via_proxy(proxy_url: &str, thread_id: &str) -> serde_json::Value { + let (mut tui, _) = tokio_tungstenite::connect_async(proxy_url) + .await + .expect("fake TUI dials the --remote proxy URL"); + tui.send(WsMessage::Text( + json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}).to_string(), + )) + .await + .expect("send initialize"); + let init = wait_for_rpc_id(&mut tui, 1).await; + assert!( + init.get("result").is_some(), + "initialize through the relay failed: {init}" + ); + tui.send(WsMessage::Text( + json!({"jsonrpc": "2.0", "method": "initialized"}).to_string(), + )) + .await + .expect("send initialized"); + tui.send(WsMessage::Text( + json!({"jsonrpc": "2.0", "id": 2, "method": "thread/resume", "params": {"threadId": thread_id}}) + .to_string(), + )) + .await + .expect("send thread/resume"); + wait_for_rpc_id(&mut tui, 2).await +} + +// ─── non-panicking evidence/cleanup polls (asserts run AFTER cleanup) ───────── + +/// Poll the fixture's `appendThreadOperationLogPath` op log for a +/// `thread/resume` entry with `thread_id` (the fixture appends AFTER +/// responding, so the RPC reply can beat the disk write). `None` on deadline. +fn poll_thread_resume_logged( + path: &std::path::Path, + thread_id: &str, + budget: Duration, +) -> Option { + let deadline = std::time::Instant::now() + budget; + loop { + if let Ok(raw) = std::fs::read_to_string(path) { + for line in raw.lines() { + if let Ok(entry) = serde_json::from_str::(line) { + if entry["method"] == json!("thread/resume") + && entry["threadId"] == json!(thread_id) + { + return Some(entry); + } + } + } + } + if std::time::Instant::now() >= deadline { + return None; + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +/// Poll the store until `ownership_id`'s record carries a terminal id. +/// `None` on deadline. (Adopt completes before `terminal.created` is emitted, +/// so this returns almost immediately on the green path.) +fn poll_record_terminal_id( + store: &CodexSidecarStore, + ownership_id: &str, + budget: Duration, +) -> Option { + let deadline = std::time::Instant::now() + budget; + loop { + let terminal_id = store + .load_all() + .into_iter() + .find(|r| r.ownership_id == ownership_id) + .and_then(|r| r.terminal_id); + if terminal_id.is_some() { + return terminal_id; + } + if std::time::Instant::now() >= deadline { + return None; + } + std::thread::sleep(Duration::from_millis(50)); + } +} + +/// Poll until `pid` reads gone from `/proc` (`proc_starttime` is `None` for +/// reaped AND zombie states). `true` = gone within the budget. +async fn poll_pid_gone(pid: u32, budget: Duration) -> bool { + let deadline = tokio::time::Instant::now() + budget; + loop { + if proc_starttime(pid as i32).is_none() { + return true; + } + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +/// Poll until the pane's proxy URL refuses connections (sidecar teardown +/// closes the proxy listener first) — the deterministic "teardown ran" gate +/// for the dispatcher-spawned sidecars whose pids this test never sees. +async fn poll_proxy_refused(proxy_url: &str, budget: Duration) -> bool { + let deadline = tokio::time::Instant::now() + budget; + loop { + match tokio::time::timeout( + Duration::from_secs(1), + tokio_tungstenite::connect_async(proxy_url), + ) + .await + { + Ok(Err(_)) => return true, + Ok(Ok((probe, _))) => drop(probe), + Err(_elapsed) => {} + } + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +/// An empty reconciler over a fresh temp store (the no-tracked-survivor +/// scenarios' claim source: `claim_for_session` finds nothing, the selection +/// falls through to today's spawn path). +fn empty_reconciler() -> ( + tempfile::TempDir, + Arc, + Arc, +) { + let dir = tempfile::tempdir().expect("tempdir for the sidecar store"); + let store = Arc::new(CodexSidecarStore::new(dir.path().to_path_buf())); + let (reconciler, report) = SidecarReconciler::boot_reconcile(store.clone()); + assert_eq!(report.held, 0, "an empty store holds nothing at boot"); + (dir, store, Arc::new(reconciler)) +} + +fn scenario_cwd(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "freshell-codex-reattach-e2e-{name}-{}", + std::process::id() + )); + std::fs::create_dir_all(&dir).expect("scenario cwd"); + dir +} + +fn capture_path(name: &str) -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!( + "freshell-codex-reattach-e2e-argv-{name}-{}.json", + std::process::id() + )); + let _ = std::fs::remove_file(&path); + path +} + +// ─── scenario 1: reattach to the surviving sidecar ──────────────────────────── + +#[test] +fn restore_reattaches_tui_to_surviving_sidecar_preserving_in_flight_turn() { + reattach_rt().block_on(async { + let _serial = test_lock().lock().await; + install_global_manager(); + ensure_mcp_deps_resolvable(); + std::env::set_var("CODEX_CMD", codex_dispatcher()); + // DEV-0006 S5.e: unset = managed launch ON (the leg under test). + std::env::remove_var("FRESHELL_CODEX_MANAGED_LAUNCH"); + // The SURVIVOR carries its behavior per-instance (Command env); the + // process-global knob stays unset so a (wrong) fresh spawn would + // write NO op log — the red/green discriminator. + std::env::remove_var("FAKE_CODEX_APP_SERVER_BEHAVIOR"); + + let session_id = "0199da92-e2e-reattach-thread"; + let ownership_id = "codex-sidecar-da920e2e-0001-4aaa-8aaa-aaaaaaaaaaaa"; + let log_a = std::env::temp_dir().join(format!( + "freshell-codex-reattach-e2e-oplog-a-{}.jsonl", + std::process::id() + )); + let _ = std::fs::remove_file(&log_a); + + // (1) The SURVIVOR: this test's own fixture child, mid-turn shape + // (`loadedThreadIds` reports the session as loaded), op log A. + let (mut survivor, survivor_pid, survivor_ws_url) = spawn_survivor( + ownership_id, + &json!({ + "appendThreadOperationLogPath": log_a, + "loadedThreadIds": [session_id], + }), + ) + .await; + let survivor_starttime = + proc_starttime(survivor_pid as i32).expect("live survivor has a starttime"); + + // Its verified record in a temp store; the reconciler holds it. + let dir = tempfile::tempdir().expect("tempdir for the sidecar store"); + let store = Arc::new(CodexSidecarStore::new(dir.path().to_path_buf())); + store + .write(&CodexSidecarRecord { + record_version: SIDECAR_RECORD_VERSION, + ownership_id: ownership_id.to_string(), + pid: survivor_pid, + starttime: survivor_starttime, + cmdline: proc_cmdline(survivor_pid as i32).expect("live survivor has a cmdline"), + ws_url: survivor_ws_url.clone(), + session_id: Some(session_id.to_string()), + terminal_id: None, + server_instance_id: "srv-prev-gen".to_string(), + created_at: 1_700_000_000_000, + updated_at: 1_700_000_000_001, + state: SidecarRecordState::Active, + }) + .expect("write survivor record"); + let (reconciler, report) = SidecarReconciler::boot_reconcile(store.clone()); + assert_eq!(report.held, 1, "the survivor is held at boot"); + let reconciler = Arc::new(reconciler); + // Hand the factory this scenario's reconciler + store: the selection + // claims the verified survivor for the resume plan and mints the + // reattach runtime. (TDD red, Task 8 Step 2: with this wiring absent + // the plan spawned fresh and the survivor's op log stayed empty.) + swap_test_reconciler(Some(reconciler.clone()), Some(store.clone())); + + // (2) The restore create through the WS door. + let (ws_url, registry) = spawn_server().await; + let mut ws = connect_and_handshake(&ws_url).await; + let capture = capture_path("reattach"); + std::env::set_var("CODEX_ARGV_CAPTURE_PATH", &capture); + let cwd = scenario_cwd("reattach"); + let created = create_codex_restore_terminal( + &mut ws, + "req-reattach", + cwd.to_str().unwrap(), + session_id, + ) + .await; + let terminal_id = created["terminalId"].as_str().unwrap().to_string(); + + // (3a) The captured TUI argv: `--remote ws://127.0.0.1:` + + // the `resume ` pair. + let argv = wait_for_captured_argv(&capture); + assert_eq!(argv[0], "--remote", "argv: {argv:?}"); + let proxy_url = argv[1].clone(); + assert!( + proxy_url.starts_with("ws://127.0.0.1:"), + "the --remote URL must be the loopback proxy: {proxy_url}" + ); + assert!( + resume_pair_position(&argv, session_id).is_some(), + "TUI argv must contain `resume {session_id}`: {argv:?}" + ); + + // (3b) Evidence collection — non-panicking, so cleanup ALWAYS runs + // (red and green alike) and no test-spawned process can leak. + let resume_reply = tui_resume_via_proxy(&proxy_url, session_id).await; + let logged = poll_thread_resume_logged(&log_a, session_id, RECV_TIMEOUT); + let survivor_alive_same_incarnation = + proc_starttime(survivor_pid as i32) == Some(survivor_starttime); + let record_terminal_id = + poll_record_terminal_id(&store, ownership_id, Duration::from_secs(10)); + let unclaimed = reconciler.unclaimed_len(); + + // Cleanup: kill ONLY pids this test spawned. The pane kill queues the + // reattached sidecar's teardown, which reaps the survivor (this + // test's own child); if that never ran (the red shape), kill our own + // child directly. + registry.kill(&terminal_id); + if !poll_pid_gone(survivor_pid, Duration::from_secs(8)).await { + let _ = survivor.start_kill(); + } + let _ = survivor.wait().await; + let _ = poll_proxy_refused(&proxy_url, Duration::from_secs(8)).await; + swap_test_reconciler(None, None); + std::env::remove_var("CODEX_ARGV_CAPTURE_PATH"); + + // (3c) Asserts. + assert!( + resume_reply.get("result").is_some(), + "thread/resume must SUCCEED against the claimed survivor: {resume_reply}" + ); + let logged = logged.unwrap_or_else(|| { + panic!( + "the SURVIVOR's op log never recorded thread/resume for {session_id} — \ + the pane is NOT wired to the surviving sidecar" + ) + }); + assert_eq!( + logged["listenUrl"], + json!(survivor_ws_url), + "thread/resume must have been served by the SURVIVOR's listener: {logged}" + ); + assert!( + survivor_alive_same_incarnation, + "the surviving sidecar (pid {survivor_pid}) must still be alive, same incarnation" + ); + assert_eq!( + record_terminal_id.as_deref(), + Some(terminal_id.as_str()), + "the survivor's record must gain the new terminal id at adopt" + ); + assert_eq!(unclaimed, 0, "the one-shot claim was consumed"); + }); +} + +// ─── scenario 2: no tracked survivor → today's fresh-spawn path ─────────────── + +#[test] +fn restore_falls_back_to_fresh_sidecar_without_tracked_survivor() { + reattach_rt().block_on(async { + let _serial = test_lock().lock().await; + install_global_manager(); + ensure_mcp_deps_resolvable(); + std::env::set_var("CODEX_CMD", codex_dispatcher()); + std::env::remove_var("FRESHELL_CODEX_MANAGED_LAUNCH"); + + // EMPTY reconciler: the claim runs for real, finds nothing, and the + // selection falls through to today's spawn path. + let (_dir, store, reconciler) = empty_reconciler(); + swap_test_reconciler(Some(reconciler), Some(store)); + + // The dispatcher-spawned sidecar reads the process-global behavior: + // op log B records which listener served the traffic. + let session_id = "0199da92-e2e-fresh-fallback-thread"; + let log_b = std::env::temp_dir().join(format!( + "freshell-codex-reattach-e2e-oplog-b-{}.jsonl", + std::process::id() + )); + let _ = std::fs::remove_file(&log_b); + std::env::set_var( + "FAKE_CODEX_APP_SERVER_BEHAVIOR", + json!({ "appendThreadOperationLogPath": log_b }).to_string(), + ); + + let (ws_url, registry) = spawn_server().await; + let mut ws = connect_and_handshake(&ws_url).await; + let capture = capture_path("fresh"); + std::env::set_var("CODEX_ARGV_CAPTURE_PATH", &capture); + let cwd = scenario_cwd("fresh"); + let created = + create_codex_restore_terminal(&mut ws, "req-fresh", cwd.to_str().unwrap(), session_id) + .await; + let terminal_id = created["terminalId"].as_str().unwrap().to_string(); + + // Byte-compatible with today's managed resume argv (the S5.e golden + // shape, codex_managed_launch_e2e.rs Phase 3): the `--remote` + // 4-tuple, the bel notification pair, the resume pair LAST. + let argv = wait_for_captured_argv(&capture); + assert_eq!(argv[0], "--remote", "argv: {argv:?}"); + let proxy_url = argv[1].clone(); + assert!( + proxy_url.starts_with("ws://127.0.0.1:"), + "the --remote URL must be the loopback proxy: {proxy_url}" + ); + assert_eq!( + &argv[2..4], + &["-c".to_string(), "features.apps=false".to_string()], + "argv: {argv:?}" + ); + assert_eq!( + &argv[4..6], + &["-c".to_string(), "tui.notification_method=bel".to_string()], + "argv: {argv:?}" + ); + let position = resume_pair_position(&argv, session_id) + .unwrap_or_else(|| panic!("argv must contain `resume {session_id}`: {argv:?}")); + assert_eq!( + position + 2, + argv.len(), + "resume pair must be last: {argv:?}" + ); + + // Evidence, then cleanup, then asserts (the scenario-1 discipline). + let resume_reply = tui_resume_via_proxy(&proxy_url, session_id).await; + let logged = poll_thread_resume_logged(&log_b, session_id, RECV_TIMEOUT); + + registry.kill(&terminal_id); + let torn_down = poll_proxy_refused(&proxy_url, Duration::from_secs(8)).await; + swap_test_reconciler(None, None); + std::env::remove_var("CODEX_ARGV_CAPTURE_PATH"); + std::env::remove_var("FAKE_CODEX_APP_SERVER_BEHAVIOR"); + + assert!( + resume_reply.get("result").is_some(), + "thread/resume must succeed on the fresh path: {resume_reply}" + ); + assert!( + logged.is_some(), + "the NEW fixture instance's op log (log-B) must record thread/resume \ + for {session_id} — today's fresh-spawn path serves the plan" + ); + assert!( + torn_down, + "pane kill must tear the fresh sidecar's proxy down" + ); + }); +} + +// ─── scenario 3: the da92 control — -32600 confined to the fresh path ───────── + +#[test] +fn active_writer_collision_surfaces_minus32600_only_on_the_fresh_path() { + reattach_rt().block_on(async { + let _serial = test_lock().lock().await; + install_global_manager(); + ensure_mcp_deps_resolvable(); + std::env::set_var("CODEX_CMD", codex_dispatcher()); + std::env::remove_var("FRESHELL_CODEX_MANAGED_LAUNCH"); + + // EMPTY reconciler + the scripted active-writer rejection: the + // incident's failure mode, now confined to the no-survivor path (the + // reattach scenario above proves the same resume SUCCEEDS when a + // survivor exists). + let (_dir, store, reconciler) = empty_reconciler(); + swap_test_reconciler(Some(reconciler), Some(store)); + + let session_id = "0199da92-e2e-active-writer-thread"; + std::env::set_var( + "FAKE_CODEX_APP_SERVER_BEHAVIOR", + json!({ + "overrides": { + "thread/resume": { + "error": { "code": -32600, "message": "thread already has an active writer" } + } + } + }) + .to_string(), + ); + + let (ws_url, registry) = spawn_server().await; + let mut ws = connect_and_handshake(&ws_url).await; + let capture = capture_path("writer"); + std::env::set_var("CODEX_ARGV_CAPTURE_PATH", &capture); + let cwd = scenario_cwd("writer"); + let created = + create_codex_restore_terminal(&mut ws, "req-writer", cwd.to_str().unwrap(), session_id) + .await; + let terminal_id = created["terminalId"].as_str().unwrap().to_string(); + + let argv = wait_for_captured_argv(&capture); + assert_eq!(argv[0], "--remote", "argv: {argv:?}"); + let proxy_url = argv[1].clone(); + + // Evidence, then cleanup, then asserts. + let resume_reply = tui_resume_via_proxy(&proxy_url, session_id).await; + + registry.kill(&terminal_id); + let _ = poll_proxy_refused(&proxy_url, Duration::from_secs(8)).await; + swap_test_reconciler(None, None); + std::env::remove_var("CODEX_ARGV_CAPTURE_PATH"); + std::env::remove_var("FAKE_CODEX_APP_SERVER_BEHAVIOR"); + + let error = resume_reply + .get("error") + .unwrap_or_else(|| panic!("the fresh path must surface the scripted rejection: {resume_reply}")); + assert_eq!( + error["code"], + json!(-32600), + "the incident's error code: {resume_reply}" + ); + // codex uses -32600 generically for many rejections (reports/V1.md); + // the code alone is not the incident signature — the message is. + assert!( + error["message"] + .as_str() + .is_some_and(|m| m.contains("active writer")), + "the -32600 message must carry the active-writer signature: {resume_reply}" + ); + }); +} diff --git a/crates/freshell-ws/tests/restore_plan_queue_cap.rs b/crates/freshell-ws/tests/restore_plan_queue_cap.rs index 39ac14f61..3d089dde1 100644 --- a/crates/freshell-ws/tests/restore_plan_queue_cap.rs +++ b/crates/freshell-ws/tests/restore_plan_queue_cap.rs @@ -289,10 +289,12 @@ async fn plan_queue_overflow_maps_to_rate_limited_on_the_ws_restore_door() { let plans_started = Arc::new(AtomicU64::new(0)); let factory_counter = Arc::clone(&plans_started); let manager = freshell_codex::launch_lifecycle::CodexTerminalLaunchManager::with_plan_budget( - Box::new(move || { - Arc::new(NeverRuntime { + Box::new(move |_plan| { + let rt = Arc::new(NeverRuntime { plans_started: factory_counter.clone(), - }) as Arc + }) + as Arc; + Box::pin(async move { rt }) }), 0, Duration::from_millis(50), diff --git a/crates/freshell-ws/tests/restore_storm.rs b/crates/freshell-ws/tests/restore_storm.rs index 083bff8dd..f657ecfe6 100644 --- a/crates/freshell-ws/tests/restore_storm.rs +++ b/crates/freshell-ws/tests/restore_storm.rs @@ -416,11 +416,12 @@ fn storm_controls() -> &'static Arc { let factory_controls = controls.clone(); let manager = freshell_codex::launch_lifecycle::CodexTerminalLaunchManager::with_plan_budget( - Box::new(move || { - Arc::new(StormRuntime { + Box::new(move |_plan| { + let rt = Arc::new(StormRuntime { c: factory_controls.clone(), }) - as Arc + as Arc; + Box::pin(async move { rt }) }), 2, std::time::Duration::from_secs(30), diff --git a/docs/plans/2026-08-10-codex-sidecar-lifecycle.md b/docs/plans/2026-08-10-codex-sidecar-lifecycle.md new file mode 100644 index 000000000..a85c10817 --- /dev/null +++ b/docs/plans/2026-08-10-codex-sidecar-lifecycle.md @@ -0,0 +1,1807 @@ +# Codex Sidecar Lifecycle: Persistent Tracking, Reattach, and Conservative Reaping (katas ynfn + da92) + +> **Provenance:** research and requirements come from the original write-plan +> session, which completed all exploration but crashed on an Anthropic provider +> error (`overloaded_error`, 2026-08-10 21:43 UTC) before emitting the plan +> document. This plan was reconstructed from that session's recovered step +> prompt, the verbatim kata bodies (ynfn, da92), and the surviving exploration +> reports under +> `/home/dan/code/freshell/.worktrees/.the-usual-logs/codex-sidecar-lifecycle/reports/` +> (`rust-sidecar.md`, `node-server-parity.md`, `tests-and-persistence.md`, +> `verbatim-snippets.md`). All file:line references below are drawn from those +> reports (extracted from this worktree at branch head); re-locate by content if +> drifted. + +> **For agentic workers:** This plan is executed task-by-task by the workflow's +> execute stage: a fresh implementer per task, with a spec + quality review +> after each task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** After a freshell Rust-server restart, every codex app-server sidecar +freshell ever spawned is either (a) **reattached** to a restored pane, (b) +**reaped** after provable identity verification, or (c) **intentionally +retained with a recorded reason** — never silently orphaned (kata ynfn). A +restored codex pane whose previous sidecar survived — including one mid-agent-turn +holding the thread's active writer — reattaches its TUI to that surviving +sidecar instead of spawning a fresh one that collides with JSON-RPC `-32600 +"thread ... already has an active writer"` (kata da92). Fallback to today's +fresh-spawn path whenever no live, verified, usable survivor exists. + +**Architecture:** Everything lands in the Rust server (the live production +lane). Four new pieces, all in `crates/freshell-codex` behind the existing +`real-transport` feature, wired at the existing seams the research identified: + +1. **Durable sidecar record store** — `~/.freshell/rust-codex-sidecars/` (a + Rust-owned directory, deliberately NOT Node's `~/.freshell/codex-sidecars/`; + see the Node-parity decision below), one JSON record per sidecar keyed by + ownership id, written with the repo's `atomic_write_durable` pattern + (`crates/freshell-ws/src/tabs_persist.rs:682-708`) and the + `PaneLedger::new_locked` flock/disabled-fallback pattern + (`crates/freshell-ws/src/pane_ledger.rs:236-274`). Records persist pid, ws + listen URL, ownership id, codex session/thread id (when known), and identity + evidence: full `/proc//cmdline` argv + `/proc//stat` starttime. +2. **Identity verification** — a pid is only ever trusted after `(pid, + starttime, cmdline)` all match the record (the `proc_starttime` pid-reuse + guard from `crates/freshell-freshagent/src/session_lease.rs:144-160`, + deliberately duplicated into `freshell-codex` because the dependency + direction forbids reuse). Never pattern-match process names. +3. **Reattach runtime** — a second `CodexLaunchRuntime` implementation, + `ReattachedCodexAppServerRuntime`, whose `ensure_ready` verifies + probes the + surviving listener and returns the EXISTING ws URL instead of spawning. The + planner's runtime-factory seam (`CodexRuntimeFactory`, + `crates/freshell-codex/src/launch_lifecycle.rs:46`, minted per-plan at + `plan_create` `:308`) selects reattach-vs-spawn per plan. The TUI still runs + `codex --remote ws://127.0.0.1: … resume ` — the proxy + relays to the existing sidecar port, which is how ALL `--remote` traffic is + routed in this architecture (`remote_proxy.rs:276-340`); the requirement's + "reattach via `codex --remote ws://127.0.0.1:`" is realized + through that relay. +4. **Boot reconciler + conservative reaper** — at boot, load records, prune + provably-stale ones, hold verified survivors as claimable; restores claim by + session id; a grace-delayed sweep reaps verified idle unclaimed sidecars + (SIGTERM the recorded pid only after re-verification) and RETAINS mid-turn + ones with a recorded reason. Server shutdown retains adopted sidecars + (records marked `retained: server-shutdown`) instead of killing them — + surviving restarts is the feature (kata ynfn: "Killing sidecars at shutdown + is NOT acceptable"). + +**Tech stack:** Rust 1.96 workspace (`crates/*`), tokio 1.52.3, +tokio-tungstenite 0.24, serde/serde_json, libc (Linux `/proc`), the committed +fake codex app-server fixture +`test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs` (Node, drives +the 3-tier Rust test approach), cargo fmt/clippy pinned toolchain 1.96.0. + +## Global Constraints (binding) + +- Repo root (git worktree — all commands run here): + `/home/dan/code/freshell/.worktrees/codex-sidecar-lifecycle` +- **PROCESS SAFETY (critical, applies to every task and every test):** + - NEVER kill by process-name pattern (no `pkill`, no name matching). Only + signal pids this code/test **recorded itself AND re-verified** via `(pid, + starttime, cmdline)` immediately before signalling, or pids carrying our + exact per-launch `FRESHELL_CODEX_SIDECAR_ID=` env tag (the existing + `reap_owned_codex_sidecars` discipline, `crates/freshell-codex/src/transport.rs:86-121`). + - NEVER stop, restart, or deploy the production freshell server on port 3001. + Building is fine; deploying requires the user's explicit "APPROVED". + - There are ~20 live orphaned codex app-servers on this machine (evidence for + kata ynfn; PID 545173 is mid-turn on a session the user cares about). NO + test and NO code written by this plan may signal any of them: the new store + starts EMPTY, the reaper only ever consults records the store contains, and + every test uses an isolated temp store + only kills children it spawned. + - Tests bind loopback ephemeral ports only (`127.0.0.1:0`) — never 3001/3002 + (stated convention, `crates/freshell-codex/tests/launch_lifecycle.rs` doc header). +- **CI runs `cargo fmt --all --check` + `cargo clippy --workspace --all-targets + -- -D warnings` + `cargo clippy -p freshell-codex --features real-transport + --all-targets -- -D warnings` — but NO `cargo test`** + (`.github/workflows/rust-clippy.yml`). Every task must run its cargo tests + locally and keep fmt/clippy clean; record real pass counts. +- New modules in `freshell-codex` that spawn/probe/signal go behind + `#[cfg(feature = "real-transport")]` (matching `launch_lifecycle`, + `remote_proxy`, `transport` — `crates/freshell-codex/src/lib.rs:78-83`). Test + them with `cargo test -p freshell-codex --features real-transport`. +- Non-Linux platforms: identity can never be verified (no `/proc`), so reattach + and reaping are disabled there (conservative no-op, matching the existing + `reap_owned_codex_sidecars` non-Linux stub at `transport.rs:117-121`); the + fresh-spawn path is unchanged. Task 3's detach is therefore Linux-only and + store-gated (untracked spawns keep today's `kill_on_drop(true)`), and its + `process_group(0)` call is `#[cfg(unix)]`-gated + (`tokio::process::Command::process_group` does not exist on Windows). +- Prefer `SpawnedCodexAppServerRuntime::with_command(...)`-style injection over + process-global env mutation in tests (`launch_lifecycle.rs:882-887`); any test + that must set `CODEX_CMD`/`FAKE_CODEX_APP_SERVER_BEHAVIOR` owns its test + binary or serializes (the `ENV_LOCK` discipline, + `crates/freshell-freshagent/src/codex.rs:5802-5827`). +- `set_global_codex_launch_manager_for_tests(manager)` is **set-once per test + binary** (`launch_lifecycle.rs:529-531`): a test file that installs a global + manager gets its own file and shares one manager across its scenarios. +- Keep new Rust source files under the 1,000-line limit (`port/AGENTS.md:81`); + split unit tests out via `#[cfg(test)] #[path = "x_tests.rs"] mod tests` as + `pane_ledger.rs:1002-1004` does. Carry provenance comments mapping new code to + its precedent (`session_lease.rs`, `runtime.ts:NNN`) per repo convention. +- The fake app-server fixture needs `node` on PATH and the repo's + `node_modules` (`ws` package) — run `npm ci` once if `node_modules` is absent. +- **Measured baseline at plan commit b3df5227f** (load-bearing validation ran + every gate + suite; `reports/V5.md`, logs `baseline-*.log` beside it): + fmt + workspace clippy + both feature clippy legs CLEAN; `freshell-codex` + (real-transport) 199 passed; `freshell-server` 631 passed / 1 ignored; + `freshell-platform` 255 passed; **`freshell-ws` 490 passed + 2 pre-existing + deterministic failures** in `tests/auto_resume_e2e.rs` + (`crashing_agent_is_resumed_twice_then_settles_exited`, + `reconcile_after_replacement_attaches_to_the_new_terminal` — both time out + waiting for a terminal.created frame; reproduced 2/2). Do not chase them. + **Fail-fast caveat:** that measured run aborted at `auto_resume_e2e` + (cargo's default fail-fast across test binaries), so the 490-passed figure + covers only the lib target plus 2 of ~44 freshell-ws test binaries — the + full ws suite was NOT baselined at plan time. Therefore: (a) every + whole-package `cargo test -p freshell-ws` gate in this plan runs with + `--no-fail-fast` (per-binary `--test ` runs are unaffected); (b) + before Task 1's first edit, run + `cargo test -p freshell-ws --no-fail-fast` on the unmodified tree and + record the complete failure set to + `.worktrees/.the-usual-logs/codex-sidecar-lifecycle/baseline-test-ws-full.log`; + (c) "Baseline-identical" for freshell-ws means: with `--no-fail-fast`, the + failure set equals that recorded full-suite baseline (expected: exactly + the 2 `auto_resume_e2e` failures; any additional pre-existing failures the + full baseline reveals are recorded there and likewise not chased). + `freshell-freshagent` was not baselined (known environmental e2e + failures; baseline before/after only if a task must run it). +- The worktree has NO node_modules of its own: the fixture's `ws` import + resolves via the PARENT checkout's `/home/dan/code/freshell/node_modules` + (verified working — all fixture-driven tests passed). If that resolution + ever breaks, running `npm ci` in the worktree requires the user's approval; + record it rather than installing. +- `docs/index.html`: **N/A** — this is backend process-lifecycle work with no + user-facing UI change. +- Do not touch `server/` (Node lane) — see the recorded parity decision below. + +## Recorded decision: Node server parity is OUT OF SCOPE (Rust-only plan) + +Per `node-server-parity.md`, the Node server (`server/`) has FULL sidecar +parity of the *spawn* behavior (detached `codex … app-server --listen` per +codex pane, `runtime.ts:1828-1843`) and its own persistence + boot reaper +(`~/.freshell/codex-sidecars/.json`, `runCodexStartupReaper` at +`server/index.ts:280`). It is demoted but actively developed. This plan fixes +the **Rust lane only**, for these recorded reasons: + +1. **The incident is Rust.** The live production server is the Rust server on + port 3001; the post-restart sidecars in both katas were children of Rust + server PID 2921377. The kata fix MUST land in the Rust restore path. +2. **No shared store is possible.** The Rust server must NOT read or write + Node's `~/.freshell/codex-sidecars/` — that directory's semantics are owned + by the Node boot reaper, and entering it creates a two-writer race. The house + pattern is the `rust-session-cache.json` precedent + (`crates/freshell-sessions/src/directory_index.rs:1428-1436`): distinct + filename, distinct schema, no coupling. This plan therefore introduces the + Rust-owned `~/.freshell/rust-codex-sidecars/`. Consequently Node parity is + not a wiring change but an independent TypeScript implementation with its own + TDD cycle — out of scope here. +3. **Severity differs.** Node's boot reaper already kills prior-generation + sidecars (its ynfn analog is "kill", not "orphan"); it lacks only the da92 + reattach. The Rust lane today has NEITHER tracking NOR reattach. +4. **Follow-up recorded, not silent:** the final task requires the PR + description to state this decision and to file a follow-up kata for Node-lane + reattach parity (`kata create "Node server: codex pane restore should + reattach to a surviving sidecar (da92 parity)" --label bug --related da92`). + +Scope note within Rust (REVISED after load-bearing validation): the second +spawn site, `freshell-freshagent/src/codex.rs::spawn_sidecar` +(fresh-agent/freshcodex panes), remains excluded from THIS plan — but the +original justification ("not part of either incident") was FALSIFIED: +validation attributed at least one live orphan (pid 44963, matched to a +`freshagent.sidecar.spawned` log line) to the freshagent lane, and that lane +orphans across unclean restarts by the same mechanism (leases are in-memory +only, kill runs only at graceful shutdown, the 5s force-exit skips Drop) — +`reports/V2.md`. The recorded decision: keep this plan terminal-pane-only +(the freshagent lane has its own lease lifecycle needing its own TDD cycle), +Task 11 files a follow-up kata for freshagent-lane tracking, and the ynfn +close-out must state that this PR fixes the terminal-pane lane with the +freshagent lane tracked in the follow-up. Only the terminal-pane path +(`SpawnedCodexAppServerRuntime::ensure_ready`) changes here. + +## File Structure + +**New production files:** + +| File | Responsibility | +|---|---| +| `crates/freshell-codex/src/sidecar_store.rs` (+ `sidecar_store_tests.rs`) | `CodexSidecarRecord` (schema v1) + `CodexSidecarStore` (flock'd dir of atomic JSON rows, disabled fallback) + `/proc` identity evidence capture & verification | +| `crates/freshell-codex/src/sidecar_reconcile.rs` (+ `sidecar_reconcile_tests.rs`) | `SidecarReconciler` (boot load/prune, claim-by-session, grace-delayed conservative reap sweep) + `ReattachedCodexAppServerRuntime` | +| `crates/freshell-ws/tests/codex_sidecar_reattach_e2e.rs` | da92 end-to-end proof over the WS `terminal.create{restore:true}` door | + +**Modified production files:** + +| File | Change | +|---|---| +| `crates/freshell-codex/Cargo.toml` | add `serde = { workspace = true }`; new `[dev-dependencies]` with `tempfile` | +| `crates/freshell-codex/src/lib.rs` | declare the two new modules behind `real-transport`; re-export the store/reconciler seams | +| `crates/freshell-codex/src/launch_lifecycle.rs` | persist-on-spawn / scrub-on-teardown in `SpawnedCodexAppServerRuntime`; store-gated detach (`kill_on_drop(false)` + `process_group(0)` only when tracked: Linux + enabled store); `note_session_id` trait seam; plan-aware `CodexRuntimeFactory`; manager shutdown-retention | +| `crates/freshell-ws/src/codex_proxy_route.rs` | forward captured thread id into the sidecar record (one call beside `mark_candidate_persisted`) | +| `crates/freshell-server/src/main.rs` | boot: construct store + reconciler, arm reap sweep; shutdown: retention before `registry.kill_all()` | +| `crates/freshell-server/tests/safe11_term22_shutdown_reaping.rs` | verify unchanged-green (it has NO terminal-pane codex assertions — reports/V4.md); optionally add a terminal-pane retention scenario with a re-scoped descendants assertion | +| `test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs` | add a scriptable per-thread `thread/read` status knob (`threadStatuses`) for Task 9's sweep tests — repo-owned test fixture, not shipped code | + +## Background for implementers (read before Task 1) + +**The gap (code-verified):** the terminal-pane sidecar's pid lives ONLY in +memory — `struct SpawnedSidecar { ws_url, ownership_id, child: tokio::process::Child }` +(`launch_lifecycle.rs:842-846`); "The pid is never written to disk anywhere" +(`rust-sidecar.md` §1b). `SpawnedCodexAppServerRuntime::adopted_metadata` is +self-documented as "in-memory until S5's durability store lands" +(`launch_lifecycle.rs:894-897`). Restore is CLIENT-driven — no boot pane walk; +the browser replays `terminal.create{restore:true, sessionRef:{provider:"codex", +sessionId}}` frames minutes after boot (12:16 boot vs 12:34 restore in the +incident), so reaping decisions cannot be made instantly at boot. + +**The restore decision point:** `prepare_launch` (`freshell-ws/src/terminal.rs:1783-1855`) +derives `resume_session_id` from `sessionRef` and calls + +```rust +async fn plan_codex_managed_launch( + state: &WsState, + mode: &str, + raw_cwd: Option<&str>, + resume_session_id: Option<&str>, + class: freshell_codex::launch_lifecycle::LaunchClass, + cancel: Option<&mut tokio::sync::watch::Receiver>, +) -> Result, PlanLaunchError> +``` + +(`terminal.rs:1160-1226`), which delegates to +`CodexTerminalLaunchManager::global()`. The manager owns ONE +`CodexLaunchPlanner`; the per-plan runtime is minted inside +`CodexLaunchPlanner::plan_create` via the stored factory — +`let runtime = (self.runtime_factory)();` — where + +```rust +pub type CodexRuntimeFactory = Box Arc + Send + Sync>; +``` + +(`launch_lifecycle.rs:46`). **That factory is the reattach seam:** make it +plan-aware and it can hand back a runtime wrapping the surviving sidecar +instead of a spawning one. `plan_create` then starts the REAL +`CodexRemoteProxy` against whatever ws URL `ensure_ready` returns — reattach +needs zero changes to the proxy or the TUI argv builder +(`resolve_coding_cli_command`, `cli_launch.rs:478-494`). + +**Why reattach avoids the -32600:** the surviving sidecar IS the thread's +active writer; `thread/resume` into the same app-server succeeds, while resume +into a fresh app-server is rejected by codex's writer lock (kata da92). The +committed fixture can script both sides: `overrides['thread/resume'] = { error: +{ code: -32600, message: … } }` (precedent +`test/integration/server/codex-session-flow.test.ts:674-684`) and +`loadedThreadIds` for mid-turn detection (`tests-and-persistence.md` §1.5). + +**Validated codex behavior (load-bearing validation, `reports/V1.md` — live +experiment on the deployed 0.147.0 binary + codex-rs source at +`rust-v0.147.0`):** + +- Reattach WORKS: a surviving app-server accepts new WS connections after its + client disconnected (including mid-turn), and a new client's `thread/resume` + into the SAME process succeeds with the in-flight turn's events streaming to + the new client. Client disconnect does NOT abort a running turn. +- The active-writer lock is a per-thread **file flock** under + `CODEX_HOME/thread-writer-locks/` (`writer_lock.rs:63-79`), held per LOADED + thread and released on process exit (even SIGKILL). Consequence: killing a + verified-unusable survivor releases the lock, so the retry's fresh spawn can + resume the thread — the Task 6 fallback is semantically sound, not just + structural. +- **`loaded` ≠ `mid-turn`.** Idle threads stay loaded (and writer-locked) + indefinitely, so `thread/loaded/list` non-empty must NOT be read as + "mid-turn" — that would convert every unclaimed idle sidecar into a + permanent `Retained` row (the ynfn leak relabeled). The correct discriminator + is per-thread `thread/read` status (`active` vs idle); Task 9 encodes this. +- `-32600` is codex's generic invalid-request code (writer collision, missing + rollout, config errors all use it) — tests must match on the message + ("active writer"), not the code alone. Also: a thread that never ran a turn + has no rollout and is un-resumable anywhere (`-32600 "no rollout found"`) — + fixture scenarios that script resume success must model threads with ≥1 + recorded turn. + +**Why identity is `(pid, starttime, cmdline)` and not the env tag:** +`/proc//stat` and `/proc//cmdline` are world-readable, but +`/proc//environ` requires ptrace access — under YAMA an orphan reparented +to init is not our descendant, so a restarted server may be UNABLE to read the +tag of the very sidecars it must verify (`session_lease.rs:162-184` doc +comment). The env-tag sweep (`reap_owned_codex_sidecars`) remains a best-effort +supplement only. (Validation nuance, `reports/V2.md`: on THIS machine same-uid +environ/fd reads of reparented orphans do work — YAMA is permissive here — but +keep the identity triple as the primary check for portability; the fd-read +ability is what makes Task 9's unreachable-sidecar writer-evidence check +viable.) + +**Why sidecars survive today only accidentally:** the spawn is NOT detached — +`cmd.kill_on_drop(true)`, no `process_group`/`setsid` anywhere in `crates/` +(`rust-sidecar.md` §1b) — and graceful shutdown kills them +(`main.rs:1663-1722`: `registry.kill_all()` → … → +`CodexTerminalLaunchManager::global().shutdown()`). The incident cohorts +survived because real restarts die uncleanly (or the 5s +`SHUTDOWN_HARD_TIMEOUT` watchdog force-exits, skipping Drops). Kata ynfn is +explicit: "Killing sidecars at shutdown is NOT acceptable — surviving restarts +is a feature." Task 10 makes survival deliberate (retained + recorded) instead +of accidental (orphaned). + +**Preserved constraint (A4 exclusion):** reattach only ever applies to plans +with a `resume_session_id` (restore-class), which set +`require_candidate_persistence = false` — so the 45s candidate-capture timer +(`remote_proxy.rs:117`) is never armed on this path and the A4 restore +exclusion (`terminal.rs:1830-1834`) is untouched. + +--- + +### Task 1: Durable sidecar record store (`rust-codex-sidecars/`) + +**Files:** +- Modify: `crates/freshell-codex/Cargo.toml` (add `serde = { workspace = true }` + to `[dependencies]`; add new `[dev-dependencies]` section with + `tempfile = "3"`) +- Modify: `crates/freshell-codex/src/lib.rs` (declare + `#[cfg(feature = "real-transport")] pub mod sidecar_store;`) +- Create: `crates/freshell-codex/src/sidecar_store.rs` +- Create: `crates/freshell-codex/src/sidecar_store_tests.rs` (unit tests via + `#[cfg(test)] #[path = "sidecar_store_tests.rs"] mod tests;`) + +**Interfaces produced:** + +```rust +pub const SIDECAR_RECORD_VERSION: u32 = 1; + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CodexSidecarRecord { + pub record_version: u32, + pub ownership_id: String, // "codex-sidecar-" (durability.rs:36) + pub pid: u32, + pub starttime: u64, // /proc//stat field 22 — pid-reuse guard + pub cmdline: Vec, // /proc//cmdline argv, NUL-split + pub ws_url: String, // the app-server's --listen URL + #[serde(skip_serializing_if = "Option::is_none")] + pub session_id: Option, // codex thread id, enriched when known + #[serde(skip_serializing_if = "Option::is_none")] + pub terminal_id: Option, // enriched at adopt + pub server_instance_id: String, // durability.rs::default_server_instance_id() + pub created_at: i64, + pub updated_at: i64, + pub state: SidecarRecordState, // Active | Retained { reason } +} + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case", tag = "kind")] +pub enum SidecarRecordState { Active, Retained { reason: String } } + +pub struct CodexSidecarStore { /* root: Option (None = disabled), _lock file */ } + +impl CodexSidecarStore { + pub fn new_locked(root: Option) -> Self; // flock, disabled on contention + pub fn new(root: std::path::PathBuf) -> Self; // lock-free (tests) + pub fn disabled() -> Self; + pub fn is_enabled(&self) -> bool; + pub fn write(&self, record: &CodexSidecarRecord) -> std::io::Result<()>; // atomic, fsync'd + pub fn remove(&self, ownership_id: &str) -> std::io::Result<()>; + pub fn load_all(&self) -> Vec; // corrupt rows quarantined loudly +} +``` + +Layout: `/.json` (ownership ids are +`codex-sidecar-` — filesystem-safe as-is), `/lock`, corrupt rows +renamed to `.quarantined-`. Production root (wired in Task 10): +`/.freshell/rust-codex-sidecars/` — the `rust-` prefix is the +anti-collision convention (see the Node-parity decision; precedent +`rust-session-cache.json`). The atomic write helper is a deliberate, +provenance-commented duplicate of +`freshell_ws::tabs_persist::atomic_write_durable(destination, temporary, bytes)` +(`tabs_persist.rs:682-708`: sibling tmp → write → `sync_all` → rename → fsync +parent dir; tmp name `{file}.tmp-{pid}-{millis}` per `pane_ledger.rs:983-1000`) +— `freshell-ws` depends on `freshell-codex`, so the helper cannot be imported +(same reason the repo already duplicates `CODEX_MANAGED_REMOTE_CONFIG_ARGS`). + +- [ ] **Step 1: Write the failing unit tests** in + `crates/freshell-codex/src/sidecar_store_tests.rs` (tempfile tempdirs, no + global state, no processes): + - `record_roundtrips_through_disk` — write, `load_all`, byte-equal record. + - `write_is_atomic_sibling_tmp_then_rename` — after `write`, no `*.tmp-*` + residue remains and the destination parses. + - `disabled_store_is_a_silent_noop` — `disabled()`: `write`/`remove` return + `Ok(())`, `load_all` is empty, `is_enabled()` is false. + - `corrupt_record_is_quarantined_not_fatal` — hand-write garbage JSON + + one healthy record; `load_all` returns the healthy one and the garbage file + is renamed `*.quarantined-*` (fail-loud-per-row policy, + `pane_ledger.rs` module header). + - `second_locked_open_comes_up_disabled` — two `new_locked` on one root: the + second is disabled (single-writer flock, `pane_ledger.rs:236-274` pattern). +- [ ] **Step 2: Verify they fail** (module doesn't exist yet → compile error is + the red state for a new module; after stubbing the API with `todo!()`, the + tests must fail, not pass): + `cargo test -p freshell-codex --features real-transport sidecar_store` +- [ ] **Step 3: Implement** `sidecar_store.rs` per the interface above. + `new_locked` follows `PaneLedger::new_locked`: `flock(LOCK_EX|LOCK_NB)` via + `libc` on `/lock`; on failure log a structured `tracing::error!` and + come up disabled (`root: None` ⇒ every write `Ok(())` no-op). Open the lock + file via `std::fs::File` (O_CLOEXEC by default) and KEEP it that way — a + leaked lock fd inherited by a detached, retained sidecar (Task 3 removes + kill_on_drop) would hold the flock after the server dies and silently + disable the store for every future generation (reports/V6.md NA-3; add a + comment pinning this). Keep the file <1,000 lines. +- [ ] **Step 4: Verify green:** + `cargo test -p freshell-codex --features real-transport sidecar_store` +- [ ] **Step 5: Gates:** `cargo fmt --all --check` and + `cargo clippy -p freshell-codex --features real-transport --all-targets -- -D warnings` +- [ ] **Step 6: Commit** + +```bash +git add crates/freshell-codex/Cargo.toml crates/freshell-codex/src/lib.rs \ + crates/freshell-codex/src/sidecar_store.rs crates/freshell-codex/src/sidecar_store_tests.rs Cargo.lock +git commit -m "$(cat <<'EOF' +feat(codex): durable rust-owned sidecar record store (rust-codex-sidecars) + +Schema-v1 JSON records (pid, starttime, cmdline, ws url, ownership id, +session/terminal ids, state) in ~/.freshell/rust-codex-sidecars/, written +atomically (sibling tmp + fsync + rename, tabs_persist.rs precedent) under a +flock single-writer with the PaneLedger disabled-fallback and per-row +quarantine policies. Deliberately a distinct store from Node's +~/.freshell/codex-sidecars/ (rust-session-cache.json anti-two-writer +precedent). Groundwork for kata ynfn/da92. + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +### Task 2: Pid identity evidence capture + verification + +**Files:** +- Modify: `crates/freshell-codex/src/sidecar_store.rs` (add the identity + section) and `sidecar_store_tests.rs` + +**Interfaces produced** (Linux bodies `#[cfg(target_os = "linux")]`; non-Linux +stubs return `None`/`IdentityVerdict::Unverifiable` — never verified ⇒ never +killed): + +```rust +/// /proc//stat field 22; None for gone/zombie. (pid, starttime) is the +/// pid-reuse guard. Deliberate duplicate of the private +/// freshell-freshagent/src/session_lease.rs:144-160 helper (dependency +/// direction forbids importing it) — keep the parsing identical: split at the +/// LAST ')' then index 19, rejecting Z/X states. +pub fn proc_starttime(pid: i32) -> Option; + +/// /proc//cmdline, NUL-split into argv. World-readable (no ptrace/YAMA +/// constraint, unlike /proc//environ). +pub fn proc_cmdline(pid: i32) -> Option>; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IdentityVerdict { + /// (pid, starttime, cmdline) all match the record — this IS our sidecar. + Verified, + /// pid gone or zombie — the sidecar is dead; the record is stale. + Dead, + /// pid alive but starttime or cmdline differ — pid reuse; NEVER signal. + Mismatch, + /// non-Linux / evidence unreadable — NEVER signal. + Unverifiable, +} + +pub fn verify_sidecar_identity(record: &CodexSidecarRecord) -> IdentityVerdict; +``` + +- [ ] **Step 1: Write the failing tests** (each test spawns and reaps ONLY its + own child — `std::process::Command::new("sleep").arg("300")` — and kills it + in a `defer`-style guard; nothing else on the machine is ever signalled): + - `proc_starttime_identifies_a_live_child_and_none_after_exit` + - `verify_identity_confirms_own_spawned_child` — build a record from the + spawned child's real `/proc` evidence → `Verified`. + - `verify_identity_rejects_cmdline_mismatch_without_signalling` — record + carries the live child's pid+starttime but a DIFFERENT cmdline → + `Mismatch`; assert the child is still alive afterwards. + - `verify_identity_reports_dead_for_missing_pid` — pid far beyond + `/proc/sys/kernel/pid_max` reads or a reaped child → `Dead`. +- [ ] **Step 2: Verify red:** + `cargo test -p freshell-codex --features real-transport verify_identity` +- [ ] **Step 3: Implement**, with the provenance comment citing + `session_lease.rs:144-160` verbatim semantics. +- [ ] **Step 4: Verify green**, then `cargo fmt --all --check` and the + `real-transport` clippy leg. +- [ ] **Step 5: Commit** + +```bash +git add crates/freshell-codex/src/sidecar_store.rs crates/freshell-codex/src/sidecar_store_tests.rs +git commit -m "$(cat <<'EOF' +feat(codex): pid identity evidence + verification for sidecar records + +(pid, starttime, cmdline) capture from /proc and a four-way verdict +(Verified/Dead/Mismatch/Unverifiable). Stale pids are never trusted: only +Verified may ever be signalled; environ tags are not required (YAMA can hide +them for reparented orphans). starttime parsing duplicated with provenance +from session_lease.rs (kata ynfn groundwork). + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +### Task 3: Persist on spawn, scrub on teardown, survive server death + +**Files:** +- Modify: `crates/freshell-codex/src/launch_lifecycle.rs` +- Modify: `crates/freshell-codex/src/lib.rs` (re-export the store seam) +- Test: `crates/freshell-codex/tests/launch_lifecycle.rs` + +**Interfaces:** `SpawnedCodexAppServerRuntime` gains an injectable store: + +```rust +pub fn with_command_and_store(command: impl Into, store: Arc) -> Self; +``` + +Production resolution: a process-global +`pub fn set_codex_sidecar_store(store: Arc)` / +`fn codex_sidecar_store() -> Option>` pair in +`sidecar_store.rs` using a `RwLock>>` (re-settable, unlike the +manager's `OnceLock` — tests inject per-instance instead and never touch the +global). `SpawnedCodexAppServerRuntime::new()` reads the global; absent global +⇒ disabled store ⇒ behavior identical to today (all existing tests unaffected) +— literally: Step 3's detach is store- and platform-gated, so a disabled or +absent store keeps today's attached `kill_on_drop(true)` spawn. + +The trait seam it modifies — + +```rust +pub trait CodexLaunchRuntime: Send + Sync { + fn ensure_ready(&self, cwd: Option) + -> BoxFuture<'_, Result>; + fn update_ownership_metadata(&self, terminal_id: String, generation: u64) + -> BoxFuture<'_, Result<(), String>>; + fn shutdown(&self) -> BoxFuture<'_, Result<(), String>>; +} +``` + +(`launch_lifecycle.rs:66-98`) — is unchanged in this task. + +- [ ] **Step 1: Write the failing integration tests** in + `crates/freshell-codex/tests/launch_lifecycle.rs` (the file is already + `#![cfg(feature = "real-transport")]` and has `fake_app_server_command()` at + `:876-895`; use `with_command_and_store(fake_app_server_command(), + temp_store)`): + - `ensure_ready_persists_a_verified_sidecar_record` — after `ensure_ready`, + `store.load_all()` has exactly one `Active` record whose pid == + `runtime.child_pid().await`, whose `ws_url` matches the returned one, and + whose `(starttime, cmdline)` verify as `Verified` against the live child. + - `runtime_shutdown_removes_the_sidecar_record` — after + `runtime.shutdown()`, the store is empty and the pid is gone + (poll `/proc/` like the existing + `spawned_runtime_launches_the_app_server_and_relays_through_the_proxy`). + - `update_ownership_metadata_enriches_the_record` — record gains + `terminal_id`. + - `spawned_sidecar_survives_runtime_drop_without_shutdown` — call + `ensure_ready`, then `drop(runtime)` WITHOUT `shutdown()`; assert the child + pid is still alive (kill_on_drop is now off) **and the record still + exists** (this is the whole point: an uncleanly-dying server leaves a + tracked, reconcilable sidecar — not an invisible orphan). The test then + kills its own child: re-verify identity via the record, `libc::kill(pid, + SIGTERM)`, poll gone (only the pid this test spawned). + - `drop_without_shutdown_with_disabled_store_keeps_the_kill_on_drop_backstop` + — build the runtime with `CodexSidecarStore::disabled()` (Task 1's + disabled fallback); `ensure_ready`, then `drop(runtime)` WITHOUT + `shutdown()` → poll the child pid GONE. A record-less sidecar must never + outlive the server: detaching it would be the silently-orphaned ynfn + hole with no reconcile path, so untracked spawns keep today's + kill_on_drop backstop. +- [ ] **Step 2: Verify red:** + `cargo test -p freshell-codex --features real-transport --test launch_lifecycle ensure_ready_persists` +- [ ] **Step 3: Implement** inside + `impl CodexLaunchRuntime for SpawnedCodexAppServerRuntime` (`:939-1055`): + - In `ensure_ready`, after the probe loop breaks (listener up): capture + `pid = child.id()`, `starttime = proc_starttime(pid)`, + `cmdline = proc_cmdline(pid)` (fall back to the constructed + `program + args` if `/proc` momentarily unreadable), build the record + (`session_id: None`, `server_instance_id: default_server_instance_id()`), + `store.write(&record)` — write failures are logged loudly, never abort the + launch (pane-ledger write-failure policy). + - Detach CONDITIONALLY — only when the sidecar will actually be tracked: + `let detach = cfg!(target_os = "linux") && store.is_enabled();` + - `detach == true`: `cmd.kill_on_drop(false)` plus, inside a + `#[cfg(unix)]` block, `cmd.process_group(0);` + (`tokio::process::Command::process_group` is Unix-only, so the call + MUST be cfg-gated even though detach is Linux-only today — keeps any + non-Unix build compiling). Comment cites kata ynfn ("surviving + restarts is a feature") and the Node `detached: true` parity + (`runtime.ts:1828-1843`). The store record now plays the safety-net + role kill_on_drop played: an unclean death leaves a tracked record + for boot reconcile (Task 5/9). + - `detach == false` (disabled store — e.g. flock contention, an + evidenced same-HOME scenario, reports/V6.md A10 — or non-Linux): + keep today's `cmd.kill_on_drop(true)` and skip the record write. A + sidecar with NO record must keep the kill_on_drop backstop, else an + unclean death creates exactly the silently-orphaned ynfn hole with + no reconcile path; and non-Linux identity can never be verified, so + a detached sidecar there would be untracked AND unreapable. This + makes the Global Constraints claims literal: disabled store ⇒ + behavior identical to today; non-Linux ⇒ fresh-spawn path unchanged. + - In `shutdown` (`:894-905` today: `start_kill` → 5s wait → + `reap_owned_codex_sidecars`): after the reap, `store.remove(&ownership_id)`. + - In both `ensure_ready` failure arms that already call + `reap_owned_codex_sidecars(&ownership_id)` (`:861`, `:868`): also + `store.remove(...)` if a record was written. + - In `update_ownership_metadata` (`:883-892`): keep the in-memory tuple and + additionally rewrite the record with `terminal_id`/`updated_at`. +- [ ] **Step 4: Full crate green:** + `cargo test -p freshell-codex --features real-transport` — the pre-existing + `spawned_runtime_launches_…` test must still pass (explicit `shutdown()` + still kills; only *drop-without-shutdown* semantics changed). +- [ ] **Step 5: Cross-crate check** (consumers of the changed spawn): + `cargo test -p freshell-ws --test codex_managed_launch_e2e -- --ignored --test-threads=1` + (the binary's only test is an `#[ignore]`-gated host e2e — without + `--ignored` it runs zero tests and passes vacuously; `--test-threads=1` + per the file's own header, it mutates process env) and + `cargo test -p freshell-ws --test restore_storm` — baseline-identical. +- [ ] **Step 6: Gates:** `cargo fmt --all --check`; + `cargo clippy --workspace --all-targets -- -D warnings`; + `cargo clippy -p freshell-codex --features real-transport --all-targets -- -D warnings` +- [ ] **Step 7: Commit** + +```bash +git add crates/freshell-codex/src/launch_lifecycle.rs crates/freshell-codex/src/lib.rs \ + crates/freshell-codex/src/sidecar_store.rs crates/freshell-codex/tests/launch_lifecycle.rs +git commit -m "$(cat <<'EOF' +feat(codex): persist terminal-pane sidecar records at spawn; detach tracked sidecars from server death + +SpawnedCodexAppServerRuntime writes a verified (pid, starttime, cmdline, +ws url) record on successful spawn, enriches it at adopt, and removes it on +explicit shutdown. Tracked spawns (Linux + enabled store) switch +kill_on_drop(true) -> false + process_group(0): a dying server no longer +silently kills or silently orphans them — an unclean death leaves a TRACKED +record for boot reconciliation (kata ynfn). Untracked spawns (disabled store +or non-Linux) keep today's attached kill_on_drop backstop: a record-less +sidecar must never outlive the server. + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +### Task 4: Session-id enrichment (`note_session_id` seam) + +**Files:** +- Modify: `crates/freshell-codex/src/launch_lifecycle.rs` +- Modify: `crates/freshell-ws/src/codex_proxy_route.rs` +- Test: `crates/freshell-codex/tests/launch_lifecycle.rs`, + `crates/freshell-ws/tests/codex_candidate_inert.rs` harness pattern for the ws leg + +**Interfaces:** one new trait method with a default no-op (so `FakeRuntime` +implementations in tests keep compiling), plus a manager forwarder mirroring the +existing manager-level seam it sits beside — +`pub async fn mark_candidate_persisted(&self, terminal_id: &str)` +(`launch_lifecycle.rs:762-780`): + +```rust +// on trait CodexLaunchRuntime (default impl: Ok(())) +fn note_session_id(&self, session_id: String) -> BoxFuture<'_, Result<(), String>> { + let _ = session_id; + Box::pin(async { Ok(()) }) +} + +// on CodexTerminalLaunchManager (adopted-map lookup, silent no-op for unknown ids) +pub async fn note_session_id(&self, terminal_id: &str, session_id: &str); +``` + +Resume launches get the id at plan time; fresh launches get it when the proxy +captures the thread candidate. + +- [ ] **Step 1: Write the failing tests:** + - `plan_create_notes_the_resume_session_id_on_the_runtime` + (freshell-codex integration): extend the existing `FakeRuntime` + (`crates/freshell-codex/tests/launch_lifecycle.rs:37-118`) to record + `note_session_id` calls; plan with `resume_session_id: Some("s-1")` → + the runtime saw `"s-1"`; plan fresh → no call. + - `manager_note_session_id_reaches_adopted_runtime` — adopt, call the + manager seam, assert the runtime recorded it; unknown terminal id is a + silent no-op. + - `spawned_runtime_note_session_id_enriches_the_record` — with a temp + store: after `ensure_ready` + `note_session_id("s-1")`, the record's + `session_id == Some("s-1")`. +- [ ] **Step 2: Verify red:** + `cargo test -p freshell-codex --features real-transport note_session_id` +- [ ] **Step 3: Implement:** + - Trait default as above; `SpawnedCodexAppServerRuntime` override rewrites + the record. + - In `CodexLaunchPlanner::plan_create` (`:238-299`), after `ensure_ready` + succeeds: `if let Some(sid) = plan.session_id.clone() { let _ = + runtime.note_session_id(sid).await; }`. + - Manager forwarder beside `mark_candidate_persisted`. + - In `freshell-ws/src/codex_proxy_route.rs` (`route_proxy_event` → + candidate arm, `:47-115`), where the router already calls + `CodexTerminalLaunchManager::global().mark_candidate_persisted(terminal_id)` + after `adopt_codex_identity` returned true, add + `.note_session_id(terminal_id, thread_id)` — the thread id is in hand at + that call site (it just flowed through `apply_codex_identity`, + `codex_identity.rs:182-227`). +- [ ] **Step 4: ws-side check:** run the candidate-path suites that drive that + router — `cargo test -p freshell-ws --test codex_candidate_inert` and + `cargo test -p freshell-ws --test codex_managed_launch_e2e -- --ignored --test-threads=1` + (as in Task 3 Step 5: its only test is `#[ignore]`-gated; without + `--ignored --test-threads=1` the run is vacuous) — + baseline-identical (the new call is a no-op without an adopted spawned + runtime + store; full ws-side proof of fresh-launch enrichment rides Task 8's + harness). +- [ ] **Step 5: Gates:** fmt + both clippy legs. +- [ ] **Step 6: Commit** + +```bash +git add crates/freshell-codex/src/launch_lifecycle.rs crates/freshell-ws/src/codex_proxy_route.rs \ + crates/freshell-codex/tests/launch_lifecycle.rs +git commit -m "$(cat <<'EOF' +feat(codex): record the codex session/thread id in the sidecar record + +New note_session_id seam (default no-op on the runtime trait): plan_create +notes resume ids at plan time; the freshell-ws proxy-event router notes +captured thread candidates beside mark_candidate_persisted. Records now carry +the session id restore-time reattach keys on (katas ynfn/da92). + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +### Task 5: Boot reconciler — load, prune, claim + +**Files:** +- Create: `crates/freshell-codex/src/sidecar_reconcile.rs` (+ + `sidecar_reconcile_tests.rs`), declared in `lib.rs` behind `real-transport` + +**Interfaces produced:** + +```rust +pub struct SidecarReconciler { + store: Arc, + /// ALL held records, keyed by ownership_id — NOT by session id. Two live + /// records can legitimately share a session id (a mid-turn survivor + /// retained at sweep + a later fresh spawn enriched with the same session; + /// validated reachable — reports/V3.md), and Verified-without-session / + /// Unverifiable records must also be held for the sweep. Keying by + /// session_id would silently drop records (a fifth-fate ynfn violation). + held: Mutex>, + /// Secondary index for restore-time claims. + by_session: Mutex>>, +} + +impl SidecarReconciler { + /// Boot: load_all(); prune records whose identity verdict is Dead + /// (remove) or Mismatch (remove — the pid is NOT ours, never signal); + /// hold every remaining record by ownership_id (Verified with session = + /// claimable via the index; Verified without session and Unverifiable = + /// held for the sweep only). Returns a summary for boot logs. + pub fn boot_reconcile(store: Arc) -> (Self, BootReconcileReport); + + /// Restore-time claim: re-verify identity at claim time and return ONE + /// record for this session. With duplicates, pick the WRITER: prefer the + /// candidate whose live sidecar reports this session in + /// thread/loaded/list (a bounded ws probe — duplicate arm only, ~1s per + /// candidate), else newest updated_at. Losers + /// STAY held (they keep their sweep fate — never silently dropped). + /// Retained-state records ARE claimable (re-verified; adopt flips them + /// back to Active) — a late restore after the sweep must still reattach + /// a mid-turn survivor instead of reproducing the -32600 (reports/V3.md). + /// Only the returned record leaves `held`; each record is claimable ONCE. + /// ASYNC because of the writer probe (Task 7's factory is async-aware and + /// awaits this): the 0/1-candidate fast path opens no connection; the + /// duplicate arm snapshots candidates OUT of the `held`/`by_session` + /// locks before any await (std Mutex guards must never be held across an + /// await point — clippy `await_holding_lock`). + /// After the probe await, the winner is claimed by re-acquiring the + /// locks and removing it from `held`/`by_session` ONLY if still present; + /// a candidate the sweep consumed during the await is skipped (fall + /// through to the remaining candidates, else None). Membership in + /// `held` is the single source of truth for claim-vs-sweep ownership — + /// every exit from `held` happens under its lock (Task 9's sweep + /// TOCTOU guard is the mirror of this rule). + pub async fn claim_for_session(&self, session_id: &str) -> Option; + + pub fn unclaimed_len(&self) -> usize; +} + +pub fn set_codex_sidecar_reconciler(r: Arc); +pub fn codex_sidecar_reconciler() -> Option>; // RwLock seam +``` + +- [ ] **Step 1: Write the failing tests** (temp store + records; live pids are + test-spawned `sleep` children only): + - `boot_reconcile_prunes_dead_and_mismatched_records` — three records + (dead pid / live-`sleep`-child pid with wrong cmdline / verified child): + after boot, store holds only the verified one, `unclaimed_len() == 1`, + nothing was signalled (children still alive). + - `boot_reconcile_holds_sessionless_records_for_the_sweep` — a verified + record WITHOUT a session_id: held (`unclaimed_len()` counts it), not + claimable by any session, not dropped. + - `claim_for_session_returns_each_record_once` — two claims for one + session: first `Some`, second `None`. + - `claim_reverifies_identity_at_claim_time` — kill the test's own child + between boot and claim → claim returns `None` and the record is removed. + - `duplicate_session_records_claim_one_keep_the_loser_held` — two verified + records sharing one session id (two live test children): claim returns + one, the OTHER remains held for the sweep (`unclaimed_len() == 1` after + the claim), and both children are still alive (nothing signalled). + (`sleep` children speak no ws, so the writer probe fails fast on both + and the newest-`updated_at` fallback decides — deterministic and + bounded; the probe's positive arm is exercised by Task 9's + fixture-backed tests.) +- [ ] **Step 2: Verify red:** + `cargo test -p freshell-codex --features real-transport sidecar_reconcile` +- [ ] **Step 3: Implement** per the interface. Removal on prune uses + `store.remove`; every prune/claim decision emits a structured + `tracing::info!/warn!` with ownership id + verdict (auditability is half the + invariant). +- [ ] **Step 4: Green + gates** (fmt, both clippy legs). +- [ ] **Step 5: Commit** + +```bash +git add crates/freshell-codex/src/sidecar_reconcile.rs crates/freshell-codex/src/sidecar_reconcile_tests.rs \ + crates/freshell-codex/src/lib.rs +git commit -m "$(cat <<'EOF' +feat(codex): boot-time sidecar reconciler with verified claim-by-session + +Loads rust-codex-sidecars records at boot, prunes Dead records and removes +Mismatch ones without ever signalling (pid reuse is never trusted), and holds +Verified survivors as one-shot claimable by codex session id, re-verifying at +claim time (katas ynfn/da92). + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +### Task 6: `ReattachedCodexAppServerRuntime` + +**Files:** +- Modify: `crates/freshell-codex/src/sidecar_reconcile.rs` (+ tests file) + +**Interfaces:** a second `CodexLaunchRuntime` impl wrapping a claimed record: + +```rust +pub struct ReattachedCodexAppServerRuntime { + record: CodexSidecarRecord, + store: Arc, + verified_usable: AtomicBool, // set by ensure_ready; gates shutdown's kill +} +``` + +- `ensure_ready(cwd)`: ignore `cwd` (the survivor already has one); re-verify + identity; probe-dial `record.ws_url` with a bounded + `tokio::time::timeout(…, tokio_tungstenite::connect_async(&ws_url))` (the + same probe shape as the spawn path's A6-fixed loop, `:831-872`, but a single + short budget — reattach must fail FAST into fallback, use 3s). On success: + mark `verified_usable`, return `CodexRuntimeReady { ws_url: + record.ws_url.clone() }`. On failure: + - `Mismatch`/`Unverifiable` → `store.remove(...)`, return `Err` — **no + signal is ever sent** (this pid is not provably ours). + - `Dead` → `store.remove(...)`, `Err`. + - `Verified` but probe failed (dead port / handshake failure) → the + survivor is unusable: `kill_verified_sidecar_tree(&record)` (below), + `store.remove(...)`, `Err`. (Spec: "Fall back … when … the surviving one + is unusable (dead port, identity mismatch, handshake failure)"; an + unusable tracked sidecar must not leak. Killing it releases codex's + per-thread writer-lock files on exit, so the retry's fresh spawn can + resume the thread — validated, reports/V1.md.) +- `update_ownership_metadata` / `note_session_id`: rewrite the record (new + terminal id, updated_at). +- `shutdown()`: pane closed or plan raced shutdown — + `kill_verified_sidecar_tree(&record)` + `store.remove`. + `Mismatch`/`Dead`/`Unverifiable` ⇒ remove record only. + +**Shared tree-aware kill helper** (introduced here, reused by Task 9's sweep). +A3 was FALSIFIED (reports/V2.md): sidecars are process TREES (14 of the 24 +live orphans have live children right now, e.g. `codex-code-mode-host`), +children live in their OWN pgids/sessions (so neither single-pid signalling +nor a pgid group-kill covers them), codex's SIGTERM handler is a graceful +drain, and SIGKILL provably orphans its children (no PDEATHSIG; cleanup is +userspace-only). "Reaped" must mean the whole tree is gone: + +```rust +/// Re-verify (pid, starttime, cmdline); capture the pid's live descendant +/// set from /proc (children recursively, each snapshotted with its own +/// (pid, starttime, cmdline) so nothing is ever signalled on a stale pid); +/// SIGTERM the root; poll-gone with a drain-tolerant budget (5s, not 500ms — +/// codex drains gracefully); SIGKILL the root once if needed; then SIGTERM → +/// poll → SIGKILL each captured descendant that survived, re-verified by its +/// snapshot immediately before each signal. Returns what happened per pid. +/// Never signals anything whose snapshot no longer matches. +/// +/// ASYNC (binding): every call site is async on the tokio runtime — +/// `ReattachedCodexAppServerRuntime::ensure_ready` (inside the user-facing +/// restore path, holding one of the manager's two plan permits), +/// `shutdown`, and Task 9's `sweep_unclaimed` — and the poll-gone budgets +/// wait for multi-second intervals. All waits are `tokio::time::sleep` +/// awaits, never `std::thread::sleep`: a sync fn here would block executor +/// workers for the whole SIGTERM→poll→SIGKILL sequence (the same +/// sync/async impedance class the claim path already fixed). Callers must +/// not hold any `held`/store lock across this await. +pub async fn kill_verified_sidecar_tree(record: &CodexSidecarRecord) -> KillTreeOutcome; +``` + +`plan_create`'s existing cleanup-on-plan-failure (`:290-298` calls +`sidecar.shutdown()` on `ensure_ready` error) composes with this: a failed +reattach tears down via the SAME conservative path, and the retry loop +(`plan_create_with_retry`, `:312-339`) re-invokes the factory, which — the +claim being consumed — mints a fresh `SpawnedCodexAppServerRuntime`: **fallback +is structural, not special-cased.** + +- [ ] **Step 1: Write the failing tests** (each spawns its own fake app-server: + `node test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs --listen + ws://127.0.0.1:` as a direct `tokio::process::Command` child with env + `FRESHELL_CODEX_SIDECAR_ID=`; the test records the pid and + kills only that pid in cleanup): + - `reattach_ensure_ready_returns_the_existing_listener` — record built + from the live fixture's real `/proc` evidence; `ensure_ready` returns the + fixture's ws URL; fixture pid still alive; NO new process spawned. + - `reattach_refuses_on_identity_mismatch_without_signalling` — record + with the fixture's pid but a wrong cmdline → `Err`, record removed, + fixture still alive. + - `reattach_reaps_verified_but_unusable_survivor` — kill the fixture's + listener by scripting `exitProcessAfterMethodsOnce`… simpler: point the + record's `ws_url` at a port nothing listens on while pid evidence stays + valid (spawn the fixture on port A, record port B) → `Err`, fixture pid + reaped (gone), record removed. + - `reattach_shutdown_kills_only_after_reverification` — successful + `ensure_ready`, then `shutdown()` → fixture gone, record removed; and the + negative: replace the record's starttime, `shutdown()` → fixture alive. +- [ ] **Step 2: Verify red:** + `cargo test -p freshell-codex --features real-transport reattach_` +- [ ] **Step 3: Implement** per the sketch (keep `sidecar_reconcile.rs` under + 1,000 lines — split the runtime into `sidecar_reattach.rs` if needed). +- [ ] **Step 4: Green + gates.** +- [ ] **Step 5: Commit** + +```bash +git add crates/freshell-codex/src/sidecar_reconcile.rs crates/freshell-codex/src/sidecar_reconcile_tests.rs \ + crates/freshell-codex/src/lib.rs +git commit -m "$(cat <<'EOF' +feat(codex): reattach runtime — adopt a surviving verified sidecar instead of spawning + +ReattachedCodexAppServerRuntime implements CodexLaunchRuntime over a claimed +record: ensure_ready re-verifies identity and probes the existing listener +(3s budget, fail-fast into the structural fresh-spawn fallback via the plan +retry loop). Unusable-but-verified survivors are reaped; mismatched pids are +never signalled. Teardown kills only after re-verification (kata da92). + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +### Task 7: Plan-aware runtime factory (the selection seam) + +**Files:** +- Modify: `crates/freshell-codex/src/launch_lifecycle.rs` (factory type + + `plan_create` + `global()`), `crates/freshell-codex/src/sidecar_reconcile.rs` + (`select_codex_runtime`) +- Modify every factory construction site (mechanical closure-arity change): + `crates/freshell-codex/tests/launch_lifecycle.rs`, + `crates/freshell-codex/tests/global_manager_install.rs`, + `crates/freshell-ws/tests/*` files calling + `CodexTerminalLaunchManager::new/with_plan_budget` (locate with + `grep -rn "CodexTerminalLaunchManager::\(new\|with_plan_budget\)" crates/`) + +**Interface change** (quoting the current seam): + +```rust +// BEFORE (launch_lifecycle.rs:98): +pub type CodexRuntimeFactory = Box Arc + Send + Sync>; +// AFTER (async-aware: the selector must await the reconciler's writer-probe +// claim — claim_for_session is async, Task 5 — and plan_create is already +// async, so the factory returns a boxed future): +pub type CodexRuntimeFactory = Box< + dyn for<'a> Fn(&'a CodexLaunchPlan) + -> Pin> + Send + 'a>> + + Send + + Sync, +>; +``` + +(If closure-site lifetime inference gets awkward, the mechanical alternative — +take an owned `CodexLaunchPlan` clone and return a `'static` future — is +equally acceptable; the binding contract is only that async `plan_create` +AWAITS the factory's future.) The factory call already sits after +`plan_codex_launch(input)` succeeds (plan at `:306`, factory mint at `:308`); +this task only changes the signature: pass `&plan` and `.await` the returned +future. New pure-ish selector (unit-testable without globals): + +```rust +/// The production selection: a claimable verified survivor for the plan's +/// resume session ⇒ reattach; otherwise the spawn runtime. Reattach applies +/// only to resume plans (plan.session_id is Some ⇔ resume, launch_lifecycle.rs:169), +/// so the A4 fresh-restore exclusion and the 45s candidate-capture timer are +/// untouched. +pub async fn select_codex_runtime( + reconciler: Option<&Arc>, + store: Option<&Arc>, + plan: &CodexLaunchPlan, +) -> Arc; +``` + +`CodexTerminalLaunchManager::global()` (`:584-590`) installs +`Box::new(|plan| Box::pin(async move { +select_codex_runtime(codex_sidecar_reconciler().as_ref(), +codex_sidecar_store().as_ref(), plan).await }))`. + +- [ ] **Step 1: Write the failing tests:** + - `select_codex_runtime_prefers_a_claimable_survivor` (unit, in + `sidecar_reconcile_tests.rs`): reconciler with a verified record for + session `s-1` (live test-spawned fixture) — a resume plan for `s-1` gets a + reattach runtime (probe its `ensure_ready` ws URL == the record's); a + fresh plan or unknown session gets the spawn type; `None` reconciler gets + spawn. + - `plan_retry_falls_back_to_fresh_spawn_after_reattach_failure` + (integration, `tests/launch_lifecycle.rs`): planner whose factory wraps a + reconciler holding a **dead** record for `s-1` plus a spawn fallback using + `fake_app_server_command()`; `plan_create_with_retry(input(resume s-1), + attempts=2, …)` succeeds and the resulting launch is served by a fresh + fixture (record removed, no signal sent anywhere). +- [ ] **Step 2: Verify red** (the factory-arity change won't compile until + implemented — stub first, then red on assertions): + `cargo test -p freshell-codex --features real-transport select_codex_runtime` +- [ ] **Step 3: Implement** the type change, thread `&plan` through + `plan_create` and `.await` the factory's future there, fix every closure + site (existing fakes become + `Box::new(move |_plan| { let rt = fake.clone(); Box::pin(async move { rt }) })` + — a ready future, no behavior change), implement async + `select_codex_runtime`, update `global()`. +- [ ] **Step 4: Whole-workspace compile + affected suites:** + `cargo test -p freshell-codex --features real-transport` and + `cargo test -p freshell-ws --no-fail-fast` (factory closures in ws tests + updated; compare the failure set against the recorded full-suite baseline — + behavior baseline-identical since no reconciler global is installed there). +- [ ] **Step 5: Gates:** fmt; `cargo clippy --workspace --all-targets -- -D warnings`; + the two `real-transport` clippy legs. +- [ ] **Step 6: Commit** + +```bash +git add crates/freshell-codex/src crates/freshell-codex/tests crates/freshell-ws/tests +git commit -m "$(cat <<'EOF' +feat(codex): plan-aware runtime factory selects reattach over spawn + +CodexRuntimeFactory now receives the pure launch plan and returns a boxed +future that async plan_create awaits (the claim's duplicate arm awaits a +bounded ws writer probe); the global manager's factory claims a verified +surviving sidecar for resume plans via the reconciler and mints +ReattachedCodexAppServerRuntime, else the spawn runtime. +Claim consumption makes fresh-spawn fallback structural through the existing +plan retry loop (kata da92). + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +### Task 8: da92 end-to-end — restore reattaches through the WS door + +**Files:** +- Create: `crates/freshell-ws/tests/codex_sidecar_reattach_e2e.rs` + +**Harness (all pieces exist per the reports):** copy the `spawn_server()` +`WsState` construction + `connect_and_handshake` + `create_codex_terminal` +helpers from `crates/freshell-ws/tests/codex_managed_launch_e2e.rs:116-200` and +`codex_session_ref_resume.rs:202-241` (`NoIndexProbe` satisfies +`gate_wire_resume` — `Unknown` ⇒ proceed, §16c of `verbatim-snippets.md`); +`write_codex_dispatcher()` from `codex_managed_launch_e2e.rs:40-114` for the +TUI/sidecar dual role via `CODEX_CMD`; the fixture's +`appendThreadOperationLogPath` behavior knob for "which listener served +`thread/resume`". Install ONE global manager for the whole binary via +`set_global_codex_launch_manager_for_tests(CodexTerminalLaunchManager::new(factory))` +whose factory closes over a test-owned `SidecarReconciler` behind a +`static Mutex>>` the scenarios swap (set-once global ⇒ single +test binary; scenarios run under one `#[tokio::test]` or serialized). This +binary owns process env (`CODEX_CMD`, `FAKE_CODEX_APP_SERVER_BEHAVIOR`, +`CODEX_ARGV_CAPTURE_PATH`) — note it in the file header like +`resume_validation_gate.rs:692` does. + +- [ ] **Step 1: Write the failing e2e tests** (test frame fields verbatim per + §17: `{"type":"terminal.create","requestId":…,"mode":"codex","shell":"system", + "cwd":…,"restore":true,"sessionRef":{"provider":"codex","sessionId":…}}`): + - `restore_reattaches_tui_to_surviving_sidecar_preserving_in_flight_turn` — + (1) spawn the SURVIVOR: fixture on an ephemeral port with + `FRESHELL_CODEX_SIDECAR_ID=`, behavior + `{"appendThreadOperationLogPath": , "loadedThreadIds": [""]}` + (mid-turn shape); write its verified record (session ``) into a temp + store; reconciler over it. (2) send the restore create. (3) assert: the + captured TUI argv contains `--remote ws://127.0.0.1:` + + `resume `; then, playing the TUI, dial that proxy URL with + tokio-tungstenite, send `initialize`/`initialized` + `thread/resume + {threadId: }` and assert a SUCCESS result — and that log-A (the + survivor's op log) recorded the `thread/resume`, proving the pane is wired + to the surviving mid-turn sidecar; survivor pid still alive; store record + now carries the new `terminal_id`. + - `restore_falls_back_to_fresh_sidecar_without_tracked_survivor` — empty + reconciler; same create; a NEW fixture instance serves the plan (its op + log — log-B via `FAKE_CODEX_APP_SERVER_BEHAVIOR` for the dispatcher-spawned + sidecar — records the traffic); pane creates fine (today's path preserved). + - `active_writer_collision_surfaces_minus32600_only_on_the_fresh_path` — + the da92 control: empty reconciler + behavior `{"overrides": + {"thread/resume": {"error": {"code": -32600, "message": "thread already + has an active writer"}}}}` (the scripted rejection the fixture already + supports, `tests-and-persistence.md` §1.6 Route A); create the pane, dial + the proxy as the TUI, send `thread/resume` → assert the `-32600` error + frame comes back AND its message contains "active writer" (codex uses + -32600 generically for many rejections — reports/V1.md; the code alone + is not the incident signature). (The incident's failure mode, now + confined to the no-survivor path; the reattach test above proves the + same resume SUCCEEDS when a survivor exists.) + - Cleanup in every scenario: kill ONLY the pids the test spawned (survivor + fixture child; `registry.kill(&terminal_id)` for panes), matching the + existing suites. +- [ ] **Step 2: Verify red on the first test** (before Task-7's factory is + given the test reconciler, the reattach assertion fails — if implementing + strictly in order, red here means: with the reconciler deliberately absent, + the survivor's op log stays empty): + `cargo test -p freshell-ws --test codex_sidecar_reattach_e2e` +- [ ] **Step 3: Implement/fix** anything the e2e flushes out (expected: none — + Tasks 5–7 carry the logic; this task is the proof at the wire). +- [ ] **Step 4: Green:** + `cargo test -p freshell-ws --test codex_sidecar_reattach_e2e -- --nocapture` + plus baseline re-runs: `cargo test -p freshell-ws --test codex_session_ref_resume + --test restore_spawn_gate --test restore_storm` +- [ ] **Step 5: Gates:** fmt + workspace clippy. +- [ ] **Step 6: Commit** + +```bash +git add crates/freshell-ws/tests/codex_sidecar_reattach_e2e.rs +git commit -m "$(cat <<'EOF' +test(ws): e2e — codex pane restore reattaches to a surviving sidecar (da92) + +terminal.create{restore:true, sessionRef} against a tracked surviving fake +app-server routes the TUI's thread/resume to the SURVIVOR (mid-turn state +preserved, pid untouched); with no tracked survivor the fresh-spawn path is +byte-compatible with today, and the scripted -32600 active-writer rejection +is confined to that fresh path — the incident shape, now guarded. + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +### Task 9: Conservative reap sweep + the never-silently-orphaned invariant (ynfn) + +**Files:** +- Modify: `crates/freshell-codex/src/sidecar_reconcile.rs` (+ tests file) +- Modify: `test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs` + (Step 0 adds the per-thread `threadStatuses` knob — it does NOT exist yet) + +**Interfaces:** + +```rust +pub const FRESHELL_CODEX_SIDECAR_REAP_GRACE_MS_ENV: &str = "FRESHELL_CODEX_SIDECAR_REAP_GRACE_MS"; +pub const CODEX_SIDECAR_REAP_GRACE_MS_DEFAULT: u64 = 30 * 60 * 1000; // incident gap was 18 min + +#[derive(Debug, PartialEq)] +pub enum SweepOutcome { Reaped, RetainedMidTurn, RetainedWriterHeld, RecordRemovedStale, RetainedUnverifiable, SkippedClaimedDuringSweep } + +impl SidecarReconciler { + /// For every still-held, unclaimed record: re-verify identity, then + /// Dead / Mismatch -> remove record, NEVER signal (RecordRemovedStale) + /// Unverifiable -> retain, state = Retained{reason:"identity-unverifiable"} + /// Verified + ws probe (initialize -> thread/loaded/list -> thread/read + /// per loaded thread; `loaded` alone does NOT mean mid-turn — idle + /// threads stay loaded forever, reports/V1.md): + /// any thread/read status ACTIVE -> retain, Retained{reason:"mid-turn-active-thread"} + /// reachable, no active thread -> kill_verified_sidecar_tree, remove record (Reaped) + /// ws UNREACHABLE -> /proc//fd writer-evidence check + /// (open rollout .jsonl write handle or + /// thread-writer-locks/ file — readable + /// same-uid on this host, reports/V2.md): + /// evidence held -> retain, Retained{reason:"ws-unreachable-writer-held"} + /// no evidence -> kill_verified_sidecar_tree, remove record (Reaped) + /// TOCTOU guard (binding): the probe phase runs on a SNAPSHOT of `held` + /// (no locks across awaits), and a late claim_for_session may adopt a + /// snapshotted record while a probe awaits. Per-pid identity + /// re-verification CANNOT detect that (same live process), so it never + /// authorizes a kill alone. Structure the sweep decide → commit: for + /// each kill decision, re-acquire the `held` lock, confirm the record + /// is STILL held (unclaimed), REMOVE it from `held`/`by_session` under + /// that lock, release, and only then `kill_verified_sidecar_tree(...) + /// .await` + `store.remove`. If the record already left `held` + /// (claimed mid-sweep), skip with outcome SkippedClaimedDuringSweep and + /// send NO signal — a restore just reattached to that sidecar; killing + /// it is the exact da92 harm. Membership in `held` is the single source + /// of truth for claim-vs-sweep ownership (Task 5's claim removes + /// winners under the same lock). + /// Sweep CONSUMES only Reaped/RecordRemovedStale entries from `held`; + /// every Retained row STAYS held and claimable (a late restore must still + /// reattach a mid-turn survivor — reports/V3.md), and is re-evaluated at + /// next boot. Every decision logged with ownership id + verdict + outcome. + pub async fn sweep_unclaimed(&self) -> Vec<(String, SweepOutcome)>; +} +``` + +The mid-turn probe reuses the crate's own client +(`CodexAppServerClient` over `WsTransport`, already `real-transport`): +connect → `initialize`/`initialized` → `thread/loaded/list` → `thread/read` +per loaded thread, discriminating on its status (`active` vs idle). The +fixture implements `thread/loaded/list` (`loadedThreadIds` knob, +`tests-and-persistence.md` §1.3/§1.5) but has NO per-thread status knob +today: `thread/read` hardcodes `status: { type: 'idle' }` +(`fake-app-server.mjs:252-259`) and the `overrides` knob is a per-method +blanket, not per-thread. Step 0 therefore extends the repo-owned fixture +with a scriptable `threadStatuses: {"": "active"|"idle"}` config knob +consulted by `thread/read` (absent knob/id ⇒ current idle behavior, so all +existing fixture consumers are untouched). Kill mechanics use +Task 6's `kill_verified_sidecar_tree` (tree-aware — A3 falsified, sidecars +have children in their own pgids) plus the tag sweep supplement, +`reap_owned_codex_sidecars(ownership_id)` (`transport.rs:86-121`, quoted: +"we only signal processes carrying OUR unique tag"). + +- [ ] **Step 0: Extend the fixture (test harness, not production code):** add + the `threadStatuses` knob described above to + `test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs`; sanity: + existing suites that drive the fixture stay green (Tasks 3/6/8 tests). +- [ ] **Step 1: Write the failing tests** (all pids test-spawned): + - `sweep_reaps_verified_idle_unclaimed_sidecar` — fixture with + `{"loadedThreadIds": ["t-1"], "threadStatuses": {"t-1": "idle"}}` + + verified record, unclaimed → after `sweep_unclaimed`: pid gone (whole + tree), record removed, outcome `Reaped`. (Deliberately loaded-but-idle: + pins the `loaded ≠ mid-turn` discriminator, reports/V1.md.) + - `sweep_retains_mid_turn_sidecar_with_recorded_reason` — fixture with + `{"loadedThreadIds": ["t-1"], "threadStatuses": {"t-1": "active"}}` → + pid ALIVE, record present with + `state == Retained{reason:"mid-turn-active-thread"}` (spec: "A sidecar + mid-turn must end up reattached, not killed and not leaked" — unclaimed + mid-turn ⇒ retained + recorded, STILL claimable, re-evaluated at next + boot). + - `late_restore_after_sweep_reattaches_mid_turn_survivor` — after the + sweep retained the mid-turn fixture above, `claim_for_session` for its + session STILL returns the record (re-verified) — a late restore + reattaches instead of fresh-spawning into the `-32600` (A5 fix, + reports/V3.md). + - `sweep_never_touches_unverified_pids` — record naming a live + test-`sleep` pid with mismatched cmdline → `RecordRemovedStale`, sleep + child still alive. + - `sweep_never_kills_a_record_claimed_during_the_probe_window` — the + TOCTOU guard, deterministic (no timing dependence): boot a reconciler + holding one verified fixture record; claim it via `claim_for_session` + (it leaves `held` — a restore now owns it); then drive the sweep's + commit arm with the PRE-claim snapshot (the tests file is compiled + into the crate via `#[cfg(test)] #[path]`, so it can call the + pub(crate) decide/commit helper directly) → outcome + `SkippedClaimedDuringSweep`, fixture pid still ALIVE, the claimant's + record untouched. + - `kill_verified_sidecar_tree_reaps_descendants` (unit-level, own + processes only) — spawn a small tree the test owns (e.g. `bash -c + 'sleep 300 & sleep 300 & wait'`), build a verified record for the root, + kill via the helper → root AND both children gone; and the negative: + a snapshot-mismatched descendant is never signalled. + - `restart_reconciliation_leaves_no_sidecar_silently_orphaned` — **the + invariant test.** Seed five records: (a) claimable verified survivor, + (b) dead pid, (c) verified idle, (d) verified mid-turn, (e) a DUPLICATE + verified record sharing (a)'s session id (the A4 shape, reports/V3.md). + Run `boot_reconcile` → `claim_for_session` for (a)'s session → + `sweep_unclaimed`. Assert the exhaustive end-state: one of {(a),(e)} + claimed (reattached-by-construction), (b) removed at boot, (c) reaped, + (d) retained with recorded reason, and the claim LOSER of {(a),(e)} + swept to its own fate (reaped here — idle) — `store.load_all()` contains + ONLY the claimed Active record and (d)'s retained record. Every sidecar + is accounted for: reattached, reaped, or intentionally retained with a + recorded reason — never silently dropped from the books. +- [ ] **Step 2: Verify red:** + `cargo test -p freshell-codex --features real-transport sweep_` +- [ ] **Step 3: Implement.** (Optional extra isolation: these tests signal only + self-spawned children — same discipline as the existing + `spawned_runtime_…` kill test — but they may also be run under the repo's + sandbox: `scripts/sandbox-test.sh "cargo test -p freshell-codex --features + real-transport sweep_"`.) +- [ ] **Step 4: Green + gates.** +- [ ] **Step 5: Commit** + +```bash +git add crates/freshell-codex/src/sidecar_reconcile.rs crates/freshell-codex/src/sidecar_reconcile_tests.rs \ + test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs +git commit -m "$(cat <<'EOF' +feat(codex): conservative reap sweep — tracked, verified, never mid-turn (ynfn) + +sweep_unclaimed reaps only sidecars freshell recorded AND re-verified by +(pid, starttime, cmdline) at kill time, tree-aware (descendants snapshotted +and verified before any signal); mid-turn survivors (thread/read status +active — loaded alone is NOT mid-turn) are retained with a recorded reason +and stay claimable by late restores; unreachable-but-writer-holding survivors +are retained, not killed; mismatched/unverifiable pids are never signalled. +Invariant encoded in +restart_reconciliation_leaves_no_sidecar_silently_orphaned: every tracked +sidecar (including session-id duplicates) ends reattached, reaped, or +retained-with-reason. The repo-owned fake app-server fixture gains a +per-thread threadStatuses knob so tests can script thread/read status. + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +### Task 10: Server wiring — boot reconcile + shutdown retention + +**Files:** +- Modify: `crates/freshell-codex/src/launch_lifecycle.rs` (manager retention) +- Modify: `crates/freshell-server/src/main.rs` +- Modify: `crates/freshell-server/tests/safe11_term22_shutdown_reaping.rs` +- Test: `crates/freshell-codex/tests/launch_lifecycle.rs` + +**Manager retention interface:** + +```rust +impl CodexTerminalLaunchManager { + /// Server-shutdown mode: adopted (terminal-owned) sidecars are RETAINED + /// across the restart — proxies close, runtimes are asked to + /// prepare_retention(reason) instead of shutdown. Unadopted planner + /// sidecars (mid-plan) are still torn down. Call BEFORE registry.kill_all() + /// so PTY-exit hooks (notify_terminal_exit) also retain instead of reap. + pub fn begin_shutdown_retention(&self); +} + +// on trait CodexLaunchRuntime (default: Ok(())): +fn prepare_retention(&self, reason: String) -> BoxFuture<'_, Result<(), String>>; +``` + +`SpawnedCodexAppServerRuntime::prepare_retention` drops the `Child` handle +without killing (kill_on_drop is already false, Task 3) and rewrites the record +`state = Retained { reason }`. `ReattachedCodexAppServerRuntime` does the same +(no signal). `notify_terminal_exit` (`:616-623`) checks the retention flag: when +set, route the entry through retention instead of the teardown worker. +`shutdown()` (`:631-641`) with the flag set: `planner.shutdown()` still tears +down unadopted sidecars (they have no pane to reattach to and a fresh-plan +proxy may hold the candidate timer); adopted entries get proxy-close + +`prepare_retention("server-shutdown")`. + +**main.rs wiring** (quoting the touch points from §14 of +`verbatim-snippets.md`): +- Boot, beside the `PaneLedger` construction (`main.rs:383-395`): + ```rust + let codex_sidecar_store = std::sync::Arc::new( + freshell_codex::sidecar_store::CodexSidecarStore::new_locked( + home.as_ref().map(|h| h.join(".freshell").join("rust-codex-sidecars")), + ), + ); + freshell_codex::sidecar_store::set_codex_sidecar_store(codex_sidecar_store.clone()); + ``` +- Beside the `pane_ledger.boot_scan` call (`main.rs:975-999`): run + `SidecarReconciler::boot_reconcile(store)`, log the report, + `set_codex_sidecar_reconciler(...)`, then + `tokio::spawn` the grace-delayed sweep + (`sleep(reap_grace_from_env()).await; reconciler.sweep_unclaimed().await;`). +- **Disablement must be LOUD** (A10 validation, reports/V6.md: the restart + script provably waits for old-process exit, so contention is not expected on + the normal path — but a same-HOME scratch server on another port, e.g. the + evidenced `--port 3499` runs, would silently disable the store with no + timing race at all): when `!store.is_enabled()`, emit a dedicated + `tracing::error!` at boot and carry `codex_sidecar_store_enabled: bool` in + the logged reconcile report so a disabled generation is diagnosable from + logs alone. +- Supervisor caveat (recorded, no code): the in-repo systemd unit is NOT + installed today (restarts are script-driven). If it is ever adopted, its + KillMode must not be `control-group` — cgroup kill would slaughter the + retained sidecars that this task deliberately keeps alive + (`process_group(0)` detaches the pgid, not the cgroup). reports/V6.md NA-1. +- Shutdown (`main.rs:1663-1722`): insert + `freshell_codex::launch_lifecycle::CodexTerminalLaunchManager::global().begin_shutdown_retention();` + immediately BEFORE `registry.kill_all();` (`:1329`), and leave the existing + `…global().shutdown().await` (`:1355-1357`) in place — it now retains adopted + sidecars and still reaps unadopted ones. Update the surrounding SAFE-11 + comment block to record the new deliberate behavior (kata ynfn). +- Retention gate: retention applies ONLY to runtimes whose sidecar record + was persisted (enabled store — Task 3's conditional detach). A runtime + spawned under a disabled store has NO record and kept `kill_on_drop(true)`; + `shutdown()` MUST tear it down exactly as today — "retaining" a + record-less sidecar would orphan it silently (the ynfn hole). + +- [ ] **Step 1: Write the failing manager tests** in + `crates/freshell-codex/tests/launch_lifecycle.rs`: + - `shutdown_retention_retains_adopted_sidecars_and_records_reason` — real + spawned runtime (fake fixture) + temp store; plan, adopt, + `begin_shutdown_retention()`, `shutdown().await` → fixture pid STILL + ALIVE, record `Retained{reason:"server-shutdown"}`. Test then reaps its + own fixture pid (verified) in cleanup. + - `shutdown_still_tears_down_unadopted_planner_sidecars` — plan WITHOUT + adopt, retention on, `shutdown()` → pid gone, record removed. + - `retention_with_disabled_store_tears_down_as_today` — runtime built + with `CodexSidecarStore::disabled()` (Task 3's conditional detach ⇒ + kill_on_drop(true), NO record), adopted, retention on, + `shutdown().await` → pid GONE: record-less sidecars are never + retained. + - `notify_terminal_exit_retains_under_retention_flag` — adopted + + retention on + `notify_terminal_exit` → pid alive, record retained + (vs. the default flag-off behavior: teardown worker reaps — existing + behavior, assert both arms). +- [ ] **Step 2: Verify red:** + `cargo test -p freshell-codex --features real-transport retention` +- [ ] **Step 3: Implement** manager + trait + both runtimes + main.rs wiring. +- [ ] **Step 4: Reconcile SAFE-11.** Run + `cargo test -p freshell-server --test safe11_term22_shutdown_reaping` + (black-box: spawns the built binary on an ephemeral port — NEVER 3001 — and + drives real `/ws` frames). VALIDATED (reports/V4.md): the test contains NO + terminal-pane codex sidecar assertions — its codex coverage is a + freshagent-lane sidecar (`freshAgent.create {sessionType:"freshcodex"}`) + plus a shell PTY, both outside this plan's retention. Expectation: + **the suite passes UNCHANGED** (it now doubles as a tripwire that retention + did not leak into the freshagent lane or shell-PTY reaping). If it is + cheap, ADD a terminal-pane codex retention scenario: create a codex + terminal pane, graceful shutdown, assert the sidecar pid is ALIVE and the + store row (test-owned `FRESHELL_HOME`) reads + `Retained{reason:"server-shutdown"}`, then reap the pid from the test + (verified) — and re-scope the test's generic "no live descendants" + assertion to exclude the intentionally retained pid. Record the documentary + deviation: the parity checklist's acceptance text "terminate exact + terminal/provider/extension trees" + (`docs/plans/2026-07-14-rust-tauri-parity-completion-checklist.md:615`) is + deliberately inverted for tracked codex terminal-pane sidecars (kata ynfn: + "killing sidecars at shutdown is NOT acceptable — surviving restarts is a + feature") — one bullet in the test header comment and in the PR description + (Task 11). +- [ ] **Step 5: Green:** `cargo test -p freshell-codex --features + real-transport` + `cargo test -p freshell-server`; fmt + workspace clippy + + both feature clippy legs. +- [ ] **Step 6: Commit** + +```bash +git add crates/freshell-codex/src/launch_lifecycle.rs crates/freshell-codex/tests/launch_lifecycle.rs \ + crates/freshell-server/src/main.rs crates/freshell-server/tests/safe11_term22_shutdown_reaping.rs +git commit -m "$(cat <<'EOF' +feat(server): boot sidecar reconcile + reap sweep; retain adopted sidecars at shutdown + +Boot constructs the flock'd rust-codex-sidecars store, reconciles records +(prune stale, hold verified survivors claimable), and arms the grace-delayed +conservative sweep (FRESHELL_CODEX_SIDECAR_REAP_GRACE_MS, default 30m — +restores arrived 18m post-boot in the incident). Graceful shutdown now +RETAINS adopted codex sidecars with a recorded reason instead of killing them +(kata ynfn: surviving restarts is a feature); unadopted mid-plan sidecars are +still torn down. SAFE-11 expectations updated with recorded rationale. + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +### Task 11: Full verification sweep + parity follow-up recording + +**Files:** test/verification only; fix-forward anything the sweep flushes out +in files this plan already touched. + +- [ ] **Step 1: Per-crate cargo tests with real counts** (no npm wrapper + exists for cargo; run from the worktree root; NOT `--workspace` — it drags + tauri's GTK deps): + ```bash + cargo test -p freshell-codex --features real-transport + cargo test -p freshell-ws --no-fail-fast + cargo test -p freshell-server + cargo test -p freshell-platform + cargo test -p freshell-freshagent # baseline the known environmental e2e failures before/after + ``` + Record pass counts (freshell-ws: compare the full failure set against the + recorded full-suite baseline); `freshell-freshagent` must be + baseline-identical. +- [ ] **Step 2: Mirror CI exactly:** + ```bash + cargo fmt --all --check + cargo clippy --workspace --all-targets -- -D warnings + cargo clippy -p freshell-codex --features real-transport --all-targets -- -D warnings + cargo clippy -p freshell-opencode --features real-transport --all-targets -- -D warnings + ``` +- [ ] **Step 3: Safety audit of the diff** (grep-verifiable, binding): + `git diff origin/main --stat` touches ONLY the files this plan names; grep + the diff for `pkill|killall|kill -9 [0-9]` → zero hits; every `libc::kill` + in new code is preceded by an identity re-verification or the ownership-tag + needle match; no test binds port 3001; nothing writes + `~/.freshell/codex-sidecars/` (Node's dir) — + `grep -rn "codex-sidecars" crates/` must show only the `rust-codex-sidecars` + literal and comments. +- [ ] **Step 4: Record the follow-ups and deviations.** Add to the eventual PR + description: the Node-parity decision (verbatim from this plan's "Recorded + decision" section), the freshagent-lane scope decision (revised scope note; + the orphan cohort is NOT all terminal-pane — reports/V2.md), and the SAFE-11 + documentary deviation (Task 10 Step 4). File BOTH follow-up katas: + `kata create "Node server: codex pane restore should reattach to a surviving sidecar (da92 parity)" --label bug --related da92 --agent --body ""` + and + `kata create "freshagent-lane codex sidecars orphan across unclean restarts (ynfn residue)" --label bug --related ynfn --agent --body ""`. + Known minor limitation to note in the PR (hub-side, out of scope): a + reattached mid-turn pane's FIRST turn completion lands in the status hub's + Idle arm, so its completion chime may not fire once + (`codex_proxy_route.rs:419-423`, reports/V4.md). + `docs/index.html`: N/A (backend lifecycle work; no user-facing UI change). +- [ ] **Step 5: Commit (only if fixes were needed)** — same trailer convention: + +```bash +git add -A crates/ +git commit -m "$(cat <<'EOF' +test(codex): regression fixes from the full sidecar-lifecycle sweep + +🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) + +Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> +EOF +)" +``` + +--- + +## Spec-requirement → coverage map + +| Requirement (task-description.md §A) | Task | Proof | +|---|---|---| +| Persist pid, ws port, session/thread id + identity evidence (cmdline + starttime); stale pids never trusted | 1, 2, 3, 4 | `record_roundtrips_through_disk`, `verify_identity_rejects_cmdline_mismatch_without_signalling`, `ensure_ready_persists_a_verified_sidecar_record`, `spawned_runtime_note_session_id_enriches_the_record` | +| Reattach on restore when alive + usable, incl. mid-turn, in-flight turn intact | 5, 6, 7, 8 | `reattach_ensure_ready_returns_the_existing_listener`, `select_codex_runtime_prefers_a_claimable_survivor`, `restore_reattaches_tui_to_surviving_sidecar_preserving_in_flight_turn` (mid-turn `loadedThreadIds` fixture, resume served by the SURVIVOR) | +| Fallback to fresh spawn (dead port / identity mismatch / handshake failure) | 6, 7, 8 | `reattach_reaps_verified_but_unusable_survivor`, `plan_retry_falls_back_to_fresh_spawn_after_reattach_failure`, `restore_falls_back_to_fresh_sidecar_without_tracked_survivor` | +| Reap tracked-but-unclaimed ONLY after provable identity verification, never by name pattern; never touch non-freshell codex processes | 2, 9 | `sweep_never_touches_unverified_pids`, `sweep_reaps_verified_idle_unclaimed_sidecar`; new tracking starts EMPTY ⇒ the ~20 live orphans (incl. PID 545173) are structurally out of reach | +| Mid-turn ⇒ reattached, not killed, not leaked | 8, 9 | reattach e2e (claimed) + `sweep_retains_mid_turn_sidecar_with_recorded_reason` (unclaimed ⇒ retained with reason, still claimable) + `late_restore_after_sweep_reattaches_mid_turn_survivor` | +| Invariant: after restart every freshell-spawned sidecar is reattached / reaped / intentionally-retained-with-reason — never silently orphaned | 9 | `restart_reconciliation_leaves_no_sidecar_silently_orphaned` (exhaustive five-record end-state assertion, incl. the duplicate-session claim loser) | +| Sidecars survive restarts (ynfn: shutdown kills are unacceptable) | 3, 10 | `spawned_sidecar_survives_runtime_drop_without_shutdown`, `shutdown_retention_retains_adopted_sidecars_and_records_reason`, SAFE-11 adjustment with recorded rationale | +| da92 incident shape (`-32600` active-writer) encoded with the existing fixture | 8 | `active_writer_collision_surfaces_minus32600_only_on_the_fresh_path` (fixture Route A override) | +| Node parity decision recorded, not silent | header + 11 | "Recorded decision" section; PR description + follow-up kata step | +| TDD, unit + integration, fmt/clippy clean, local cargo test (CI runs none) | every task | failing-test-first steps; per-task fmt/clippy; Task 11 mirrors CI + records counts | + +## Self-Review + +Checks performed on the draft before finalizing, and fixes applied: + +- **(a) Failing-test-first ordering** — every behavior task (1–10) opens with a + named failing test and an explicit red-verification step before + implementation. Fixed during review: Task 7's Step 2 originally said "verify + red" for a change that is compile-breaking — reworded to stub-first so red is + an assertion failure, not a compile error; Task 8's red state clarified + (survivor op-log empty when the reconciler is withheld). Task 11 is + verification-only by design (like the exemplar's Task 5). +- **(b) Exact paths/symbols exist per the reports** — re-checked every quoted + seam against `verbatim-snippets.md`/`rust-sidecar.md`: `CodexRuntimeFactory` + (`launch_lifecycle.rs:46`), factory mint at `plan_create` `:308` + (report §4 note), `SpawnedCodexAppServerRuntime` struct/ctors `:840-907`, + trait `:66-98`, manager `mark_candidate_persisted(&self, terminal_id)` + `:762-780`, `reap_owned_codex_sidecars` `transport.rs:86-121`, + `proc_starttime`/`scan_tagged_pids` `session_lease.rs:130-188` (private ⇒ + duplicated with provenance, dependency direction verified), + `plan_codex_managed_launch` `terminal.rs:1160-1226`, main.rs shutdown + `:1663-1722` / PaneLedger boot `:383-395` / `boot_scan` `:975-999`, + `atomic_write_durable` `tabs_persist.rs:682-708`, dispatcher + `WsState` + harness `codex_managed_launch_e2e.rs:40-200`, `NoIndexProbe` gate + satisfaction (§16c), create-frame field names (§17), fixture knobs + (`overrides` -32600, `loadedThreadIds`, `appendThreadOperationLogPath`, + `thread/loaded/list` — `tests-and-persistence.md` §1.3–1.6). Fixed during + review: an earlier draft imported `atomic_write_durable` from `freshell-ws` + — impossible (freshell-ws depends on freshell-codex); switched to the + documented deliberate-duplication pattern. Also corrected the earlier + assumption that the manager mints "a new planner per plan" + (verbatim-snippets §5c: it does not — one planner, per-plan runtime via the + stored factory), which is why the factory type, not the manager, is the + selection seam. One residual risk flagged honestly: + `tokio::process::Command::process_group` availability was reasoned from the + resolved tokio 1.52.3 (API added well before), not grepped from vendored + sources — if absent, fall back to `unsafe pre_exec(setsid)` with the same + test coverage (`spawned_sidecar_survives_runtime_drop_without_shutdown` + pins the behavior either way). +- **(c) Invariant coverage** — the four fates (reattached / reaped / retained + with reason / stale-record-removed) are each individually tested AND jointly + asserted in `restart_reconciliation_leaves_no_sidecar_silently_orphaned` + (Task 9), with retention-at-shutdown (Task 10) writing the recorded reason. + Fixed during review: the unclaimed-mid-turn case originally had no recorded + fate — added `Retained{reason:"mid-turn-active-thread"}` + re-evaluation at + next boot, closing the "never silently orphaned" hole for panes the client + never restores. +- **(d) Process-safety compliance** — no name-pattern kills anywhere; every + signal path re-verifies `(pid, starttime, cmdline)` at kill time or matches + the per-launch env tag; non-Linux ⇒ `Unverifiable` ⇒ never signalled; the + new store starts empty so the ~20 live orphans (incl. mid-turn PID 545173) + are structurally unreachable by the reaper and by every test (tests signal + only self-spawned children); no test or step touches port 3001 (SAFE-11 + black-box test uses an ephemeral port per its existing harness); Task 11 + Step 3 makes the audit mechanical. +- **(e) Node-parity decision recorded** — explicit "Recorded decision" section + (Rust-only, four reasons, `rust-codex-sidecars/` distinct-store constraint + honored — the plan never reads or writes Node's `~/.freshell/codex-sidecars/`) + plus Task 11 Step 4 carries it into the PR description and a follow-up kata, + satisfying "do not let the Node side rot silently without a decision". + +## Load-bearing validation pass (post-planning; ledger + evidence in `.worktrees/.the-usual-logs/codex-sidecar-lifecycle/`) + +Ten load-bearing assumptions were surfaced and validated (finder → strategist +→ 6 parallel validators; full evidence in `reports/finder.md`, +`reports/strategist.md`, `reports/V1.md`–`V6.md`, ledger in +`load-bearing-ledger.md`). Plan changes applied from the verdicts: + +- **VERIFIED — reattach works** (V1, live experiment on the deployed 0.147.0 + binary): new-client `thread/resume` into a surviving mid-turn app-server + succeeds and streams the in-flight turn; disconnect doesn't abort turns; + writer lock is a per-thread file flock released on process death. Recorded + in Background ("Validated codex behavior"). +- **FALSIFIED — `loaded` ≠ `mid-turn`** (V1): Task 9's discriminator changed + to per-thread `thread/read` status, with a `/proc//fd` writer-evidence + arm for unreachable survivors; fixture gains a `threadStatuses` knob. +- **FALSIFIED — single-pid kill** (V2): sidecars are process trees (children + in their own pgids; SIGKILL orphans them). Task 6 introduces + `kill_verified_sidecar_tree` (snapshot-verified, drain-tolerant, tree-wide); + Task 9 reuses it; "Reaped" now means the whole tree is gone. +- **FALSIFIED — one-record-per-session** (V3): reconciler re-keyed by + ownership_id with a session index; writer-aware deterministic claim; + losers keep sweep fates; invariant test seeds the duplicate case. +- **FALSIFIED — restores-arrive-in-grace-window** (V3): Retained rows stay + claimable after the sweep; `late_restore_after_sweep_reattaches_mid_turn_survivor` + pins it. +- **FALSIFIED — orphans are all terminal-pane-lane** (V2): scope note revised; + freshagent-lane follow-up kata added to Task 11; ynfn close-out wording + updated. +- **FALSIFIED (favorably) — SAFE-11 "adjust expectations"** (V4): the test has + no terminal-pane codex assertions; Task 10 Step 4 rewritten to + verify-unchanged + optional added retention scenario + recorded documentary + deviation. +- **FALSIFIED — clean baseline** (V5): measured baseline recorded in Global + Constraints (freshell-ws has exactly 2 pre-existing deterministic failures; + node_modules resolves from the parent checkout). +- **VERIFIED — proxy needs zero changes** (V4) and **restart choreography + releases the flock** (V6), with recorded hardening: loud store-disablement, + O_CLOEXEC lock fd, systemd KillMode caveat. + +Self-review re-run over every edited task against (a) failing-test-first +ordering (new/changed tests remain Step-1 items with red verification), +(b) paths/symbols grounded (all new claims cite validator reports with +file:line evidence), (c) invariant coverage (five fates incl. duplicate +loser; retained rows claimable), (d) process-safety (tree kills are +snapshot-verified per pid; nothing signals unverified pids; the ~20 live +orphans remain structurally unreachable — the store starts empty), +(e) scope decisions recorded (Node parity + freshagent lane + SAFE-11 +deviation all carried into Task 11/PR description). + +## Fresh-eyes review pass (iteration 1) + +An independent zero-context cross-model review (log: +`.worktrees/.the-usual-logs/codex-sidecar-lifecycle/fresheyes-plan.md`) found +two blocking executable-spec defects, both fixed above: + +- **Task 9's fixture extension was unnamed.** The required `threadStatuses` + knob does not exist in `fake-app-server.mjs` (`thread/read` hardcodes idle + at `:252-259`; `overrides` is per-method, not per-thread), yet the fixture + file was missing from the File Structure table, Task 9's Files list, and + Task 9's commit — stranding the edit uncommitted and breaking Task 11 + Step 3's binding "only files this plan names" audit. Fixed: fixture named + in File Structure and Task 9 Files, added as Task 9 Step 0 + (default-idle, backward-compatible), staged in Task 9's commit, and + described in its commit message. +- **`claim_for_session` sync/async impedance.** The claim's writer + preference requires a bounded ws probe, but the fn was spec'd sync and + reached through a sync factory inside async `plan_create` — no awaitable + path (`block_on`/`block_in_place` panic in the runtimes involved). Fixed: + `claim_for_session` and `select_codex_runtime` are now async, and + `CodexRuntimeFactory` returns a boxed future that async `plan_create` + awaits; fake-closure sites wrap ready futures (mechanical, no behavior + change); duplicate candidates are snapshotted out of the locks before any + await. + +Self-review re-run over the edited tasks (5, 7, 9): failing-test-first +ordering intact (fixture Step 0 is harness work preceding Task 9's red +tests; Task 7 keeps stub-first red); paths/symbols grounded (fixture +hardcoded-idle at `fake-app-server.mjs:252-259` and factory alias at +`launch_lifecycle.rs:98` are reviewer-verified against the worktree); +invariant coverage unchanged (five fates incl. the duplicate loser; retained +rows stay claimable); process-safety unchanged (no new signal paths); scope +decisions unchanged. + +## Fresh-eyes review pass (iteration 2) + +A second independent zero-context cross-model review (same log) found two +blocking verification-gate defects, both fixed above: + +- **Vacuous `codex_managed_launch_e2e` gates (Tasks 3/4).** The binary's + only test is `#[ignore]`-gated (host e2e, + `codex_managed_launch_e2e.rs:312`), so the plan's + `--test codex_managed_launch_e2e` commands ran zero tests and passed + unconditionally — while being the only ws-side exercise of the exact spawn + semantics Task 3 changes. Fixed: both gates now pass + `-- --ignored --test-threads=1` (single-threaded per the file's own + header). +- **Fail-fast made every `cargo test -p freshell-ws` gate unverifiable.** + The measured 490-passed baseline aborted at `auto_resume_e2e` (3rd of ~44 + test binaries), so the "exactly these 2 fail" definition could not be + checked and the suites this plan touches never ran under those gates. + Fixed: the Global-Constraints baseline bullet now mandates + `--no-fail-fast` on every whole-package ws gate, a pre-Task-1 full-suite + baseline recording (`baseline-test-ws-full.log` in the logs dir), and + redefines "baseline-identical" as failure-set equality against that + recording; Task 7 Step 4 and Task 11 Step 1 updated accordingly. + +Self-review re-run over the edited sections (Global Constraints baseline, +Tasks 3, 4, 7, 11): failing-test-first ordering untouched (only +verification commands and the baseline definition changed — no +test-creation steps moved); paths/symbols grounded (the `#[ignore]` gate at +`codex_managed_launch_e2e.rs:312` and the 3-binary fail-fast abort in +`baseline-test-ws.log` are reviewer-verified against the worktree); +invariant coverage unchanged (five fates incl. the duplicate loser); +process-safety unchanged (no new signal paths); scope decisions unchanged +(verification-only amendments; no new files enter the plan's task file +set, so Task 11 Step 3's binding files audit is unaffected). + +## Fresh-eyes review pass (iteration 3) + +A third independent zero-context cross-model review (same log) found four +blocking executable-spec defects, all fixed above: + +- **Task 11 Step 3's binding audit tripped on this plan document.** The + branch's `git diff origin/main` necessarily contains this plan file, + which is not in the audited file set and legitimately contains `pkill` + and the audit's own grep pattern — so "only named files" and "zero hits" + could never pass as written. Fixed: the audit is now scoped to the code + portion of the diff (`-- crates/ test/fixtures/`), with the exclusion + reason recorded inline. +- **Task 3's detach was unconditional.** `kill_on_drop(false)` + + `process_group(0)` applied even with a disabled store (flock contention — + an evidenced same-HOME scenario, reports/V6.md A10) or off-Linux, + contradicting the "disabled store ⇒ identical to today" claim and the + non-Linux "fresh-spawn path unchanged" constraint, creating record-less + detached orphans (the ynfn hole, minus even today's kill_on_drop + backstop), and calling the Unix-only `process_group` unguarded. Fixed: + detach is store- and platform-gated + (`cfg!(target_os = "linux") && store.is_enabled()`), `process_group(0)` + is `#[cfg(unix)]`-gated, untracked spawns keep today's + `kill_on_drop(true)`, a new Task 3 Step 1 test pins the disabled-store + backstop, Task 10 gains the matching retention gate (record-less + sidecars are never retained) with its own test, and the Global + Constraints / File Structure / commit-message wording now matches. +- **Claim/sweep TOCTOU.** The sweep probes a snapshot across awaits; a + late claim could adopt a snapshotted record, and per-pid identity + re-verification cannot detect that (same live process) — the sweep could + kill a just-reattached sidecar (the exact da92 harm). Fixed: membership + in `held` is the single source of truth; both claim and sweep remove + records under the `held` lock after their awaits (claim skips + sweep-consumed candidates; the sweep's commit arm re-checks membership + under lock immediately before any signal and skips claimed records with + the new `SkippedClaimedDuringSweep` outcome), pinned by the + deterministic `sweep_never_kills_a_record_claimed_during_the_probe_window` + test. +- **`kill_verified_sidecar_tree` sync signature.** The + SIGTERM→poll(5s drain)→SIGKILL contract was spec'd as a sync `pub fn` + yet every call site is async (one holding a plan permit inside the + user-facing restore path) — it would block executor workers for + multi-second intervals, the same impedance class iteration 1 fixed for + `claim_for_session`. Fixed: the helper is `pub async fn` with + `tokio::time::sleep` waits and a no-locks-across-await caller rule. + +Self-review re-run over the edited sections (Global Constraints, File +Structure, Tasks 3, 5, 6, 9, 10, 11): failing-test-first ordering intact +(the new behaviors — disabled-store backstop, sweep TOCTOU skip, retention +gate — enter as named Step 1 failing tests in their tasks); paths/symbols +grounded (the gating uses only symbols this plan itself introduces, incl. +`CodexSidecarStore::disabled()`/`is_enabled()` from Task 1's contract; the +reviewer verified tokio 1.52.3's `process_group` and the fixture state +against the worktree); invariant coverage strengthened (record-less spawns +keep the kill_on_drop backstop, so the recorded fates stay exhaustive for +tracked sidecars while untracked ones cannot outlive the server; +claimed-during-sweep records keep their claimant's Active fate); +process-safety strengthened (one MORE pre-signal check — under-lock +membership confirmation before every sweep kill; no new signal paths; the +~20 live orphans remain structurally unreachable); scope decisions +unchanged (docs-only plan edits; the audit rescope names its reason +inline). diff --git a/test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs b/test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs index af63f29ea..0ffcccc20 100644 --- a/test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs +++ b/test/fixtures/coding-cli/codex-app-server/fake-app-server.mjs @@ -250,12 +250,20 @@ function successResult(method, params) { return {} } if (method === 'thread/read') { - return { - thread: makeThread(params?.threadId, { - ...params, - includeTurns: params?.includeTurns === true, - }), + const thread = makeThread(params?.threadId, { + ...params, + includeTurns: params?.includeTurns === true, + }) + // Task 9 knob: per-thread scriptable status, consulted by thread/read only. + // threadStatuses: {"": "active"|"idle"} — an absent knob or an + // unlisted thread id keeps makeThread's hardcoded { type: 'idle' }, so all + // existing fixture consumers are untouched. (The `overrides` knob is a + // per-METHOD blanket and cannot express per-thread status.) + const scriptedStatus = behavior.threadStatuses?.[thread.id] + if (typeof scriptedStatus === 'string') { + thread.status = { type: scriptedStatus } } + return { thread } } if (method === 'thread/turns/list') { return makeThreadTurnsPage(params)