diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index d1d59d3ceb..a01af557e2 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -177,9 +177,11 @@ BUSY_TURN_MAX_SECS=${FM_BUSY_TURN_MAX_SECS:-3600} # A crew that declared a pause is idling on a known external wait, so its stale # pane is absorbed rather than wedge-escalated. # A captain-held or paused crew whose agent has confidently exited uses the same -# bounded cadence, while a live or ambiguously read agent still surfaces once; a -# secondmate earns the cadence on its declaration alone, because its endpoint -# liveness is deliberately never read (pause_state_class owns that split). +# bounded cadence, while a live or ambiguously read agent surfaces once per +# declaration and then joins it; a secondmate earns the cadence on its declaration +# alone, because its endpoint liveness is deliberately never read +# (pause_state_class owns that split, and the declaration - not the pane hash - +# owns how long the cadence holds). # These cases re-surface once for a recheck every PAUSE_RESURFACE_SECS - far # longer than the wedge threshold, but finite so a forgotten hold cannot rot invisibly. PAUSE_RESURFACE_SECS=${FM_PAUSE_RESURFACE_SECS:-$FM_PAUSE_RESURFACE_SECS_DEFAULT} @@ -490,8 +492,23 @@ clear_pause_tracking() { # # After fm-crew-state has fallen back to stopped or unknown, paused classification is # recovered only for a confidently dead ordinary crew, or for a secondmate, whose # endpoint liveness this function deliberately never reads. +# +# The ordinary-crew liveness gate is ONE-SHOT per declared wait, and .paused- +# is what makes it one-shot: an ordinary crew whose agent is still live (or read +# ambiguously) may have declared a wait while actually parked at a decision gate, so +# the FIRST stale sighting of that declaration is surfaced for inspection. Once a +# caller has put the key on the bounded cadence, the gate is spent and liveness can +# no longer re-arm it. Re-reading it on every evaluation is what made a declared +# wait surface a bare stale wake for every pane redraw: the answer `none` is a +# caller's "no declared wait here" signal, so the changed-hash caller cleared the +# cadence flag and its long-cadence throttle, and the next stable hash surfaced as a +# first sighting all over again. The declaration itself, not the pane hash, now owns +# how long the cadence holds - .paused- is dropped only when the last status +# line stops declaring a wait or the crew is provably working again, so the cadence +# survives further declaring appends and watcher restarts while a genuinely new +# captain-relevant line still clears it and surfaces through the terminal path. pause_state_class() { # - local win=$1 task=$2 key last recheck_file class agent_alive kind + local win=$1 task=$2 key last recheck_file class agent_alive kind on_cadence key=$(window_key "$win") last=$(last_status_line "$STATE/$task.status") recheck_file="$STATE/.paused-rechecked-$key" @@ -500,19 +517,11 @@ pause_state_class() { # crew_absorb_class "$task" return fi - # Read once past the declared-wait gate and reused by both liveness gates below, - # so a mate's stale poll costs one metadata scan rather than one per gate, and the - # far more common no-declaration path above still costs none. - kind=$(window_kind "$win") - if [ -e "$STATE/.paused-$key" ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then - if [ "$kind" != secondmate ]; then - agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown - if [ "$agent_alive" != dead ]; then - rm -f "$recheck_file" - printf 'none' - return - fi - fi + if [ -e "$STATE/.paused-$key" ]; then on_cadence=0; else on_cadence=1; fi + # Already on the cadence and reconciled within the last wedge window: the poll that + # covers the overwhelming majority of a long wait re-reads nothing at all - not the + # crew state, not the window's metadata, not the backend. + if [ "$on_cadence" -eq 0 ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then printf 'paused' return fi @@ -522,22 +531,28 @@ pause_state_class() { # printf 'working' return fi - if [ "$kind" != secondmate ]; then - agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown - if [ "$agent_alive" != dead ]; then - rm -f "$recheck_file" - printf 'none' - return + # The one-shot liveness gate, and the only reader of the window's kind, so a poll + # that never reaches it costs no metadata scan. + if [ "$on_cadence" -ne 0 ]; then + kind=$(window_kind "$win") + if [ "$kind" != secondmate ]; then + agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown + if [ "$agent_alive" != dead ]; then + rm -f "$recheck_file" + printf 'none' + return + fi fi fi # Recover paused classification for a declared wait that authoritative crew state - # could not name. Reaching here already proves the only two admissible cases: an + # could not name. Reaching here already proves the only three admissible cases: an # ordinary crew whose agent the gate above confirmed dead, so no live decision gate - # is being silenced, or a secondmate, whose endpoint liveness is deliberately never - # read and so cannot supply that confirmation. Without the mate case a mate's - # captain hold - which has no current-state mapping and so arrives as `none` - - # would be silenced by every caller rather than taking the bounded re-surface - # cadence, and a forgotten hold would rot invisibly. + # is being silenced; a secondmate, whose endpoint liveness is deliberately never + # read and so cannot supply that confirmation; or a key already on the bounded + # cadence, whose one live-agent inspection has been spent. Without the mate case a + # mate's captain hold - which has no current-state mapping and so arrives as + # `none` - would be silenced by every caller rather than taking the bounded + # re-surface cadence, and a forgotten hold would rot invisibly. [ "$class" = none ] && class=paused case "$class" in paused) date +%s > "$recheck_file" ;; diff --git a/docs/architecture.md b/docs/architecture.md index f5857d5601..4616991e32 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -26,8 +26,9 @@ A concurrent replacement remains armed, every non-merged or invalid observation No-verb wakes, such as `working:` notes and bare turn-ended signals, are benign only when `bin/fm-crew-state.sh` reports positive evidence that the crew is still working: an actively running no-mistakes step attributed to that crew's current code, or an exact busy verdict from the semantic busy-state contract. A `kind=secondmate` task's status signal is the parent-directed reply stream and is never absorbed as provably working; only its bare turn-ended signal retains the ordinary absorb rule. A crew that declares `paused:` for a known external wait, or carries a verified `captain-held` transfer, is separately absorbed while idle and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge. -For an ordinary crew that has stopped, the normal-mode watcher first surfaces one stale wake, then applies that same cadence to an unchanged `paused:` or durable `captain-held` endpoint only when the backend confidently reports its agent dead. -Live or inconclusive liveness remains fail-open at that initial surface, and a secondmate's endpoint liveness is still never read at all; a mate is admitted to that same cadence only to serve a declared wait's bounded re-surface, so a forgotten pause or captain hold on a mate cannot rot invisibly. +An ordinary crew whose backend confidently reports its agent dead joins that cadence immediately, while live or inconclusive liveness first surfaces one stale wake and then joins it too. +That inspection is spent once per declaration rather than re-read on every evaluation, so the declaration - not the pane hash - decides how long the cadence holds, and a still-declared wait keeps it across pane redraws, further declaring appends, and watcher restarts until its last status line stops declaring a wait or its crew is provably working again. +A secondmate's endpoint liveness is still never read at all; a mate is admitted to that same cadence only to serve a declared wait's bounded re-surface, so a forgotten pause or captain hold on a mate cannot rot invisibly. Its initial normal-mode status signal still surfaces through the no-verb path, while away mode self-handles that routine signal and owns the later recheck. Fresh stale panes use the same current-state read before trusting the status log, so an active run or a proven busy worker outranks an old captain-relevant status-log line left behind before validation. No-change heartbeats are also benign. diff --git a/tests/fm-watch-triage.test.sh b/tests/fm-watch-triage.test.sh index 1bee5cab9b..8b3f39c667 100755 --- a/tests/fm-watch-triage.test.sh +++ b/tests/fm-watch-triage.test.sh @@ -1112,6 +1112,169 @@ test_exited_declared_pause_is_bounded_but_live_gate_surfaces() { pass "exited declared-pause and captain-held panes use bounded pause cadence while a live decision gate still surfaces once" } +# One watcher restart over a crew that is idling on a declared wait: run a round, +# and let it either absorb (three completed poll cycles - the changed hash, the +# first repeat, and the poll that classifies the now-stable hash) or exit on a +# wake, acknowledging the queue so the next round starts clean. Output is +# APPENDED so a caller can count wakes across every round of a scenario. +# 1 only when an intentional stop could not be acknowledged. +watch_paused_round() { # + local state=$1 fakebin=$2 out=$3 window=$4 capture=$5 pid + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_FAKE_TMUX_CURRENT_COMMAND=grok \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=3600 FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" >> "$out" & + pid=$! + if wait_poll_cycle "$state" "$pid" && wait_poll_cycle "$state" "$pid" \ + && wait_poll_cycle "$state" "$pid"; then + reap "$pid" + return 0 + fi + wait "$pid" 2>/dev/null || true + ack_stopped_cycle "$state" +} + +# A fully armed declared-wait cadence for : a stably stale idle pane, a +# current paused: declaration whose signature the signal scan has already seen, +# and every pause marker the watcher would hold after the wait joined the long +# cadence. The captain-relevant boundary cases start from exactly this state. +arm_pause_cadence() { # + local state=$1 capture=$2 statusf=$3 window=$4 key=$5 pane_hash=$6 + printf 'idle awaiting the upstream release\n' > "$capture" + printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/armed.meta" + printf 'paused: waiting on the upstream vendor release\n' > "$statusf" + printf '%s' "$(seen_sig "$statusf")" > "$state/.seen-armed_status" + printf '%s' "$pane_hash" > "$state/.hash-$key" + printf '1\n' > "$state/.count-$key" + : > "$state/.paused-$key" + date +%s > "$state/.paused-rechecked-$key" + date +%s > "$state/.paused-resurfaced-$key" +} + +# The watcher run the captain-relevant boundary cases drive: the same live agent +# and long cadence as a real declared wait, so only the captain-relevant line +# differs between the armed fixture and the wake it must still produce. +watch_armed_pause() { # + local state=$1 fakebin=$2 out=$3 window=$4 capture=$5 + PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \ + FM_FAKE_TMUX_CURRENT_COMMAND=grok \ + FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \ + FM_PAUSE_RESURFACE_SECS=3600 FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \ + FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" & +} + +# Regression: the bounded pause cadence is owned by the DECLARATION, not by the +# pane hash. A crew idling on a declared wait keeps re-rendering - a ticking +# clock, a token counter, its own status echo - and every redraw used to re-arm +# the one-per-declaration live-agent inspection, because pause_state_class read +# agent liveness on every evaluation and its `none` answer is a caller's "no +# declared wait here" signal: the changed-hash caller then cleared the cadence +# flag and its long-cadence throttle, so the next stable hash surfaced as a first +# sighting all over again (windows cycling bare stale wakes every few minutes +# against a current paused: line, 2026-08-22). The declaration never changes +# here, so exactly one stale wake may be surfaced across every churn, every +# further declaring append, and every watcher restart. +test_declared_pause_cadence_survives_pane_churn_and_restarts() { + local dir state fakebin out capture_file statusf window key round stales + dir=$(make_case declared-pause-churn); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/churn.status" + window="test:fm-churn" + printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/churn.meta" + printf 'paused: waiting on the upstream vendor release\n' > "$statusf" + printf '%s' "$(seen_sig "$statusf")" > "$state/.seen-churn_status" + key=$(printf '%s' "$window" | tr ':/.' '___') + # The agent is LIVE throughout (a paused crew normally still has its agent up), + # which is exactly the case the one-shot inspection covers. + export FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting on the upstream vendor release' + + # Phase A: five watcher restarts, each seeing a differently-rendered idle pane. + round=1 + while [ "$round" -le 5 ]; do + printf 'idle awaiting the upstream release\ntokens %s\n' "$round" > "$capture_file" + watch_paused_round "$state" "$fakebin" "$out" "$window" "$capture_file" \ + || fail "could not acknowledge the pause-churn round $round stop" + round=$((round + 1)) + done + stales=$(grep -c '^stale:' "$out" || true) + [ "$stales" -eq 1 ] \ + || fail "a churning declared pause surfaced $stales stale wakes across five restarts (expected the single live-agent inspection)" + + # Phase B: the crew keeps appending to its still-current declaration. Each + # append's .seen-* signature is primed so the signal scan cannot pre-empt the + # stale path being asserted here; the append must not re-arm it either. + printf 'idle awaiting the upstream release\ntokens settled\n' > "$capture_file" + round=1 + while [ "$round" -le 3 ]; do + printf 'paused: still waiting on the upstream vendor release (check %s)\n' "$round" >> "$statusf" + printf '%s' "$(seen_sig "$statusf")" > "$state/.seen-churn_status" + watch_paused_round "$state" "$fakebin" "$out" "$window" "$capture_file" \ + || fail "could not acknowledge the pause-append round $round stop" + round=$((round + 1)) + done + stales=$(grep -c '^stale:' "$out" || true) + [ "$stales" -eq 1 ] \ + || fail "declaring status appends re-armed the stale path ($stales stale wakes in total)" + [ -e "$state/.paused-$key" ] \ + || fail "the declared-wait cadence marker was lost across pane churn, appends and restarts" + [ ! -e "$state/.stale-since-$key" ] || fail "a declared wait started the wedge timer" + grep -F 'possible wedge' "$out" >/dev/null && fail "a declared wait was mislabeled a possible wedge" + unset FM_FAKE_CREW_STATE + pass "a current declared wait holds its long cadence across pane churn, declaring appends and watcher restarts" +} + +# The safety boundary the pause cadence must never cross: an armed cadence is +# bookkeeping about a declared wait, never a suppressor for the captain-relevant +# line that ends it. With the cadence fully armed and the agent live, each +# terminal verb must still wake immediately - through the signal path when its +# append is unseen, and through the stale path when the signal scan has already +# been satisfied. +test_captain_relevant_line_breaks_an_armed_pause_cadence() { + local dir state fakebin out capture_file statusf window key pane_hash pid slug line + window="test:fm-armed" + key=$(printf '%s' "$window" | tr ':/.' '___') + pane_hash=$(hash_text "idle awaiting the upstream release") + export FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting on the upstream vendor release' + while IFS='|' read -r slug line; do + [ -n "$slug" ] || continue + + # Signal path: the append is unseen, so the captain-relevant line must wake + # firstmate on the spot. + dir=$(make_case "armed-pause-signal-$slug"); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/armed.status" + arm_pause_cadence "$state" "$capture_file" "$statusf" "$window" "$key" "$pane_hash" + printf '%s\n' "$line" >> "$statusf" + watch_armed_pause "$state" "$fakebin" "$out" "$window" "$capture_file" + pid=$! + wait_for_exit "$pid" 100 || { reap "$pid"; fail "an armed pause cadence swallowed a $slug append on the signal path"; } + grep -F "$statusf" "$state/.wake-queue" >/dev/null \ + || fail "a $slug append behind an armed pause cadence was not queued: $(cat "$state/.wake-queue" 2>/dev/null)" + + # Stale path: the same append with its signature already seen, so only the + # stale classifier can surface it. It must, and it must drop the cadence. + dir=$(make_case "armed-pause-stale-$slug"); state="$dir/state"; fakebin="$dir/fakebin" + out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/armed.status" + arm_pause_cadence "$state" "$capture_file" "$statusf" "$window" "$key" "$pane_hash" + printf '%s\n' "$line" >> "$statusf" + printf '%s' "$(seen_sig "$statusf")" > "$state/.seen-armed_status" + watch_armed_pause "$state" "$fakebin" "$out" "$window" "$capture_file" + pid=$! + wait_for_exit "$pid" 100 || { reap "$pid"; fail "an armed pause cadence swallowed a $slug line on the stale path"; } + grep -F "stale: $window" "$out" >/dev/null \ + || fail "a $slug line behind an armed pause cadence did not surface as a stale wake: $(cat "$out")" + grep -F 'awaiting external' "$out" >/dev/null \ + && fail "a $slug line was rechecked on the pause cadence instead of surfacing" + [ ! -e "$state/.paused-$key" ] \ + || fail "a $slug line left the declared-wait cadence armed" + done <<'VERBS' +needs-decision|needs-decision [key=api-shape]: pick the sync or the async client +blocked|blocked: the upstream vendor token expired +done|done: PR https://example.test/x/pull/9 checks green +VERBS + unset FM_FAKE_CREW_STATE + pass "needs-decision, blocked and done still wake immediately through an armed declared-wait cadence" +} + test_secondmate_paused_resurfaces_in_normal_mode() { local dir state fakebin out capture_file statusf window key pane_hash sig pid back dir=$(make_case secondmate-paused-resurface); state="$dir/state"; fakebin="$dir/fakebin" @@ -2641,6 +2804,8 @@ test_busy_declared_pause_is_rechecked_not_wedge_escalated test_nonterminal_stale_not_working_surfaced test_nonterminal_stale_paused_absorbed_then_resurfaced test_exited_declared_pause_is_bounded_but_live_gate_surfaces +test_declared_pause_cadence_survives_pane_churn_and_restarts +test_captain_relevant_line_breaks_an_armed_pause_cadence test_secondmate_paused_resurfaces_in_normal_mode test_secondmate_captain_held_resurfaces_in_normal_mode test_secondmate_nonpaused_stale_remains_suppressed