fix(watch): hold the declared-wait pause cadence across pane churn

bin/fm-watch.sh

@@ -177,9 +177,11 @@ BUSY_TURN_MAX_SECS=${FM_BUSY_TURN_MAX_SECS:-3600}
 # A crew that declared a pause is idling on a known external wait, so its stale
 # pane is absorbed rather than wedge-escalated.
 # A captain-held or paused crew whose agent has confidently exited uses the same
-# bounded cadence, while a live or ambiguously read agent still surfaces once; a
-# secondmate earns the cadence on its declaration alone, because its endpoint
-# liveness is deliberately never read (pause_state_class owns that split).
+# bounded cadence, while a live or ambiguously read agent surfaces once per
+# declaration and then joins it; a secondmate earns the cadence on its declaration
+# alone, because its endpoint liveness is deliberately never read
+# (pause_state_class owns that split, and the declaration - not the pane hash -
+# owns how long the cadence holds).
 # These cases re-surface once for a recheck every PAUSE_RESURFACE_SECS - far
 # longer than the wedge threshold, but finite so a forgotten hold cannot rot invisibly.
 PAUSE_RESURFACE_SECS=${FM_PAUSE_RESURFACE_SECS:-$FM_PAUSE_RESURFACE_SECS_DEFAULT}
@@ -490,8 +492,23 @@ clear_pause_tracking() {  # <window-key>
 # After fm-crew-state has fallen back to stopped or unknown, paused classification is
 # recovered only for a confidently dead ordinary crew, or for a secondmate, whose
 # endpoint liveness this function deliberately never reads.
+#
+# The ordinary-crew liveness gate is ONE-SHOT per declared wait, and .paused-<key>
+# is what makes it one-shot: an ordinary crew whose agent is still live (or read
+# ambiguously) may have declared a wait while actually parked at a decision gate, so
+# the FIRST stale sighting of that declaration is surfaced for inspection. Once a
+# caller has put the key on the bounded cadence, the gate is spent and liveness can
+# no longer re-arm it. Re-reading it on every evaluation is what made a declared
+# wait surface a bare stale wake for every pane redraw: the answer `none` is a
+# caller's "no declared wait here" signal, so the changed-hash caller cleared the
+# cadence flag and its long-cadence throttle, and the next stable hash surfaced as a
+# first sighting all over again. The declaration itself, not the pane hash, now owns
+# how long the cadence holds - .paused-<key> is dropped only when the last status
+# line stops declaring a wait or the crew is provably working again, so the cadence
+# survives further declaring appends and watcher restarts while a genuinely new
+# captain-relevant line still clears it and surfaces through the terminal path.
 pause_state_class() {  # <window> <task>
-  local win=$1 task=$2 key last recheck_file class agent_alive kind
+  local win=$1 task=$2 key last recheck_file class agent_alive kind on_cadence
   key=$(window_key "$win")
   last=$(last_status_line "$STATE/$task.status")
   recheck_file="$STATE/.paused-rechecked-$key"
@@ -500,19 +517,11 @@ pause_state_class() {  # <window> <task>
     crew_absorb_class "$task"
     return
   fi
-  # Read once past the declared-wait gate and reused by both liveness gates below,
-  # so a mate's stale poll costs one metadata scan rather than one per gate, and the
-  # far more common no-declaration path above still costs none.
-  kind=$(window_kind "$win")
-  if [ -e "$STATE/.paused-$key" ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then
-    if [ "$kind" != secondmate ]; then
-      agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown
-      if [ "$agent_alive" != dead ]; then
-        rm -f "$recheck_file"
-        printf 'none'
-        return
-      fi
-    fi
+  if [ -e "$STATE/.paused-$key" ]; then on_cadence=0; else on_cadence=1; fi
+  # Already on the cadence and reconciled within the last wedge window: the poll that
+  # covers the overwhelming majority of a long wait re-reads nothing at all - not the
+  # crew state, not the window's metadata, not the backend.
+  if [ "$on_cadence" -eq 0 ] && [ "$(age_of "$recheck_file")" -lt "$STALE_ESCALATE_SECS" ]; then
     printf 'paused'
     return
   fi
@@ -522,22 +531,28 @@ pause_state_class() {  # <window> <task>
     printf 'working'
     return
   fi
-  if [ "$kind" != secondmate ]; then
-    agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown
-    if [ "$agent_alive" != dead ]; then
-      rm -f "$recheck_file"
-      printf 'none'
-      return
+  # The one-shot liveness gate, and the only reader of the window's kind, so a poll
+  # that never reaches it costs no metadata scan.
+  if [ "$on_cadence" -ne 0 ]; then
+    kind=$(window_kind "$win")
+    if [ "$kind" != secondmate ]; then
+      agent_alive=$(fm_backend_agent_alive "$(window_backend "$win")" "$win" 2>/dev/null) || agent_alive=unknown
+      if [ "$agent_alive" != dead ]; then
+        rm -f "$recheck_file"
+        printf 'none'
+        return
+      fi
     fi
   fi
   # Recover paused classification for a declared wait that authoritative crew state
-  # could not name. Reaching here already proves the only two admissible cases: an
+  # could not name. Reaching here already proves the only three admissible cases: an
   # ordinary crew whose agent the gate above confirmed dead, so no live decision gate
-  # is being silenced, or a secondmate, whose endpoint liveness is deliberately never
-  # read and so cannot supply that confirmation. Without the mate case a mate's
-  # captain hold - which has no current-state mapping and so arrives as `none` -
-  # would be silenced by every caller rather than taking the bounded re-surface
-  # cadence, and a forgotten hold would rot invisibly.
+  # is being silenced; a secondmate, whose endpoint liveness is deliberately never
+  # read and so cannot supply that confirmation; or a key already on the bounded
+  # cadence, whose one live-agent inspection has been spent. Without the mate case a
+  # mate's captain hold - which has no current-state mapping and so arrives as
+  # `none` - would be silenced by every caller rather than taking the bounded
+  # re-surface cadence, and a forgotten hold would rot invisibly.
   [ "$class" = none ] && class=paused
   case "$class" in
     paused) date +%s > "$recheck_file" ;;

docs/architecture.md

@@ -26,8 +26,9 @@ A concurrent replacement remains armed, every non-merged or invalid observation
 No-verb wakes, such as `working:` notes and bare turn-ended signals, are benign only when `bin/fm-crew-state.sh` reports positive evidence that the crew is still working: an actively running no-mistakes step attributed to that crew's current code, or an exact busy verdict from the semantic busy-state contract.
 A `kind=secondmate` task's status signal is the parent-directed reply stream and is never absorbed as provably working; only its bare turn-ended signal retains the ordinary absorb rule.
 A crew that declares `paused:` for a known external wait, or carries a verified `captain-held` transfer, is separately absorbed while idle and re-surfaced only on the longer pause cadence, rather than being treated as a possible wedge.
-For an ordinary crew that has stopped, the normal-mode watcher first surfaces one stale wake, then applies that same cadence to an unchanged `paused:` or durable `captain-held` endpoint only when the backend confidently reports its agent dead.
-Live or inconclusive liveness remains fail-open at that initial surface, and a secondmate's endpoint liveness is still never read at all; a mate is admitted to that same cadence only to serve a declared wait's bounded re-surface, so a forgotten pause or captain hold on a mate cannot rot invisibly.
+An ordinary crew whose backend confidently reports its agent dead joins that cadence immediately, while live or inconclusive liveness first surfaces one stale wake and then joins it too.
+That inspection is spent once per declaration rather than re-read on every evaluation, so the declaration - not the pane hash - decides how long the cadence holds, and a still-declared wait keeps it across pane redraws, further declaring appends, and watcher restarts until its last status line stops declaring a wait or its crew is provably working again.
+A secondmate's endpoint liveness is still never read at all; a mate is admitted to that same cadence only to serve a declared wait's bounded re-surface, so a forgotten pause or captain hold on a mate cannot rot invisibly.
 Its initial normal-mode status signal still surfaces through the no-verb path, while away mode self-handles that routine signal and owns the later recheck.
 Fresh stale panes use the same current-state read before trusting the status log, so an active run or a proven busy worker outranks an old captain-relevant status-log line left behind before validation.
 No-change heartbeats are also benign.

tests/fm-watch-triage.test.sh

@@ -1112,6 +1112,169 @@ test_exited_declared_pause_is_bounded_but_live_gate_surfaces() {
   pass "exited declared-pause and captain-held panes use bounded pause cadence while a live decision gate still surfaces once"
 }
 
+# One watcher restart over a crew that is idling on a declared wait: run a round,
+# and let it either absorb (three completed poll cycles - the changed hash, the
+# first repeat, and the poll that classifies the now-stable hash) or exit on a
+# wake, acknowledging the queue so the next round starts clean. Output is
+# APPENDED so a caller can count wakes across every round of a scenario.
+# 1 only when an intentional stop could not be acknowledged.
+watch_paused_round() {  # <state> <fakebin> <out> <window> <capture-file>
+  local state=$1 fakebin=$2 out=$3 window=$4 capture=$5 pid
+  PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \
+    FM_FAKE_TMUX_CURRENT_COMMAND=grok \
+    FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
+    FM_PAUSE_RESURFACE_SECS=3600 FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
+    FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" >> "$out" &
+  pid=$!
+  if wait_poll_cycle "$state" "$pid" && wait_poll_cycle "$state" "$pid" \
+    && wait_poll_cycle "$state" "$pid"; then
+    reap "$pid"
+    return 0
+  fi
+  wait "$pid" 2>/dev/null || true
+  ack_stopped_cycle "$state"
+}
+
+# A fully armed declared-wait cadence for <window>: a stably stale idle pane, a
+# current paused: declaration whose signature the signal scan has already seen,
+# and every pause marker the watcher would hold after the wait joined the long
+# cadence. The captain-relevant boundary cases start from exactly this state.
+arm_pause_cadence() {  # <state> <capture-file> <status-file> <window> <key> <pane-hash>
+  local state=$1 capture=$2 statusf=$3 window=$4 key=$5 pane_hash=$6
+  printf 'idle awaiting the upstream release\n' > "$capture"
+  printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/armed.meta"
+  printf 'paused: waiting on the upstream vendor release\n' > "$statusf"
+  printf '%s' "$(seen_sig "$statusf")" > "$state/.seen-armed_status"
+  printf '%s' "$pane_hash" > "$state/.hash-$key"
+  printf '1\n' > "$state/.count-$key"
+  : > "$state/.paused-$key"
+  date +%s > "$state/.paused-rechecked-$key"
+  date +%s > "$state/.paused-resurfaced-$key"
+}
+
+# The watcher run the captain-relevant boundary cases drive: the same live agent
+# and long cadence as a real declared wait, so only the captain-relevant line
+# differs between the armed fixture and the wake it must still produce.
+watch_armed_pause() {  # <state> <fakebin> <out> <window> <capture-file>
+  local state=$1 fakebin=$2 out=$3 window=$4 capture=$5
+  PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture" \
+    FM_FAKE_TMUX_CURRENT_COMMAND=grok \
+    FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
+    FM_PAUSE_RESURFACE_SECS=3600 FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
+    FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
+}
+
+# Regression: the bounded pause cadence is owned by the DECLARATION, not by the
+# pane hash. A crew idling on a declared wait keeps re-rendering - a ticking
+# clock, a token counter, its own status echo - and every redraw used to re-arm
+# the one-per-declaration live-agent inspection, because pause_state_class read
+# agent liveness on every evaluation and its `none` answer is a caller's "no
+# declared wait here" signal: the changed-hash caller then cleared the cadence
+# flag and its long-cadence throttle, so the next stable hash surfaced as a first
+# sighting all over again (windows cycling bare stale wakes every few minutes
+# against a current paused: line, 2026-08-22). The declaration never changes
+# here, so exactly one stale wake may be surfaced across every churn, every
+# further declaring append, and every watcher restart.
+test_declared_pause_cadence_survives_pane_churn_and_restarts() {
+  local dir state fakebin out capture_file statusf window key round stales
+  dir=$(make_case declared-pause-churn); state="$dir/state"; fakebin="$dir/fakebin"
+  out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/churn.status"
+  window="test:fm-churn"
+  printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/churn.meta"
+  printf 'paused: waiting on the upstream vendor release\n' > "$statusf"
+  printf '%s' "$(seen_sig "$statusf")" > "$state/.seen-churn_status"
+  key=$(printf '%s' "$window" | tr ':/.' '___')
+  # The agent is LIVE throughout (a paused crew normally still has its agent up),
+  # which is exactly the case the one-shot inspection covers.
+  export FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting on the upstream vendor release'
+
+  # Phase A: five watcher restarts, each seeing a differently-rendered idle pane.
+  round=1
+  while [ "$round" -le 5 ]; do
+    printf 'idle awaiting the upstream release\ntokens %s\n' "$round" > "$capture_file"
+    watch_paused_round "$state" "$fakebin" "$out" "$window" "$capture_file" \
+      || fail "could not acknowledge the pause-churn round $round stop"
+    round=$((round + 1))
+  done
+  stales=$(grep -c '^stale:' "$out" || true)
+  [ "$stales" -eq 1 ] \
+    || fail "a churning declared pause surfaced $stales stale wakes across five restarts (expected the single live-agent inspection)"
+
+  # Phase B: the crew keeps appending to its still-current declaration. Each
+  # append's .seen-* signature is primed so the signal scan cannot pre-empt the
+  # stale path being asserted here; the append must not re-arm it either.
+  printf 'idle awaiting the upstream release\ntokens settled\n' > "$capture_file"
+  round=1
+  while [ "$round" -le 3 ]; do
+    printf 'paused: still waiting on the upstream vendor release (check %s)\n' "$round" >> "$statusf"
+    printf '%s' "$(seen_sig "$statusf")" > "$state/.seen-churn_status"
+    watch_paused_round "$state" "$fakebin" "$out" "$window" "$capture_file" \
+      || fail "could not acknowledge the pause-append round $round stop"
+    round=$((round + 1))
+  done
+  stales=$(grep -c '^stale:' "$out" || true)
+  [ "$stales" -eq 1 ] \
+    || fail "declaring status appends re-armed the stale path ($stales stale wakes in total)"
+  [ -e "$state/.paused-$key" ] \
+    || fail "the declared-wait cadence marker was lost across pane churn, appends and restarts"
+  [ ! -e "$state/.stale-since-$key" ] || fail "a declared wait started the wedge timer"
+  grep -F 'possible wedge' "$out" >/dev/null && fail "a declared wait was mislabeled a possible wedge"
+  unset FM_FAKE_CREW_STATE
+  pass "a current declared wait holds its long cadence across pane churn, declaring appends and watcher restarts"
+}
+
+# The safety boundary the pause cadence must never cross: an armed cadence is
+# bookkeeping about a declared wait, never a suppressor for the captain-relevant
+# line that ends it. With the cadence fully armed and the agent live, each
+# terminal verb must still wake immediately - through the signal path when its
+# append is unseen, and through the stale path when the signal scan has already
+# been satisfied.
+test_captain_relevant_line_breaks_an_armed_pause_cadence() {
+  local dir state fakebin out capture_file statusf window key pane_hash pid slug line
+  window="test:fm-armed"
+  key=$(printf '%s' "$window" | tr ':/.' '___')
+  pane_hash=$(hash_text "idle awaiting the upstream release")
+  export FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting on the upstream vendor release'
+  while IFS='|' read -r slug line; do
+    [ -n "$slug" ] || continue
+
+    # Signal path: the append is unseen, so the captain-relevant line must wake
+    # firstmate on the spot.
+    dir=$(make_case "armed-pause-signal-$slug"); state="$dir/state"; fakebin="$dir/fakebin"
+    out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/armed.status"
+    arm_pause_cadence "$state" "$capture_file" "$statusf" "$window" "$key" "$pane_hash"
+    printf '%s\n' "$line" >> "$statusf"
+    watch_armed_pause "$state" "$fakebin" "$out" "$window" "$capture_file"
+    pid=$!
+    wait_for_exit "$pid" 100 || { reap "$pid"; fail "an armed pause cadence swallowed a $slug append on the signal path"; }
+    grep -F "$statusf" "$state/.wake-queue" >/dev/null \
+      || fail "a $slug append behind an armed pause cadence was not queued: $(cat "$state/.wake-queue" 2>/dev/null)"
+
+    # Stale path: the same append with its signature already seen, so only the
+    # stale classifier can surface it. It must, and it must drop the cadence.
+    dir=$(make_case "armed-pause-stale-$slug"); state="$dir/state"; fakebin="$dir/fakebin"
+    out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/armed.status"
+    arm_pause_cadence "$state" "$capture_file" "$statusf" "$window" "$key" "$pane_hash"
+    printf '%s\n' "$line" >> "$statusf"
+    printf '%s' "$(seen_sig "$statusf")" > "$state/.seen-armed_status"
+    watch_armed_pause "$state" "$fakebin" "$out" "$window" "$capture_file"
+    pid=$!
+    wait_for_exit "$pid" 100 || { reap "$pid"; fail "an armed pause cadence swallowed a $slug line on the stale path"; }
+    grep -F "stale: $window" "$out" >/dev/null \
+      || fail "a $slug line behind an armed pause cadence did not surface as a stale wake: $(cat "$out")"
+    grep -F 'awaiting external' "$out" >/dev/null \
+      && fail "a $slug line was rechecked on the pause cadence instead of surfacing"
+    [ ! -e "$state/.paused-$key" ] \
+      || fail "a $slug line left the declared-wait cadence armed"
+  done <<'VERBS'
+needs-decision|needs-decision [key=api-shape]: pick the sync or the async client
+blocked|blocked: the upstream vendor token expired
+done|done: PR https://example.test/x/pull/9 checks green
+VERBS
+  unset FM_FAKE_CREW_STATE
+  pass "needs-decision, blocked and done still wake immediately through an armed declared-wait cadence"
+}
+
 test_secondmate_paused_resurfaces_in_normal_mode() {
   local dir state fakebin out capture_file statusf window key pane_hash sig pid back
   dir=$(make_case secondmate-paused-resurface); state="$dir/state"; fakebin="$dir/fakebin"
@@ -2641,6 +2804,8 @@ test_busy_declared_pause_is_rechecked_not_wedge_escalated
 test_nonterminal_stale_not_working_surfaced
 test_nonterminal_stale_paused_absorbed_then_resurfaced
 test_exited_declared_pause_is_bounded_but_live_gate_surfaces
+test_declared_pause_cadence_survives_pane_churn_and_restarts
+test_captain_relevant_line_breaks_an_armed_pause_cadence
 test_secondmate_paused_resurfaces_in_normal_mode
 test_secondmate_captain_held_resurfaces_in_normal_mode
 test_secondmate_nonpaused_stale_remains_suppressed