From 7191726441265d1273fe911b3ed83a448edba25f Mon Sep 17 00:00:00 2001 From: knowttl Date: Tue, 18 Aug 2026 10:54:55 -0700 Subject: [PATCH 1/2] feat(disk): reclaim build scratch on teardown and alarm on low disk Pooled build worktrees accumulate git-ignored install trees: `treehouse return` resets tracked content but leaves node_modules and build output in place, so every returned pool slot keeps the full install tree its last task built. Nothing watched free space either, so the shared disk filled silently until a build failed. Two independent safeguards: - bin/fm-reclaim-build-scratch.sh removes regenerable build scratch from a finished worktree, gated on the path being git-ignored, its basename being a known regenerable name, and the worktree having no modified tracked files. bin/fm-teardown.sh calls it only after its landed-work refusals have passed and the worktree's processes are reaped, and a refusal never blocks teardown. - bin/fm-guard.sh alarms, loudly and rate-limited per home, when the watched filesystem's free space drops below the configured headroom. It warns only: nothing is deleted and no spawn is paused. Threshold and watched path come from the gitignored config/disk-guard. --- AGENTS.md | 1 + bin/fm-guard.sh | 84 +++++++++++++++ bin/fm-reclaim-build-scratch.sh | 79 ++++++++++++++ bin/fm-teardown.sh | 14 +++ docs/configuration.md | 14 +++ tests/fm-guard-disk-space.test.sh | 136 ++++++++++++++++++++++++ tests/fm-reclaim-build-scratch.test.sh | 141 +++++++++++++++++++++++++ tests/fm-teardown.test.sh | 61 +++++++++++ 8 files changed, 530 insertions(+) create mode 100755 bin/fm-reclaim-build-scratch.sh create mode 100755 tests/fm-guard-disk-space.test.sh create mode 100755 tests/fm-reclaim-build-scratch.test.sh diff --git a/AGENTS.md b/AGENTS.md index 80e3c0a5b5..96e5825123 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -73,6 +73,7 @@ config/backend runtime session-provider backend override for new tasks; LOCAL, config/calm Pi Calm presentation preference; LOCAL, gitignored, and not inherited; see docs/configuration.md "Pi Calm preference" config/startup-memory-budget primary-authoritative per-home startup-memory budget; LOCAL, gitignored, materialized as 7,500 estimated tokens by locked primary bootstrap and inherited into secondmate homes; see docs/configuration.md "Startup memory budget" config/herdr-presentation-spaces optional "off" opt-out from, or "on" opt-in to, Herdr's default-on disposable single-task visual projection, which is unconfigured-default-on only at or above a Herdr version floor; LOCAL, gitignored; inherited by secondmate homes; see docs/herdr-backend.md "Presentation spaces" +config/disk-guard optional free-space headroom and watched filesystem for the low-disk alarm; LOCAL, gitignored; absent means 20 GiB on "/"; see docs/configuration.md "Disk-space guard" config/trace-context optional presence flag enabling default-off native W3C trace-context propagation to spawned agents; LOCAL, gitignored; inherited by secondmate homes; see docs/configuration.md "Trace context propagation" and docs/trace-context.md config/cmux-socket-password optional cmux control-socket password; LOCAL, gitignored; read fresh on every cmux CLI call and passed through without ever overriding an operator's own ambient CMUX_SOCKET_PASSWORD when absent (docs/cmux-backend.md "Setup") config/wedge-alarm optional away-mode wedge-alarm active-alert directives; LOCAL, gitignored; absent means auto (macOS Notification Center when available); see docs/wedge-alarm.md diff --git a/bin/fm-guard.sh b/bin/fm-guard.sh index 21d6da3ed8..abaa224387 100755 --- a/bin/fm-guard.sh +++ b/bin/fm-guard.sh @@ -5,6 +5,12 @@ # First, always warn if the firstmate primary checkout (FM_ROOT) is on a named # non-default branch, because that means firstmate-on-itself work landed in the # primary instead of an isolated worktree. +# Second, always alarm when the watched filesystem's free space has fallen below +# the configured headroom, so a fill is caught before it silently fails a build or +# a deploy. That alarm warns only - it never deletes anything and never pauses +# work - and is rate-limited to once per FM_DISK_GUARD_REPEAT_SECS per FM_HOME, +# re-arming as soon as free space recovers. config/disk-guard owns the threshold +# and the watched path (docs/configuration.md "Disk-space guard"). # Then, if a task is in flight (a state/.meta exists) or X-mode relay # polling is active (state/x-watch.check.sh exists) and supervision is not # healthy, prints a loud, clearly delimited banner so the agent cannot skim past @@ -46,6 +52,12 @@ CONTINUE_LINE=${FM_GUARD_CONTINUE_LINE:-This is a supervision warning only; the # Cleared when the home leaves the unhealthy state so a later episode re-arms. STALE_BANNER_MARKER="$STATE/.guard-watcher-stale-banner" +# Volatile, home-scoped disk alarm marker: one line = the epoch of the last full +# alarm. Removed when free space recovers so the next fill alarms immediately. +DISK_ALARM_MARKER="$STATE/.guard-disk-space-alarm" +DISK_REPEAT_SECS=${FM_DISK_GUARD_REPEAT_SECS:-3600} +case "$DISK_REPEAT_SECS" in ''|*[!0-9]*) DISK_REPEAT_SECS=3600 ;; esac + # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" # shellcheck source=bin/fm-tangle-lib.sh @@ -121,6 +133,46 @@ fm_guard_clear_stale_banner() { rm -f "$STALE_BANNER_MARKER" 2>/dev/null || true } +# Free-space threshold and watched path from config/disk-guard: the first +# non-empty, non-comment line is " []". An absent file means +# the built-in defaults; an unusable threshold is reported and falls back rather +# than silently disabling the alarm. A threshold of 0 disables it. +DISK_MIN_GIB=20 +DISK_PATH=/ +fm_guard_read_disk_config() { + local line min path + line=$(grep -v -e '^[[:space:]]*#' -e '^[[:space:]]*$' "$CONFIG/disk-guard" 2>/dev/null | head -1 || true) + [ -n "$line" ] || return 0 + read -r min path _ <&2 + ;; + *) DISK_MIN_GIB=$min ;; + esac + [ -z "$path" ] || DISK_PATH=$path +} + +# Available 1024-byte blocks on the filesystem holding $1, or empty when it +# cannot be read (POSIX df -Pk reports the same units everywhere). +fm_guard_disk_free_kib() { + df -Pk "$1" 2>/dev/null | awk 'NR == 2 { print $4 }' +} + +# Rate limit: the full alarm prints when no alarm was recorded, when the record +# is unreadable, or when it is older than the repeat window. +fm_guard_disk_alarm_due() { + local last now + last=$(cat "$DISK_ALARM_MARKER" 2>/dev/null || true) + last=${last%$'\n'} + case "$last" in ''|*[!0-9]*) return 0 ;; esac + now=$(date +%s) + [ "$((now - last))" -ge "$DISK_REPEAT_SECS" ] +} + # Worktree-tangle alarm, checked FIRST and independent of in-flight tasks: the # firstmate PRIMARY checkout (FM_ROOT) must stay on its default branch. If a # crewmate's branch/commits landed here instead of in its own isolated worktree, @@ -148,6 +200,38 @@ if [ -n "$tangle_branch" ]; then } >&2 fi +# Low-disk alarm, checked independently of in-flight tasks: pooled build +# worktrees and their install trees share one filesystem, and a fill fails the +# next build or deploy with an error that looks like anything but a full disk. +fm_guard_read_disk_config +if [ "$DISK_MIN_GIB" -gt 0 ]; then + disk_free_kib=$(fm_guard_disk_free_kib "$DISK_PATH") + disk_alarm= + if [ -z "$disk_free_kib" ]; then + disk_alarm=$(printf 'cannot read free space on %s (check config/disk-guard).' "$DISK_PATH") + elif [ "$disk_free_kib" -lt "$((DISK_MIN_GIB * 1024 * 1024))" ]; then + disk_alarm=$(awk -v k="$disk_free_kib" -v p="$DISK_PATH" -v m="$DISK_MIN_GIB" \ + 'BEGIN { printf "%s has %.1fG free, below the %sG headroom.", p, k / 1048576, m }') + fi + if [ -n "$disk_alarm" ]; then + if fm_guard_disk_alarm_due; then + [ "$READ_ONLY" -eq 1 ] || date +%s >"$DISK_ALARM_MARKER" 2>/dev/null || true + drule='━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━' + { + printf '●%s\n' "$drule" + printf '● LOW DISK SPACE - BUILDS AND DEPLOYS ARE AT RISK\n' + printf '● %s\n' "$disk_alarm" + printf '● A full disk fails a build or deploy with an error that names anything but the disk.\n' + printf '● Free space before dispatching more build work; finished worktrees are the usual hoard.\n' + printf '● %s\n' "$CONTINUE_LINE" + printf '●%s\n' "$drule" + } >&2 + fi + elif [ "$READ_ONLY" -ne 1 ]; then + rm -f "$DISK_ALARM_MARKER" 2>/dev/null || true + fi +fi + # Compute supervision need and watcher-beacon freshness via the shared # grace-based predicate (bin/fm-supervision-lib.sh). Act when work, an event # source, or an X-mode relay poll needs supervision. diff --git a/bin/fm-reclaim-build-scratch.sh b/bin/fm-reclaim-build-scratch.sh new file mode 100755 index 0000000000..28c47a64e7 --- /dev/null +++ b/bin/fm-reclaim-build-scratch.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Reclaim regenerable build scratch (node_modules, compiled output, tool caches) +# from a worktree that is finished with, so pooled worktrees do not accumulate +# gigabytes of install trees on the disk that holds the pool. `treehouse return` +# resets tracked content but leaves git-IGNORED trees in place, so a returned +# pool slot keeps every install tree its last task built. +# +# Usage: fm-reclaim-build-scratch.sh +# Prints one summary line on stdout. Exit 0 = reclaimed (possibly nothing), +# 1 = refused (nothing was removed), 2 = usage error. +# +# Removal is gated on three independent conditions, all of which must hold: +# 1. is a git worktree whose TRACKED files are unmodified. +# A tracked modification means live or unfinished work, so this refuses and +# removes nothing. This is a second, independent check: whether the task's +# work has LANDED is owned by bin/fm-teardown.sh, which runs its landed-work +# test (and its process reap) before ever calling this. +# 2. The path is reported IGNORED by git itself, so no tracked file, no +# untracked-but-unignored file, and nothing in the index can ever be a +# candidate. +# 3. Its basename is in the regenerable-scratch list below. Being ignored is +# not sufficient: an ignored `.env`, credential, or local data directory is +# precious, and only names whose contents a plain install or build step +# recreates are removed. +# A symlink, or a directory whose resolved path escapes the worktree, is skipped: +# scratch is reclaimed in place or not at all. +set -eu + +# Regenerable when git-ignored: a package install tree, a build output tree, or a +# tool cache that the next install/build/test run recreates. +SCRATCH_NAMES="node_modules .next .nuxt .turbo dist build target __pycache__ .pytest_cache .mypy_cache .ruff_cache" + +usage() { + echo "usage: fm-reclaim-build-scratch.sh " >&2 + exit 2 +} + +refuse() { + echo "reclaim: REFUSED: $1" >&2 + exit 1 +} + +[ "$#" -eq 1 ] || usage +[ -n "$1" ] || usage +[ -d "$1" ] || refuse "no worktree directory at $1" + +ROOT_DIR=$(cd "$1" && pwd -P) +git -C "$ROOT_DIR" rev-parse --is-inside-work-tree >/dev/null 2>&1 \ + || refuse "$ROOT_DIR is not a git worktree" +STATUS=$(git -C "$ROOT_DIR" status --porcelain --untracked-files=no 2>/dev/null) \ + || refuse "cannot inspect $ROOT_DIR for tracked modifications" +[ -z "$STATUS" ] || refuse "$ROOT_DIR has modified tracked files; leaving all scratch in place" + +removed=0 +while IFS= read -r -d '' rel; do + rel=${rel%/} + [ -n "$rel" ] || continue + case " $SCRATCH_NAMES " in + *" ${rel##*/} "*) ;; + *) continue ;; + esac + path="$ROOT_DIR/$rel" + if [ -L "$path" ] || [ ! -d "$path" ]; then + continue + fi + real=$(cd "$path" && pwd -P) || continue + case "$real" in + "$ROOT_DIR"/*) ;; + *) continue ;; + esac + rm -rf "$real" + removed=$((removed + 1)) +done < <(git -C "$ROOT_DIR" ls-files --ignored --exclude-standard --others --directory -z) + +if [ "$removed" -eq 0 ]; then + echo "reclaim: no regenerable build scratch under $ROOT_DIR" +else + echo "reclaim: removed $removed regenerable build-scratch path(s) under $ROOT_DIR" +fi diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 4178217c91..465ccd7580 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -133,6 +133,14 @@ # root still exists, so the account's healthy LaunchAgent worker and every # live remote secondmate worker are out of scope. Best effort: a sweep # failure never blocks this teardown. +# Fix 4 - reclaim regenerable build scratch. `treehouse return` resets tracked +# content but leaves git-ignored trees in place, so each pooled worktree keeps +# the full install tree its last task built (measured 2026-08-18: ~1.5G of +# node_modules per frontend worktree, never reclaimed, filling the pool disk +# until a build failed). bin/fm-reclaim-build-scratch.sh owns the removal +# rules and its own independent safety gate; it runs only here, after every +# landed-work refusal above has passed and the worktree's processes are +# reaped, and its refusal never blocks teardown. set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -2385,6 +2393,12 @@ fi if [ "$KIND" != secondmate ]; then conclude_task_no_mistakes_run "$WT" reap_task_worktree_processes worktree "$WT" "$TASK_TMP" + # Fix 4 (see script header): reclaim regenerable build scratch before the + # worktree goes back to the pool. Best effort - its own refusal, or any + # failure, never blocks this teardown. + if [ -d "$WT" ]; then + "$SCRIPT_DIR/fm-reclaim-build-scratch.sh" "$WT" >&2 || true + fi fi # Fix 3 (see script header): sweep remote job workers abandoned by an already diff --git a/docs/configuration.md b/docs/configuration.md index d948709753..9cc3ee3f04 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -117,6 +117,19 @@ An absent file means `auto`, i.e. default-on on macOS: the alarm exists precisel A missing or failing channel logs and falls through to the next, never crashing the daemon. See [`wedge-alarm.md`](wedge-alarm.md) for the current channel reference, [`verification/supervision.md`](verification/supervision.md#wedge-alarm-channels) for active evidence, and [`examples/wedge-alarm`](examples/wedge-alarm) for a copyable config. +## Disk-space guard (config/disk-guard) + +Pooled build worktrees and their install trees share one filesystem, and a fill fails the next build or deploy with an error that names anything but the disk. +`bin/fm-guard.sh` therefore alarms whenever the watched filesystem's free space is below the configured headroom, on every fleet action that already runs the watcher and tangle guards. +The alarm warns only: it never deletes a cache and never pauses a spawn. +It is rate-limited to once per `FM_DISK_GUARD_REPEAT_SECS` (default 3600) per home, and re-arms as soon as free space recovers. + +`config/disk-guard` (local, gitignored) holds one non-empty, non-comment line: ` []`. +An absent file means 20 GiB of headroom on `/`; a threshold of `0` disables the alarm; an unusable threshold is reported and falls back to the default rather than disabling the check silently. +A watched path whose free space cannot be read alarms as well, so a mistyped mount point is visible rather than quietly unmonitored. + +Reclaiming that space when a task finishes is a separate, independent safeguard: `bin/fm-teardown.sh` calls `bin/fm-reclaim-build-scratch.sh` once a torn-down task's landed-work checks have passed, and that script's header owns which paths it will and will not remove. + ## Trace context propagation (config/trace-context / FM_TRACE_CONTEXT) The optional local, gitignored `config/trace-context` presence flag enables default-off native W3C trace-context propagation. @@ -545,6 +558,7 @@ FM_STARTUP_NETWORK_TIMEOUT=120 # seconds bounding the whole deferred network s FM_TASKS_AXI_COMPATIBLE= # internal one-hop handoff of an already-computed tasks-axi compatibility verdict (0 or 1); consumed when bin/fm-tasks-axi-lib.sh is sourced FM_GUARD_READ_ONLY=0 # internal/read-only guard mode: keep alarms but suppress drain, supervision repair, and checkout repair commands FM_GUARD_CONTINUE_LINE='This is a supervision warning only; the guarded operation WILL still run.' # banner continuation line; fm-send.sh overrides it to name the requested message specifically +FM_DISK_GUARD_REPEAT_SECS=3600 # seconds between repeats of the low-disk alarm in one home; see "Disk-space guard" FM_POLL=15 # seconds between watcher poll cycles FM_HEARTBEAT=600 # base seconds between heartbeat scans; no-change heartbeats are absorbed while idle FM_HEARTBEAT_MAX=7200 # heartbeat backoff cap diff --git a/tests/fm-guard-disk-space.test.sh b/tests/fm-guard-disk-space.test.sh new file mode 100755 index 0000000000..f8eebdbda0 --- /dev/null +++ b/tests/fm-guard-disk-space.test.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Behavior tests for fm-guard's low-disk alarm. +# +# The alarm is the early-detection half of the disk-hygiene safeguards: pooled +# build worktrees fill the filesystem they share, and a fill fails the next +# build or deploy with an error that names anything but the disk. These cases +# pin the threshold decision (alarms below the configured headroom, silent +# above it), the configuration contract in config/disk-guard, the rate limit and +# its re-arm on recovery, and the warning-only promise: the guard never removes +# anything and always exits 0. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP_ROOT=$(fm_test_tmproot fm-guard-disk-space) +ALARM_LINE='LOW DISK SPACE' +MARKER=.guard-disk-space-alarm + +# A guard home with no in-flight work: the disk alarm is independent of the +# fleet, so every case below starts from an otherwise silent guard. +make_home() { + local name=$1 dir + dir="$TMP_ROOT/$name" + mkdir -p "$dir/home/state" "$dir/home/config" "$dir/root" + git init -q -b main "$dir/root" + printf '%s\n' "$dir" +} + +# Run the guard against that home; all guard output is on stderr. +run_guard() { + local dir=$1 + shift + ( cd "$dir" && env "$@" FM_ROOT_OVERRIDE="$dir/root" FM_HOME="$dir/home" \ + "$ROOT/bin/fm-guard.sh" 2>&1 ) +} + +# A threshold no real filesystem satisfies, so "below headroom" is deterministic +# wherever the suite runs. +IMPOSSIBLE_GIB=999999999 + +test_threshold_decides_the_alarm() { + local dir out + dir=$(make_home threshold) + + printf '%s\n' "$IMPOSSIBLE_GIB" > "$dir/home/config/disk-guard" + out=$(run_guard "$dir") + assert_contains "$out" "$ALARM_LINE" "free space below the threshold must alarm" + + rm -f "$dir/home/state/$MARKER" + printf '%s\n' "1" > "$dir/home/config/disk-guard" + out=$(run_guard "$dir") + assert_not_contains "$out" "$ALARM_LINE" "free space above the threshold must stay silent" + + rm -f "$dir/home/state/$MARKER" + printf '%s\n' "0" > "$dir/home/config/disk-guard" + out=$(run_guard "$dir") + assert_not_contains "$out" "$ALARM_LINE" "a zero threshold must disable the alarm" + + rm -f "$dir/home/config/disk-guard" + out=$(run_guard "$dir") + assert_not_contains "$out" "$ALARM_LINE" "an absent config must not alarm on a healthy filesystem" + pass "disk guard: alarms below the configured headroom and stays silent above it" +} + +test_config_selects_path_and_reports_invalid_threshold() { + local dir out + dir=$(make_home config) + + printf '%s\n' "5 /no/such/mount/point" > "$dir/home/config/disk-guard" + out=$(run_guard "$dir") + assert_contains "$out" "$ALARM_LINE" "an unreadable watched path must alarm, not go quietly unmonitored" + assert_contains "$out" "/no/such/mount/point" "the alarm must name the watched path" + + rm -f "$dir/home/state/$MARKER" + printf '%s\n%s\n' "# headroom in GiB" "plenty" > "$dir/home/config/disk-guard" + out=$(run_guard "$dir") + assert_contains "$out" "invalid free-space threshold" "an unusable threshold must be reported" + assert_not_contains "$out" "$ALARM_LINE" "an unusable threshold must fall back to the default, not alarm falsely" + pass "disk guard: config selects the watched path and an unusable threshold is reported" +} + +test_alarm_is_rate_limited_and_rearms_on_recovery() { + local dir out + dir=$(make_home ratelimit) + printf '%s\n' "$IMPOSSIBLE_GIB" > "$dir/home/config/disk-guard" + + out=$(run_guard "$dir") + assert_contains "$out" "$ALARM_LINE" "the first low-disk alarm must print" + out=$(run_guard "$dir") + assert_not_contains "$out" "$ALARM_LINE" "a repeat inside the window must stay quiet" + + out=$(run_guard "$dir" FM_DISK_GUARD_REPEAT_SECS=0) + assert_contains "$out" "$ALARM_LINE" "the alarm must repeat once its window elapses" + + # Recovery re-arms: a healthy filesystem must clear the record so the next + # fill alarms immediately instead of waiting out the window. + printf '%s\n' "1" > "$dir/home/config/disk-guard" + out=$(run_guard "$dir") + assert_absent "$dir/home/state/$MARKER" "recovery must clear the rate-limit record" + printf '%s\n' "$IMPOSSIBLE_GIB" > "$dir/home/config/disk-guard" + out=$(run_guard "$dir") + assert_contains "$out" "$ALARM_LINE" "a fresh fill after recovery must alarm immediately" + pass "disk guard: alarm is rate-limited per home and re-arms after recovery" +} + +test_alarm_only_warns() { + local dir out code + dir=$(make_home warn-only) + printf '%s\n' "$IMPOSSIBLE_GIB" > "$dir/home/config/disk-guard" + mkdir -p "$dir/home/cache" + printf 'x\n' > "$dir/home/cache/keep" + + out=$(run_guard "$dir") && code=0 || code=$? + expect_code 0 "$code" "the guard must warn without failing the guarded operation" + assert_contains "$out" "$ALARM_LINE" "the warn-only case must still alarm" + assert_present "$dir/home/cache/keep" "the guard must never delete anything to free space" + pass "disk guard: the alarm warns only - it exits 0 and deletes nothing" +} + +test_read_only_session_does_not_write_state() { + local dir out + dir=$(make_home read-only) + printf '%s\n' "$IMPOSSIBLE_GIB" > "$dir/home/config/disk-guard" + + out=$(run_guard "$dir" FM_GUARD_READ_ONLY=1) + assert_contains "$out" "$ALARM_LINE" "a read-only session must still report a low disk" + assert_absent "$dir/home/state/$MARKER" "a read-only session must not write the rate-limit record" + pass "disk guard: a read-only session reports the alarm without mutating home state" +} + +test_threshold_decides_the_alarm +test_config_selects_path_and_reports_invalid_threshold +test_alarm_is_rate_limited_and_rearms_on_recovery +test_alarm_only_warns +test_read_only_session_does_not_write_state diff --git a/tests/fm-reclaim-build-scratch.test.sh b/tests/fm-reclaim-build-scratch.test.sh new file mode 100755 index 0000000000..2f2639bf9e --- /dev/null +++ b/tests/fm-reclaim-build-scratch.test.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# Behavior tests for fm-reclaim-build-scratch. +# +# Reclaiming a finished worktree's install trees is only safe if it can never +# reach anything a rebuild does not recreate. These cases pin both halves of +# that contract: regenerable git-ignored build scratch is removed, while tracked +# files, untracked-but-unignored files, ignored-but-precious files, escaping +# symlinks, and any worktree still carrying modified tracked work are left +# exactly as they were. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP_ROOT=$(fm_test_tmproot fm-reclaim-build-scratch) +RECLAIM="$ROOT/bin/fm-reclaim-build-scratch.sh" +fm_git_identity fmtest fmtest@example.invalid + +# A committed worktree carrying one of everything the script must decide about: +# scratch to reclaim, and neighbours it must never touch. +make_worktree() { + local name=$1 wt + wt="$TMP_ROOT/$name" + git init -q -b main "$wt" + printf '%s\n' 'node_modules/' 'dist/' '.venv/' '.env' 'local-data/' > "$wt/.gitignore" + + mkdir -p "$wt/src" "$wt/node_modules/pkg" "$wt/frontend/node_modules" "$wt/dist" \ + "$wt/.venv" "$wt/local-data" + printf 'source\n' > "$wt/src/app.js" + printf 'installed\n' > "$wt/node_modules/pkg/index.js" + printf 'installed\n' > "$wt/frontend/node_modules/dep.js" + printf 'built\n' > "$wt/dist/bundle.js" + printf 'venv\n' > "$wt/.venv/pyvenv.cfg" + printf 'secret\n' > "$wt/.env" + printf 'rows\n' > "$wt/local-data/db.sqlite" + printf 'notes\n' > "$wt/scratch-notes.txt" + + git -C "$wt" add -A >/dev/null + git -C "$wt" commit -qm init + printf '%s\n' "$wt" +} + +test_removes_regenerable_scratch_only() { + local wt out code + wt=$(make_worktree removes) + + out=$("$RECLAIM" "$wt") && code=0 || code=$? + expect_code 0 "$code" "reclaiming a clean worktree must succeed" + assert_contains "$out" "removed 3" "the summary must report the reclaimed paths" + + assert_absent "$wt/node_modules" "an ignored install tree must be reclaimed" + assert_absent "$wt/frontend/node_modules" "a nested ignored install tree must be reclaimed" + assert_absent "$wt/dist" "ignored build output must be reclaimed" + + assert_present "$wt/src/app.js" "tracked source must survive" + assert_present "$wt/.env" "an ignored credential file is not regenerable and must survive" + assert_present "$wt/local-data/db.sqlite" "an ignored local data directory must survive" + assert_present "$wt/.venv/pyvenv.cfg" "an ignored directory outside the scratch list must survive" + assert_present "$wt/scratch-notes.txt" "committed notes must survive" + pass "reclaim: removes git-ignored regenerable build scratch and nothing else" +} + +test_leaves_untracked_and_unignored_work_alone() { + local wt + wt=$(make_worktree untracked) + # Untracked but NOT ignored: unfiled work in progress, never scratch. Both a + # loose file and a directory whose name is on the scratch list. + printf 'draft\n' > "$wt/src/wip.js" + mkdir -p "$wt/build" + printf 'hand written\n' > "$wt/build/notes.md" + + "$RECLAIM" "$wt" >/dev/null + assert_present "$wt/src/wip.js" "untracked work in progress must survive" + assert_present "$wt/build/notes.md" "an untracked directory git does not ignore must survive even when its name is scratch-shaped" + assert_absent "$wt/node_modules" "reclaiming must still remove genuine ignored scratch" + pass "reclaim: untracked-but-unignored files are never scratch" +} + +test_refuses_a_worktree_with_modified_tracked_work() { + local wt out code + wt=$(make_worktree dirty) + printf 'edited\n' >> "$wt/src/app.js" + + out=$("$RECLAIM" "$wt" 2>&1) && code=0 || code=$? + expect_code 1 "$code" "a worktree with modified tracked work must be refused" + assert_contains "$out" "REFUSED" "the refusal must be explicit" + assert_present "$wt/node_modules/pkg/index.js" "a refused worktree must keep every path, scratch included" + assert_present "$wt/dist/bundle.js" "a refused worktree must keep its build output too" + pass "reclaim: refuses a worktree still carrying modified tracked work" +} + +test_refuses_a_non_worktree_target() { + local out code plain + plain="$TMP_ROOT/not-a-repo" + mkdir -p "$plain/node_modules" + printf 'x\n' > "$plain/node_modules/dep.js" + + out=$("$RECLAIM" "$plain" 2>&1) && code=0 || code=$? + expect_code 1 "$code" "a directory that is not a git worktree must be refused" + assert_contains "$out" "REFUSED" "the refusal must be explicit" + assert_present "$plain/node_modules/dep.js" "nothing outside a git worktree may be removed" + + out=$("$RECLAIM" "$TMP_ROOT/missing-entirely" 2>&1) && code=0 || code=$? + expect_code 1 "$code" "a missing directory must be refused" + + out=$("$RECLAIM" 2>&1) && code=0 || code=$? + expect_code 2 "$code" "a missing argument must be a usage error" + pass "reclaim: refuses a non-worktree, a missing target, and a missing argument" +} + +test_never_follows_scratch_out_of_the_worktree() { + local wt outside + wt=$(make_worktree symlink) + outside="$TMP_ROOT/shared-store" + mkdir -p "$outside" + printf 'shared\n' > "$outside/dep.js" + rm -rf "$wt/node_modules" + ln -s "$outside" "$wt/node_modules" + + "$RECLAIM" "$wt" >/dev/null + assert_present "$outside/dep.js" "a symlinked scratch path must never be followed out of the worktree" + assert_present "$wt/node_modules" "the symlink itself must be left in place" + assert_absent "$wt/dist" "genuine in-worktree scratch must still be reclaimed" + pass "reclaim: scratch is reclaimed in place, never through a symlink out of the worktree" +} + +test_is_idempotent() { + local wt out + wt=$(make_worktree idempotent) + "$RECLAIM" "$wt" >/dev/null + out=$("$RECLAIM" "$wt") + assert_contains "$out" "no regenerable build scratch" "a second run must be a clean no-op" + pass "reclaim: a repeat run is a clean no-op" +} + +test_removes_regenerable_scratch_only +test_leaves_untracked_and_unignored_work_alone +test_refuses_a_worktree_with_modified_tracked_work +test_refuses_a_non_worktree_target +test_never_follows_scratch_out_of_the_worktree +test_is_idempotent diff --git a/tests/fm-teardown.test.sh b/tests/fm-teardown.test.sh index a0815a967e..e8512b3a6a 100755 --- a/tests/fm-teardown.test.sh +++ b/tests/fm-teardown.test.sh @@ -2591,6 +2591,65 @@ EOF pass "the run abort and the leaked-process reap both complete before the destructive worktree return" } +# Give the worktree a committed .gitignore plus one ignored install tree and one +# ignored build-output tree, the shape a finished frontend task leaves behind. +add_build_scratch() { + local case_dir=$1 + printf '%s\n%s\n' 'node_modules/' 'dist/' > "$case_dir/wt/.gitignore" + git -C "$case_dir/wt" add -- .gitignore + git -C "$case_dir/wt" -c user.email=t@t -c user.name=t commit -q -m "ignore build scratch" + mkdir -p "$case_dir/wt/node_modules/pkg" "$case_dir/wt/dist" + printf 'installed\n' > "$case_dir/wt/node_modules/pkg/index.js" + printf 'built\n' > "$case_dir/wt/dist/bundle.js" +} + +test_landed_teardown_reclaims_build_scratch_before_the_worktree_returns() { + local case_dir rc + case_dir=$(make_case reclaim-landed) + write_meta "$case_dir" no-mistakes ship + add_build_scratch "$case_dir" + land_shippable_commit "$case_dir" + + # Snapshot, at the moment of the destructive return, whether the scratch was + # already gone: the pool slot must not go back carrying the install tree. + cat > "$case_dir/fakebin/treehouse" <> "$case_dir/order.log"; fi +exit 0 +EOF + chmod +x "$case_dir/fakebin/treehouse" + + rc=0 + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" || rc=$? + expect_code 0 "$rc" "reclaim-landed: teardown should succeed for landed work" + assert_grep "reclaim-already-happened" "$case_dir/order.log" \ + "reclaim-landed: build scratch was still present when the worktree was returned" + assert_absent "$case_dir/wt/dist" "reclaim-landed: ignored build output should have been reclaimed" + assert_present "$case_dir/wt/.gitignore" "reclaim-landed: tracked files must survive reclamation" + pass "teardown of landed work reclaims the worktree's build scratch before returning it" +} + +test_refused_teardown_leaves_build_scratch_untouched() { + local case_dir rc + case_dir=$(make_case reclaim-refused) + write_meta "$case_dir" no-mistakes ship + add_build_scratch "$case_dir" + # Committed but on no remote and in no merged PR: teardown must refuse, and + # nothing - scratch included - may be touched in a worktree that still holds + # unlanded work. + wt_commit "$case_dir" "unlanded work" + + rc=0 + run_teardown "$case_dir" > "$case_dir/stdout" 2> "$case_dir/stderr" || rc=$? + expect_code 1 "$rc" "reclaim-refused: teardown should refuse unlanded work" + grep -q REFUSED "$case_dir/stderr" || fail "reclaim-refused: no REFUSED line in stderr" + assert_present "$case_dir/wt/node_modules/pkg/index.js" \ + "reclaim-refused: a refused teardown must not reclaim anything" + assert_present "$case_dir/wt/dist/bundle.js" \ + "reclaim-refused: a refused teardown must not remove build output either" + pass "a teardown refused for unlanded work never reclaims that worktree's build scratch" +} + test_local_only_fork_remote_allows test_teardown_prompts_tasks_axi_done_when_compatible test_teardown_manual_backend_prompts_hand_edit_even_when_tasks_axi_present @@ -2649,3 +2708,5 @@ test_process_spawned_during_grace_is_reaped_on_later_pass test_persistent_scan_refuses_after_bounded_retries test_process_exit_during_identity_lookup_does_not_refuse test_run_abort_precedes_process_reap_precedes_worktree_removal +test_landed_teardown_reclaims_build_scratch_before_the_worktree_returns +test_refused_teardown_leaves_build_scratch_untouched From bf59f733374f9b94c784a2b97b93bc70355b2dd9 Mon Sep 17 00:00:00 2001 From: knowttl Date: Tue, 18 Aug 2026 11:08:46 -0700 Subject: [PATCH 2/2] no-mistakes(document): document disk-space guard and build-scratch reclaim script --- docs/architecture.md | 2 +- docs/scripts.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index d11cc74904..edab0b411f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -86,7 +86,7 @@ It suppresses failed-looking closes when the same identity-matched watcher is he Cursor's `bin/fm-turnend-guard-cursor.sh` hook is the same between-turns shape in one synchronous step: it parks the awaited `stop` hook on the arm wrapper and translates an actionable close into one `followup_message`, with a generation baton that makes an older park still running after the next `stop` claim stand down instead of leaking a stale duplicate wake. The existing turn-end guard remains the final backstop for every harness-engine protocol, with pi-signed sharing Pi's protocol, the `--claude` mode cooperating with the auto-arm claim, and Cursor's `--cursor` mode rendering a block as one bounded follow-up because its `stop` step cannot be blocked. Its `--restart` mode signals only the watcher recorded in the current home's `state/.watch.lock`, so restarting one home cannot kill sibling secondmate watchers. -A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if the primary checkout is tangled, if work, process-event sources, or Relay polling has an unhealthy model-aware supervision verdict, or if queued wakes are waiting to be drained. +A pull-based guard (`bin/fm-guard.sh`) warns through supervision tool output if the primary checkout is tangled, if the watched filesystem's free space is below the configured headroom (docs/configuration.md "Disk-space guard"), if work, process-event sources, or Relay polling has an unhealthy model-aware supervision verdict, or if queued wakes are waiting to be drained. The drain script calls that guard after presenting the queue; records remain durable, and may keep the queued-wakes warning visible, until the exact generation-bound acknowledgement printed by the drain succeeds after handling. It leads with a prominent bordered tangle banner, while `bin/fm-guard.sh` owns the watcher-down banner and reminder policy so repeated guarded commands stay noisy without reprinting the full banner in the same episode. On every verified primary harness, tracked hook integration gives the primary session a push-based backstop: when work, a process-event source, or Relay polling needs supervision and no identity-matched watcher lock with a fresh beacon is live, blocking-capable Stop hooks block and nonblocking turn-end integrations force one bounded follow-up. diff --git a/docs/scripts.md b/docs/scripts.md index e94ccb0e16..c3fd3ada23 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -34,7 +34,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-test-run.sh` | Behavior-test runner: selection, portable lanes, proven-isolated `--jobs`, coverage guard, timing/JSON | | `fm-test-isolation-proof.sh` | Concurrent isolation proof and proven-isolated candidate set owner | | `fm-ensure-agents-md.sh` | Ensure a project's real `AGENTS.md`, its `CLAUDE.md` `@AGENTS.md` pointer, and the canonical self-governance section | -| `fm-guard.sh` | Warn on primary-checkout tangles, pending queued wakes, and unhealthy supervision | +| `fm-guard.sh` | Warn on primary-checkout tangles, low root-disk free space, pending queued wakes, and unhealthy supervision | | `fm-primary-scope-lib.sh` | Shared marker-or-plain-checkout primary-home predicate for tracked hooks | | `fm-session-lock-lib.sh` | Shared session-lock harness identity (ancestry walk and holder liveness) for fm-lock.sh and the Claude Stop auto-arm | | `fm-claude-stop-autoarm.sh` | Claude Stop `asyncRewake` hook owning tokenless watcher continuity with single-flight exit-2 rewake (docs/watcher-continuity.md) | @@ -108,6 +108,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-pr-merge.sh` | Record PR metadata, then merge a task's canonical full GitHub URL | | `fm-promote.sh` | Promote a scout task in place to a protected ship task with an explicit delivery mode | | `fm-teardown.sh` | Fail-closed teardown: return landed ship worktrees, require completed scout deliverables, retire secondmate homes | +| `fm-reclaim-build-scratch.sh` | Fail-closed reclaim of a finished worktree's regenerable build scratch (git-ignored install trees, build output, tool caches) | | `fm-harness.sh` | Detect the running harness and resolve crew or secondmate harness, model, and effort | | `fm-lock.sh` | Per-home firstmate session lock | | `fm-x-lib.sh` | Shared Relay config, relay, and reply-threading helpers |