diff --git a/.agents/skills/goal/SKILL.md b/.agents/skills/goal/SKILL.md new file mode 100644 index 00000000000..53ab14be34f --- /dev/null +++ b/.agents/skills/goal/SKILL.md @@ -0,0 +1,94 @@ +--- +name: goal +description: >- + Take one large objective, hold it durably, and drive it to completion through repeated verified passes of the gauntlet loop. + Use when the captain invokes /goal, or asks to set up, run, resume, check, or close a long-running objective that will outlive this session - "make X real", "keep working on X until it is done", "set up a loop for X", "where is X up to". + Owns goal creation, gate design, the eight-stage loop, the automation handoff, and the fail-closed completion verdict. + Not for ordinary task dispatch: a single change that a worker can finish and land is a backlog item, not a goal. +user-invocable: true +metadata: + internal: true +--- + +# goal + +A goal is an objective too large for one task, too long-lived for one session, and too important to be tracked in conversation memory. +This skill turns such an objective into a durable record that survives restarts, drives itself through repeated verified passes, and refuses to report success until evidence says so. + +`bin/fm-goal.sh` owns every command, path, and record format. +Read its header before the first use in a session; this skill owns the judgement, not the mechanics. + +## When this is the right shape + +Use a goal when all three hold. + +- The objective needs many tasks over many sessions, so no single brief can carry it. +- "Done" is a verdict that must be computed from evidence, not asserted by whoever finished last. +- Progress must survive a context reset, a crash, and a change of harness. + +Use the backlog instead when a single worker can finish and land the change. +Adding a goal for ordinary work buys durability nobody needed and costs a second place to look. + +## The gauntlet loop + +The default stage list is `ingest, verify, interpret, merge, render, evaluate, improve, archive`. +It is a complete cycle: gather the inputs, check they are trustworthy, understand what they mean, integrate them, produce something, score it, refine it, and record it. +A loop that skips `verify` or `evaluate` is not a gauntlet, it is a treadmill. + +Stages are configurable per goal with `--stages`, because a research goal and a production goal do not have the same shape. +Keep the list short enough that a full pass is a meaningful unit of progress. +The loop is deliberately circular: reaching the last stage wraps to the first and journals a completed pass, because a goal that is not yet closed always has another pass to run. + +## Gates are the whole point + +A gate is an executable check that answers one closed question about the objective with an exit code. +Gates are what make the completion verdict a fact rather than an opinion, so design them before running the loop, not after. + +Every gate must satisfy all of these. + +- It answers one question, and its exit code is the whole answer: `0` passes, anything else fails. +- It is runnable unattended, with no prompt, no network dependency it cannot tolerate losing, and a bounded runtime. +- It fails closed. A gate that cannot determine the answer must fail, never pass. A gate that passes when its subject is missing is worse than no gate. +- It reads state and reports; it does not fix, install, push, or merge. A gate that changes the thing it measures cannot measure it. +- Its output is the evidence a sceptic would want, because that output is what gets captured when it fails. + +Register a gate with `gate-add`, which records its exact bytes. +If those bytes later change, the gate reports `UNBOUND` and refuses to run, and any result recorded against the previous version stops counting. +That is deliberate: an unattended loop that can quietly rewrite its own success condition is not a verification loop. +Re-registering a gate is normal and expected; re-proving it afterwards is mandatory. + +State the gates in the captain's terms when you propose them, and get agreement on the gate set before spending work against it. +A goal whose gates were never agreed will finish and satisfy nobody. + +## Running a goal + +1. **Frame it.** Write the objective to a file and create the goal with `new`, passing that file to `--objective`. The objective text is the contract; keep it specific enough that a stranger could tell whether it was met. +2. **Design the gates.** Propose the gate set to the captain, then register each one with `gate-add`. A goal with no gates can never pass, by design. +3. **Run passes.** Do the work for the current stage, record what happened with `note`, then `advance`. Never advance a stage whose work did not happen; a journal that records passes nobody made is worse than an empty one. +4. **Prove it.** Run `gate-run` at the end of each full pass, and read the failures rather than the summary line. The evidence file for a failing gate is the actual finding. +5. **Close it.** `close` refuses unless every gate has a current pass. Take the refusal as the answer, never as an obstacle to work around. + +## Escalation + +Bring the captain in for the gate set before work starts against it, for any gate that has failed the same way across three consecutive passes, and for any objective the evidence shows is unreachable as written. +A goal that keeps looping without its failing gate moving is not progress, and reporting the loop as activity misrepresents it. +Routine passes, individual gate runs, and stage advances are not captain-facing events. + +## Automation handoff + +A goal can be driven by a recurring trigger so it continues without the captain re-asking. +Before arming anything, inventory the automation that already exists in the target project and say what you found. +Adding a loop on top of unmapped automation is how two schedulers end up fighting over one repository, and an existing unattended loop that pushes or resets a working tree makes that compounding damage rather than duplicated effort. + +The arming rules are narrow on purpose. + +- One owner. Exactly one trigger drives one goal. Never arm a second while the first is live. +- The automated pass runs the loop and the gates, and it may write evidence and journal entries. It does not merge, push to a default branch, discard unlanded work, or answer a decision that was escalated to the captain. +- A gate that fails is a stop, not a retry. Escalate the failure with its evidence rather than looping on it. +- Anything destructive, irreversible, or security-sensitive stays with the captain regardless of what the loop concludes. + +## Reporting + +Report a goal in outcomes: what the objective is, which gates pass, which do not, what the failing evidence says, and what decision is needed next. +`export-json` gives the machine-readable form for a status page or a dashboard. +Never present a passing verdict without naming the gate set it passed, because the verdict means exactly as much as the gates do and no more. diff --git a/AGENTS.md b/AGENTS.md index 67ec0d69609..9a9eceb10a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -486,6 +486,9 @@ Unresolved decisions discovered by investigations or visual reviews follow `deci Update the backlog on every dispatch, completion, and decision for a work item. Re-evaluate queued work after every teardown and heartbeat, dispatching items only when dependencies and time gates have cleared. +An objective too large for one work item and too long-lived for one session is a goal rather than a backlog entry. +Load the `goal` skill when the captain invokes `/goal`, or asks to set up, run, resume, check, or close such an objective; the backlog still holds the individual work items that goal spawns. + `.tasks.toml`, `docs/configuration.md`, and current `tasks-axi --help` own the backlog schema, compatibility, retention, and routine command syntax. Use compatible `tasks-axi` when the configured backend selects it and the documented manual path otherwise; keep only the configured recent Done entries. `secondmate-provisioning` and `bin/fm-backlog-handoff.sh` own cross-home handoff safety. diff --git a/README.md b/README.md index 8ed5226b171..2474f215cb1 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,7 @@ Claude and grok use the slash form shown here; codex uses the same names with `$ | `/afk` | Enter away-mode supervision: the sub-supervisor self-handles routine notifications in bash, escalates captain-relevant events and bounded declared-external-wait rechecks as batched digests, and actively alerts if delivery gets stuck while you step away | | `/ahoy` | Recap visible session events since the prior real captain message plus visibly unanswered captain decisions, then guide the captain through any open decisions one at a time in agent-judged impact order; fall back to Bearings when invoked as the session's first real captain message | | `/bearings` | Generate a concise four-section chat digest from bounded local fleet and registered-secondmate state; use `/bearings file` to also replace today's dated report in `data/`, and add `include PRs` when live PR enrichment is wanted | +| `/goal` | Hold one long-running objective in a durable ledger and drive it through repeated verified passes of the gauntlet loop, with content-bound gates that decide completion from evidence rather than assertion | | `/updatefirstmate` | Self-update the running firstmate and its secondmates to the latest from origin with fast-forward-only pulls, then re-read instructions and nudge secondmates | | `/stow` | Sweep the session for uncaptured durable knowledge, persist the open work records this session knows are unfiled or now wrong, curate tiered startup memory with decay and cold archival, enforce each home's budget or surface the required decision, cascade to registered second mates, and report what is safe to reset | diff --git a/bin/fm-goal.sh b/bin/fm-goal.sh new file mode 100755 index 00000000000..f88d18ba84b --- /dev/null +++ b/bin/fm-goal.sh @@ -0,0 +1,607 @@ +#!/usr/bin/env bash +# fm-goal.sh - durable goal ledger and gauntlet-loop driver. +# +# A goal is a long-running objective that outlives any one session: it holds its +# own objective text, an ordered stage list, an append-only journal, and a set of +# content-bound verification gates. The loop is the point. A goal advances one +# stage at a time, every pass is recorded, and the goal cannot be closed until +# every registered gate passes, so "done" is a verdict this script computes from +# evidence rather than a claim an agent makes. +# +# Gates are content-bound on purpose. A gate script must be registered with +# `gate-add`, which records its exact bytes; if those bytes later change, every +# run and every verdict refuses until it is registered again. An unattended loop +# that can silently rewrite its own success condition is not a verification loop, +# so this refuses rather than trusting whatever is on disk at run time. +# +# Nothing here reaches the network, spawns an agent, or writes outside the goal +# directory. This script keeps the record; the caller does the work. +# +# Layout, under $FM_HOME/data/goals//: +# goal.meta key=value record: id, title, created, status, stage, stages +# objective.md the objective text, verbatim +# journal.log append-only TSV: +# gates/.sh a registered gate, mode 0700, exit 0 = pass +# gates/.trust its recorded sha256 binding +# evidence/ per-run gate output, newest kept +# +# Usage: +# fm-goal.sh new --title [--objective <file>|-] [--stages <csv>] +# fm-goal.sh list +# fm-goal.sh show <goal-id> +# fm-goal.sh stage <goal-id> +# fm-goal.sh advance <goal-id> [--note <text>] +# fm-goal.sh note <goal-id> --note <text> +# fm-goal.sh gate-add <goal-id> <gate-name> <script-path> +# fm-goal.sh gate-list <goal-id> +# fm-goal.sh gate-run <goal-id> [--gate <gate-name>] +# fm-goal.sh verdict <goal-id> +# fm-goal.sh close <goal-id> +# fm-goal.sh export-json <goal-id> +# fm-goal.sh --help +# +# Exit codes: 0 success or PASS verdict, 1 refusal or FAIL verdict, 2 usage error. +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" +FM_HOME="${FM_HOME:-$FM_ROOT}" +GOALS_ROOT="${FM_GOALS_OVERRIDE:-$FM_HOME/data/goals}" + +# The captain's gauntlet loop: ingest, verify, interpret, merge, render, +# evaluate, improve, archive. Any ordered stage list works; this is the default +# because it is a complete cycle - gather, check, understand, integrate, +# produce, score, refine, record - and a loop that skips verify or evaluate is +# not a gauntlet. +DEFAULT_STAGES='ingest,verify,interpret,merge,render,evaluate,improve,archive' + +GATE_TRUST_VERSION='fm-goal-gate-v1' +EVIDENCE_KEEP=50 + +die() { + printf 'error: %s\n' "$1" >&2 + exit "${2:-1}" +} + +usage() { + sed -n '2,45p' "$SCRIPT_DIR/fm-goal.sh" | sed 's/^# \{0,1\}//' +} + +# A goal id is a filename, so keep it to a conservative portable set and refuse +# anything that could traverse a path or collide case-insensitively. +goal_id_valid() { + local id=$1 + case $id in + ''|.|..) return 1 ;; + *[!a-z0-9-]*) return 1 ;; + -*|*-) return 1 ;; + esac + [ "${#id}" -le 64 ] +} + +gate_name_valid() { + local name=$1 + case $name in + ''|.|..) return 1 ;; + *[!a-z0-9-]*) return 1 ;; + -*|*-) return 1 ;; + esac + [ "${#name}" -le 48 ] +} + +goal_dir() { + printf '%s/%s\n' "$GOALS_ROOT" "$1" +} + +require_goal() { + local id=$1 dir + goal_id_valid "$id" || die "invalid goal id: $id" 2 + dir=$(goal_dir "$id") + [ -d "$dir" ] && [ ! -L "$dir" ] || die "no such goal: $id" + [ -f "$dir/goal.meta" ] || die "goal record is unreadable: $id" + printf '%s\n' "$dir" +} + +sha256_of() { + local file=$1 + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$file" 2>/dev/null | awk '{print $1}' + elif command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$file" 2>/dev/null | awk '{print $1}' + else + return 1 + fi +} + +meta_get() { + local dir=$1 key=$2 line + while IFS= read -r line || [ -n "$line" ]; do + case $line in + "$key="*) printf '%s\n' "${line#"$key"=}"; return 0 ;; + esac + done < "$dir/goal.meta" + return 1 +} + +# Rewrite one key in place. The whole record is small, so a read-modify-write +# through a temp file in the same directory keeps the update atomic. +meta_set() { + local dir=$1 key=$2 value=$3 tmp line found=0 + tmp=$(mktemp "$dir/.goal.meta.XXXXXX") || return 1 + while IFS= read -r line || [ -n "$line" ]; do + case $line in + "$key="*) printf '%s=%s\n' "$key" "$value" >> "$tmp"; found=1 ;; + *) printf '%s\n' "$line" >> "$tmp" ;; + esac + done < "$dir/goal.meta" + [ "$found" -eq 1 ] || printf '%s=%s\n' "$key" "$value" >> "$tmp" + chmod 0600 "$tmp" || { rm -f -- "$tmp"; return 1; } + mv -f -- "$tmp" "$dir/goal.meta" +} + +# Journal entries are TSV, so strip the two characters that would break the +# format rather than silently writing a record that cannot be parsed back. +sanitize_field() { + printf '%s' "$1" | tr '\t\n' ' ' +} + +journal_append() { + local dir=$1 event=$2 note=$3 stage + stage=$(meta_get "$dir" stage || printf '%s' '-') + printf '%s\t%s\t%s\t%s\n' "$(date -u +%s)" "$stage" "$event" "$(sanitize_field "$note")" \ + >> "$dir/journal.log" +} + +stage_list() { + local dir=$1 + meta_get "$dir" stages | tr ',' '\n' +} + +stage_index() { + local dir=$1 want=$2 i=0 s + while IFS= read -r s; do + [ -n "$s" ] || continue + if [ "$s" = "$want" ]; then + printf '%s\n' "$i" + return 0 + fi + i=$((i + 1)) + done < <(stage_list "$dir") + return 1 +} + +stage_count() { + local dir=$1 n=0 s + while IFS= read -r s; do + [ -n "$s" ] || continue + n=$((n + 1)) + done < <(stage_list "$dir") + printf '%s\n' "$n" +} + +stage_at() { + local dir=$1 want=$2 i=0 s + while IFS= read -r s; do + [ -n "$s" ] || continue + if [ "$i" -eq "$want" ]; then + printf '%s\n' "$s" + return 0 + fi + i=$((i + 1)) + done < <(stage_list "$dir") + return 1 +} + +# --- commands --------------------------------------------------------------- + +cmd_new() { + local id='' title='' objective='' stages="$DEFAULT_STAGES" dir first + [ "$#" -ge 1 ] || die "new needs a goal id" 2 + id=$1 + shift + goal_id_valid "$id" || die "invalid goal id: $id (use lowercase letters, digits, and inner dashes)" 2 + while [ "$#" -gt 0 ]; do + case $1 in + --title) [ "$#" -ge 2 ] || die "--title needs a value" 2; title=$2; shift 2 ;; + --objective) [ "$#" -ge 2 ] || die "--objective needs a value" 2; objective=$2; shift 2 ;; + --stages) [ "$#" -ge 2 ] || die "--stages needs a value" 2; stages=$2; shift 2 ;; + *) die "unknown option for new: $1" 2 ;; + esac + done + [ -n "$title" ] || die "new needs --title" 2 + case $stages in + *[!a-z0-9,-]*) die "invalid --stages value: use a comma-separated lowercase list" 2 ;; + ''|,*|*,) die "invalid --stages value: use a comma-separated lowercase list" 2 ;; + esac + dir=$(goal_dir "$id") + [ ! -e "$dir" ] || die "goal already exists: $id" + umask 077 + mkdir -p "$dir/gates" "$dir/evidence" || die "cannot create goal directory" + first=${stages%%,*} + { + printf 'id=%s\n' "$id" + printf 'title=%s\n' "$(sanitize_field "$title")" + printf 'created=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'status=%s\n' open + printf 'stage=%s\n' "$first" + printf 'stages=%s\n' "$stages" + } > "$dir/goal.meta" || die "cannot write goal record" + chmod 0600 "$dir/goal.meta" + if [ -n "$objective" ]; then + if [ "$objective" = '-' ]; then + cat > "$dir/objective.md" + else + [ -f "$objective" ] || die "objective file not found: $objective" + cp -- "$objective" "$dir/objective.md" || die "cannot copy objective" + fi + else + printf '# %s\n\nNo objective text recorded yet.\n' "$title" > "$dir/objective.md" + fi + chmod 0600 "$dir/objective.md" + : > "$dir/journal.log" + chmod 0600 "$dir/journal.log" + journal_append "$dir" created "$title" + printf 'created: %s (stage %s)\n' "$id" "$first" +} + +cmd_list() { + local dir id status stage title + [ -d "$GOALS_ROOT" ] || { printf 'no goals\n'; return 0; } + local found=0 + for dir in "$GOALS_ROOT"/*/; do + [ -f "${dir}goal.meta" ] || continue + found=1 + dir=${dir%/} + id=$(meta_get "$dir" id || printf '?') + status=$(meta_get "$dir" status || printf '?') + stage=$(meta_get "$dir" stage || printf '?') + title=$(meta_get "$dir" title || printf '') + printf '%-28s %-8s %-12s %s\n' "$id" "$status" "$stage" "$title" + done + [ "$found" -eq 1 ] || printf 'no goals\n' +} + +cmd_show() { + local id=$1 dir + dir=$(require_goal "$id") || exit 1 + printf '== goal %s ==\n' "$id" + cat "$dir/goal.meta" + printf '\n-- objective --\n' + cat "$dir/objective.md" + printf '\n-- gates --\n' + cmd_gate_list "$id" + printf '\n-- journal (last 20) --\n' + tail -n 20 "$dir/journal.log" 2>/dev/null || true +} + +cmd_stage() { + local id=$1 dir + dir=$(require_goal "$id") || exit 1 + meta_get "$dir" stage +} + +cmd_advance() { + local id='' note='' dir cur idx count next + [ "$#" -ge 1 ] || die "advance needs a goal id" 2 + id=$1 + shift + while [ "$#" -gt 0 ]; do + case $1 in + --note) [ "$#" -ge 2 ] || die "--note needs a value" 2; note=$2; shift 2 ;; + *) die "unknown option for advance: $1" 2 ;; + esac + done + dir=$(require_goal "$id") || exit 1 + [ "$(meta_get "$dir" status)" = open ] || die "goal is not open: $id" + cur=$(meta_get "$dir" stage) || die "goal has no stage" + idx=$(stage_index "$dir" "$cur") || die "current stage is not in the stage list: $cur" + count=$(stage_count "$dir") + journal_append "$dir" stage-done "$note" + idx=$(( (idx + 1) % count )) + next=$(stage_at "$dir" "$idx") || die "cannot resolve the next stage" + meta_set "$dir" stage "$next" || die "cannot record the next stage" + if [ "$idx" -eq 0 ]; then + journal_append "$dir" loop-complete 'stage list wrapped; a full pass finished' + fi + journal_append "$dir" stage-start '' + printf 'advanced: %s -> %s\n' "$cur" "$next" +} + +cmd_note() { + local id='' note='' dir + [ "$#" -ge 1 ] || die "note needs a goal id" 2 + id=$1 + shift + while [ "$#" -gt 0 ]; do + case $1 in + --note) [ "$#" -ge 2 ] || die "--note needs a value" 2; note=$2; shift 2 ;; + *) die "unknown option for note: $1" 2 ;; + esac + done + [ -n "$note" ] || die "note needs --note" 2 + dir=$(require_goal "$id") || exit 1 + journal_append "$dir" note "$note" + printf 'noted: %s\n' "$id" +} + +cmd_gate_add() { + local id=$1 name=$2 src=$3 dir dest trust hash + dir=$(require_goal "$id") || exit 1 + gate_name_valid "$name" || die "invalid gate name: $name" 2 + [ -f "$src" ] && [ ! -L "$src" ] || die "gate script is not a regular file: $src" + dest="$dir/gates/$name.sh" + trust="$dir/gates/$name.trust" + umask 077 + cp -- "$src" "$dest" || die "cannot install gate: $name" + chmod 0700 "$dest" || die "cannot set gate mode: $name" + hash=$(sha256_of "$dest") || die "cannot hash gate: $name" + printf '%s\n%s\n' "$GATE_TRUST_VERSION" "$hash" > "$trust" || die "cannot bind gate: $name" + chmod 0600 "$trust" + journal_append "$dir" gate-add "$name $hash" + printf 'registered gate: %s (%s)\n' "$name" "$hash" +} + +cmd_gate_list() { + local id=$1 dir gate name found=0 + dir=$(require_goal "$id") || exit 1 + for gate in "$dir"/gates/*.sh; do + [ -f "$gate" ] || continue + found=1 + name=$(basename "$gate" .sh) + if gate_bound "$dir" "$name"; then + printf '%-24s bound\n' "$name" + else + printf '%-24s UNBOUND (refuses to run until re-registered)\n' "$name" + fi + done + [ "$found" -eq 1 ] || printf 'no gates registered\n' +} + +# A gate is bound when its recorded hash matches its current bytes. Anything +# else - missing binding, wrong version line, edited script - is unbound, and an +# unbound gate never runs and never counts as a pass. +gate_bound() { + local dir=$1 name=$2 trust gate version recorded actual + gate="$dir/gates/$name.sh" + trust="$dir/gates/$name.trust" + [ -f "$gate" ] && [ ! -L "$gate" ] || return 1 + [ -f "$trust" ] && [ ! -L "$trust" ] || return 1 + version=$(sed -n '1p' "$trust") + recorded=$(sed -n '2p' "$trust") + [ "$version" = "$GATE_TRUST_VERSION" ] || return 1 + [ -n "$recorded" ] || return 1 + actual=$(sha256_of "$gate") || return 1 + [ "$actual" = "$recorded" ] +} + +prune_evidence() { + local dir=$1 keep=$EVIDENCE_KEEP file n=0 + while IFS= read -r file; do + n=$((n + 1)) + [ "$n" -gt "$keep" ] || continue + rm -f -- "$file" + done < <(ls -1t "$dir"/evidence/*.log 2>/dev/null) +} + +run_one_gate() { + local dir=$1 name=$2 log stamp hash rc=0 + stamp=$(date -u +%s) + log="$dir/evidence/$stamp-$name.log" + if ! gate_bound "$dir" "$name"; then + journal_append "$dir" gate-refused "$name unbound" + printf '%-24s REFUSED (unbound)\n' "$name" + return 1 + fi + hash=$(sha256_of "$dir/gates/$name.sh") || return 1 + ( FM_GOAL_ID="$(meta_get "$dir" id)" \ + FM_GOAL_DIR="$dir" \ + FM_GOAL_STAGE="$(meta_get "$dir" stage)" \ + "$dir/gates/$name.sh" ) > "$log" 2>&1 || rc=$? + chmod 0600 "$log" 2>/dev/null || true + # The hash goes in the record so a later verdict can tell whether the result + # belongs to the gate that is on disk now. Re-registering a gate must not + # inherit the previous version's pass. + if [ "$rc" -eq 0 ]; then + journal_append "$dir" gate-pass "$name $hash" + printf '%-24s PASS\n' "$name" + return 0 + fi + journal_append "$dir" gate-fail "$name $hash rc=$rc" + printf '%-24s FAIL (rc=%s, evidence/%s)\n' "$name" "$rc" "$(basename "$log")" + return 1 +} + +cmd_gate_run() { + local id='' only='' dir gate name failed=0 ran=0 + [ "$#" -ge 1 ] || die "gate-run needs a goal id" 2 + id=$1 + shift + while [ "$#" -gt 0 ]; do + case $1 in + --gate) [ "$#" -ge 2 ] || die "--gate needs a value" 2; only=$2; shift 2 ;; + *) die "unknown option for gate-run: $1" 2 ;; + esac + done + dir=$(require_goal "$id") || exit 1 + for gate in "$dir"/gates/*.sh; do + [ -f "$gate" ] || continue + name=$(basename "$gate" .sh) + [ -z "$only" ] || [ "$only" = "$name" ] || continue + ran=$((ran + 1)) + run_one_gate "$dir" "$name" || failed=$((failed + 1)) + done + prune_evidence "$dir" + if [ "$ran" -eq 0 ]; then + [ -z "$only" ] || die "no such gate: $only" + printf 'no gates registered\n' + return 0 + fi + printf '%s gate(s) run, %s failed\n' "$ran" "$failed" + [ "$failed" -eq 0 ] +} + +# The verdict is computed from the last recorded result per gate, so it reports +# what the evidence says rather than re-running anything. A gate that has never +# run, or whose bytes changed since it ran, is not a pass. +cmd_verdict() { + local id=$1 dir gate name last current failed=0 total=0 + dir=$(require_goal "$id") || exit 1 + for gate in "$dir"/gates/*.sh; do + [ -f "$gate" ] || continue + name=$(basename "$gate" .sh) + total=$((total + 1)) + if ! gate_bound "$dir" "$name"; then + printf '%-24s UNBOUND\n' "$name" + failed=$((failed + 1)) + continue + fi + current=$(sha256_of "$gate") || current='' + # Only a result recorded against the bytes now on disk counts. A result + # from a superseded version of the gate reads as never run. + last=$(awk -F'\t' -v g="$name" -v h="$current" ' + ($3=="gate-pass" || $3=="gate-fail") { + split($4, f, " ") + if (f[1]==g && f[2]==h) r = ($3=="gate-pass") ? "PASS" : "FAIL" + } + END { print r }' "$dir/journal.log" 2>/dev/null) + case ${last:-} in + PASS) printf '%-24s PASS\n' "$name" ;; + FAIL) printf '%-24s FAIL\n' "$name"; failed=$((failed + 1)) ;; + *) printf '%-24s NEVER RUN\n' "$name"; failed=$((failed + 1)) ;; + esac + done + if [ "$total" -eq 0 ]; then + printf 'verdict: NO GATES\n' + return 1 + fi + if [ "$failed" -eq 0 ]; then + printf 'verdict: PASS (%s gates)\n' "$total" + return 0 + fi + printf 'verdict: FAIL (%s of %s gates not passing)\n' "$failed" "$total" + return 1 +} + +cmd_close() { + local id=$1 dir + dir=$(require_goal "$id") || exit 1 + if ! cmd_verdict "$id" > /dev/null 2>&1; then + cmd_verdict "$id" || true + die "refusing to close $id: the gates do not all pass" + fi + meta_set "$dir" status closed || die "cannot record closure" + journal_append "$dir" closed 'all gates passing' + printf 'closed: %s\n' "$id" +} + +# Machine-readable export for a dashboard or a status page. Deliberately a +# separate command so the human-facing output above stays free to change. +cmd_export_json() { + local id=$1 dir + dir=$(require_goal "$id") || exit 1 + python3 - "$dir" <<'PY' +import json, os, signal, sys + +# Piping this into head is normal; exit quietly instead of dumping a traceback. +try: + signal.signal(signal.SIGPIPE, signal.SIG_DFL) +except (AttributeError, ValueError): + pass + +d = sys.argv[1] +meta = {} +with open(os.path.join(d, "goal.meta"), encoding="utf-8") as fh: + for line in fh: + line = line.rstrip("\n") + if "=" in line: + k, v = line.split("=", 1) + meta[k] = v + +journal = [] +jpath = os.path.join(d, "journal.log") +if os.path.exists(jpath): + with open(jpath, encoding="utf-8") as fh: + for line in fh: + parts = line.rstrip("\n").split("\t") + if len(parts) == 4: + journal.append( + {"ts": parts[0], "stage": parts[1], "event": parts[2], "note": parts[3]} + ) + +gates = {} +for entry in journal: + if entry["event"] in ("gate-pass", "gate-fail"): + fields = entry["note"].split(" ") + if len(fields) >= 2: + gates[fields[0]] = ("PASS" if entry["event"] == "gate-pass" else "FAIL", fields[1]) + +import hashlib + + +def digest(path): + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +rows = [] +gdir = os.path.join(d, "gates") +if os.path.isdir(gdir): + for name in sorted(n[:-3] for n in os.listdir(gdir) if n.endswith(".sh")): + current = digest(os.path.join(gdir, name + ".sh")) + trust = os.path.join(gdir, name + ".trust") + bound = False + if os.path.exists(trust): + with open(trust, encoding="utf-8") as fh: + lines = fh.read().splitlines() + bound = len(lines) >= 2 and lines[0] == "fm-goal-gate-v1" and lines[1] == current + recorded = gates.get(name) + # A result only counts when it was recorded against the bytes on disk now. + if not bound: + result = "UNBOUND" + elif recorded and recorded[1] == current: + result = recorded[0] + else: + result = "NEVER RUN" + rows.append({"name": name, "result": result, "bound": bound}) + +meta["stages"] = meta.get("stages", "").split(",") if meta.get("stages") else [] +json.dump( + { + "goal": meta, + "gates": rows, + "verdict": "PASS" if rows and all(r["result"] == "PASS" for r in rows) else "FAIL", + "journal": journal, + }, + sys.stdout, + indent=2, +) +sys.stdout.write("\n") +PY +} + +main() { + [ "$#" -ge 1 ] || { usage; exit 2; } + local cmd=$1 + shift + case $cmd in + --help|-h|help) usage ;; + new) cmd_new "$@" ;; + list) cmd_list ;; + show) [ "$#" -eq 1 ] || die "show needs exactly one goal id" 2; cmd_show "$1" ;; + stage) [ "$#" -eq 1 ] || die "stage needs exactly one goal id" 2; cmd_stage "$1" ;; + advance) cmd_advance "$@" ;; + note) cmd_note "$@" ;; + gate-add) [ "$#" -eq 3 ] || die "gate-add needs <goal-id> <gate-name> <script>" 2; cmd_gate_add "$1" "$2" "$3" ;; + gate-list) [ "$#" -eq 1 ] || die "gate-list needs exactly one goal id" 2; cmd_gate_list "$1" ;; + gate-run) cmd_gate_run "$@" ;; + verdict) [ "$#" -eq 1 ] || die "verdict needs exactly one goal id" 2; cmd_verdict "$1" ;; + close) [ "$#" -eq 1 ] || die "close needs exactly one goal id" 2; cmd_close "$1" ;; + export-json) [ "$#" -eq 1 ] || die "export-json needs exactly one goal id" 2; cmd_export_json "$1" ;; + *) die "unknown command: $cmd" 2 ;; + esac +} + +main "$@" diff --git a/docs/documentation-audiences.json b/docs/documentation-audiences.json index 4889467bd41..2160b93eaf6 100644 --- a/docs/documentation-audiences.json +++ b/docs/documentation-audiences.json @@ -152,6 +152,10 @@ "path": ".agents/skills/fmx-respond/SKILL.md", "audience": "agent-runtime" }, + { + "path": ".agents/skills/goal/SKILL.md", + "audience": "agent-runtime" + }, { "path": ".agents/skills/harness-adapters/SKILL.md", "audience": "agent-runtime" diff --git a/tests/fm-goal.test.sh b/tests/fm-goal.test.sh new file mode 100755 index 00000000000..f0d0bdc5312 --- /dev/null +++ b/tests/fm-goal.test.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# Behavior tests for bin/fm-goal.sh. +# +# The properties under test are the ones a long-running unattended loop depends +# on: a gate cannot pass unless its recorded bytes match what is on disk, a +# verdict cannot inherit a superseded gate's result, and a goal cannot close +# while any gate is not passing. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TMP_ROOT=$(fm_test_tmproot fm-goal) +GOAL="$ROOT/bin/fm-goal.sh" +export FM_GOALS_OVERRIDE="$TMP_ROOT/goals" + +write_gate() { + local path=$1 code=$2 + printf '#!/usr/bin/env bash\nexit %s\n' "$code" > "$path" + chmod 0755 "$path" +} + +# Run a command, discard its output, and echo its exit code for expect_code. +code_of() { + "$@" >/dev/null 2>&1 + printf '%s\n' "$?" +} + +test_new_creates_record_and_first_stage() { + local out + out=$("$GOAL" new alpha --title "First goal" --stages one,two,three 2>&1) \ + || fail "fm-goal.sh new failed" + assert_contains "$out" "created: alpha" "new did not report creation" + assert_present "$FM_GOALS_OVERRIDE/alpha/goal.meta" "goal.meta was not written" + assert_present "$FM_GOALS_OVERRIDE/alpha/objective.md" "objective.md was not written" + assert_grep "status=open" "$FM_GOALS_OVERRIDE/alpha/goal.meta" "new goal is not open" + assert_contains "$("$GOAL" stage alpha)" "one" "first stage is not the first entry of --stages" + pass "fm-goal.sh: new creates a durable record starting at the first stage" +} + +test_new_refuses_duplicate_and_bad_ids() { + expect_code 1 "$(code_of "$GOAL" new alpha --title Duplicate)" "duplicate goal id" + expect_code 2 "$(code_of "$GOAL" new ../escape --title Traversal)" "path traversal id" + expect_code 2 "$(code_of "$GOAL" new UPPER --title Uppercase)" "uppercase id" + expect_code 2 "$(code_of "$GOAL" new no-title-goal)" "missing title" + assert_absent "$FM_GOALS_OVERRIDE/../escape" "a traversing goal id created a directory outside the goals root" + pass "fm-goal.sh: new refuses duplicates, path traversal, and missing titles" +} + +test_advance_cycles_and_journals() { + "$GOAL" new cycler --title "Cycles" --stages a,b >/dev/null 2>&1 || fail "setup goal failed" + assert_contains "$("$GOAL" advance cycler --note first)" "a -> b" "advance did not move to the next stage" + assert_contains "$("$GOAL" advance cycler --note second)" "b -> a" "advance did not wrap to the first stage" + assert_grep "loop-complete" "$FM_GOALS_OVERRIDE/cycler/journal.log" "wrapping did not record a completed pass" + assert_grep "first" "$FM_GOALS_OVERRIDE/cycler/journal.log" "advance did not record its note" + pass "fm-goal.sh: advance cycles the stage list and journals every pass" +} + +test_journal_note_cannot_break_the_record_format() { + local lines + "$GOAL" new tsvsafe --title "TSV" >/dev/null 2>&1 || fail "setup goal failed" + "$GOAL" note tsvsafe --note "$(printf 'has\ta tab\nand a newline')" >/dev/null 2>&1 \ + || fail "note with control characters failed" + lines=$(grep -c "" "$FM_GOALS_OVERRIDE/tsvsafe/journal.log") + [ "$lines" -eq 2 ] || fail "a note containing a newline wrote $lines journal lines, expected 2" + pass "fm-goal.sh: a note containing tabs or newlines cannot corrupt the journal" +} + +test_verdict_without_gates_fails_closed() { + "$GOAL" new nogates --title "No gates" >/dev/null 2>&1 || fail "setup goal failed" + expect_code 1 "$(code_of "$GOAL" verdict nogates)" "verdict with no gates" + assert_contains "$("$GOAL" verdict nogates 2>&1)" "NO GATES" "verdict did not name the missing gates" + expect_code 1 "$(code_of "$GOAL" close nogates)" "close with no gates" + pass "fm-goal.sh: a goal with no gates never reports a passing verdict" +} + +test_gate_run_reports_pass_and_fail() { + local out + "$GOAL" new gated --title "Gated" >/dev/null 2>&1 || fail "setup goal failed" + write_gate "$TMP_ROOT/pass.sh" 0 + write_gate "$TMP_ROOT/fail.sh" 3 + "$GOAL" gate-add gated good "$TMP_ROOT/pass.sh" >/dev/null || fail "gate-add of a passing gate failed" + "$GOAL" gate-add gated bad "$TMP_ROOT/fail.sh" >/dev/null || fail "gate-add of a failing gate failed" + out=$("$GOAL" gate-run gated 2>&1) && fail "gate-run reported success with a failing gate" + assert_contains "$out" "good" "gate-run did not report the passing gate" + assert_contains "$out" "FAIL" "gate-run did not report the failing gate" + expect_code 1 "$(code_of "$GOAL" close gated)" "close with a failing gate" + pass "fm-goal.sh: gate-run reports each gate and close refuses while one fails" +} + +test_gate_evidence_is_captured() { + local log + "$GOAL" new evidence --title "Evidence" >/dev/null 2>&1 || fail "setup goal failed" + cat > "$TMP_ROOT/loud.sh" <<'GATE' +#!/usr/bin/env bash +echo "gate saw stage $FM_GOAL_STAGE" +exit 1 +GATE + chmod 0755 "$TMP_ROOT/loud.sh" + "$GOAL" gate-add evidence loud "$TMP_ROOT/loud.sh" >/dev/null || fail "gate-add failed" + "$GOAL" gate-run evidence >/dev/null 2>&1 + log=$(find "$FM_GOALS_OVERRIDE/evidence/evidence" -name '*-loud.log' | head -n 1) + [ -n "$log" ] || fail "no evidence file was written for a failing gate" + assert_grep "gate saw stage ingest" "$log" "the gate did not receive the current stage in its environment" + pass "fm-goal.sh: a gate run captures output as evidence and receives the goal stage" +} + +# The core safety property. An unattended loop that can rewrite its own success +# condition is not a verification loop, so edited bytes must refuse to run. +test_edited_gate_refuses_to_run() { + local out + "$GOAL" new tamper --title "Tamper" >/dev/null 2>&1 || fail "setup goal failed" + write_gate "$TMP_ROOT/strict.sh" 1 + "$GOAL" gate-add tamper strict "$TMP_ROOT/strict.sh" >/dev/null || fail "gate-add failed" + write_gate "$FM_GOALS_OVERRIDE/tamper/gates/strict.sh" 0 + assert_contains "$("$GOAL" gate-list tamper)" "UNBOUND" "an edited gate was still reported as bound" + out=$("$GOAL" gate-run tamper 2>&1) && fail "an edited gate was allowed to pass" + assert_contains "$out" "REFUSED" "gate-run did not refuse the edited gate" + assert_contains "$("$GOAL" verdict tamper 2>&1)" "UNBOUND" "verdict counted an edited gate as a result" + expect_code 1 "$(code_of "$GOAL" close tamper)" "close with an edited gate" + pass "fm-goal.sh: a gate edited after registration refuses to run and blocks closure" +} + +# A gate replaced with new bytes must be re-proven. Inheriting the previous +# version's pass would let a rewrite launder an unverified success. +test_reregistered_gate_does_not_inherit_a_pass() { + "$GOAL" new stale --title "Stale" >/dev/null 2>&1 || fail "setup goal failed" + write_gate "$TMP_ROOT/v1.sh" 0 + cat > "$TMP_ROOT/v2.sh" <<'GATE' +#!/usr/bin/env bash +# a different implementation with the same outcome +exit 0 +GATE + chmod 0755 "$TMP_ROOT/v2.sh" + "$GOAL" gate-add stale check "$TMP_ROOT/v1.sh" >/dev/null || fail "gate-add v1 failed" + "$GOAL" gate-run stale >/dev/null 2>&1 || fail "gate-run of a passing gate failed" + "$GOAL" verdict stale >/dev/null 2>&1 || fail "verdict did not pass after a passing run" + "$GOAL" gate-add stale check "$TMP_ROOT/v2.sh" >/dev/null || fail "gate-add v2 failed" + assert_contains "$("$GOAL" verdict stale 2>&1)" "NEVER RUN" \ + "a re-registered gate inherited the previous version's pass" + expect_code 1 "$(code_of "$GOAL" close stale)" "close after re-registering a gate" + "$GOAL" gate-run stale >/dev/null 2>&1 || fail "gate-run of the re-registered gate failed" + "$GOAL" close stale >/dev/null 2>&1 || fail "close refused after every gate passed" + pass "fm-goal.sh: a re-registered gate must be re-proven before it counts" +} + +test_close_requires_every_gate_and_records_status() { + "$GOAL" new closer --title "Closer" >/dev/null 2>&1 || fail "setup goal failed" + write_gate "$TMP_ROOT/green.sh" 0 + "$GOAL" gate-add closer green "$TMP_ROOT/green.sh" >/dev/null || fail "gate-add failed" + expect_code 1 "$(code_of "$GOAL" close closer)" "close before any gate ran" + "$GOAL" gate-run closer >/dev/null 2>&1 || fail "gate-run failed" + "$GOAL" close closer >/dev/null 2>&1 || fail "close refused with every gate passing" + assert_grep "status=closed" "$FM_GOALS_OVERRIDE/closer/goal.meta" "close did not record the status" + pass "fm-goal.sh: close requires a run pass from every gate and records the status" +} + +test_export_json_reports_gate_state() { + local json + "$GOAL" new exporter --title "Exporter" --stages one,two >/dev/null 2>&1 || fail "setup goal failed" + write_gate "$TMP_ROOT/exp.sh" 0 + "$GOAL" gate-add exporter e1 "$TMP_ROOT/exp.sh" >/dev/null || fail "gate-add failed" + "$GOAL" gate-run exporter >/dev/null 2>&1 || fail "gate-run failed" + json=$("$GOAL" export-json exporter) || fail "export-json failed" + printf '%s' "$json" | python3 -c 'import json,sys; d=json.load(sys.stdin); assert d["verdict"]=="PASS", d["verdict"]; assert d["goal"]["stages"]==["one","two"], d["goal"]["stages"]; assert d["gates"][0]["result"]=="PASS", d["gates"]' \ + || fail "export-json did not describe the goal, stages, and gate result" + pass "fm-goal.sh: export-json emits the goal, its stages, and each gate result" +} + +test_unknown_goal_and_command_are_refused() { + expect_code 1 "$(code_of "$GOAL" show does-not-exist)" "unknown goal" + expect_code 2 "$(code_of "$GOAL" not-a-command)" "unknown command" + expect_code 2 "$(code_of "$GOAL" advance alpha --bogus value)" "unknown option" + pass "fm-goal.sh: unknown goals, commands, and options are refused" +} + +test_new_creates_record_and_first_stage +test_new_refuses_duplicate_and_bad_ids +test_advance_cycles_and_journals +test_journal_note_cannot_break_the_record_format +test_verdict_without_gates_fails_closed +test_gate_run_reports_pass_and_fail +test_gate_evidence_is_captured +test_edited_gate_refuses_to_run +test_reregistered_gate_does_not_inherit_a_pass +test_close_requires_every_gate_and_records_status +test_export_json_reports_gate_state +test_unknown_goal_and_command_are_refused