Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 99 additions & 17 deletions bin/aas.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,13 @@ const STAGE_NAMES = ["decide", "act", "prove"];
const DEMO_FLAG_OPTIONS = new Set(["--dispute", "--json"]);
const DEMO_VALUE_OPTIONS = new Set(["--response", "--fault"]);

export class UsageError extends Error {
constructor(message) {
super(message);
this.name = "UsageError";
}
}

function has(args, name) {
return args.includes(name);
}
Expand All @@ -108,41 +115,83 @@ function option(args, name, fallback = null) {
return index >= 0 ? args[index + 1] ?? fallback : fallback;
}

function isHelpFlag(value) {
return value === "--help" || value === "-h";
}

function isHelpToken(value) {
return value === "help" || isHelpFlag(value);
}

function demoRequestsHelp(args) {
for (let index = 0; index < args.length; index += 1) {
const name = args[index];
if (isHelpFlag(name)) return true;
if (DEMO_VALUE_OPTIONS.has(name)) index += 1;
}
return false;
}

function validateDemoArgs(args) {
const seen = new Set();
for (let index = 0; index < args.length; index += 1) {
const name = args[index];
if (!DEMO_FLAG_OPTIONS.has(name) && !DEMO_VALUE_OPTIONS.has(name)) {
throw new Error(`Unsupported demo option: ${name}`);
if (typeof name === "string" && name.startsWith("-")) {
throw new UsageError(`Unsupported demo option: ${name}`);
}
throw new UsageError(`Unexpected argument: ${name}`);
}
if (seen.has(name)) throw new Error(`Duplicate demo option: ${name}`);
if (seen.has(name)) throw new UsageError(`Duplicate demo option: ${name}`);
seen.add(name);
if (DEMO_VALUE_OPTIONS.has(name)) {
const value = args[index + 1];
if (value === undefined || value.startsWith("--")) throw new Error(`Missing value for demo option: ${name}`);
if (value === undefined || value.startsWith("-")) throw new UsageError(`Missing value for demo option: ${name}`);
if (value.trim() === "") throw new UsageError(`Empty value for demo option: ${name}`);
index += 1;
}
}
}

function printHelp() {
process.stdout.write(`Agent Action Stack
export function helpText() {
return `Agent Action Stack

Usage:
aas demo [--response pass|fail] [--fault none|duplicate|...] [--dispute] [--json]
aas demo [--response pass|fail] [--fault none|duplicate] [--dispute] [--json]
aas help

Options:
--response pass|fail Policy fixture to evaluate (default: pass)
--fault none|duplicate Rail demo fault (default: none)
--dispute Force MandateBound prove after a settled act
--json Print the run report as JSON
-h, --help Show this help

Flow:
decide -> constitutional-agent-testbench evaluate
on pass -> consequence-rail demo refund
on dispute -> mandatebound simulate
on dispute -> mandatebound simulate --scenario operator

First-time setup:
npm run bootstrap

Each run is written to .out/runs/<run-id>. The .out/latest.json pointer identifies
the most recent complete bundle.
`);
Missing child tools fail closed with a bootstrap hint. Each run is written to
.out/runs/<run-id>. The .out/latest.json pointer identifies the most recent
complete bundle.

Exit codes:
0 completed run (including fail-closed policy denial)
1 stage or environment error
2 usage error
`;
}

function printHelp(stream = process.stdout) {
stream.write(helpText());
}

function missingChildTool(label) {
return new Error(`Missing ${label}. Run: npm run bootstrap`);
}

/** @returns {ChildResult} */
Expand Down Expand Up @@ -266,6 +315,10 @@ export function runDecide(
) {
const policyPath = join(fixturesDir, "policy.json");
const pythonPath = join(depsDir, "constitutional-agent-testbench", "src");
const decideCli = join(pythonPath, "constitutional_agent_testbench", "cli.py");
if (runner === runCapture && !existsSync(decideCli)) {
throw missingChildTool("decide CLI (deps/constitutional-agent-testbench/src/constitutional_agent_testbench/cli.py)");
}
const env = { ...process.env, PYTHONPATH: pythonPath, PYTHONUTF8: "1" };
let lastError = null;
for (const [bin, prefix] of pythonCandidates()) {
Expand Down Expand Up @@ -298,6 +351,9 @@ export function runAct(
{ depsDir = DEFAULT_PATHS.deps, runner = runCapture } = {},
) {
const crctl = join(depsDir, "consequence-rail", "cmd", "crctl.js");
if (runner === runCapture && !existsSync(crctl)) {
throw missingChildTool("act CLI (deps/consequence-rail/cmd/crctl.js)");
}
const args = ["demo", "refund", "--json"];
if (fault && fault !== "none") args.push("--fault", fault);
const result = runner(process.execPath, [crctl, ...args], {
Expand All @@ -322,7 +378,13 @@ export function runProve(
scenario,
{ depsDir = DEFAULT_PATHS.deps, runner = runCapture } = {},
) {
if (typeof scenario !== "string" || scenario.trim() === "") {
throw new Error("prove requires a non-empty scenario");
}
const cli = join(depsDir, "mandatebound", "dist", "cli.js");
if (runner === runCapture && !existsSync(cli)) {
throw missingChildTool("prove CLI (deps/mandatebound/dist/cli.js)");
}
const result = runner(
process.execPath,
[cli, "simulate", "--scenario", scenario],
Expand Down Expand Up @@ -469,7 +531,9 @@ export async function runDemo(args = [], options = {}) {
...(options.paths ?? {}),
};
const responseName = option(args, "--response", "pass");
if (responseName !== "pass" && responseName !== "fail") throw new Error("--response must be pass or fail");
if (responseName !== "pass" && responseName !== "fail") {
throw new UsageError("--response must be pass or fail");
}
const fault = option(args, "--fault", "none");
const forceDispute = has(args, "--dispute");
const asJson = has(args, "--json");
Expand Down Expand Up @@ -601,21 +665,39 @@ export async function runDemo(args = [], options = {}) {
}
}

function writeCliError(error, { asJson = false, usage = false } = {}) {
if (asJson) {
process.stderr.write(`${JSON.stringify({ error: { message: error.message } })}\n`);
return;
}
process.stderr.write(`${error.message}\n`);
if (usage) process.stderr.write("Try `aas help` for usage.\n");
}

export async function main(argv = process.argv.slice(2)) {
const command = argv[0] ?? "help";
if (command === "help" || command === "--help" || command === "-h") {
const asJson = has(argv, "--json");
if (isHelpToken(command) || (command === "demo" && demoRequestsHelp(argv.slice(1)))) {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Help hides malformed demo commands

A standalone help flag makes main skip validation of every other demo argument. Malformed commands then print help and exit successfully.

Prompt for agents
Validate demo arguments before taking the help-success path, while preserving the rule that --help or -h used as a value remains a missing-value error. A standalone help flag can request help only when all remaining arguments are structurally valid. Add tests combining help with unknown, unexpected, and duplicate options so malformed demo commands retain exit code 2.
Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

printHelp();
process.exitCode = 0;
return;
}
if (command !== "demo") {
printHelp();
writeCliError(new UsageError(`Unknown command: ${command}`), { asJson, usage: true });
if (!asJson) printHelp(process.stderr);
process.exitCode = 2;
return;
}
const result = await runDemo(argv.slice(1));
if (has(argv, "--json")) process.stdout.write(`${JSON.stringify(result.report, null, 2)}\n`);
else printHuman(result.report, result.bundleDir);
process.exitCode = result.exitCode;
try {
const result = await runDemo(argv.slice(1));
if (asJson) process.stdout.write(`${JSON.stringify(result.report, null, 2)}\n`);
else printHuman(result.report, result.bundleDir);
process.exitCode = result.exitCode;
} catch (error) {
const usage = error instanceof UsageError;
writeCliError(error, { asJson, usage });
process.exitCode = usage ? 2 : 1;
}
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
Expand Down
106 changes: 105 additions & 1 deletion test/stack.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import test from "node:test";
import { fileURLToPath } from "node:url";
import { loadComponentLock, inspectDependencyDirectory, npmInvocation } from "../scripts/bootstrap.mjs";
import {
helpText,
main,
parseJsonOutput,
persistRunBundle,
printHuman,
Expand Down Expand Up @@ -386,11 +388,113 @@ test("component provenance resolver rejects a missing dependency", () => {
assert.throws(() => resolveComponentProvenance(tempRoot(), LOCK), /Missing deps/);
});

test("demo arguments reject unknown, duplicate, and missing-value options", async () => {
test("demo arguments reject unknown, duplicate, missing-value, and empty options", async () => {
await assert.rejects(() => runDemo(["--unknown"], stubOptions(tempRoot())), /Unsupported/);
await assert.rejects(() => runDemo(["extra"], stubOptions(tempRoot())), /Unexpected argument/);
await assert.rejects(() => runDemo(["--response"], stubOptions(tempRoot())), /Missing value/);
await assert.rejects(() => runDemo(["--fault", ""], stubOptions(tempRoot())), /Empty value/);
await assert.rejects(() => runDemo(["--response", " "], stubOptions(tempRoot())), /Empty value/);
await assert.rejects(
() => runDemo(["--response", "pass", "--response", "fail"], stubOptions(tempRoot())),
/Duplicate/,
);
});

test("prove rejects an empty scenario before spawning a child", () => {
const runner = () => {
throw new Error("should not spawn");
};
assert.throws(() => runProve("", { runner }), /non-empty scenario/);
assert.throws(() => runProve(" ", { runner }), /non-empty scenario/);
});

test("missing child tools fail closed with a bootstrap hint", () => {
const depsDir = tempRoot();
assert.throws(
() => runDecide("unused", { depsDir }),
/Missing decide CLI \(deps\/constitutional-agent-testbench\/src\/constitutional_agent_testbench\/cli.py\)/,
);
assert.throws(
() => runAct("none", { depsDir }),
/Missing act CLI \(deps\/consequence-rail\/cmd\/crctl.js\)/,
);
assert.throws(
() => runProve("operator", { depsDir }),
/Missing prove CLI \(deps\/mandatebound\/dist\/cli.js\)/,
);
});

async function captureMain(argv) {
const stdout = [];
const stderr = [];
const originalStdout = process.stdout.write;
const originalStderr = process.stderr.write;
const originalExitCode = process.exitCode;
process.stdout.write = (chunk, encoding, callback) => {
stdout.push(String(chunk));
if (typeof encoding === "function") encoding();
else if (typeof callback === "function") callback();
return true;
};
process.stderr.write = (chunk, encoding, callback) => {
stderr.push(String(chunk));
if (typeof encoding === "function") encoding();
else if (typeof callback === "function") callback();
return true;
};
process.exitCode = undefined;
try {
await main(argv);
return { stdout: stdout.join(""), stderr: stderr.join(""), exitCode: process.exitCode ?? 0 };
} finally {
process.stdout.write = originalStdout;
process.stderr.write = originalStderr;
process.exitCode = originalExitCode;
}
}

test("CLI help covers usage, help flags, and exit codes", () => {
const text = helpText();
assert.match(text, /aas demo \[--response pass\|fail\]/);
assert.match(text, /-h, --help/);
assert.match(text, /simulate --scenario operator/);
assert.match(text, /Exit codes:/);
});

test("CLI prints help for help tokens and demo --help", async () => {
for (const argv of [[], ["help"], ["--help"], ["-h"], ["demo", "--help"], ["demo", "-h"]]) {
const result = await captureMain(argv);
assert.equal(result.exitCode, 0, `expected help exit 0 for ${JSON.stringify(argv)}`);
assert.equal(result.stdout, helpText());
assert.equal(result.stderr, "");
}
});

test("CLI reports unknown commands, flags, and empty values as usage errors", async () => {
const unknown = await captureMain(["nope"]);
assert.equal(unknown.exitCode, 2);
assert.equal(unknown.stdout, "");
assert.match(unknown.stderr, /Unknown command: nope/);
assert.match(unknown.stderr, /Try `aas help` for usage/);
assert.match(unknown.stderr, /Usage:/);

const flag = await captureMain(["demo", "--wat"]);
assert.equal(flag.exitCode, 2);
assert.match(flag.stderr, /Unsupported demo option: --wat/);
assert.match(flag.stderr, /Try `aas help` for usage/);
assert.equal(flag.stdout, "");

const empty = await captureMain(["demo", "--fault", ""]);
assert.equal(empty.exitCode, 2);
assert.match(empty.stderr, /Empty value for demo option: --fault/);

const jsonUsage = await captureMain(["demo", "--unknown", "--json"]);
assert.equal(jsonUsage.exitCode, 2);
assert.deepEqual(JSON.parse(jsonUsage.stderr), {
error: { message: "Unsupported demo option: --unknown" },
});

const helpAsValue = await captureMain(["demo", "--response", "--help"]);
assert.equal(helpAsValue.exitCode, 2);
assert.match(helpAsValue.stderr, /Missing value for demo option: --response/);
});
Loading