diff --git a/.github/workflows/cli-real-harness-smoke.yml b/.github/workflows/cli-real-harness-smoke.yml index 513b7c3a..4e0e0343 100644 --- a/.github/workflows/cli-real-harness-smoke.yml +++ b/.github/workflows/cli-real-harness-smoke.yml @@ -50,6 +50,12 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} PROSE_CODEX_APPROVAL_POLICY: never PROSE_CODEX_SANDBOX_MODE: danger-full-access + PROSE_SMOKE_CODEX_MODEL: ${{ vars.PROSE_SMOKE_CODEX_MODEL }} + PROSE_SMOKE_CODEX_REASONING_EFFORT: ${{ vars.PROSE_SMOKE_CODEX_REASONING_EFFORT }} + PROSE_SMOKE_CODEX_MODEL_PATTERN: ${{ vars.PROSE_SMOKE_CODEX_MODEL_PATTERN }} + PROSE_SMOKE_CLAUDE_MODEL: ${{ vars.PROSE_SMOKE_CLAUDE_MODEL }} + PROSE_SMOKE_CLAUDE_REASONING_EFFORT: ${{ vars.PROSE_SMOKE_CLAUDE_REASONING_EFFORT }} + PROSE_SMOKE_CLAUDE_MODEL_PATTERN: ${{ vars.PROSE_SMOKE_CLAUDE_MODEL_PATTERN }} steps: - name: Skip unrequested harness if: ${{ env.SHOULD_RUN != 'true' }} diff --git a/tools/cli/README.md b/tools/cli/README.md index e0dca6a4..3b201ee7 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -75,6 +75,7 @@ Examples: prose run std/evals/inspector prose run std/evals/prose-contributor -- subjects: 20260406-201439-1a3369 prose run std/evals/inspector --harness codex-sdk +prose run std/evals/inspector --model gpt-5.4 --reasoning-effort high prose run co/systems/company-repo-checker --harness claude-sdk PROSE_HARNESS=claude-sdk prose run std/evals/inspector ``` @@ -88,6 +89,10 @@ PROSE_HARNESS=claude-sdk prose run std/evals/inspector - `mock` echoes prompts for tests and local smoke checks. Select a harness with `--harness ` or `PROSE_HARNESS`. +Override the selected harness model with `--model `. For reasoning +controls, use `--reasoning-effort `; Codex accepts `minimal`, `low`, +`medium`, `high`, or `xhigh`, and Claude accepts `low`, `medium`, `high`, or +`max`. OpenProse commands are allowed to run from non-git directories. Codex SDK harness runs leave Codex sandbox and approval policy controls to Codex and the @@ -101,7 +106,9 @@ For externally sandboxed CI environments, Codex harnesses also honor To keep `workspace-write` sandboxing while granting narrow extra capabilities, Codex harnesses also honor `PROSE_CODEX_ADD_DIR` as a comma-separated list of additional writable directories and `PROSE_CODEX_NETWORK` (`true` or `false`) -for outbound network access. +for outbound network access. Codex harnesses also honor `PROSE_CODEX_MODEL` and +`PROSE_CODEX_REASONING_EFFORT`; command-line `--model` and +`--reasoning-effort` values take precedence. ## Skill Setup diff --git a/tools/cli/scripts/smoke-harness.mjs b/tools/cli/scripts/smoke-harness.mjs index 4163c92d..3b265f23 100644 --- a/tools/cli/scripts/smoke-harness.mjs +++ b/tools/cli/scripts/smoke-harness.mjs @@ -4,13 +4,16 @@ import { spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; const scriptDir = dirname(fileURLToPath(import.meta.url)); const cliDir = resolve(scriptDir, ".."); const harnesses = ["codex-sdk", "claude-sdk"]; const okToken = "PROSE_HARNESS_SMOKE_OK"; const skillSentinel = "PROSE_SKILL_BOOTSTRAP_VISIBLE"; +const claudeReasoningEfforts = ["low", "medium", "high", "max"]; +// The current Claude Code path still sends enabled thinking when effort is set. +const claudeThinkingType = "enabled"; const usage = `Usage: node scripts/smoke-harness.mjs [options] @@ -22,6 +25,17 @@ Options: --timeout Per-harness timeout in milliseconds (default: 180000) --keep-temp Keep temporary HOME/workspace directories for inspection -h, --help Show this help + +Environment: + PROSE_SMOKE_MODEL Explicit model override for all harnesses + PROSE_SMOKE_REASONING_EFFORT Explicit reasoning effort override for all harnesses + PROSE_SMOKE_MODEL_PATTERN Regex used to filter discovered models for all harnesses + PROSE_SMOKE_CODEX_MODEL Explicit Codex model override + PROSE_SMOKE_CODEX_REASONING_EFFORT Explicit Codex reasoning effort override + PROSE_SMOKE_CODEX_MODEL_PATTERN Regex used to filter discovered Codex models + PROSE_SMOKE_CLAUDE_MODEL Explicit Claude model override + PROSE_SMOKE_CLAUDE_REASONING_EFFORT Explicit Claude reasoning effort override + PROSE_SMOKE_CLAUDE_MODEL_PATTERN Regex used to filter discovered Claude models `; function parseArgs(argv) { @@ -89,11 +103,216 @@ function requiredSecret(harness) { return harness.startsWith("claude") ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"; } +function harnessEnvPrefix(harness) { + return harness === "codex-sdk" ? "CODEX" : "CLAUDE"; +} + +function envValue(name) { + const value = process.env[name]; + return value === undefined || value.trim() === "" ? undefined : value.trim(); +} + +function harnessEnvValue(harness, suffix) { + const prefix = harnessEnvPrefix(harness); + return envValue(`PROSE_SMOKE_${prefix}_${suffix}`) ?? envValue(`PROSE_SMOKE_${suffix}`); +} + +function compilePattern(pattern, envName) { + if (!pattern) { + return undefined; + } + try { + return new RegExp(pattern, "i"); + } catch (error) { + throw new Error(`${envName} must be a valid JavaScript regex: ${error instanceof Error ? error.message : error}`); + } +} + +function candidateText(...values) { + return values.filter((value) => typeof value === "string" && value.length > 0).join("\n"); +} + +function preferredEffort(levels, requested, label) { + if (requested !== undefined) { + if (levels.includes(requested)) { + return requested; + } + throw new Error(`${label} does not support --reasoning-effort ${requested}. Supported: ${levels.join(", ")}`); + } + if (levels.includes("low")) { + return "low"; + } + const [first] = levels; + if (first === undefined) { + throw new Error(`${label} does not advertise any supported reasoning effort levels`); + } + return first; +} + +async function smokeControlArgs(harness) { + const model = harnessEnvValue(harness, "MODEL"); + const reasoningEffort = harnessEnvValue(harness, "REASONING_EFFORT"); + const discovered = + harness === "codex-sdk" + ? discoverCodexControls({ model, reasoningEffort }) + : await discoverClaudeControls({ model, reasoningEffort }); + const selected = await discovered; + const args = []; + + if (selected.model) { + args.push("--model", selected.model); + } + if (selected.reasoningEffort) { + args.push("--reasoning-effort", selected.reasoningEffort); + } + + return args; +} + +function discoverCodexControls({ model, reasoningEffort }) { + const pattern = compilePattern(harnessEnvValue("codex-sdk", "MODEL_PATTERN"), "PROSE_SMOKE_CODEX_MODEL_PATTERN"); + const codexBin = join(cliDir, "node_modules", ".bin", process.platform === "win32" ? "codex.cmd" : "codex"); + const result = run(codexBin, ["debug", "models"], { + cwd: cliDir, + env: process.env, + maxBuffer: 32 * 1024 * 1024, + timeout: 60_000, + }); + let catalog; + try { + catalog = JSON.parse(result.stdout); + } catch (error) { + throw new Error(`Could not parse Codex model catalog JSON: ${error instanceof Error ? error.message : error}`); + } + + const models = Array.isArray(catalog.models) ? catalog.models : []; + const candidates = models + .map((entry) => { + const levels = Array.isArray(entry.supported_reasoning_levels) + ? entry.supported_reasoning_levels + .map((level) => level?.effort) + .filter((level) => typeof level === "string" && level.length > 0) + : []; + return { + displayName: entry.display_name, + model: typeof entry.slug === "string" ? entry.slug : undefined, + searchText: candidateText(entry.slug, entry.display_name, entry.description), + levels, + supportedInApi: entry.supported_in_api !== false, + visible: entry.visibility !== "hidden", + }; + }) + .filter((entry) => entry.model && entry.supportedInApi && entry.visible && entry.levels.length > 0); + + const selected = selectDiscoveredModel(candidates, { model, pattern, provider: "Codex", reasoningEffort }); + return { + model: selected.model, + reasoningEffort: preferredEffort(selected.levels, reasoningEffort, `Codex model ${selected.model}`), + }; +} + +async function discoverClaudeControls({ model, reasoningEffort }) { + const pattern = compilePattern(harnessEnvValue("claude-sdk", "MODEL_PATTERN"), "PROSE_SMOKE_CLAUDE_MODEL_PATTERN"); + const models = model === undefined ? await listClaudeModels() : [await retrieveClaudeModel(model)]; + const candidates = models + .map((entry) => { + const effort = entry.capabilities?.effort; + const thinking = entry.capabilities?.thinking; + const levels = claudeReasoningEfforts.filter((level) => effort?.[level]?.supported === true); + return { + displayName: entry.display_name, + model: typeof entry.id === "string" ? entry.id : undefined, + searchText: candidateText(entry.id, entry.display_name), + levels, + supportedInApi: effort?.supported === true && thinking?.types?.[claudeThinkingType]?.supported === true, + visible: true, + }; + }) + .filter((entry) => entry.model && entry.supportedInApi && entry.levels.length > 0); + + const selected = selectDiscoveredModel(candidates, { + model, + pattern, + provider: "Claude", + requirement: `${claudeThinkingType} thinking and reasoning effort support compatible with the current Claude Code path`, + }); + return { + model: selected.model, + reasoningEffort: preferredEffort(selected.levels, reasoningEffort, `Claude model ${selected.model}`), + }; +} + +function selectDiscoveredModel(candidates, { model, pattern, provider, requirement = "reasoning effort support" }) { + if (model !== undefined) { + const selected = candidates.find((candidate) => candidate.model === model); + if (selected === undefined && candidates.length === 1) { + return candidates[0]; + } + if (selected === undefined) { + throw new Error(`${provider} model ${model} was not found or does not advertise ${requirement}`); + } + return selected; + } + + const filtered = pattern === undefined ? candidates : candidates.filter((candidate) => pattern.test(candidate.searchText)); + const [selected] = filtered; + if (selected === undefined) { + const suffix = pattern === undefined ? "" : ` matching ${pattern}`; + throw new Error(`No ${provider} models with ${requirement} were discovered${suffix}`); + } + return selected; +} + +async function listClaudeModels() { + const models = []; + let afterId; + do { + const url = new URL("https://api.anthropic.com/v1/models"); + url.searchParams.set("limit", "100"); + if (afterId) { + url.searchParams.set("after_id", afterId); + } + const page = await fetchClaudeJson(url); + if (Array.isArray(page.data)) { + models.push(...page.data); + } + afterId = page.has_more === true ? page.last_id : undefined; + } while (afterId); + return models; +} + +async function retrieveClaudeModel(model) { + const url = new URL(`https://api.anthropic.com/v1/models/${encodeURIComponent(model)}`); + return fetchClaudeJson(url); +} + +async function fetchClaudeJson(url) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 30_000); + try { + const response = await fetch(url, { + headers: { + "anthropic-version": "2023-06-01", + "x-api-key": process.env.ANTHROPIC_API_KEY, + }, + signal: controller.signal, + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`Anthropic Models API returned ${response.status}: ${body.slice(0, 500)}`); + } + return response.json(); + } finally { + clearTimeout(timeout); + } +} + function run(command, args, options) { const result = spawnSync(command, args, { cwd: options.cwd, encoding: "utf8", env: options.env ?? process.env, + maxBuffer: options.maxBuffer ?? 1024 * 1024, timeout: options.timeout, }); @@ -170,7 +389,7 @@ Verifies the harness can see the preloaded OpenProse skill text. ); } -function smokeHarness(harness, options) { +async function smokeHarness(harness, options) { const secret = requiredSecret(harness); if (!process.env[secret]) { throw new Error(`Missing required secret: ${secret}`); @@ -194,11 +413,16 @@ function smokeHarness(harness, options) { XDG_CONFIG_HOME: join(home, ".config"), }; - const result = run( - process.execPath, - [options.cli, "run", "smoke.prose.md", "--harness", harness], - { cwd: workspace, env, timeout: options.timeout }, - ); + const controlArgs = await smokeControlArgs(harness); + if (controlArgs.length > 0) { + process.stderr.write(`Using ${harness} model controls: ${controlArgs.join(" ")}\n`); + } + + const result = run(process.execPath, [options.cli, "run", "smoke.prose.md", "--harness", harness, ...controlArgs], { + cwd: workspace, + env, + timeout: options.timeout, + }); const output = `${result.stdout}\n${result.stderr}`; if (!output.includes(okToken)) { @@ -215,7 +439,7 @@ function smokeHarness(harness, options) { } } -function main() { +async function main() { const options = parseArgs(process.argv.slice(2)); if (options.help) { process.stdout.write(usage); @@ -224,13 +448,19 @@ function main() { for (const harness of requestedHarnesses(options.harness)) { process.stderr.write(`Smoking ${harness}...\n`); - smokeHarness(harness, options); + await smokeHarness(harness, options); } } -try { - main(); -} catch (error) { - process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); - process.exitCode = 1; +const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href; + +if (isMain) { + try { + await main(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } } + +export { discoverClaudeControls }; diff --git a/tools/cli/src/commands/base.ts b/tools/cli/src/commands/base.ts index 6f3dde68..3db7da5a 100644 --- a/tools/cli/src/commands/base.ts +++ b/tools/cli/src/commands/base.ts @@ -16,6 +16,11 @@ export interface SkillPreflightOptions { export type SkillPreflight = (options: SkillPreflightOptions) => Promise; export type SkillBootstrapLoader = (options: SkillPreflightOptions) => Promise; +export interface HarnessControlOptions { + model?: string; + reasoningEffort?: string; +} + export interface ForwardRunOptions { command: CommandName; argv: readonly string[]; @@ -87,7 +92,7 @@ function isOclifExit(error: unknown): boolean { } export async function runForwardedProseCommand(options: ForwardRunOptions): Promise { - const { harness, args } = splitHarnessArgs(options.argv, options.env, options.command); + const { harness, args, harnessOptions } = splitHarnessArgs(options.argv, options.env, options.command); const prompt = canonicalPrompt(options.command, args); if (shouldRunSkillPreflight(options)) { await runSkillPreflight(harness, options); @@ -106,6 +111,10 @@ export async function runForwardedProseCommand(options: ForwardRunOptions): Prom }), cwd: options.cwd, env: { ...options.env }, + ...(harnessOptions.model === undefined ? {} : { model: harnessOptions.model }), + ...(harnessOptions.reasoningEffort === undefined + ? {} + : { reasoningEffort: harnessOptions.reasoningEffort }), stdout: options.stdout, stderr: options.stderr, ...(options.signal === undefined ? {} : { signal: options.signal }), @@ -174,8 +183,9 @@ export function splitHarnessArgs( argv: readonly string[], env: Readonly>, command: CommandName = "run", -): { harness: HarnessName | string; args: string[] } { +): { harness: HarnessName | string; args: string[]; harnessOptions: HarnessControlOptions } { const args: string[] = []; + const harnessOptions: HarnessControlOptions = {}; let harness = env.PROSE_HARNESS || "codex-sdk"; for (let index = 0; index < argv.length; index += 1) { @@ -208,17 +218,55 @@ export function splitHarnessArgs( continue; } + if (arg === "--model") { + const value = argv[index + 1]; + if (!value || value === "--") { + throw new CommandModelError("Missing value for --model.", usageFor(command)); + } + harnessOptions.model = value; + index += 1; + continue; + } + + if (arg.startsWith("--model=")) { + const value = arg.slice("--model=".length); + if (!value) { + throw new CommandModelError("Missing value for --model.", usageFor(command)); + } + harnessOptions.model = value; + continue; + } + + if (arg === "--reasoning-effort") { + const value = argv[index + 1]; + if (!value || value === "--") { + throw new CommandModelError("Missing value for --reasoning-effort.", usageFor(command)); + } + harnessOptions.reasoningEffort = value; + index += 1; + continue; + } + + if (arg.startsWith("--reasoning-effort=")) { + const value = arg.slice("--reasoning-effort=".length); + if (!value) { + throw new CommandModelError("Missing value for --reasoning-effort.", usageFor(command)); + } + harnessOptions.reasoningEffort = value; + continue; + } + args.push(arg); } - return { harness, args }; + return { harness, args, harnessOptions }; } export function normalizeEntrypointArgv( argv: readonly string[], ): string[] { const normalized: string[] = []; - let harness: string | undefined; + const commandOptions: string[] = []; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -229,26 +277,15 @@ export function normalizeEntrypointArgv( normalized.push(...argv.slice(index)); break; } - if (arg === "--harness") { - const value = argv[index + 1]; - if (value && value !== "--") { - harness = value; - index += 1; - continue; - } - } - if (arg.startsWith("--harness=")) { - const value = arg.slice("--harness=".length); - if (value) { - harness = value; - continue; - } + const consumedOption = consumePreCommandOption(argv, index); + if (consumedOption !== undefined) { + commandOptions.push(...consumedOption.args); + index = consumedOption.index; + continue; } normalized.push(arg); if (!arg.startsWith("-")) { - if (harness !== undefined) { - normalized.push("--harness", harness); - } + normalized.push(...commandOptions); normalized.push(...argv.slice(index + 1)); break; } @@ -256,6 +293,35 @@ export function normalizeEntrypointArgv( return normalized; } +function consumePreCommandOption( + argv: readonly string[], + index: number, +): { args: string[]; index: number } | undefined { + const arg = argv[index]; + if (arg === undefined) { + return undefined; + } + + for (const option of ["--harness", "--model", "--reasoning-effort"]) { + if (arg === option) { + const value = argv[index + 1]; + if (value && value !== "--") { + return { args: [option, value], index: index + 1 }; + } + return undefined; + } + + if (arg.startsWith(`${option}=`)) { + const value = arg.slice(option.length + 1); + if (value) { + return { args: [arg], index }; + } + } + } + + return undefined; +} + function forwardProcessSignals(controller: AbortController): () => void { const onSignal = (signal: NodeJS.Signals) => { if (!controller.signal.aborted) { diff --git a/tools/cli/src/commands/index.ts b/tools/cli/src/commands/index.ts index 62d104e3..b5a02b66 100644 --- a/tools/cli/src/commands/index.ts +++ b/tools/cli/src/commands/index.ts @@ -46,10 +46,11 @@ const forwardCommandDefinitions = { command: "run", examples: [ "<%= config.bin %> run std/evals/inspector --harness codex-sdk", + "<%= config.bin %> run std/evals/inspector --model gpt-5.4 --reasoning-effort high", "<%= config.bin %> run co/systems/company-repo-checker", ], summary: "Run an OpenProse service or system.", - usage: "run [inputs...] [--harness ]", + usage: "run [inputs...] [--harness ] [--model ] [--reasoning-effort ]", }, test: { command: "test", diff --git a/tools/cli/src/harnesses/claude-sdk.ts b/tools/cli/src/harnesses/claude-sdk.ts index 0c6a7cc4..63007d74 100644 --- a/tools/cli/src/harnesses/claude-sdk.ts +++ b/tools/cli/src/harnesses/claude-sdk.ts @@ -3,9 +3,12 @@ import type { Harness, HarnessRunOptions } from "./types.js"; export type ClaudeSdkQuery = typeof import("@anthropic-ai/claude-agent-sdk").query; export type ClaudeSdkQueryResult = ReturnType; +type ClaudeQueryOptions = NonNullable[0]["options"]>; export type ClaudeSdkMessage = ClaudeSdkQueryResult extends AsyncIterable ? Message : never; export type ClaudeSdkQueryLike = (...args: Parameters) => ClaudeSdkQueryResult | Promise; +const CLAUDE_REASONING_EFFORTS = ["low", "medium", "high", "max"] as const; + export interface ClaudeSdkHarnessOptions { query?: ClaudeSdkQueryLike; } @@ -32,6 +35,10 @@ export function createClaudeSdkHarness(options: ClaudeSdkHarnessOptions = {}): H : { additionalDirectories: runOptions.additionalDirectories }), ...(runOptions.cwd === undefined ? {} : { cwd: runOptions.cwd }), ...(runOptions.env === undefined ? {} : { env: runOptions.env }), + ...(runOptions.model === undefined ? {} : { model: runOptions.model }), + ...(runOptions.reasoningEffort === undefined + ? {} + : claudeReasoningOptions(runOptions.reasoningEffort)), includePartialMessages: true, settingSources: ["user", "project"], stderr: (chunk: string) => runOptions.stderr.write(chunk), @@ -88,6 +95,20 @@ export function createClaudeSdkHarness(options: ClaudeSdkHarnessOptions = {}): H }; } +function claudeReasoningOptions(value: string): Pick { + return { + effort: claudeReasoningEffort(value), + }; +} + +function claudeReasoningEffort(value: string): NonNullable { + if (CLAUDE_REASONING_EFFORTS.includes(value as (typeof CLAUDE_REASONING_EFFORTS)[number])) { + return value as NonNullable; + } + + throw new Error(`--reasoning-effort for claude-sdk must be one of: ${CLAUDE_REASONING_EFFORTS.join(", ")}`); +} + function writeClaudeMessage( message: ClaudeSdkMessage, runOptions: HarnessRunOptions, diff --git a/tools/cli/src/harnesses/codex-options.ts b/tools/cli/src/harnesses/codex-options.ts index 89ab33d0..93ccc12d 100644 --- a/tools/cli/src/harnesses/codex-options.ts +++ b/tools/cli/src/harnesses/codex-options.ts @@ -2,22 +2,44 @@ import type { CodexThreadOptions } from "./types.js"; const CODEX_SANDBOX_MODES = ["read-only", "workspace-write", "danger-full-access"] as const; const CODEX_APPROVAL_POLICIES = ["never", "on-request", "on-failure", "untrusted"] as const; +const CODEX_REASONING_EFFORTS = ["minimal", "low", "medium", "high", "xhigh"] as const; + +export interface CodexRuntimeOverrides { + model?: string; + reasoningEffort?: string; +} export function codexThreadRuntimeOptions( env: Record | undefined, additionalDirectories: readonly string[] = [], + overrides: CodexRuntimeOverrides = {}, ): Pick< CodexThreadOptions, - "additionalDirectories" | "approvalPolicy" | "networkAccessEnabled" | "sandboxMode" | "skipGitRepoCheck" + | "additionalDirectories" + | "approvalPolicy" + | "model" + | "modelReasoningEffort" + | "networkAccessEnabled" + | "sandboxMode" + | "skipGitRepoCheck" > { const sandboxMode = codexEnvOption("PROSE_CODEX_SANDBOX_MODE", CODEX_SANDBOX_MODES, env); const approvalPolicy = codexEnvOption("PROSE_CODEX_APPROVAL_POLICY", CODEX_APPROVAL_POLICIES, env); + const model = codexStringOption("PROSE_CODEX_MODEL", env, overrides.model); + const modelReasoningEffort = codexEnvOption( + overrides.reasoningEffort === undefined ? "PROSE_CODEX_REASONING_EFFORT" : "--reasoning-effort", + CODEX_REASONING_EFFORTS, + env, + overrides.reasoningEffort, + ); const envAdditionalDirectories = codexEnvList("PROSE_CODEX_ADD_DIR", env); const networkAccessEnabled = codexEnvBoolean("PROSE_CODEX_NETWORK", env); const mergedAdditionalDirectories = [...additionalDirectories, ...envAdditionalDirectories]; return { skipGitRepoCheck: true, + ...(model === undefined ? {} : { model }), + ...(modelReasoningEffort === undefined ? {} : { modelReasoningEffort }), ...(sandboxMode === undefined ? {} : { sandboxMode }), ...(approvalPolicy === undefined ? {} : { approvalPolicy }), ...(networkAccessEnabled === undefined ? {} : { networkAccessEnabled }), @@ -37,8 +59,9 @@ function codexEnvOption( name: string, allowedValues: T, env: Record | undefined, + override?: string, ): T[number] | undefined { - const value = env?.[name] ?? process.env[name]; + const value = override ?? env?.[name] ?? process.env[name]; if (value === undefined || value === "") { return undefined; } @@ -50,6 +73,19 @@ function codexEnvOption( throw new Error(`${name} must be one of: ${allowedValues.join(", ")}`); } +function codexStringOption( + name: string, + env: Record | undefined, + override?: string, +): string | undefined { + const value = override ?? env?.[name] ?? process.env[name]; + if (value === undefined || value === "") { + return undefined; + } + + return value; +} + function codexEnvList(name: string, env: Record | undefined): string[] { const value = env?.[name] ?? process.env[name]; if (value === undefined || value === "") { diff --git a/tools/cli/src/harnesses/codex-sdk.ts b/tools/cli/src/harnesses/codex-sdk.ts index 129064a8..264a3cf2 100644 --- a/tools/cli/src/harnesses/codex-sdk.ts +++ b/tools/cli/src/harnesses/codex-sdk.ts @@ -21,7 +21,10 @@ export function createCodexSdkHarness(options: CodexSdkHarnessOptions = {}): Har const env = definedEnv(runOptions.env); const codex = await factory(codexClientOptions(env, runOptions.systemPromptAppend)); const thread = codex.startThread( - codexThreadOptions(runOptions.cwd, env, runOptions.additionalDirectories), + codexThreadOptions(runOptions.cwd, env, runOptions.additionalDirectories, { + ...(runOptions.model === undefined ? {} : { model: runOptions.model }), + ...(runOptions.reasoningEffort === undefined ? {} : { reasoningEffort: runOptions.reasoningEffort }), + }), ); const { events } = await thread.runStreamed( prompt, @@ -48,8 +51,9 @@ function codexThreadOptions( cwd: string | undefined, env: Record | undefined, additionalDirectories: readonly string[] | undefined, + overrides: { model?: string; reasoningEffort?: string }, ) { - const runtimeOptions = codexThreadRuntimeOptions(env, additionalDirectories); + const runtimeOptions = codexThreadRuntimeOptions(env, additionalDirectories, overrides); const options = { ...(cwd === undefined ? {} : { workingDirectory: cwd }), ...runtimeOptions, diff --git a/tools/cli/src/harnesses/types.ts b/tools/cli/src/harnesses/types.ts index 2e97a95c..c2053fb9 100644 --- a/tools/cli/src/harnesses/types.ts +++ b/tools/cli/src/harnesses/types.ts @@ -18,6 +18,8 @@ export interface HarnessRunOptions { additionalDirectories?: string[]; cwd?: string; env?: Record; + model?: string; + reasoningEffort?: string; signal?: AbortSignal; systemPromptAppend?: string; stdout: WritableStreamLike; diff --git a/tools/cli/src/prose/command-model.ts b/tools/cli/src/prose/command-model.ts index dd4aea25..9829fe7c 100644 --- a/tools/cli/src/prose/command-model.ts +++ b/tools/cli/src/prose/command-model.ts @@ -37,7 +37,7 @@ export const supportedCommands = [ const usageByCommand: Record = { compile: "prose compile [path] [--out ]", - run: "prose run [inputs...]", + run: "prose run [inputs...] [--harness ] [--model ] [--reasoning-effort ]", lint: "prose lint ", preflight: "prose preflight ", test: "prose test ", diff --git a/tools/cli/tests/cli/cli.test.ts b/tools/cli/tests/cli/cli.test.ts index 2b04fb0d..720f23e6 100644 --- a/tools/cli/tests/cli/cli.test.ts +++ b/tools/cli/tests/cli/cli.test.ts @@ -257,6 +257,18 @@ describe("Oclif entrypoint helpers", () => { ]); }); + it("normalizes pre-command model controls for Oclif dispatch", () => { + expect( + normalizeEntrypointArgv([ + "--model", + "gpt-5.4", + "--reasoning-effort=high", + "run", + "flow.prose.md", + ]), + ).toEqual(["run", "--model", "gpt-5.4", "--reasoning-effort=high", "flow.prose.md"]); + }); + it("does not consume literal harness-looking args after --", () => { expect(normalizeEntrypointArgv(["run", "flow.prose.md", "--", "--harness", "literal"])).toEqual([ "run", @@ -284,12 +296,30 @@ describe("harness argument splitting", () => { expect(parsed.args).toEqual(["./flows/needs review.prose.md", "--topic", "two words"]); }); + it("removes model controls while preserving run inputs", () => { + const parsed = splitHarnessArgs( + ["./flow.prose.md", "--model", "gpt-5.4", "--topic", "two words", "--reasoning-effort=high"], + {}, + ); + + expect(parsed.harness).toBe("codex-sdk"); + expect(parsed.args).toEqual(["./flow.prose.md", "--topic", "two words"]); + expect(parsed.harnessOptions).toEqual({ model: "gpt-5.4", reasoningEffort: "high" }); + }); + it("keeps --harness literal after --", () => { const parsed = splitHarnessArgs(["./flow.prose.md", "--", "--harness", "literal"], { PROSE_HARNESS: "mock" }); expect(parsed.harness).toBe("mock"); expect(parsed.args).toEqual(["./flow.prose.md", "--", "--harness", "literal"]); }); + + it("keeps model-looking args literal after --", () => { + const parsed = splitHarnessArgs(["./flow.prose.md", "--", "--model", "literal"], {}); + + expect(parsed.args).toEqual(["./flow.prose.md", "--", "--model", "literal"]); + expect(parsed.harnessOptions).toEqual({}); + }); }); describe("runForwardedProseCommand", () => { @@ -322,6 +352,31 @@ describe("runForwardedProseCommand", () => { expect(io.stderr).toBe("err"); }); + it("passes model controls to the selected harness without adding prompt args", async () => { + const io = memoryStreams(); + const seen: unknown[] = []; + const harness: Harness = { + name: "mock", + async run(prompt, options) { + seen.push(prompt, { model: options.model, reasoningEffort: options.reasoningEffort }); + return 0; + }, + }; + + const exitCode = await runForwardedProseCommand({ + command: "run", + argv: ["flow.prose.md", "--model", "gpt-5.4", "--reasoning-effort", "high"], + cwd: "/repo", + env: {}, + stdout: io.streams.stdout, + stderr: io.streams.stderr, + harnessFactory: () => harness, + }); + + expect(exitCode).toBe(0); + expect(seen).toEqual(["prose run flow.prose.md", { model: "gpt-5.4", reasoningEffort: "high" }]); + }); + it("forwards compile prompts through the selected harness", async () => { const io = memoryStreams(); const seen: string[] = []; diff --git a/tools/cli/tests/harnesses/harnesses.test.ts b/tools/cli/tests/harnesses/harnesses.test.ts index 8dc53dc1..9339fdcb 100644 --- a/tools/cli/tests/harnesses/harnesses.test.ts +++ b/tools/cli/tests/harnesses/harnesses.test.ts @@ -163,7 +163,9 @@ describe("codex-sdk harness", () => { env: { PROSE_CODEX_ADD_DIR: " /var/lib/prose, /tmp/grant-finder ,,", PROSE_CODEX_APPROVAL_POLICY: "never", + PROSE_CODEX_MODEL: "gpt-5.4-mini", PROSE_CODEX_NETWORK: "true", + PROSE_CODEX_REASONING_EFFORT: "low", PROSE_CODEX_SANDBOX_MODE: "danger-full-access", }, }); @@ -171,15 +173,47 @@ describe("codex-sdk harness", () => { expect(exitCode).toBe(0); expect(starts).toEqual([ { - additionalDirectories: ["/skills/open-prose", "/var/lib/prose", "/tmp/grant-finder"], - approvalPolicy: "never", - networkAccessEnabled: true, - sandboxMode: "danger-full-access", - skipGitRepoCheck: true, + additionalDirectories: ["/skills/open-prose", "/var/lib/prose", "/tmp/grant-finder"], + approvalPolicy: "never", + model: "gpt-5.4-mini", + modelReasoningEffort: "low", + networkAccessEnabled: true, + sandboxMode: "danger-full-access", + skipGitRepoCheck: true, }, ]); }); + test("command model settings override Codex environment fallbacks", async () => { + const io = memoryStreams(); + const starts: unknown[] = []; + const factory: CodexSdkFactory = () => ({ + startThread: (options) => { + starts.push(options); + return { + runStreamed: async () => ({ + events: events([ + { type: "item.completed", item: { id: "item-1", type: "agent_message", text: "sdk output" } }, + ]), + }), + }; + }, + }); + + const exitCode = await createCodexSdkHarness({ factory }).run("prose run inspector.prose.md", { + ...io.options, + env: { + PROSE_CODEX_MODEL: "gpt-5.4-mini", + PROSE_CODEX_REASONING_EFFORT: "low", + }, + model: "gpt-5.4", + reasoningEffort: "high", + }); + + expect(exitCode).toBe(0); + expect(starts).toEqual([{ model: "gpt-5.4", modelReasoningEffort: "high", skipGitRepoCheck: true }]); + }); + test("rejects invalid Codex network setting", async () => { const io = memoryStreams(); const harness = createCodexSdkHarness({ @@ -239,6 +273,8 @@ describe("claude-sdk harness", () => { additionalDirectories: ["/skills/open-prose"], cwd: "/repo", env: { A: "B" }, + model: "claude-opus-4-6", + reasoningEffort: "high", systemPromptAppend: bootstrap, }); @@ -249,7 +285,9 @@ describe("claude-sdk harness", () => { options: expect.objectContaining({ additionalDirectories: ["/skills/open-prose"], cwd: "/repo", + effort: "high", env: { A: "B" }, + model: "claude-opus-4-6", systemPrompt: { type: "preset", preset: "claude_code", diff --git a/tools/cli/tests/prose/command-model.test.ts b/tools/cli/tests/prose/command-model.test.ts index 37296ae3..ed99583e 100644 --- a/tools/cli/tests/prose/command-model.test.ts +++ b/tools/cli/tests/prose/command-model.test.ts @@ -42,15 +42,17 @@ describe("command model", () => { ); }); + const runUsage = + "prose run [inputs...] [--harness ] [--model ] [--reasoning-effort ]"; const validationCases: Array<[Parameters, string, string]> = [ [["compile", ["one", "two"]], "Unexpected argument 'two'", "prose compile [path] [--out ]"], [["compile", ["--out"]], "Missing value for --out", "prose compile [path] [--out ]"], [["compile", ["--out="]], "Missing value for --out", "prose compile [path] [--out ]"], [["compile", ["--json"]], "Unexpected option '--json'", "prose compile [path] [--out ]"], [["compile", ["--out", "dist", "--out", "other"]], "Duplicate option", "prose compile [path] [--out ]"], - [["run", []], "Missing required argument ", "prose run [inputs...]"], - [["run", ["system.md"]], "Expected ", "prose run [inputs...]"], - [["run", ["script.prose"]], "Expected ", "prose run [inputs...]"], + [["run", []], "Missing required argument ", runUsage], + [["run", ["system.md"]], "Expected ", runUsage], + [["run", ["script.prose"]], "Expected ", runUsage], [["inspect", []], "Missing required argument ", "prose inspect "], [["lint", ["system.md"]], "Expected ", "prose lint "], [["preflight", ["system.md"]], "Expected ", "prose preflight "], diff --git a/tools/cli/tests/scripts/smoke-harness.test.mjs b/tools/cli/tests/scripts/smoke-harness.test.mjs new file mode 100644 index 00000000..8b50aded --- /dev/null +++ b/tools/cli/tests/scripts/smoke-harness.test.mjs @@ -0,0 +1,101 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { discoverClaudeControls } from "../../scripts/smoke-harness.mjs"; + +const envKeys = [ + "PROSE_SMOKE_MODEL", + "PROSE_SMOKE_MODEL_PATTERN", + "PROSE_SMOKE_REASONING_EFFORT", + "PROSE_SMOKE_CLAUDE_MODEL", + "PROSE_SMOKE_CLAUDE_MODEL_PATTERN", + "PROSE_SMOKE_CLAUDE_REASONING_EFFORT", +]; + +afterEach(() => { + vi.unstubAllGlobals(); + for (const key of envKeys) { + delete process.env[key]; + } +}); + +describe("smoke-harness Claude control discovery", () => { + it("requires enabled thinking because that is what current Claude Code sends with effort", async () => { + mockClaudeModel({ + id: "claude-adaptive-only", + display_name: "Claude adaptive only", + capabilities: { + effort: { + supported: true, + low: { supported: true }, + }, + thinking: { + types: { + adaptive: { supported: true }, + enabled: { supported: false }, + }, + }, + }, + }); + + await expect(discoverClaudeControls({ model: "claude-adaptive-only" })).rejects.toThrow( + "enabled thinking and reasoning effort support", + ); + }); + + it("selects a model with enabled thinking and harness-supported effort", async () => { + mockClaudeModel({ + id: "claude-enabled", + display_name: "Claude enabled", + capabilities: { + effort: { + supported: true, + low: { supported: true }, + xhigh: { supported: true }, + }, + thinking: { + types: { + enabled: { supported: true }, + }, + }, + }, + }); + + await expect(discoverClaudeControls({ model: "claude-enabled" })).resolves.toEqual({ + model: "claude-enabled", + reasoningEffort: "low", + }); + }); + + it("does not treat xhigh as Claude-compatible because the harness rejects it", async () => { + mockClaudeModel({ + id: "claude-max", + display_name: "Claude max", + capabilities: { + effort: { + supported: true, + max: { supported: true }, + xhigh: { supported: true }, + }, + thinking: { + types: { + enabled: { supported: true }, + }, + }, + }, + }); + + await expect(discoverClaudeControls({ model: "claude-max", reasoningEffort: "xhigh" })).rejects.toThrow( + "Claude model claude-max does not support --reasoning-effort xhigh. Supported: max", + ); + }); +}); + +function mockClaudeModel(model) { + vi.stubGlobal( + "fetch", + vi.fn(async () => ({ + ok: true, + json: async () => model, + })), + ); +} diff --git a/tools/cli/vitest.config.ts b/tools/cli/vitest.config.ts index 651c9797..bcc3b205 100644 --- a/tools/cli/vitest.config.ts +++ b/tools/cli/vitest.config.ts @@ -3,6 +3,6 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "node", - include: ["tests/**/*.test.ts"] + include: ["tests/**/*.test.ts", "tests/**/*.test.mjs"] } });