[codex] Add CLI model option pass-through

.github/workflows/cli-real-harness-smoke.yml

@@ -50,6 +50,12 @@ jobs:
       ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
       PROSE_CODEX_APPROVAL_POLICY: never
       PROSE_CODEX_SANDBOX_MODE: danger-full-access
+      PROSE_SMOKE_CODEX_MODEL: ${{ vars.PROSE_SMOKE_CODEX_MODEL }}
+      PROSE_SMOKE_CODEX_REASONING_EFFORT: ${{ vars.PROSE_SMOKE_CODEX_REASONING_EFFORT }}
+      PROSE_SMOKE_CODEX_MODEL_PATTERN: ${{ vars.PROSE_SMOKE_CODEX_MODEL_PATTERN }}
+      PROSE_SMOKE_CLAUDE_MODEL: ${{ vars.PROSE_SMOKE_CLAUDE_MODEL }}
+      PROSE_SMOKE_CLAUDE_REASONING_EFFORT: ${{ vars.PROSE_SMOKE_CLAUDE_REASONING_EFFORT }}
+      PROSE_SMOKE_CLAUDE_MODEL_PATTERN: ${{ vars.PROSE_SMOKE_CLAUDE_MODEL_PATTERN }}
     steps:
       - name: Skip unrequested harness
         if: ${{ env.SHOULD_RUN != 'true' }}

tools/cli/README.md

@@ -75,6 +75,7 @@ Examples:
 prose run std/evals/inspector
 prose run std/evals/prose-contributor -- subjects: 20260406-201439-1a3369
 prose run std/evals/inspector --harness codex-sdk
+prose run std/evals/inspector --model gpt-5.4 --reasoning-effort high
 prose run co/systems/company-repo-checker --harness claude-sdk
 PROSE_HARNESS=claude-sdk prose run std/evals/inspector
 ```
@@ -88,6 +89,10 @@ PROSE_HARNESS=claude-sdk prose run std/evals/inspector
 - `mock` echoes prompts for tests and local smoke checks.
 
 Select a harness with `--harness <name>` or `PROSE_HARNESS`.
+Override the selected harness model with `--model <name>`. For reasoning
+controls, use `--reasoning-effort <level>`; Codex accepts `minimal`, `low`,
+`medium`, `high`, or `xhigh`, and Claude accepts `low`, `medium`, `high`, or
+`max`.
 
 OpenProse commands are allowed to run from non-git directories. Codex SDK
 harness runs leave Codex sandbox and approval policy controls to Codex and the
@@ -101,7 +106,9 @@ For externally sandboxed CI environments, Codex harnesses also honor
 To keep `workspace-write` sandboxing while granting narrow extra capabilities,
 Codex harnesses also honor `PROSE_CODEX_ADD_DIR` as a comma-separated list of
 additional writable directories and `PROSE_CODEX_NETWORK` (`true` or `false`)
-for outbound network access.
+for outbound network access. Codex harnesses also honor `PROSE_CODEX_MODEL` and
+`PROSE_CODEX_REASONING_EFFORT`; command-line `--model` and
+`--reasoning-effort` values take precedence.
 
 ## Skill Setup
 

tools/cli/scripts/smoke-harness.mjs

@@ -4,13 +4,16 @@ import { spawnSync } from "node:child_process";
 import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
+import { fileURLToPath, pathToFileURL } from "node:url";
 
 const scriptDir = dirname(fileURLToPath(import.meta.url));
 const cliDir = resolve(scriptDir, "..");
 const harnesses = ["codex-sdk", "claude-sdk"];
 const okToken = "PROSE_HARNESS_SMOKE_OK";
 const skillSentinel = "PROSE_SKILL_BOOTSTRAP_VISIBLE";
+const claudeReasoningEfforts = ["low", "medium", "high", "max"];
+// The current Claude Code path still sends enabled thinking when effort is set.
+const claudeThinkingType = "enabled";
 
 const usage = `Usage: node scripts/smoke-harness.mjs [options]
 
@@ -22,6 +25,17 @@ Options:
   --timeout <ms>     Per-harness timeout in milliseconds (default: 180000)
   --keep-temp        Keep temporary HOME/workspace directories for inspection
   -h, --help         Show this help
+
+Environment:
+  PROSE_SMOKE_MODEL                       Explicit model override for all harnesses
+  PROSE_SMOKE_REASONING_EFFORT            Explicit reasoning effort override for all harnesses
+  PROSE_SMOKE_MODEL_PATTERN               Regex used to filter discovered models for all harnesses
+  PROSE_SMOKE_CODEX_MODEL                 Explicit Codex model override
+  PROSE_SMOKE_CODEX_REASONING_EFFORT      Explicit Codex reasoning effort override
+  PROSE_SMOKE_CODEX_MODEL_PATTERN         Regex used to filter discovered Codex models
+  PROSE_SMOKE_CLAUDE_MODEL                Explicit Claude model override
+  PROSE_SMOKE_CLAUDE_REASONING_EFFORT     Explicit Claude reasoning effort override
+  PROSE_SMOKE_CLAUDE_MODEL_PATTERN        Regex used to filter discovered Claude models
 `;
 
 function parseArgs(argv) {
@@ -89,11 +103,216 @@ function requiredSecret(harness) {
 	return harness.startsWith("claude") ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY";
 }
 
+function harnessEnvPrefix(harness) {
+	return harness === "codex-sdk" ? "CODEX" : "CLAUDE";
+}
+
+function envValue(name) {
+	const value = process.env[name];
+	return value === undefined || value.trim() === "" ? undefined : value.trim();
+}
+
+function harnessEnvValue(harness, suffix) {
+	const prefix = harnessEnvPrefix(harness);
+	return envValue(`PROSE_SMOKE_${prefix}_${suffix}`) ?? envValue(`PROSE_SMOKE_${suffix}`);
+}
+
+function compilePattern(pattern, envName) {
+	if (!pattern) {
+		return undefined;
+	}
+	try {
+		return new RegExp(pattern, "i");
+	} catch (error) {
+		throw new Error(`${envName} must be a valid JavaScript regex: ${error instanceof Error ? error.message : error}`);
+	}
+}
+
+function candidateText(...values) {
+	return values.filter((value) => typeof value === "string" && value.length > 0).join("\n");
+}
+
+function preferredEffort(levels, requested, label) {
+	if (requested !== undefined) {
+		if (levels.includes(requested)) {
+			return requested;
+		}
+		throw new Error(`${label} does not support --reasoning-effort ${requested}. Supported: ${levels.join(", ")}`);
+	}
+	if (levels.includes("low")) {
+		return "low";
+	}
+	const [first] = levels;
+	if (first === undefined) {
+		throw new Error(`${label} does not advertise any supported reasoning effort levels`);
+	}
+	return first;
+}
+
+async function smokeControlArgs(harness) {
+	const model = harnessEnvValue(harness, "MODEL");
+	const reasoningEffort = harnessEnvValue(harness, "REASONING_EFFORT");
+	const discovered =
+		harness === "codex-sdk"
+			? discoverCodexControls({ model, reasoningEffort })
+			: await discoverClaudeControls({ model, reasoningEffort });
+	const selected = await discovered;
+	const args = [];
+
+	if (selected.model) {
+		args.push("--model", selected.model);
+	}
+	if (selected.reasoningEffort) {
+		args.push("--reasoning-effort", selected.reasoningEffort);
+	}
+
+	return args;
+}
+
+function discoverCodexControls({ model, reasoningEffort }) {
+	const pattern = compilePattern(harnessEnvValue("codex-sdk", "MODEL_PATTERN"), "PROSE_SMOKE_CODEX_MODEL_PATTERN");
+	const codexBin = join(cliDir, "node_modules", ".bin", process.platform === "win32" ? "codex.cmd" : "codex");
+	const result = run(codexBin, ["debug", "models"], {
+		cwd: cliDir,
+		env: process.env,
+		maxBuffer: 32 * 1024 * 1024,
+		timeout: 60_000,
+	});
+	let catalog;
+	try {
+		catalog = JSON.parse(result.stdout);
+	} catch (error) {
+		throw new Error(`Could not parse Codex model catalog JSON: ${error instanceof Error ? error.message : error}`);
+	}
+
+	const models = Array.isArray(catalog.models) ? catalog.models : [];
+	const candidates = models
+		.map((entry) => {
+			const levels = Array.isArray(entry.supported_reasoning_levels)
+				? entry.supported_reasoning_levels
+						.map((level) => level?.effort)
+						.filter((level) => typeof level === "string" && level.length > 0)
+				: [];
+			return {
+				displayName: entry.display_name,
+				model: typeof entry.slug === "string" ? entry.slug : undefined,
+				searchText: candidateText(entry.slug, entry.display_name, entry.description),
+				levels,
+				supportedInApi: entry.supported_in_api !== false,
+				visible: entry.visibility !== "hidden",
+			};
+		})
+		.filter((entry) => entry.model && entry.supportedInApi && entry.visible && entry.levels.length > 0);
+
+	const selected = selectDiscoveredModel(candidates, { model, pattern, provider: "Codex", reasoningEffort });
+	return {
+		model: selected.model,
+		reasoningEffort: preferredEffort(selected.levels, reasoningEffort, `Codex model ${selected.model}`),
+	};
+}
+
+async function discoverClaudeControls({ model, reasoningEffort }) {
+	const pattern = compilePattern(harnessEnvValue("claude-sdk", "MODEL_PATTERN"), "PROSE_SMOKE_CLAUDE_MODEL_PATTERN");
+	const models = model === undefined ? await listClaudeModels() : [await retrieveClaudeModel(model)];
+	const candidates = models
+		.map((entry) => {
+			const effort = entry.capabilities?.effort;
+			const thinking = entry.capabilities?.thinking;
+			const levels = claudeReasoningEfforts.filter((level) => effort?.[level]?.supported === true);
+			return {
+				displayName: entry.display_name,
+				model: typeof entry.id === "string" ? entry.id : undefined,
+				searchText: candidateText(entry.id, entry.display_name),
+				levels,
+				supportedInApi: effort?.supported === true && thinking?.types?.[claudeThinkingType]?.supported === true,
+				visible: true,
+			};
+		})
+		.filter((entry) => entry.model && entry.supportedInApi && entry.levels.length > 0);
+
+	const selected = selectDiscoveredModel(candidates, {
+		model,
+		pattern,
+		provider: "Claude",
+		requirement: `${claudeThinkingType} thinking and reasoning effort support compatible with the current Claude Code path`,
+	});
+	return {
+		model: selected.model,
+		reasoningEffort: preferredEffort(selected.levels, reasoningEffort, `Claude model ${selected.model}`),
+	};
+}
+
+function selectDiscoveredModel(candidates, { model, pattern, provider, requirement = "reasoning effort support" }) {
+	if (model !== undefined) {
+		const selected = candidates.find((candidate) => candidate.model === model);
+		if (selected === undefined && candidates.length === 1) {
+			return candidates[0];
+		}
+		if (selected === undefined) {
+			throw new Error(`${provider} model ${model} was not found or does not advertise ${requirement}`);
+		}
+		return selected;
+	}
+
+	const filtered = pattern === undefined ? candidates : candidates.filter((candidate) => pattern.test(candidate.searchText));
+	const [selected] = filtered;
+	if (selected === undefined) {
+		const suffix = pattern === undefined ? "" : ` matching ${pattern}`;
+		throw new Error(`No ${provider} models with ${requirement} were discovered${suffix}`);
+	}
+	return selected;
+}
+
+async function listClaudeModels() {
+	const models = [];
+	let afterId;
+	do {
+		const url = new URL("https://api.anthropic.com/v1/models");
+		url.searchParams.set("limit", "100");
+		if (afterId) {
+			url.searchParams.set("after_id", afterId);
+		}
+		const page = await fetchClaudeJson(url);
+		if (Array.isArray(page.data)) {
+			models.push(...page.data);
+		}
+		afterId = page.has_more === true ? page.last_id : undefined;
+	} while (afterId);
+	return models;
+}
+
+async function retrieveClaudeModel(model) {
+	const url = new URL(`https://api.anthropic.com/v1/models/${encodeURIComponent(model)}`);
+	return fetchClaudeJson(url);
+}
+
+async function fetchClaudeJson(url) {
+	const controller = new AbortController();
+	const timeout = setTimeout(() => controller.abort(), 30_000);
+	try {
+		const response = await fetch(url, {
+			headers: {
+				"anthropic-version": "2023-06-01",
+				"x-api-key": process.env.ANTHROPIC_API_KEY,
+			},
+			signal: controller.signal,
+		});
+		if (!response.ok) {
+			const body = await response.text();
+			throw new Error(`Anthropic Models API returned ${response.status}: ${body.slice(0, 500)}`);
+		}
+		return response.json();
+	} finally {
+		clearTimeout(timeout);
+	}
+}
+
 function run(command, args, options) {
 	const result = spawnSync(command, args, {
 		cwd: options.cwd,
 		encoding: "utf8",
 		env: options.env ?? process.env,
+		maxBuffer: options.maxBuffer ?? 1024 * 1024,
 		timeout: options.timeout,
 	});
 
@@ -170,7 +389,7 @@ Verifies the harness can see the preloaded OpenProse skill text.
 	);
 }
 
-function smokeHarness(harness, options) {
+async function smokeHarness(harness, options) {
 	const secret = requiredSecret(harness);
 	if (!process.env[secret]) {
 		throw new Error(`Missing required secret: ${secret}`);
@@ -194,11 +413,16 @@ function smokeHarness(harness, options) {
 			XDG_CONFIG_HOME: join(home, ".config"),
 		};
 
-		const result = run(
-			process.execPath,
-			[options.cli, "run", "smoke.prose.md", "--harness", harness],
-			{ cwd: workspace, env, timeout: options.timeout },
-		);
+		const controlArgs = await smokeControlArgs(harness);
+		if (controlArgs.length > 0) {
+			process.stderr.write(`Using ${harness} model controls: ${controlArgs.join(" ")}\n`);
+		}
+
+		const result = run(process.execPath, [options.cli, "run", "smoke.prose.md", "--harness", harness, ...controlArgs], {
+			cwd: workspace,
+			env,
+			timeout: options.timeout,
+		});
 		const output = `${result.stdout}\n${result.stderr}`;
 
 		if (!output.includes(okToken)) {
@@ -215,7 +439,7 @@ function smokeHarness(harness, options) {
 	}
 }
 
-function main() {
+async function main() {
 	const options = parseArgs(process.argv.slice(2));
 	if (options.help) {
 		process.stdout.write(usage);
@@ -224,13 +448,19 @@ function main() {
 
 	for (const harness of requestedHarnesses(options.harness)) {
 		process.stderr.write(`Smoking ${harness}...\n`);
-		smokeHarness(harness, options);
+		await smokeHarness(harness, options);
 	}
 }
 
-try {
-	main();
-} catch (error) {
-	process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
-	process.exitCode = 1;
+const isMain = process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href;
+
+if (isMain) {
+	try {
+		await main();
+	} catch (error) {
+		process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+		process.exitCode = 1;
+	}
 }
+
+export { discoverClaudeControls };