diff --git a/.github/workflows/cli-live-smoke.yml b/.github/workflows/cli-live-smoke.yml new file mode 100644 index 00000000..a36ceb1d --- /dev/null +++ b/.github/workflows/cli-live-smoke.yml @@ -0,0 +1,52 @@ +name: CLI Live Smoke + +on: + pull_request: + paths: + - "tools/cli/**" + - ".github/workflows/cli-live-smoke.yml" + push: + branches: + - main + paths: + - "tools/cli/**" + - ".github/workflows/cli-live-smoke.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: cli-live-smoke-${{ github.ref }} + cancel-in-progress: true + +jobs: + live-smoke: + name: Harness live smoke (vitest) + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: tools/cli + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: tools/cli/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Build CLI + run: npm run build + + - name: Run harness live smoke + run: npm run test:live diff --git a/tools/cli/package-lock.json b/tools/cli/package-lock.json index 4aefaf6a..21ff838d 100644 --- a/tools/cli/package-lock.json +++ b/tools/cli/package-lock.json @@ -9,7 +9,7 @@ "version": "0.2.5", "license": "MIT", "dependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.2.90", + "@anthropic-ai/claude-agent-sdk": "0.2.107", "@hono/node-server": "^1.19.14", "@oclif/core": "^4.10.6", "@openai/codex-sdk": "^0.125.0", @@ -29,13 +29,13 @@ } }, "node_modules/@anthropic-ai/claude-agent-sdk": { - "version": "0.2.90", - "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk/-/claude-agent-sdk-0.2.90.tgz", - "integrity": "sha512-up5bK0pUbthKIZtNE18WDrIYi0KNpZUhdgjGbkfH/mFQJxI6W/uE3mTiLrCX3UF0SqNl0fMtojBTZPJr2b3O4g==", + "version": "0.2.107", + "resolved": "https://registry.npmjs.org/@anthropic-ai/claude-agent-sdk/-/claude-agent-sdk-0.2.107.tgz", + "integrity": "sha512-zH5CCjvFn4A+RN0LLaqKJYEcGEg2O/Bm+tDpkBGcEKaRZOqwXkKJ2d9JmboALGSxsCAN5K0+uQxPgzk9LhiQzg==", "license": "SEE LICENSE IN README.md", "dependencies": { - "@anthropic-ai/sdk": "^0.74.0", - "@modelcontextprotocol/sdk": "^1.27.1" + "@anthropic-ai/sdk": "^0.81.0", + "@modelcontextprotocol/sdk": "^1.29.0" }, "engines": { "node": ">=18.0.0" @@ -56,9 +56,9 @@ } }, "node_modules/@anthropic-ai/sdk": { - "version": "0.74.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.74.0.tgz", - "integrity": "sha512-srbJV7JKsc5cQ6eVuFzjZO7UR3xEPJqPamHFIe29bs38Ij2IripoAhC0S5NslNbaFUYqBKypmmpzMTpqfHEUDw==", + "version": "0.92.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.92.0.tgz", + "integrity": "sha512-l653JFC83wCglH8H83t1xpgDurCyPyslYW1maPRdCsfuNuGbLvQjQ81sWd3Go3LWRm0jNspzAhuqAYV8r9joSw==", "license": "MIT", "dependencies": { "json-schema-to-ts": "^3.1.1" @@ -1433,6 +1433,7 @@ "integrity": "sha512-orrrD74MBUyK8jOAD/r0+lfa1I2MO6I+vAkmAWzMYbCcgrN4lCrmK52gRFQq/JRxfYPfonkr4b0jcY7Olqdqbw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -2109,6 +2110,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -2412,6 +2414,7 @@ "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz", "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==", "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -3336,6 +3339,7 @@ "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "~0.27.0", "get-tsconfig": "^4.7.5" @@ -3619,6 +3623,7 @@ "integrity": "sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -3766,6 +3771,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.1.tgz", "integrity": "sha512-a6ENMBBGZBsnlSebQ/eKCguSBeGKSf4O7BPnqVPmYGtpBYI7VSqoVqw+QcB7kPRjbqPwhYTpFbVj/RqNz/CT0Q==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/tools/cli/package.json b/tools/cli/package.json index 86b13a58..91f188d2 100644 --- a/tools/cli/package.json +++ b/tools/cli/package.json @@ -51,6 +51,7 @@ "build": "npm run clean && tsc -p tsconfig.build.json", "typecheck": "tsc -p tsconfig.json --noEmit", "test": "vitest --run", + "test:live": "vitest --run --config vitest.live.config.ts", "smoke:harness": "node scripts/smoke-harness.mjs", "dev": "tsx src/index.ts", "clean": "rm -rf dist", @@ -68,7 +69,7 @@ "node": ">=18.14.1" }, "dependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.2.90", + "@anthropic-ai/claude-agent-sdk": "0.2.107", "@hono/node-server": "^1.19.14", "@oclif/core": "^4.10.6", "@openai/codex-sdk": "^0.125.0", @@ -79,5 +80,8 @@ "tsx": "^4.19.2", "typescript": "^5.7.3", "vitest": "^3.2.4" + }, + "overrides": { + "@anthropic-ai/sdk": "^0.92.0" } } diff --git a/tools/cli/src/harnesses/claude-sdk.ts b/tools/cli/src/harnesses/claude-sdk.ts index dcd2d8e9..589c78cd 100644 --- a/tools/cli/src/harnesses/claude-sdk.ts +++ b/tools/cli/src/harnesses/claude-sdk.ts @@ -1,3 +1,4 @@ +import { CLAUDE_SDK_DEFAULTS, resolveClaudeModel } from "./defaults.js"; import { writeLine } from "./streams.js"; import type { Harness, HarnessRunOptions } from "./types.js"; @@ -27,6 +28,8 @@ export function createClaudeSdkHarness(options: ClaudeSdkHarnessOptions = {}): H prompt, options: { abortController, + model: resolveClaudeModel(runOptions.env), + thinking: CLAUDE_SDK_DEFAULTS.thinking, ...(runOptions.additionalDirectories === undefined ? {} : { additionalDirectories: runOptions.additionalDirectories }), diff --git a/tools/cli/src/harnesses/codex-sdk.ts b/tools/cli/src/harnesses/codex-sdk.ts index 129064a8..9512ed80 100644 --- a/tools/cli/src/harnesses/codex-sdk.ts +++ b/tools/cli/src/harnesses/codex-sdk.ts @@ -1,4 +1,5 @@ import { codexClientConfig, codexThreadRuntimeOptions } from "./codex-options.js"; +import { resolveCodexModel } from "./defaults.js"; import { writeLine } from "./streams.js"; import type { CodexSdkClientOptions, CodexSdkFactory, CodexThreadEvent, CodexThreadItem, Harness } from "./types.js"; @@ -19,9 +20,10 @@ export function createCodexSdkHarness(options: CodexSdkHarnessOptions = {}): Har async run(prompt, runOptions) { try { const env = definedEnv(runOptions.env); + const model = resolveCodexModel(runOptions.env); const codex = await factory(codexClientOptions(env, runOptions.systemPromptAppend)); const thread = codex.startThread( - codexThreadOptions(runOptions.cwd, env, runOptions.additionalDirectories), + codexThreadOptions(runOptions.cwd, env, runOptions.additionalDirectories, model), ); const { events } = await thread.runStreamed( prompt, @@ -48,14 +50,14 @@ function codexThreadOptions( cwd: string | undefined, env: Record | undefined, additionalDirectories: readonly string[] | undefined, + model: string, ) { const runtimeOptions = codexThreadRuntimeOptions(env, additionalDirectories); - const options = { + return { + model, ...(cwd === undefined ? {} : { workingDirectory: cwd }), ...runtimeOptions, }; - - return Object.keys(options).length === 0 ? undefined : options; } function codexClientOptions(env: Record | undefined, systemPromptAppend: string | undefined) { diff --git a/tools/cli/src/harnesses/defaults.ts b/tools/cli/src/harnesses/defaults.ts new file mode 100644 index 00000000..971610a3 --- /dev/null +++ b/tools/cli/src/harnesses/defaults.ts @@ -0,0 +1,16 @@ +export const CLAUDE_SDK_DEFAULTS = { + model: "claude-sonnet-4-6", + thinking: { type: "adaptive" as const }, +} as const; + +export const CODEX_SDK_DEFAULTS = { + model: "gpt-5-codex", +} as const; + +export function resolveClaudeModel(env: Record | undefined): string { + return env?.ANTHROPIC_MODEL ?? process.env.ANTHROPIC_MODEL ?? CLAUDE_SDK_DEFAULTS.model; +} + +export function resolveCodexModel(env: Record | undefined): string { + return env?.OPENAI_MODEL ?? env?.CODEX_MODEL ?? process.env.OPENAI_MODEL ?? process.env.CODEX_MODEL ?? CODEX_SDK_DEFAULTS.model; +} diff --git a/tools/cli/tests/harnesses/harnesses.test.ts b/tools/cli/tests/harnesses/harnesses.test.ts index 5d562dbc..254c3f82 100644 --- a/tools/cli/tests/harnesses/harnesses.test.ts +++ b/tools/cli/tests/harnesses/harnesses.test.ts @@ -92,7 +92,7 @@ describe("codex-sdk harness", () => { expect(exitCode).toBe(0); expect(io.stdout).toBe("sdk output\n"); - expect(starts).toEqual([{ skipGitRepoCheck: true, workingDirectory: "/repo" }]); + expect(starts).toEqual([{ model: "gpt-5-codex", skipGitRepoCheck: true, workingDirectory: "/repo" }]); expect(factoryOptions).toEqual([{ apiKey: "test", env: { OPENAI_API_KEY: "test" } }]); }); @@ -127,6 +127,7 @@ describe("codex-sdk harness", () => { expect(starts).toEqual([ { additionalDirectories: ["/skills/open-prose"], + model: "gpt-5-codex", skipGitRepoCheck: true, }, ]); @@ -166,7 +167,9 @@ describe("codex-sdk harness", () => { }); expect(exitCode).toBe(0); - expect(starts).toEqual([{ approvalPolicy: "never", sandboxMode: "danger-full-access", skipGitRepoCheck: true }]); + expect(starts).toEqual([ + { approvalPolicy: "never", model: "gpt-5-codex", sandboxMode: "danger-full-access", skipGitRepoCheck: true }, + ]); }); test("maps failed turns to stderr and nonzero exit", async () => { @@ -219,6 +222,8 @@ describe("claude-sdk harness", () => { additionalDirectories: ["/skills/open-prose"], cwd: "/repo", env: { A: "B" }, + model: "claude-sonnet-4-6", + thinking: { type: "adaptive" }, systemPrompt: { type: "preset", preset: "claude_code", @@ -229,6 +234,37 @@ describe("claude-sdk harness", () => { ]); }); + test("honors ANTHROPIC_MODEL override from runOptions.env", async () => { + const io = memoryStreams(); + const calls: unknown[] = []; + const harness = createClaudeSdkHarness({ + query: async (args) => { + calls.push(args); + return { + async *[Symbol.asyncIterator]() { + yield { type: "result", subtype: "success", result: "ok", is_error: false }; + }, + close() {}, + } as never; + }, + }); + + await harness.run("prose status", { + ...io.options, + env: { ANTHROPIC_MODEL: "claude-opus-4-7" }, + }); + + expect(calls).toEqual([ + { + prompt: "prose status", + options: expect.objectContaining({ + model: "claude-opus-4-7", + thinking: { type: "adaptive" }, + }), + }, + ]); + }); + test("streams text deltas without duplicating final result", async () => { const io = memoryStreams(); const abortControllers: AbortController[] = []; diff --git a/tools/cli/tests/live/smoke.live.test.ts b/tools/cli/tests/live/smoke.live.test.ts new file mode 100644 index 00000000..c13cf95f --- /dev/null +++ b/tools/cli/tests/live/smoke.live.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, test } from "vitest"; + +import { createClaudeSdkHarness, createCodexSdkHarness } from "../../src/harnesses/index.js"; + +const MARKER = "PROSE_LIVE_SMOKE_OK"; +const PROMPT = `Reply with exactly the literal string ${MARKER} and nothing else. Do not use any tools.`; +const TIMEOUT_MS = 90_000; + +function memoryStreams() { + let stdout = ""; + let stderr = ""; + return { + options: { + stdout: { write: (chunk: string) => void (stdout += chunk) }, + stderr: { write: (chunk: string) => void (stderr += chunk) }, + }, + get stdout() { + return stdout; + }, + get stderr() { + return stderr; + }, + }; +} + +const claudeKey = process.env.ANTHROPIC_API_KEY; +const codexKey = process.env.CODEX_API_KEY ?? process.env.OPENAI_API_KEY; + +describe("live smoke", () => { + test.runIf(claudeKey !== undefined)( + "claude-sdk reaches Anthropic and produces text", + async () => { + const io = memoryStreams(); + const exitCode = await createClaudeSdkHarness().run(PROMPT, { + ...io.options, + env: { ANTHROPIC_API_KEY: claudeKey, ...processEnvWhitelist(["HOME", "PATH"]) }, + }); + expect(exitCode).toBe(0); + expect(io.stderr).not.toMatch(/API Error|invalid_request_error/); + expect(io.stdout).toContain(MARKER); + }, + TIMEOUT_MS, + ); + + test.runIf(codexKey !== undefined)( + "codex-sdk reaches OpenAI and produces text", + async () => { + const io = memoryStreams(); + const exitCode = await createCodexSdkHarness().run(PROMPT, { + ...io.options, + env: { OPENAI_API_KEY: codexKey, ...processEnvWhitelist(["HOME", "PATH"]) }, + }); + expect(exitCode).toBe(0); + expect(io.stderr).not.toMatch(/error|failed/i); + expect(io.stdout).toContain(MARKER); + }, + TIMEOUT_MS, + ); + + test.skipIf(claudeKey !== undefined || codexKey !== undefined)( + "skipped: no live keys present", + () => { + expect(true).toBe(true); + }, + ); +}); + +function processEnvWhitelist(keys: readonly string[]): Record { + const out: Record = {}; + for (const key of keys) { + const value = process.env[key]; + if (value !== undefined) { + out[key] = value; + } + } + return out; +} diff --git a/tools/cli/vitest.config.ts b/tools/cli/vitest.config.ts index 651c9797..57f8426d 100644 --- a/tools/cli/vitest.config.ts +++ b/tools/cli/vitest.config.ts @@ -3,6 +3,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "node", - include: ["tests/**/*.test.ts"] + include: ["tests/**/*.test.ts"], + exclude: ["**/node_modules/**", "**/dist/**", "tests/live/**"] } }); diff --git a/tools/cli/vitest.live.config.ts b/tools/cli/vitest.live.config.ts new file mode 100644 index 00000000..7b437fc8 --- /dev/null +++ b/tools/cli/vitest.live.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + include: ["tests/live/**/*.test.ts"], + exclude: ["**/node_modules/**", "**/dist/**"], + testTimeout: 120_000 + } +});