diff --git a/packages/opencode/src/bench/cli.ts b/packages/opencode/src/bench/cli.ts index 2b47ec452def..5d8f9772e1f5 100644 --- a/packages/opencode/src/bench/cli.ts +++ b/packages/opencode/src/bench/cli.ts @@ -25,6 +25,7 @@ import os from "node:os" import { spawn } from "node:child_process" import { runDeepReset } from "./deep_reset" import { bootstrapRepoIfMissing } from "./bootstrap_repo" +import * as BenchTerminalError from "./terminal_error" // opencode's built-in anthropic system prompt — Bun bundles .txt as a string. // Used as the default when no --system-prompt override is passed. import PROMPT_ANTHROPIC from "../session/prompt/anthropic.txt" @@ -157,9 +158,7 @@ async function buildConfigDir(args: { const tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), `bench-${args.instanceId}-`)) await fs.mkdir(tmpRoot, { recursive: true }) - const systemPrompt = args.systemPromptPath - ? await fs.readFile(args.systemPromptPath, "utf8") - : DEFAULT_SYSTEM_PROMPT + const systemPrompt = args.systemPromptPath ? await fs.readFile(args.systemPromptPath, "utf8") : DEFAULT_SYSTEM_PROMPT const cfg: Record = { $schema: "https://opencode.ai/config.json", @@ -202,12 +201,105 @@ async function buildConfigDir(args: { // Allow the read+write tool set; disable web/skill/task to keep the // agent focused on local code editing. permission: { - // Glob-keyed `PermissionActionConfig` for file/shell access. edit: { "**": "allow" }, - bash: { "*": "allow" }, - // webfetch / websearch use a different schema (single action, not - // a glob map) and we already disable them in `tools` below — no - // need for an explicit entry here. + bash: { + "*": "allow", + + // process termination + "*killall*": "deny", + "*pkill*": "deny", + "*kill -1*": "deny", + "*kill 0*": "deny", + + // filesystem destruction + "*rm -rf /": "deny", + "*rm -rf /*": "deny", + "*rm -rf /bin*": "deny", + "*rm -rf /usr*": "deny", + "*rm -rf /etc*": "deny", + "*rm -rf /var*": "deny", + "*rm -rf /home*": "deny", + "*rm -rf /root*": "deny", + "*rm -rf /opt*": "deny", + "*rm -rf /lib*": "deny", + "*rm -rf /lib64*": "deny", + "*rm -rf /sbin*": "deny", + "*rm -rf /boot*": "deny", + "*rm -rf /dev*": "deny", + "*rm -rf /proc*": "deny", + "*rm -rf /sys*": "deny", + + // system control + // "*shutdown*": "deny", + // "*reboot*": "deny", + // "*poweroff*": "deny", + // "*halt*": "deny", + // "init 0*": "deny", + // "init 6*": "deny", + + // disk devices + "dd *of=/dev/sd*": "deny", + "dd *of=/dev/nvme*": "deny", + "dd *of=/dev/hd*": "deny", + "dd *of=/dev/null*": "deny", + + // git network + "*git fetch*": "deny", + "*git pull*": "deny", + "*git clone*": "deny", + "*git ls-remote*": "deny", + "*git remote add*": "deny", + "*git remote set-url*": "deny", + "*git remote set-head*": "deny", + "*git remote update*": "deny", + "*git remote rename*": "deny", + "*git remote set-branches*": "deny", + "*git submodule add*": "deny", + "*git submodule update*": "deny", + "*git submodule sync*": "deny", + "*git submodule init*": "deny", + "*git archive*--remote*": "deny", + "*git *://*": "deny", + "*git *@*:*": "deny", + + // git history mining + "*git log*--all*": "deny", + "*git log*--branches*": "deny", + "*git log*--remotes*": "deny", + "*git log*--walk-reflogs*": "deny", + "*git log*--grep*": "deny", + "*git rev-list*--all*": "deny", + "*git rev-list*--branches*": "deny", + "*git rev-list*--remotes*": "deny", + "*git rev-list*--grep*": "deny", + "*git shortlog*--all*": "deny", + "*git reflog*": "deny", + "*git cat-file*": "deny", + "*git fsck*": "deny", + "*git verify-pack*": "deny", + "*git unpack-objects*": "deny", + "*git cherry*": "deny", + "*git show*": "deny", + "*git merge-base*--is-ancestor*": "deny", + "*git branch*--contains*": "deny", + "*git tag*--contains*": "deny", + "*git for-each-ref*--contains*": "deny", + + // git internals (substring match on path) + "*.git/logs*": "deny", + "*.git/packed-refs*": "deny", + "*.git/ORIG_HEAD*": "deny", + "*.git/FETCH_HEAD*": "deny", + "*.git/refs*": "deny", + + // online lookups + "*curl *github.com*": "deny", + "*wget *github.com*": "deny", + "*curl *githubusercontent.com*": "deny", + "*wget *githubusercontent.com*": "deny", + "*curl *github.io*": "deny", + "*wget *github.io*": "deny", + }, }, tools: { bash: true, @@ -254,7 +346,7 @@ function runOpencode(args: { env: NodeJS.ProcessEnv opencodeBin: string agent: string -}): Promise<{ exitCode: number; stdout: string; stderr: string }> { +}): Promise<{ exitCode: number; stdout: string; stderr: string; terminalError?: BenchTerminalError.Kind }> { // Use the same bun binary that's currently running — guaranteed to exist // and avoids PATH lookup quirks under Bun's posix_spawn. const bunPath = process.execPath @@ -275,7 +367,6 @@ function runOpencode(args: { `nemo-gym/${args.modelName}`, "--format", "json", - "--dangerously-skip-permissions", "--dir", args.workspaceRoot, ], @@ -286,6 +377,13 @@ function runOpencode(args: { ) let stdout = "" let stderr = "" + let terminalError: BenchTerminalError.Kind | undefined + let terminalSignalBuffer = "" + const observeTerminalSignal = (chunk: string) => { + // Retain enough overlap to recognize a marker split across pipe chunks. + terminalSignalBuffer = (terminalSignalBuffer + chunk).slice(-256) + terminalError = BenchTerminalError.prefer(terminalError, BenchTerminalError.detect(terminalSignalBuffer)) + } // Strip bulky token-ID metadata from echoed event lines. The IDs already // live in the llm_completions dumps; leaving them in the event stream // makes each turn re-echo that turn's full-context prompt_token_ids -> @@ -309,7 +407,9 @@ function runOpencode(args: { const MAX_KEEP = 256 * 1024 // keep only a bounded tail for error reporting let lineBuf = "" child.stdout?.on("data", (b) => { - lineBuf += b.toString("utf8") + const chunk = b.toString("utf8") + observeTerminalSignal(chunk) + lineBuf += chunk let idx: number while ((idx = lineBuf.indexOf("\n")) >= 0) { const line = scrub(lineBuf.slice(0, idx)) @@ -321,13 +421,14 @@ function runOpencode(args: { }) child.stderr?.on("data", (b) => { const chunk = b.toString("utf8") + observeTerminalSignal(chunk) stderr = (stderr + chunk).slice(-MAX_KEEP) process.stderr.write(chunk) }) - child.on("close", (code) => resolve({ exitCode: code ?? 0, stdout, stderr })) + child.on("close", (code) => resolve({ exitCode: code ?? 0, stdout, stderr, terminalError })) child.on("error", (err) => { stderr += String(err) - resolve({ exitCode: 999, stdout, stderr }) + resolve({ exitCode: 999, stdout, stderr, terminalError }) }) }) } @@ -436,12 +537,19 @@ async function main() { OPENCODE_DB: ":memory:", OPENCODE_DATA: path.join(tmpRoot, "data"), OPENCODE_CONFIG: configFile, + // The benchmark already runs inside a SIF sandbox, so make that the + // security boundary. This final config override applies to subagents too. + OPENCODE_PERMISSION: JSON.stringify({ "*": "allow" }), + // Disable opencode's built-in plugin loaders; the bench harness doesn't need them. OPENCODE_PURE: "1", // Skip the dynamic env block (working dir + Today's date) in the system // prompt — keeps the RL prompt-token prefix invariant stable across turns // (a midnight rollover would otherwise shift `Today's date: ...`). OPENCODE_DISABLE_ENV_PROMPT: "1", + // Have all agent sessions report terminal states to this bench wrapper. + // This is bench-only and does not alter normal opencode runs. + [BenchTerminalError.ENV]: "1", } // Bootstrap a git repo if the SIF shipped a flat source tree (swe-bench-ext @@ -471,7 +579,7 @@ async function main() { const patch = await captureGitDiff(workspaceRoot) const benchRunTime = (Date.now() - startedAt) / 1000 - const error: string | null = result.exitCode === 0 ? null : `opencode_exit_${result.exitCode}` + const error = BenchTerminalError.toGymError(result.exitCode, result.terminalError) const outPath = await writeOutputJsonl(args.outputDir, instance.instance_id, { instance_id: instance.instance_id, test_result: { git_patch: patch }, @@ -491,7 +599,7 @@ async function main() { // child-stdio pipes from the opencode subprocess). Gym's runner treats any // non-zero apptainer exit as `Agent command failed` and discards the // already-written patch, so we MUST exit 0 deterministically on success. - process.exit(result.exitCode === 0 ? 0 : 1) + process.exit(BenchTerminalError.shouldExitSuccessfully(result.exitCode, result.terminalError) ? 0 : 1) } main().catch((err) => { diff --git a/packages/opencode/src/bench/terminal_error.ts b/packages/opencode/src/bench/terminal_error.ts new file mode 100644 index 000000000000..d8a0b64064ef --- /dev/null +++ b/packages/opencode/src/bench/terminal_error.ts @@ -0,0 +1,38 @@ +export type Kind = "max_iteration" | "context_window" + +export const ENV = "OPENCODE_BENCH_TERMINAL_SIGNALS" +export const PREFIX = "[opencode-bench-terminal] " + +export function encode(kind: Kind): string { + return PREFIX + kind +} + +/** Report terminal agent states from any session, including subagents. */ +export function report(kind: Kind): void { + if (process.env[ENV] !== "1") return + process.stderr.write(encode(kind) + "\n") +} + +export function detect(text: string): Kind | undefined { + if (text.includes(encode("context_window"))) return "context_window" + if (text.includes(encode("max_iteration"))) return "max_iteration" + return undefined +} + +/** Context overflow wins when the forced final max-step call also overflows. */ +export function prefer(current: Kind | undefined, incoming: Kind | undefined): Kind | undefined { + if (!current) return incoming + if (!incoming) return current + if (current === "context_window" || incoming === "context_window") return "context_window" + return "max_iteration" +} + +export function toGymError(exitCode: number, kind?: Kind): string | null { + if (kind === "max_iteration") return "maximum iteration reached" + if (kind === "context_window") return "context window exceeded" + return exitCode === 0 ? null : `opencode_exit_${exitCode}` +} + +export function shouldExitSuccessfully(exitCode: number, kind?: Kind): boolean { + return exitCode === 0 || kind !== undefined +} diff --git a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts index 0a0f9e56c386..fca48b9a9311 100644 --- a/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts +++ b/packages/opencode/src/provider/sdk/nemo-gym/language-model.ts @@ -85,6 +85,24 @@ interface ChatResponse { usage?: ChatResponseUsage } +function contextOverflowStreamError(message: string): string { + return JSON.stringify({ + type: "error", + error: { + code: "context_length_exceeded", + message: message.slice(0, 2_000) || "Input exceeds context window of this model", + }, + }) +} + +function isGymContextOverflowCompletion(choice: ChatResponseChoice): boolean { + // Gym's vLLM wrapper translates an upstream context-overflow HTTP 400 into + // a successful empty completion. The stable signal it returns is exactly + // this pair. `content == null` alone is not enough because valid tool-call + // completions also normally carry null assistant content. + return choice.finish_reason === "length" && choice.message?.content == null +} + // --------------------------------------------------------------------------- // Config // --------------------------------------------------------------------------- @@ -197,7 +215,13 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const choice = responseJson.choices[0] if (!choice) throw new Error("nemo-gym: empty choices in response") - const msg: ChatResponseChoice["message"] = choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) + if (isGymContextOverflowCompletion(choice)) { + throw new Error( + contextOverflowStreamError("NeMo Gym returned an empty length completion for an overlong context"), + ) + } + const msg: ChatResponseChoice["message"] = + choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) const providerSpecificFields = this._extractProviderFields(msg) const providerMetadata = this._buildProviderMetadata(providerSpecificFields) @@ -253,6 +277,11 @@ export class NemoGymLanguageModel implements LanguageModelV3 { const choice = responseJson.choices[0] if (!choice) throw new Error("nemo-gym: empty choices in response") + if (isGymContextOverflowCompletion(choice)) { + throw new Error( + contextOverflowStreamError("NeMo Gym returned an empty length completion for an overlong context"), + ) + } const msg: ChatResponseChoice["message"] = choice.message ?? ({ role: "assistant" } as ChatResponseChoice["message"]) @@ -575,7 +604,10 @@ export class NemoGymLanguageModel implements LanguageModelV3 { return md } - private _mapFinishReason(raw: string | null): { unified: "stop" | "length" | "tool-calls" | "error" | "other"; raw: string | undefined } { + private _mapFinishReason(raw: string | null): { + unified: "stop" | "length" | "tool-calls" | "error" | "other" + raw: string | undefined + } { if (!raw) return { unified: "other", raw: undefined } switch (raw) { case "stop": diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index f22da92927d2..727eaed99526 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -24,6 +24,7 @@ import { EventV2 } from "@/v2/event" import { SessionEvent } from "@/v2/session-event" import { Modelv2 } from "@/v2/model" import * as DateTime from "effect/DateTime" +import * as BenchTerminalError from "@/bench/terminal_error" const DOOM_LOOP_THRESHOLD = 3 const log = Log.create({ service: "session.processor" }) @@ -647,6 +648,7 @@ export const layer: Layer.Layer< slog.error("process", { error: errorMessage(e), stack: e instanceof Error ? e.stack : undefined }) const error = parse(e) if (MessageV2.ContextOverflowError.isInstance(error)) { + BenchTerminalError.report("context_window") ctx.needsCompaction = true yield* bus.publish(Session.Event.Error, { sessionID: ctx.sessionID, error }) return diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index f7c59fe4cba0..a16b3a08c867 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -61,6 +61,7 @@ import * as DateTime from "effect/DateTime" import { eq } from "@/storage/db" import * as Database from "@/storage/db" import { SessionTable } from "./session.sql" +import * as BenchTerminalError from "@/bench/terminal_error" // @ts-ignore globalThis.AI_SDK_LOG_WARNINGS = false @@ -1495,6 +1496,7 @@ NOTE: At any point in time through this workflow you should feel free to ask the } const maxSteps = agent.steps ?? Infinity const isLastStep = step >= maxSteps + if (isLastStep) BenchTerminalError.report("max_iteration") msgs = yield* insertReminders({ messages: msgs, agent, session }) const msg: MessageV2.Assistant = { diff --git a/packages/opencode/test/bench/terminal_error.test.ts b/packages/opencode/test/bench/terminal_error.test.ts new file mode 100644 index 000000000000..5985446408f7 --- /dev/null +++ b/packages/opencode/test/bench/terminal_error.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from "bun:test" +import * as BenchTerminalError from "@/bench/terminal_error" + +describe("bench terminal error signals", () => { + test("detects max-iteration and context-window markers", () => { + expect(BenchTerminalError.detect(`before ${BenchTerminalError.encode("max_iteration")} after`)).toBe( + "max_iteration", + ) + expect(BenchTerminalError.detect(BenchTerminalError.encode("context_window"))).toBe("context_window") + expect(BenchTerminalError.detect("ordinary opencode stderr")).toBeUndefined() + }) + + test("prefers context overflow when both terminal states occur", () => { + expect(BenchTerminalError.prefer("max_iteration", "context_window")).toBe("context_window") + expect(BenchTerminalError.prefer("context_window", "max_iteration")).toBe("context_window") + }) + + test("writes errors that Gym classifies and preserves ordinary exit errors", () => { + expect(BenchTerminalError.toGymError(0, "max_iteration")).toBe("maximum iteration reached") + expect(BenchTerminalError.toGymError(0, "context_window")).toBe("context window exceeded") + expect(BenchTerminalError.toGymError(17)).toBe("opencode_exit_17") + expect(BenchTerminalError.toGymError(0)).toBeNull() + }) + + test("keeps terminal trajectories even when opencode exits nonzero", () => { + expect(BenchTerminalError.shouldExitSuccessfully(1, "context_window")).toBeTrue() + expect(BenchTerminalError.shouldExitSuccessfully(1, "max_iteration")).toBeTrue() + expect(BenchTerminalError.shouldExitSuccessfully(1)).toBeFalse() + }) +}) diff --git a/packages/opencode/test/provider/nemo-gym/context-overflow.test.ts b/packages/opencode/test/provider/nemo-gym/context-overflow.test.ts new file mode 100644 index 000000000000..6e6117163f5b --- /dev/null +++ b/packages/opencode/test/provider/nemo-gym/context-overflow.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, mock, test } from "bun:test" +import type { LanguageModelV3CallOptions, LanguageModelV3StreamPart } from "@ai-sdk/provider" +import { NemoGymLanguageModel } from "@/provider/sdk/nemo-gym/language-model" + +async function drain(stream: ReadableStream): Promise { + const reader = stream.getReader() + const parts: LanguageModelV3StreamPart[] = [] + while (true) { + const { done, value } = await reader.read() + if (done) break + parts.push(value) + } + return parts +} + +const CALL_OPTIONS: LanguageModelV3CallOptions = { + prompt: [{ role: "user", content: [{ type: "text", text: "hi" }] }], + tools: [{ type: "function", name: "bash", inputSchema: { type: "object", properties: {} } }], +} + +describe("NemoGymLanguageModel context overflow", () => { + test("recognizes Gym's null-content length completion as context overflow", async () => { + const originalFetch = globalThis.fetch + const fetchSpy = mock( + async () => + new Response( + JSON.stringify({ + id: "chatcmpl-123", + model: "test-model", + choices: [ + { + index: 0, + finish_reason: "length", + message: { role: "assistant", content: null, tool_calls: null }, + }, + ], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ) + // @ts-expect-error test override + globalThis.fetch = fetchSpy + + try { + const model = new NemoGymLanguageModel("test-model", { + provider: "nemo-gym", + baseURL: "http://unused.invalid", + retries: Number.MAX_SAFE_INTEGER, + }) + + const parts = await drain((await model.doStream(CALL_OPTIONS)).stream) + expect(fetchSpy).toHaveBeenCalledTimes(1) + + const error = parts.find((part) => part.type === "error") + expect(error?.type).toBe("error") + if (error?.type !== "error" || typeof error.error !== "string") throw new Error("missing stream error") + expect(JSON.parse(error.error)).toMatchObject({ + type: "error", + error: { code: "context_length_exceeded" }, + }) + } finally { + globalThis.fetch = originalFetch + } + }) +})