diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e8dca7ab..a30a23781 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -493,9 +493,9 @@ jobs: # (`is_bun_runtime_crash`). The unsharded macOS control had no equivalent, # so the same crash that Linux absorbs failed the whole promotion here. # - # Keep the signature list in sync with `is_bun_runtime_crash` in - # scripts/ci/run-bun-test-batches.sh. An assertion failure still fails on - # the first attempt — only the crash signature is retried, exactly once. + # Keep the signature list in sync with `is_bun_runtime_crash` in + # scripts/ci/run-bun-test-batches.sh. An assertion failure still fails on + # the first attempt — only the crash signature is retried, exactly once. - name: Test run: | # GitHub Actions starts bash `run:` blocks with `-e`. Disable @@ -504,7 +504,14 @@ jobs: set -uo pipefail suite_log="$(mktemp -t ocx-macos-suite.XXXXXX)" for attempt in 1 2; do - bun test --isolate tests 2>&1 | tee "$suite_log" + # --timeout: Bun's default 5s per-test ceiling is the recurring flake + # class on this loaded shared runner (real retry windows + server + # round-trips exceed 5s under contention; a 10s-floor in-test + # watchdog fired at 10.16s there). 60s keeps hangs bounded (the 30m + # job timeout is the outer backstop) while removing the timing + # flakes — assertions are untouched. Pairs with the 30s CI floor in + # tests/helpers/ci-watchdog.ts. + bun test --isolate --timeout 60000 tests 2>&1 | tee "$suite_log" suite_status="${PIPESTATUS[0]}" if [ "$suite_status" -eq 0 ]; then exit 0 diff --git a/scripts/ci/run-bun-test-batches.sh b/scripts/ci/run-bun-test-batches.sh index ff4d20210..699f99cb8 100644 --- a/scripts/ci/run-bun-test-batches.sh +++ b/scripts/ci/run-bun-test-batches.sh @@ -106,7 +106,7 @@ run_test_once() { set +e timeout --signal=TERM --kill-after="${BATCH_KILL_GRACE_SECONDS}s" \ "${BATCH_TIMEOUT_SECONDS}s" \ - bun test --isolate "${files[@]}" 2>&1 | tee "$log_file" + bun test --isolate --timeout 60000 "${files[@]}" 2>&1 | tee "$log_file" status="${PIPESTATUS[0]}" set -e diff --git a/tests/bridge-lifecycle.test.ts b/tests/bridge-lifecycle.test.ts index a1cdc4299..08d491395 100644 --- a/tests/bridge-lifecycle.test.ts +++ b/tests/bridge-lifecycle.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test"; import { bridgeToResponsesSSE } from "../src/bridge"; import type { AdapterEvent } from "../src/types"; +import { watchdogMs } from "./helpers/ci-watchdog"; async function* replay(events: AdapterEvent[]): AsyncGenerator { for (const event of events) yield event; } @@ -127,7 +128,7 @@ describe("bridge stream lifecycle (RC1 / RC2)", () => { // 500ms, and a slow runner spent it. The stream is proven to close well // inside 1500ms, so the guard sits at the point where something is // genuinely wrong, and the test timeout gives the drift room. - new Promise((_, reject) => setTimeout(() => reject(new Error("stall stream did not close")), 4_000)), + new Promise((_, reject) => setTimeout(() => reject(new Error("stall stream did not close")), watchdogMs(4_000))), ]); } finally { await reader.cancel(); diff --git a/tests/ci-workflows.test.ts b/tests/ci-workflows.test.ts index 1f8865b90..5b6946f66 100644 --- a/tests/ci-workflows.test.ts +++ b/tests/ci-workflows.test.ts @@ -173,7 +173,9 @@ describe("GitHub Actions hardening", () => { // only when the shared path filter says the entire expensive suite is out of // scope (for example a docs-site-only PR). const macosSteps = (ci.jobs?.["platform-macos"] as { steps?: { run?: string }[] })?.steps ?? []; - expect(macosSteps.some(step => step.run?.includes("bun test --isolate tests"))).toBe(true); + // The 60s per-test ceiling is part of the pinned shape: dropping it silently + // restores the timing-flake class this lane kept surfacing. + expect(macosSteps.some(step => step.run?.includes("bun test --isolate --timeout 60000 tests"))).toBe(true); expect(macosSteps.some(step => step.run?.includes("--shard"))).toBe(false); // The macOS leg retries ONLY a Bun runtime crash, and only once. Bun 1.3.14 @@ -183,7 +185,7 @@ describe("GitHub Actions hardening", () => { // `scripts/ci/run-bun-test-batches.sh`. Two ways to break this silently: // drop the crash-signature guard so an assertion failure gets retried into // green, or let the retry loop swallow a repeated crash. Pin both. - const macosTestRun = macosSteps.find(step => step.run?.includes("bun test --isolate tests"))?.run ?? ""; + const macosTestRun = macosSteps.find(step => step.run?.includes("bun test --isolate --timeout 60000 tests"))?.run ?? ""; // Actions invokes multiline `run:` blocks with `bash -e`. The retry loop // must disable errexit before the crash-prone command or exit 133 aborts // the step before PIPESTATUS can be inspected and the retry can run. diff --git a/tests/helpers/ci-watchdog.ts b/tests/helpers/ci-watchdog.ts new file mode 100644 index 000000000..f09713dc9 --- /dev/null +++ b/tests/helpers/ci-watchdog.ts @@ -0,0 +1,17 @@ +/** + * CI-scaled test watchdogs. + * + * Several tests race a real local server round-trip against a short in-test + * watchdog (`setTimeout(..., reject)`). Locally 1-2 s is generous, but the + * unsharded GitHub macOS runner runs the whole suite in one pool under heavy + * CPU contention and these watchdogs were the recurring flake class there + * (server-auth WS terminal 1 s, provider-option fixture WS 2 s, …). Observed + * runner stalls exceed 10 s on that lane (a 10 s-floor watchdog fired at + * 10.16 s), so the CI floor is 30 s: the watchdog exists to bound a genuinely + * hung test, not to assert latency. Local behaviour is unchanged. Bun's own + * per-test timeout (`--timeout`, 60 s on CI) would pre-empt a 30 s watchdog, + * so the lane timeout and this floor move together. + */ +export function watchdogMs(base: number): number { + return process.env.CI === "true" ? Math.max(base, 30_000) : base; +} diff --git a/tests/native-profile-drain-server.test.ts b/tests/native-profile-drain-server.test.ts index d9c752706..d8907307a 100644 --- a/tests/native-profile-drain-server.test.ts +++ b/tests/native-profile-drain-server.test.ts @@ -24,6 +24,7 @@ import type { OcxConfig } from "../src/types"; import { fakeChatGptJwt } from "./helpers/fake-chatgpt-jwt"; import { ownedServiceHomeInspection } from "./helpers/owned-service-home-inspection"; +import { watchdogMs } from "./helpers/ci-watchdog"; /** These cases sandbox CODEX_HOME/OPENCODEX_HOME, so the installed service is not their evidence. */ const inspectNativeCodexOwnership = ownedServiceHomeInspection("native-profile drain server test"); @@ -96,7 +97,7 @@ describe("native main profile scoped server admission", () => { autoSwitchThreshold: 0, } as OcxConfig); const waitForFrame = (ws: WebSocket, needle: string) => new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error(`websocket timeout waiting for ${needle}`)), 2_000); + const timer = setTimeout(() => reject(new Error(`websocket timeout waiting for ${needle}`)), watchdogMs(2_000)); const onMessage = (event: MessageEvent) => { const text = typeof event.data === "string" ? event.data : ""; if (!text.includes(needle)) return; @@ -217,7 +218,7 @@ describe("native main profile scoped server admission", () => { url.protocol = "ws:"; const ws = new WebSocket(url, { headers } as unknown as string[]); await new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error("sideband echo timeout")), 5_000); + const timer = setTimeout(() => reject(new Error("sideband echo timeout")), watchdogMs(5_000)); ws.addEventListener("open", () => ws.send("ping"), { once: true }); ws.addEventListener("message", event => { if (String(event.data) !== "echo:ping") return; @@ -417,7 +418,7 @@ describe("native main profile scoped server admission", () => { try { await new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error("sideband websocket open timeout")), 2_000); + const timer = setTimeout(() => reject(new Error("sideband websocket open timeout")), watchdogMs(2_000)); client.addEventListener("open", () => { clearTimeout(timer); resolve(); diff --git a/tests/openai-api-virtual-models.test.ts b/tests/openai-api-virtual-models.test.ts index b14846387..e95757ea0 100644 --- a/tests/openai-api-virtual-models.test.ts +++ b/tests/openai-api-virtual-models.test.ts @@ -17,6 +17,7 @@ import { saveConfig } from "../src/config"; import { startServer } from "../src/server"; import { usageLogPath } from "../src/usage/log"; +import { watchdogMs } from "./helpers/ci-watchdog"; const moduleOriginalFetch = globalThis.fetch; const moduleOriginalHome = process.env.OPENCODEX_HOME; afterEach(() => { @@ -440,7 +441,7 @@ describe("OpenAI API Pro transport identities", () => { url.protocol = "ws:"; const ws = new WebSocket(url); return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error("OpenAI API Pro websocket timeout")), 2000); + const timer = setTimeout(() => reject(new Error("OpenAI API Pro websocket timeout")), watchdogMs(2000)); ws.addEventListener("open", () => { ws.send(JSON.stringify({ type: "response.create", model, input: "hello", reasoning: { effort: "high" } })); }, { once: true }); diff --git a/tests/openai-provider-option-e2e.test.ts b/tests/openai-provider-option-e2e.test.ts index 243e43dba..e68067541 100644 --- a/tests/openai-provider-option-e2e.test.ts +++ b/tests/openai-provider-option-e2e.test.ts @@ -17,6 +17,7 @@ import { import { homedir, tmpdir } from "node:os"; import { join, relative } from "node:path"; +import { watchdogMs } from "./helpers/ci-watchdog"; type Capture = { url: string; method: string; @@ -346,7 +347,7 @@ describe("OpenAI provider-option integration spine", () => { }); const wsTurn = (model: string) => new Promise((resolve, reject) => { const before = captures.length; - const timer = setTimeout(() => reject(new Error(`fixture websocket timeout: ${model}`)), 2_000); + const timer = setTimeout(() => reject(new Error(`fixture websocket timeout: ${model}`)), watchdogMs(2_000)); const onMessage = (event: MessageEvent) => { if (!String(event.data).includes('"type":"response.completed"')) return; clearTimeout(timer); diff --git a/tests/relay-eager.test.ts b/tests/relay-eager.test.ts index e8e30409c..22bbef697 100644 --- a/tests/relay-eager.test.ts +++ b/tests/relay-eager.test.ts @@ -10,6 +10,7 @@ import { relaySseEagerBounded, type EagerRelayHooks } from "../src/server/relay- import { createTranslatorBudget } from "../src/lib/translator-budget"; import type { RequestLogContext } from "../src/server/request-log"; +import { watchdogMs } from "./helpers/ci-watchdog"; const enc = new TextEncoder(); function sse(event: string): Uint8Array { @@ -290,7 +291,7 @@ describe("relaySseEagerBounded — inline payload rewrite (#864)", () => { await Promise.race([ done, new Promise((_, reject) => { - timeout = setTimeout(() => reject(new Error("relay cleanup timed out")), 2_000); + timeout = setTimeout(() => reject(new Error("relay cleanup timed out")), watchdogMs(2_000)); }), ]).finally(() => { if (timeout) clearTimeout(timeout); diff --git a/tests/server-auth.test.ts b/tests/server-auth.test.ts index 671baa67e..1a5f78fc7 100644 --- a/tests/server-auth.test.ts +++ b/tests/server-auth.test.ts @@ -44,6 +44,7 @@ import { configuredAdminToken } from "../src/lib/admin-secrets"; import { SYSTEM_RESTART_CAPABILITY_VERSION } from "../src/lib/system-restart-contract"; import { LOCAL_PROVIDER_RELOAD_CAPABILITY_VERSION } from "../src/lib/local-provider-reload-contract"; +import { watchdogMs } from "./helpers/ci-watchdog"; const previousApiToken = process.env.OPENCODEX_API_AUTH_TOKEN; const previousOpencodexHome = process.env.OPENCODEX_HOME; const originalGlobalFetch = globalThis.fetch; @@ -1207,7 +1208,7 @@ describe("server local API auth", () => { url.protocol = "ws:"; const ws = new WebSocket(url, { headers: { "x-opencodex-api-key": "local-secret", ...(headers ?? {}) } } as unknown as string[]); return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error("tier websocket timeout")), 5_000); + const timer = setTimeout(() => reject(new Error("tier websocket timeout")), watchdogMs(5_000)); ws.addEventListener("open", () => { ws.send(JSON.stringify({ type: "response.create", model, input: "hello" })); }, { once: true }); @@ -1441,7 +1442,7 @@ describe("server local API auth", () => { const sendFrame = async (model: string) => { const before = seen.length; const message = new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error(`sequential websocket timeout: ${model}`)), 5_000); + const timer = setTimeout(() => reject(new Error(`sequential websocket timeout: ${model}`)), watchdogMs(5_000)); const onMessage = (event: MessageEvent) => { const value = typeof event.data === "string" ? event.data : ""; if (!value.includes('"type":"response.completed"')) return; @@ -1711,7 +1712,7 @@ describe("server local API auth", () => { }, } as unknown as string[]); const wsFailure = new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error("websocket affinity timeout")), 1000); + const timer = setTimeout(() => reject(new Error("websocket affinity timeout")), watchdogMs(1000)); ws.addEventListener("open", () => { ws.send(JSON.stringify({ type: "response.create", model: "gpt-test", input: "hello" })); }, { once: true }); @@ -1799,7 +1800,7 @@ describe("server local API auth", () => { ws.addEventListener("error", () => reject(new Error("websocket failed to open")), { once: true }); }); const waitForTerminal = () => new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error("websocket terminal timeout")), 1000); + const timer = setTimeout(() => reject(new Error("websocket terminal timeout")), watchdogMs(1000)); const onMessage = (event: MessageEvent) => { const text = typeof event.data === "string" ? event.data : ""; if (text.includes('"type":"response.completed"')) { @@ -1875,7 +1876,7 @@ describe("server local API auth", () => { ws.addEventListener("error", () => reject(new Error("websocket failed to open")), { once: true }); }); const waitForTerminal = () => new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error("websocket terminal timeout")), 1000); + const timer = setTimeout(() => reject(new Error("websocket terminal timeout")), watchdogMs(1000)); const onMessage = (event: MessageEvent) => { const text = typeof event.data === "string" ? event.data : ""; if (text.includes('"type":"response.completed"')) {