ci(tests): scale timing watchdogs and per-test timeout for loaded CI runners

.github/workflows/ci.yml

@@ -493,9 +493,9 @@ jobs:
       # (`is_bun_runtime_crash`). The unsharded macOS control had no equivalent,
       # so the same crash that Linux absorbs failed the whole promotion here.
       #
-      # Keep the signature list in sync with `is_bun_runtime_crash` in
-      # scripts/ci/run-bun-test-batches.sh. An assertion failure still fails on
-      # the first attempt — only the crash signature is retried, exactly once.
+    # Keep the signature list in sync with `is_bun_runtime_crash` in
+    # scripts/ci/run-bun-test-batches.sh. An assertion failure still fails on
+    # the first attempt — only the crash signature is retried, exactly once.
       - name: Test
         run: |
           # GitHub Actions starts bash `run:` blocks with `-e`. Disable
@@ -504,7 +504,14 @@ jobs:
           set -uo pipefail
           suite_log="$(mktemp -t ocx-macos-suite.XXXXXX)"
           for attempt in 1 2; do
-            bun test --isolate tests 2>&1 | tee "$suite_log"
+            # --timeout: Bun's default 5s per-test ceiling is the recurring flake
+            # class on this loaded shared runner (real retry windows + server
+            # round-trips exceed 5s under contention; a 10s-floor in-test
+            # watchdog fired at 10.16s there). 60s keeps hangs bounded (the 30m
+            # job timeout is the outer backstop) while removing the timing
+            # flakes — assertions are untouched. Pairs with the 30s CI floor in
+            # tests/helpers/ci-watchdog.ts.
+            bun test --isolate --timeout 60000 tests 2>&1 | tee "$suite_log"
             suite_status="${PIPESTATUS[0]}"
             if [ "$suite_status" -eq 0 ]; then
               exit 0

scripts/ci/run-bun-test-batches.sh

@@ -106,7 +106,7 @@ run_test_once() {
   set +e
   timeout --signal=TERM --kill-after="${BATCH_KILL_GRACE_SECONDS}s" \
     "${BATCH_TIMEOUT_SECONDS}s" \
-    bun test --isolate "${files[@]}" 2>&1 | tee "$log_file"
+    bun test --isolate --timeout 60000 "${files[@]}" 2>&1 | tee "$log_file"
   status="${PIPESTATUS[0]}"
   set -e
 

tests/bridge-lifecycle.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from "bun:test";
 import { bridgeToResponsesSSE } from "../src/bridge";
 import type { AdapterEvent } from "../src/types";
 
+import { watchdogMs } from "./helpers/ci-watchdog";
 async function* replay(events: AdapterEvent[]): AsyncGenerator<AdapterEvent> {
   for (const event of events) yield event;
 }
@@ -127,7 +128,7 @@ describe("bridge stream lifecycle (RC1 / RC2)", () => {
         // 500ms, and a slow runner spent it. The stream is proven to close well
         // inside 1500ms, so the guard sits at the point where something is
         // genuinely wrong, and the test timeout gives the drift room.
-        new Promise<never>((_, reject) => setTimeout(() => reject(new Error("stall stream did not close")), 4_000)),
+        new Promise<never>((_, reject) => setTimeout(() => reject(new Error("stall stream did not close")), watchdogMs(4_000))),
       ]);
     } finally {
       await reader.cancel();

tests/ci-workflows.test.ts

@@ -173,7 +173,9 @@ describe("GitHub Actions hardening", () => {
     // only when the shared path filter says the entire expensive suite is out of
     // scope (for example a docs-site-only PR).
     const macosSteps = (ci.jobs?.["platform-macos"] as { steps?: { run?: string }[] })?.steps ?? [];
-    expect(macosSteps.some(step => step.run?.includes("bun test --isolate tests"))).toBe(true);
+    // The 60s per-test ceiling is part of the pinned shape: dropping it silently
+    // restores the timing-flake class this lane kept surfacing.
+    expect(macosSteps.some(step => step.run?.includes("bun test --isolate --timeout 60000 tests"))).toBe(true);
     expect(macosSteps.some(step => step.run?.includes("--shard"))).toBe(false);
 
     // The macOS leg retries ONLY a Bun runtime crash, and only once. Bun 1.3.14
@@ -183,7 +185,7 @@ describe("GitHub Actions hardening", () => {
     // `scripts/ci/run-bun-test-batches.sh`. Two ways to break this silently:
     // drop the crash-signature guard so an assertion failure gets retried into
     // green, or let the retry loop swallow a repeated crash. Pin both.
-    const macosTestRun = macosSteps.find(step => step.run?.includes("bun test --isolate tests"))?.run ?? "";
+    const macosTestRun = macosSteps.find(step => step.run?.includes("bun test --isolate --timeout 60000 tests"))?.run ?? "";
     // Actions invokes multiline `run:` blocks with `bash -e`. The retry loop
     // must disable errexit before the crash-prone command or exit 133 aborts
     // the step before PIPESTATUS can be inspected and the retry can run.

tests/helpers/ci-watchdog.ts

@@ -0,0 +1,17 @@
+/**
+ * CI-scaled test watchdogs.
+ *
+ * Several tests race a real local server round-trip against a short in-test
+ * watchdog (`setTimeout(..., reject)`). Locally 1-2 s is generous, but the
+ * unsharded GitHub macOS runner runs the whole suite in one pool under heavy
+ * CPU contention and these watchdogs were the recurring flake class there
+ * (server-auth WS terminal 1 s, provider-option fixture WS 2 s, …). Observed
+ * runner stalls exceed 10 s on that lane (a 10 s-floor watchdog fired at
+ * 10.16 s), so the CI floor is 30 s: the watchdog exists to bound a genuinely
+ * hung test, not to assert latency. Local behaviour is unchanged. Bun's own
+ * per-test timeout (`--timeout`, 60 s on CI) would pre-empt a 30 s watchdog,
+ * so the lane timeout and this floor move together.
+ */
+export function watchdogMs(base: number): number {
+  return process.env.CI === "true" ? Math.max(base, 30_000) : base;
+}

tests/native-profile-drain-server.test.ts

@@ -24,6 +24,7 @@ import type { OcxConfig } from "../src/types";
 import { fakeChatGptJwt } from "./helpers/fake-chatgpt-jwt";
 import { ownedServiceHomeInspection } from "./helpers/owned-service-home-inspection";
 
+import { watchdogMs } from "./helpers/ci-watchdog";
 /** These cases sandbox CODEX_HOME/OPENCODEX_HOME, so the installed service is not their evidence. */
 const inspectNativeCodexOwnership = ownedServiceHomeInspection("native-profile drain server test");
 
@@ -96,7 +97,7 @@ describe("native main profile scoped server admission", () => {
       autoSwitchThreshold: 0,
     } as OcxConfig);
     const waitForFrame = (ws: WebSocket, needle: string) => new Promise<string>((resolve, reject) => {
-      const timer = setTimeout(() => reject(new Error(`websocket timeout waiting for ${needle}`)), 2_000);
+      const timer = setTimeout(() => reject(new Error(`websocket timeout waiting for ${needle}`)), watchdogMs(2_000));
       const onMessage = (event: MessageEvent) => {
         const text = typeof event.data === "string" ? event.data : "";
         if (!text.includes(needle)) return;
@@ -217,7 +218,7 @@ describe("native main profile scoped server admission", () => {
       url.protocol = "ws:";
       const ws = new WebSocket(url, { headers } as unknown as string[]);
       await new Promise<void>((resolve, reject) => {
-        const timer = setTimeout(() => reject(new Error("sideband echo timeout")), 5_000);
+        const timer = setTimeout(() => reject(new Error("sideband echo timeout")), watchdogMs(5_000));
         ws.addEventListener("open", () => ws.send("ping"), { once: true });
         ws.addEventListener("message", event => {
           if (String(event.data) !== "echo:ping") return;
@@ -417,7 +418,7 @@ describe("native main profile scoped server admission", () => {
 
     try {
       await new Promise<void>((resolve, reject) => {
-        const timer = setTimeout(() => reject(new Error("sideband websocket open timeout")), 2_000);
+        const timer = setTimeout(() => reject(new Error("sideband websocket open timeout")), watchdogMs(2_000));
         client.addEventListener("open", () => {
           clearTimeout(timer);
           resolve();

tests/openai-api-virtual-models.test.ts

@@ -17,6 +17,7 @@ import { saveConfig } from "../src/config";
 import { startServer } from "../src/server";
 import { usageLogPath } from "../src/usage/log";
 
+import { watchdogMs } from "./helpers/ci-watchdog";
 const moduleOriginalFetch = globalThis.fetch;
 const moduleOriginalHome = process.env.OPENCODEX_HOME;
 afterEach(() => {
@@ -440,7 +441,7 @@ describe("OpenAI API Pro transport identities", () => {
       url.protocol = "ws:";
       const ws = new WebSocket(url);
       return new Promise<string>((resolve, reject) => {
-        const timer = setTimeout(() => reject(new Error("OpenAI API Pro websocket timeout")), 2000);
+        const timer = setTimeout(() => reject(new Error("OpenAI API Pro websocket timeout")), watchdogMs(2000));
         ws.addEventListener("open", () => {
           ws.send(JSON.stringify({ type: "response.create", model, input: "hello", reasoning: { effort: "high" } }));
         }, { once: true });

tests/openai-provider-option-e2e.test.ts

@@ -17,6 +17,7 @@ import {
 import { homedir, tmpdir } from "node:os";
 import { join, relative } from "node:path";
 
+import { watchdogMs } from "./helpers/ci-watchdog";
 type Capture = {
   url: string;
   method: string;
@@ -346,7 +347,7 @@ describe("OpenAI provider-option integration spine", () => {
       });
       const wsTurn = (model: string) => new Promise<Capture>((resolve, reject) => {
         const before = captures.length;
-        const timer = setTimeout(() => reject(new Error(`fixture websocket timeout: ${model}`)), 2_000);
+        const timer = setTimeout(() => reject(new Error(`fixture websocket timeout: ${model}`)), watchdogMs(2_000));
         const onMessage = (event: MessageEvent) => {
           if (!String(event.data).includes('"type":"response.completed"')) return;
           clearTimeout(timer);

tests/relay-eager.test.ts

@@ -10,6 +10,7 @@ import { relaySseEagerBounded, type EagerRelayHooks } from "../src/server/relay-
 import { createTranslatorBudget } from "../src/lib/translator-budget";
 import type { RequestLogContext } from "../src/server/request-log";
 
+import { watchdogMs } from "./helpers/ci-watchdog";
 const enc = new TextEncoder();
 
 function sse(event: string): Uint8Array {
@@ -290,7 +291,7 @@ describe("relaySseEagerBounded — inline payload rewrite (#864)", () => {
     await Promise.race([
       done,
       new Promise<never>((_, reject) => {
-        timeout = setTimeout(() => reject(new Error("relay cleanup timed out")), 2_000);
+        timeout = setTimeout(() => reject(new Error("relay cleanup timed out")), watchdogMs(2_000));
       }),
     ]).finally(() => {
       if (timeout) clearTimeout(timeout);

tests/server-auth.test.ts

@@ -44,6 +44,7 @@ import { configuredAdminToken } from "../src/lib/admin-secrets";
 import { SYSTEM_RESTART_CAPABILITY_VERSION } from "../src/lib/system-restart-contract";
 import { LOCAL_PROVIDER_RELOAD_CAPABILITY_VERSION } from "../src/lib/local-provider-reload-contract";
 
+import { watchdogMs } from "./helpers/ci-watchdog";
 const previousApiToken = process.env.OPENCODEX_API_AUTH_TOKEN;
 const previousOpencodexHome = process.env.OPENCODEX_HOME;
 const originalGlobalFetch = globalThis.fetch;
@@ -1207,7 +1208,7 @@ describe("server local API auth", () => {
       url.protocol = "ws:";
       const ws = new WebSocket(url, { headers: { "x-opencodex-api-key": "local-secret", ...(headers ?? {}) } } as unknown as string[]);
       return new Promise<string>((resolve, reject) => {
-        const timer = setTimeout(() => reject(new Error("tier websocket timeout")), 5_000);
+        const timer = setTimeout(() => reject(new Error("tier websocket timeout")), watchdogMs(5_000));
         ws.addEventListener("open", () => {
           ws.send(JSON.stringify({ type: "response.create", model, input: "hello" }));
         }, { once: true });
@@ -1441,7 +1442,7 @@ describe("server local API auth", () => {
       const sendFrame = async (model: string) => {
         const before = seen.length;
         const message = new Promise<string>((resolve, reject) => {
-          const timer = setTimeout(() => reject(new Error(`sequential websocket timeout: ${model}`)), 5_000);
+          const timer = setTimeout(() => reject(new Error(`sequential websocket timeout: ${model}`)), watchdogMs(5_000));
           const onMessage = (event: MessageEvent) => {
             const value = typeof event.data === "string" ? event.data : "";
             if (!value.includes('"type":"response.completed"')) return;
@@ -1711,7 +1712,7 @@ describe("server local API auth", () => {
         },
       } as unknown as string[]);
       const wsFailure = new Promise<string>((resolve, reject) => {
-        const timer = setTimeout(() => reject(new Error("websocket affinity timeout")), 1000);
+        const timer = setTimeout(() => reject(new Error("websocket affinity timeout")), watchdogMs(1000));
         ws.addEventListener("open", () => {
           ws.send(JSON.stringify({ type: "response.create", model: "gpt-test", input: "hello" }));
         }, { once: true });
@@ -1799,7 +1800,7 @@ describe("server local API auth", () => {
         ws.addEventListener("error", () => reject(new Error("websocket failed to open")), { once: true });
       });
       const waitForTerminal = () => new Promise<void>((resolve, reject) => {
-        const timer = setTimeout(() => reject(new Error("websocket terminal timeout")), 1000);
+        const timer = setTimeout(() => reject(new Error("websocket terminal timeout")), watchdogMs(1000));
         const onMessage = (event: MessageEvent) => {
           const text = typeof event.data === "string" ? event.data : "";
           if (text.includes('"type":"response.completed"')) {
@@ -1875,7 +1876,7 @@ describe("server local API auth", () => {
         ws.addEventListener("error", () => reject(new Error("websocket failed to open")), { once: true });
       });
       const waitForTerminal = () => new Promise<void>((resolve, reject) => {
-        const timer = setTimeout(() => reject(new Error("websocket terminal timeout")), 1000);
+        const timer = setTimeout(() => reject(new Error("websocket terminal timeout")), watchdogMs(1000));
         const onMessage = (event: MessageEvent) => {
           const text = typeof event.data === "string" ? event.data : "";
           if (text.includes('"type":"response.completed"')) {