From 136a76e74968bef0a270340ff486003e11263aa9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 22:46:15 +0200 Subject: [PATCH 01/35] chore: stage maintainer port for pr 1039 --- .github/workflows/maint-1039-port.yml | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/maint-1039-port.yml diff --git a/.github/workflows/maint-1039-port.yml b/.github/workflows/maint-1039-port.yml new file mode 100644 index 0000000000..c11e051014 --- /dev/null +++ b/.github/workflows/maint-1039-port.yml @@ -0,0 +1,38 @@ +name: Maintainer port PR 1039 + +on: + push: + branches: + - maint/1039-host-circuit + +permissions: + contents: write + +jobs: + port: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Port original PR commit + shell: bash + run: | + set -euxo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git fetch origin pull/1039/head:refs/remotes/origin/pr-1039 + if git cherry-pick 199382912964810cb16dc3995f3ad76b831faaf0; then + git push origin HEAD:maint/1039-host-circuit + else + echo '--- CHERRY-PICK CONFLICTS ---' + git status --short + git diff --name-only --diff-filter=U + for f in $(git diff --name-only --diff-filter=U); do + echo "--- $f ---" + git checkout --conflict=merge -- "$f" || true + grep -n -C 8 -E '^(<<<<<<<|=======|>>>>>>>)' "$f" || true + done + exit 1 + fi From 9722486a19b97c50e58dbdf9c756ea0b590b5ed4 Mon Sep 17 00:00:00 2001 From: luvs01 <27862058+luvs01@users.noreply.github.com> Date: Wed, 5 Aug 2026 10:04:29 +0900 Subject: [PATCH 02/35] feat(codex): preserve opt-in upstream host circuit --- .../ja/reference/configuration/providers.md | 3 +- .../ko/reference/configuration/providers.md | 3 +- .../docs/reference/configuration/providers.md | 3 +- .../ru/reference/configuration/providers.md | 3 +- .../reference/configuration/providers.md | 3 +- src/codex/upstream-host-health.ts | 320 ++++++++++++++++-- src/server/responses/compact.ts | 122 ++++++- src/server/responses/core.ts | 132 +++++++- src/types.ts | 5 + tests/responses-compaction-routing.test.ts | 58 ++++ tests/upstream-host-circuit.test.ts | 174 ++++++++++ 11 files changed, 769 insertions(+), 57 deletions(-) create mode 100644 tests/upstream-host-circuit.test.ts diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 82b5d2cb6f..c782ee499d 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -22,7 +22,8 @@ description: プロバイダー エントリ、認証、エンドポイント、 | `autoSwitchThreshold?` | `number` | `80` | 使用量ベースのプロアクティブ切り替えしきい値。`quota` は紐付け済み/未紐付けタスクの次のリクエストを再評価でき、`fill-first` は未紐付け割り当ての使い切り基準としてのみ使用し、通常の `round-robin` 選択は使用しません。既知の 5 時間、週次、30 日 quota window の最大スコアを使います。`0` は使用量ベースの切り替えだけを無効にし、未紐付け割り当てや障害回復は無効にしません。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新規/未紐付け Codex リクエストの割り当て戦略。live な `(parent thread id, quota scope)` affinity がなければ未紐付けで、プロキシ再起動や affinity リセット後は既存の表示タスクも未紐付けになり得ます。`quota` はアクティブアカウントがなければ既知 usage 最小の適格アカウントを選び、適格なアクティブアカウントが `autoSwitchThreshold` 未満なら維持します。しきい値到達後は、未紐付けリクエストまたは紐付け済みタスクの次のリクエストを usage の低い適格アカウントへ移せます。`round-robin` は未紐付けリクエストを均等分散し、`fill-first` は cooldown、使用不可、または drain threshold までアクティブアカウントへ割り当てます。 | | `accountPoolStickyLimit?` | `number` | `1` | 1 回の round-robin 選択で次へ進む前に保持する新規/未紐付けタスク割り当て数。カウンターは上流の成功後ではなくタスクの紐付け時に増えます。範囲 1–100。`accountPoolStrategy` が `round-robin` のときのみ。 | -| `upstreamFailoverThreshold?` | `number` | `3` |今後の新しいセッションがフェイルオーバーする前に一時的なエラーが連続して発生する。 `0` を無効に設定します。実証済みの接続前DNS/TCP到達不能障害はprovider-host単位で記録され、アカウントの健全性、クールダウン、スレッド/セッションの親和性、アクティブアカウントの選択、Poolルーティングには影響せず、この閾値にもカウントされません。 | +| `upstreamFailoverThreshold?` | `number` | `3` |今後の新しいセッションがフェイルオーバーする前に一時的なエラーが連続して発生する。 `0` を無効に設定します。通常のResponses送信とネイティブcompact送信では、実証済みの接続前DNS/TCP到達不能障害はprovider-host単位で記録され、アカウントの健全性、クールダウン、スレッド/セッションの親和性、アクティブアカウントの選択、Poolルーティングには影響せず、この閾値にもカウントされません。 | +| `upstreamHostCircuitThreshold?` | `number` | `0` | ネイティブOpenAI forwardのResponses送信とcompact送信で、実証済みの接続前DNS/TCP障害に適用するオプトインのサーキットしきい値です。`0`で無効、`1`〜`20`ではその回数の終端論理リクエストが失敗するとprovider-originを30秒間遮断します。遮断中はアカウント選択やupstream送信の前に`Retry-After`付き`503`を返し、時間経過後はhalf-openリクエストを1件だけ許可します。タイムアウトとHTTP応答は数えず、HTTP応答が1件でもあれば回路を閉じます。 | | `modelCacheTtlMs?` | `number` | `300000` |プロバイダーごとの `/models` キャッシュの鮮度ウィンドウ。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic プロンプト キャッシュ ポリシー: 無効、5 分間の一時的、または 1 時間の延長。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` |オフ |オプションのプロアクティブな OAuth 更新および Codex アカウントのウォームアップ ポリシー。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index e10536e210..6b9d040416 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -22,7 +22,8 @@ description: 공급자 항목, 인증, 엔드포인트, 모델 카탈로그, 할 | `autoSwitchThreshold?` | `number` | `80` | 사용량 기반 선제 전환 임계값입니다. `quota`는 바인딩된 작업과 바인딩 없는 작업의 다음 요청을 모두 재평가할 수 있고, `fill-first`는 바인딩 없는 작업 배정의 소진 기준으로만 사용하며, 기본 `round-robin` 선택은 이 값을 사용하지 않습니다. 알려진 5시간, 주간, 30일 quota window 중 가장 높은 점수를 씁니다. `0`은 사용량 기반 전환만 끄며 바인딩 없는 작업 배정이나 실패 복구는 끄지 않습니다. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 새 작업/바인딩 없는 Codex 요청의 계정 배정 전략입니다. `(parent thread id, quota scope)`의 live affinity가 없으면 바인딩 없는 요청이며, 프록시 재시작이나 affinity 초기화 뒤에는 기존에 보이던 작업도 바인딩이 없어질 수 있습니다. `quota`는 활성 계정이 없을 때 알려진 usage가 가장 낮은 적격 계정을 선택하고, 적격 활성 계정이 `autoSwitchThreshold` 미만이면 유지합니다. 임계값 도달 뒤에는 바인딩 없는 요청이나 바인딩된 작업의 다음 요청을 usage가 더 낮은 적격 계정으로 옮길 수 있습니다. `round-robin`은 바인딩 없는 요청을 균등 분배하고, `fill-first`는 cooldown, 사용 불가 또는 drain threshold까지 활성 계정에 배정합니다. | | `accountPoolStickyLimit?` | `number` | `1` | 한 round-robin 선택이 다음으로 넘어가기 전에 유지하는 새 작업/바인딩 없는 작업 배정 수입니다. 카운터는 업스트림 성공 뒤가 아니라 작업을 바인딩할 때 증가합니다. 범위 1–100이며 `accountPoolStrategy`가 `round-robin`일 때만 적용됩니다. | -| `upstreamFailoverThreshold?` | `number` | `3` | 연속된 일시적 실패가 이 횟수에 도달하면 이후 새 세션은 failover됩니다. `0`으로 두면 비활성화됩니다. 입증된 연결 전 DNS/TCP 도달 불가 실패는 provider-host 범위로 기록되며 계정 상태, 쿨다운, 스레드/세션 선호도, 활성 계정 선택 또는 Pool 라우팅에 영향을 주지 않고 이 임계값에도 집계되지 않습니다. | +| `upstreamFailoverThreshold?` | `number` | `3` | 연속된 일시적 실패가 이 횟수에 도달하면 이후 새 세션은 failover됩니다. `0`으로 두면 비활성화됩니다. 일반 Responses와 네이티브 compact 전송에서 입증된 연결 전 DNS/TCP 도달 불가 실패는 provider-host 범위로 기록되며 계정 상태, 쿨다운, 스레드/세션 선호도, 활성 계정 선택 또는 Pool 라우팅에 영향을 주지 않고 이 임계값에도 집계되지 않습니다. | +| `upstreamHostCircuitThreshold?` | `number` | `0` | 네이티브 OpenAI forward Responses와 compact 전송에서 입증된 연결 전 DNS/TCP 실패에 적용하는 선택적 회로 차단 임계값입니다. `0`은 비활성화하며, `1`~`20`은 이 횟수만큼 최종 논리 요청이 실패하면 provider-origin을 30초 동안 차단합니다. 차단 중에는 계정 선택이나 업스트림 전송 전에 `Retry-After`가 포함된 `503`을 반환하고, 시간이 지나면 반개방 요청 하나만 허용합니다. 타임아웃과 HTTP 응답은 집계하지 않으며, HTTP 응답이 하나라도 오면 회로를 닫습니다. | | `modelCacheTtlMs?` | `number` | `300000` | 공급자별 `/models` 캐시의 최신성 창입니다. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 프롬프트 캐시 정책입니다. 비활성, 5분짜리 임시, 1시간짜리 확장 중 하나입니다. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 꺼짐 | 선택적 선제 OAuth 갱신과 Codex 계정 워밍업 정책입니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 4289d5aec6..33eb016328 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -24,7 +24,8 @@ authenticated. | `autoSwitchThreshold?` | `number` | `80` | Usage threshold for proactive switching. `quota` can re-evaluate both bound and unbound tasks on their next request; `fill-first` uses it only as the drain point for unbound assignment; normal `round-robin` selection does not use it. The score uses the hottest known 5h, weekly, or 30d quota window. `0` disables usage-based proactive switching only, not unbound assignment or failure recovery. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Assignment strategy for new/unbound Codex requests. A request is unbound when it has no live (parent thread id, quota scope) affinity; a visible existing task can become unbound after proxy restart or affinity reset. `quota` picks the lowest-usage eligible account when no active account exists, keeps an eligible active account below `autoSwitchThreshold`, and after the threshold may move an unbound request or proactively rebind a bound task to a lower-usage eligible account. `round-robin` distributes unbound requests evenly; `fill-first` keeps assigning unbound requests to the active account until cooldown, unavailability, or the configured drain threshold. | | `accountPoolStickyLimit?` | `number` | `1` | New/unbound task assignments retained on one round-robin selection before advancing; the counter advances when a task is bound, not after an upstream success. Range 1–100. | -| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive transient failures before future new sessions fail over. Set `0` to disable. Proven pre-connection DNS/TCP reachability failures are tracked at the provider-host level: they never affect account health, cooldowns, thread/session affinity, active-account selection, or Pool routing, and never count toward this threshold. | +| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive transient failures before future new sessions fail over. Set `0` to disable. For regular Responses and native compact sends, proven pre-connection DNS/TCP reachability failures are tracked at the provider-host level: they never affect account health, cooldowns, thread/session affinity, active-account selection, or Pool routing, and never count toward this threshold. | +| `upstreamHostCircuitThreshold?` | `number` | `0` | Opt-in circuit threshold for proven pre-connection DNS/TCP failures on native OpenAI forward Responses and compact sends. `0` disables it; `1`–`20` opens a 30-second provider-origin cooldown after that many terminal logical requests. While open, requests receive `503` with `Retry-After` before account selection or upstream send; after cooldown, one half-open request is admitted. Timeouts and HTTP responses never count, and any HTTP response closes the circuit. | | `modelCacheTtlMs?` | `number` | `300000` | Freshness window for the per-provider `/models` cache. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic prompt-cache policy: disabled, 5-minute ephemeral, or 1-hour extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Optional proactive OAuth refresh and Codex-account warmup policy. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index 0cde31c851..969bae8283 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -23,7 +23,8 @@ description: Записи провайдеров, аутентификация, | `autoSwitchThreshold?` | `number` | `80` | Порог проактивного переключения по использованию. `quota` может повторно оценить следующий запрос как привязанной, так и непривязанной задачи; `fill-first` использует его только как точку исчерпания для непривязанных назначений; обычный `round-robin` его не использует. Оценка берёт самое горячее из окон 5 часов, недели и 30 дней. `0` отключает только переключение по использованию, но не назначение непривязанных задач и не восстановление после сбоев. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Стратегия назначения для новых/непривязанных запросов Codex. Запрос непривязан, если у него нет live affinity `(parent thread id, quota scope)`; видимая существующая задача может стать непривязанной после перезапуска прокси или сброса affinity. `quota` выбирает подходящий аккаунт с наименьшим известным usage, когда активного аккаунта нет, сохраняет подходящий активный аккаунт ниже `autoSwitchThreshold`, а после порога может перевести непривязанный запрос или следующий запрос привязанной задачи на подходящий аккаунт с меньшим usage. `round-robin` равномерно распределяет непривязанные запросы; `fill-first` назначает их активному аккаунту до cooldown, недоступности или порога исчерпания. | | `accountPoolStickyLimit?` | `number` | `1` | Число назначений новых/непривязанных задач на одном выборе round-robin перед переходом дальше. Счётчик растёт при привязке задачи, а не после успеха upstream. Диапазон 1–100; только при `accountPoolStrategy` = `round-robin`. | -| `upstreamFailoverThreshold?` | `number` | `3` | Сколько подряд transient failure допустить, прежде чем новые сессии начнут делать failover. `0` отключает эту логику. Доказанные ошибки доступности DNS/TCP до соединения учитываются на уровне пары «провайдер, хост» и не влияют на здоровье аккаунта, кулдауны, привязку потока/сессии, выбор активного аккаунта или маршрутизацию пула, а также не учитываются в этом пороге. | +| `upstreamFailoverThreshold?` | `number` | `3` | Сколько подряд transient failure допустить, прежде чем новые сессии начнут делать failover. `0` отключает эту логику. Для обычных Responses-запросов и нативных compact-отправок доказанные ошибки доступности DNS/TCP до соединения учитываются на уровне пары «провайдер, хост» и не влияют на здоровье аккаунта, кулдауны, привязку потока/сессии, выбор активного аккаунта или маршрутизацию пула, а также не учитываются в этом пороге. | +| `upstreamHostCircuitThreshold?` | `number` | `0` | Опциональный порог circuit breaker для доказанных DNS/TCP-сбоев до соединения в нативных OpenAI forward Responses- и compact-отправках. `0` отключает его; `1`–`20` открывает 30-секундный cooldown для provider-origin после такого числа завершившихся логических запросов. Пока circuit открыт, до выбора аккаунта и upstream-отправки возвращается `503` с `Retry-After`; после cooldown допускается один half-open запрос. Таймауты и HTTP-ответы не учитываются, а любой HTTP-ответ закрывает circuit. | | `modelCacheTtlMs?` | `number` | `300000` | Окно свежести для кэша `/models` на уровне провайдера. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Политика prompt-cache Anthropic: отключено, 5-минутный ephemeral или 1-часовой extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Необязательная политика proactive OAuth refresh и warmup'а аккаунтов Codex. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index 00a34e1262..cf1c427502 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -22,7 +22,8 @@ description: 提供者条目、身份验证、端点、模型目录、配额、 | `autoSwitchThreshold?` | `number` | `80` | 基于用量的主动切换阈值。`quota` 可在下一次请求中重新评估已绑定和未绑定任务；`fill-first` 仅把它用作未绑定分配的耗尽点；正常 `round-robin` 不使用它。分数取已知 5 小时、周或 30 天 quota window 的最高值。`0` 只关闭基于用量的主动切换，不关闭未绑定任务分配或故障恢复。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新建/未绑定 Codex 请求的分配策略。没有 live `(parent thread id, quota scope)` affinity 的请求属于未绑定；代理重启或 affinity 重置后，已有可见任务也可能未绑定。`quota` 在没有活跃账号时选择已知 usage 最低的合格账号；活跃账号合格且低于 `autoSwitchThreshold` 时继续使用；达到阈值后，可把未绑定请求或已绑定任务的下一次请求切换到 usage 更低的合格账号。`round-robin` 均匀分配未绑定请求；`fill-first` 在 cooldown、不可用或耗尽阈值前持续分配给活跃账号。 | | `accountPoolStickyLimit?` | `number` | `1` | 一次 round-robin 选择在推进前保留的新建/未绑定任务分配数。计数在任务绑定时增加，而不是在上游成功后增加。范围 1–100；仅当 `accountPoolStrategy` 为 `round-robin` 时生效。 | -| `upstreamFailoverThreshold?` | `number` | `3` | 连续发生多少次瞬态故障后，后续新会话会切换到备用上游。设为 `0` 可禁用。已证明的连接前 DNS/TCP 不可达故障按 provider-host 粒度记录，不影响账户健康、冷却、线程/会话亲和性、活动账户选择或 Pool 路由，也不会计入此阈值；未确认的失败仍归属账户。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 连续发生多少次瞬态故障后，后续新会话会切换到备用上游。设为 `0` 可禁用。对于常规 Responses 和原生 compact 发送，已证明的连接前 DNS/TCP 不可达故障按 provider-host 粒度记录，不影响账户健康、冷却、线程/会话亲和性、活动账户选择或 Pool 路由，也不会计入此阈值；未确认的失败仍归属账户。 | +| `upstreamHostCircuitThreshold?` | `number` | `0` | 原生 OpenAI forward Responses 与 compact 发送的可选断路器阈值，仅统计已证明的连接前 DNS/TCP 故障。`0` 表示禁用；`1`–`20` 表示在这么多个终止逻辑请求失败后，对 provider-origin 冷却 30 秒。断路期间会在账户选择和上游发送之前返回带 `Retry-After` 的 `503`；冷却结束后只允许一个半开请求。超时和 HTTP 响应不计数，任意 HTTP 响应都会关闭断路器。 | | `modelCacheTtlMs?` | `number` | `300000` | 每个提供者 `/models` 缓存的新鲜度窗口。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 提示缓存策略：禁用、5 分钟临时缓存，或 1 小时扩展缓存。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 关闭 | 可选的主动 OAuth 刷新与 Codex 账户预热策略。 | diff --git a/src/codex/upstream-host-health.ts b/src/codex/upstream-host-health.ts index 0ca863b0b6..a8ac8f267e 100644 --- a/src/codex/upstream-host-health.ts +++ b/src/codex/upstream-host-health.ts @@ -1,67 +1,325 @@ /** - * Observational ledger for pre-connection upstream reachability failures, - * keyed by (provider, host). Records ONLY — no circuit breaker, no admission - * change (issue #914 scope). Rotation decisions stay with account health; - * this ledger exists so a host-wide outage is visible as host-wide. + * Provider-origin reachability health and the optional logical-request circuit. * - * Retention: bounded at 128 entries; on overflow the stalest entries by - * last-touch are pruned before insert, and failure timestamps older than the - * window are reconciled away on the next record — repeated provider/base-URL - * churn cannot grow the map for the process lifetime. + * Issue #914 owns the observational ledger. The circuit is deliberately opt-in: + * a threshold of 0 preserves the ledger-only behavior. Only proven + * `connect_neutral` failures may settle a lease as host failure; timeout, reset, + * HTTP, redirect, authentication, and local failures remain outside this module. */ export const UPSTREAM_HOST_HEALTH_MAX_ENTRIES = 128; export const UPSTREAM_HOST_FAILURE_WINDOW_MS = 10 * 60_000; +export const UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS = 30_000; +export const UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD = 20; export type UpstreamHostHealthEntry = { consecutiveFailures: number; lastFailureAt: number; lastFailureCode?: string; + cooldownUntil?: number; /** Recency marker for stalest-first pruning (not health semantics). */ lastTouch: number; }; -const hostHealth = new Map(); +/** Opaque ownership token for one admitted logical request. */ +export type UpstreamHostAdmissionLease = Readonly<{ + key: string; + leaseId: symbol; + generation: number; + halfOpen: boolean; +}>; + +export type UpstreamHostAdmission = + | { kind: "admitted"; lease: UpstreamHostAdmissionLease | null } + | { kind: "blocked"; retryAfterSeconds: number }; + +type InternalUpstreamHostHealth = UpstreamHostHealthEntry & { + generation: number; + activeLeaseIds: Set; + halfOpenLeaseId?: symbol; + /** True only when the entry is owned by opt-in circuit admissions. */ + circuitManaged: boolean; +}; + +const hostHealth = new Map(); +let nextGenerationValue = 0; export function upstreamHostHealthKey(provider: string, host: string): string { return `${provider}|${host.toLowerCase()}`; } -function pruneForInsert(): void { - if (hostHealth.size < UPSTREAM_HOST_HEALTH_MAX_ENTRIES) return; - const entries = [...hostHealth.entries()].sort((a, b) => a[1].lastTouch - b[1].lastTouch); - for (const [key] of entries) { - if (hostHealth.size < UPSTREAM_HOST_HEALTH_MAX_ENTRIES) return; +export function normalizeUpstreamHostCircuitThreshold(value: unknown): number { + if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) return 0; + return Math.min(value, UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD); +} + +function nextGeneration(): number { + nextGenerationValue = nextGenerationValue >= Number.MAX_SAFE_INTEGER ? 1 : nextGenerationValue + 1; + return nextGenerationValue; +} + +function snapshot(entry: InternalUpstreamHostHealth): UpstreamHostHealthEntry { + return { + consecutiveFailures: entry.consecutiveFailures, + lastFailureAt: entry.lastFailureAt, + lastTouch: entry.lastTouch, + ...(entry.lastFailureCode !== undefined ? { lastFailureCode: entry.lastFailureCode } : {}), + ...(entry.cooldownUntil !== undefined ? { cooldownUntil: entry.cooldownUntil } : {}), + }; +} + +function newEntry(now: number, circuitManaged: boolean): InternalUpstreamHostHealth { + return { + consecutiveFailures: 0, + lastFailureAt: 0, + lastTouch: now, + generation: nextGeneration(), + activeLeaseIds: new Set(), + circuitManaged, + }; +} + +function advanceGeneration(entry: InternalUpstreamHostHealth): void { + entry.generation = nextGeneration(); + entry.activeLeaseIds.clear(); + delete entry.halfOpenLeaseId; +} + +function removeExpiredUnleased(now: number): void { + for (const [key, entry] of hostHealth) { + if (entry.activeLeaseIds.size > 0) continue; + if ( + entry.consecutiveFailures === 0 + || (entry.cooldownUntil === undefined + && now - entry.lastFailureAt > UPSTREAM_HOST_FAILURE_WINDOW_MS) + ) { + hostHealth.delete(key); + } + } +} + +function oldestUnleasedKey(now: number): string | undefined { + let preferred: [string, number] | undefined; + let cooling: [string, number] | undefined; + for (const [key, entry] of hostHealth) { + if (entry.activeLeaseIds.size > 0) continue; + const candidate: [string, number] = [key, entry.lastTouch]; + if (entry.cooldownUntil !== undefined && entry.cooldownUntil > now) { + if (!cooling || candidate[1] < cooling[1]) cooling = candidate; + } else if (!preferred || candidate[1] < preferred[1]) { + preferred = candidate; + } + } + return preferred?.[0] ?? cooling?.[0]; +} + +function pruneTo(limit: number, now: number): void { + removeExpiredUnleased(now); + while (hostHealth.size > limit) { + const key = oldestUnleasedKey(now); + if (!key) return; // Active leases may temporarily exceed the retention cap. + hostHealth.delete(key); + } +} + +function makeRoom(now: number): void { + pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES - 1, now); +} + +function issueLease( + key: string, + entry: InternalUpstreamHostHealth, + halfOpen: boolean, + now: number, +): UpstreamHostAdmissionLease { + const leaseId = Symbol(halfOpen ? "upstream-host-half-open" : "upstream-host-admission"); + entry.activeLeaseIds.add(leaseId); + entry.lastTouch = now; + if (halfOpen) entry.halfOpenLeaseId = leaseId; + return { key, leaseId, generation: entry.generation, halfOpen }; +} + +function matchingEntry(lease: UpstreamHostAdmissionLease): InternalUpstreamHostHealth | null { + const entry = hostHealth.get(lease.key); + if (!entry || entry.generation !== lease.generation || !entry.activeLeaseIds.has(lease.leaseId)) { + return null; + } + if (lease.halfOpen && entry.halfOpenLeaseId !== lease.leaseId) return null; + return entry; +} + +function settleLease(entry: InternalUpstreamHostHealth, lease: UpstreamHostAdmissionLease): void { + entry.activeLeaseIds.delete(lease.leaseId); + if (entry.halfOpenLeaseId === lease.leaseId) delete entry.halfOpenLeaseId; +} + +/** + * Admit one logical request. A disabled threshold returns a null lease and has + * byte-for-byte compatible call-site behavior with the observational ledger. + */ +export function acquireUpstreamHostAdmission( + key: string, + thresholdValue: unknown, + now = Date.now(), +): UpstreamHostAdmission { + const threshold = normalizeUpstreamHostCircuitThreshold(thresholdValue); + if (threshold === 0) return { kind: "admitted", lease: null }; + + pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); + let entry = hostHealth.get(key); + if ( + entry?.activeLeaseIds.size === 0 + && entry.cooldownUntil === undefined + && entry.consecutiveFailures > 0 + && now - entry.lastFailureAt > UPSTREAM_HOST_FAILURE_WINDOW_MS + ) { hostHealth.delete(key); + entry = undefined; + } + if (!entry) { + makeRoom(now); + entry = newEntry(now, true); + hostHealth.set(key, entry); + } else if (!entry.circuitManaged) { + // Observational history predating opt-in admission must not count toward + // opening the circuit. Start a fresh fenced generation. + entry.consecutiveFailures = 0; + entry.lastFailureAt = 0; + entry.circuitManaged = true; + delete entry.lastFailureCode; + delete entry.cooldownUntil; + advanceGeneration(entry); } + + if (entry.cooldownUntil !== undefined) { + if (entry.cooldownUntil > now) { + return { + kind: "blocked", + retryAfterSeconds: Math.max(1, Math.ceil((entry.cooldownUntil - now) / 1_000)), + }; + } + if (entry.halfOpenLeaseId !== undefined) { + return { kind: "blocked", retryAfterSeconds: 1 }; + } + advanceGeneration(entry); + return { kind: "admitted", lease: issueLease(key, entry, true, now) }; + } + + return { kind: "admitted", lease: issueLease(key, entry, false, now) }; } +/** Release an admitted request without recording transport evidence. */ +export function releaseUpstreamHostAdmission( + lease: UpstreamHostAdmissionLease | null | undefined, + now = Date.now(), +): boolean { + if (!lease) return false; + const entry = matchingEntry(lease); + if (!entry) return false; + settleLease(entry, lease); + entry.lastTouch = now; + if (entry.activeLeaseIds.size === 0 && entry.consecutiveFailures === 0) { + hostHealth.delete(lease.key); + } + pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); + return true; +} + +/** Record one terminal logical `connect_neutral` failure. */ export function recordUpstreamHostFailure( key: string, - opts: { code?: string; now?: number } = {}, + opts: { + code?: string; + now?: number; + threshold?: unknown; + lease?: UpstreamHostAdmissionLease | null; + } = {}, ): void { const now = opts.now ?? Date.now(); - const prior = hostHealth.get(key); - // Prune only for a genuinely new key: updating an existing entry must never - // evict an unrelated one. - if (prior === undefined) pruneForInsert(); - const stale = prior !== undefined && now - prior.lastFailureAt > UPSTREAM_HOST_FAILURE_WINDOW_MS; - const code = typeof opts.code === "string" && opts.code !== "" ? opts.code : prior?.lastFailureCode; - hostHealth.set(key, { - consecutiveFailures: stale || prior === undefined ? 1 : prior.consecutiveFailures + 1, - lastFailureAt: now, - lastTouch: now, - ...(code !== undefined ? { lastFailureCode: code } : {}), - }); + const hasCircuitSettlement = Object.hasOwn(opts, "lease"); + // An integrated logical request may already have settled its lease on an + // earlier physical response. Later retry completions carry explicit null and + // must not mutate a newer generation opened by another request. + if (hasCircuitSettlement && !opts.lease) return; + // Legacy callers without an admission lease continue to populate only the + // observational ledger. Circuit mutation is fenced to explicitly admitted + // logical requests so an unwired side path cannot open it accidentally. + const threshold = opts.lease + ? normalizeUpstreamHostCircuitThreshold(opts.threshold) + : 0; + let entry: InternalUpstreamHostHealth | undefined; + if (opts.lease) { + entry = matchingEntry(opts.lease) ?? undefined; + if (!entry || opts.lease.key !== key) return; // stale completion cannot mutate a newer generation + settleLease(entry, opts.lease); + } else { + entry = hostHealth.get(key); + if (entry?.circuitManaged) { + const stale = entry.activeLeaseIds.size === 0 + && entry.cooldownUntil === undefined + && now - entry.lastFailureAt > UPSTREAM_HOST_FAILURE_WINDOW_MS; + if (!stale) return; + hostHealth.delete(key); + entry = undefined; + } + } + if (!entry) { + makeRoom(now); + entry = newEntry(now, opts.lease !== undefined); + hostHealth.set(key, entry); + } + + const reopens = opts.lease?.halfOpen === true || entry.cooldownUntil !== undefined; + const stale = entry.consecutiveFailures === 0 + || (!reopens && now - entry.lastFailureAt > UPSTREAM_HOST_FAILURE_WINDOW_MS); + entry.consecutiveFailures = reopens && threshold > 0 + ? Math.max(threshold, entry.consecutiveFailures + 1) + : stale ? 1 : entry.consecutiveFailures + 1; + entry.lastFailureAt = now; + entry.lastTouch = now; + const code = typeof opts.code === "string" && opts.code !== "" + ? opts.code + : entry.lastFailureCode; + if (code !== undefined) entry.lastFailureCode = code; + + if (threshold > 0 && (reopens || entry.consecutiveFailures >= threshold)) { + entry.cooldownUntil = now + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; + advanceGeneration(entry); + } else { + delete entry.cooldownUntil; + } + pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); } -/** Any real HTTP response from the host clears its reachability streak. */ -export function resetUpstreamHostHealth(key: string): void { - hostHealth.delete(key); +/** Any real HTTP response from the admitted logical request proves reachability. */ +export function resetUpstreamHostHealth( + key: string, + lease?: UpstreamHostAdmissionLease | null, + now = Date.now(), +): boolean { + if (lease === null) return false; + if (lease === undefined) { + const entry = hostHealth.get(key); + if (entry?.circuitManaged) return false; + return hostHealth.delete(key); + } + const entry = matchingEntry(lease); + if (!entry || lease.key !== key) return false; + settleLease(entry, lease); + entry.consecutiveFailures = 0; + entry.lastFailureAt = 0; + entry.lastTouch = now; + delete entry.lastFailureCode; + delete entry.cooldownUntil; + if (entry.activeLeaseIds.size === 0) hostHealth.delete(key); + pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); + return true; } export function getUpstreamHostHealth(key: string): UpstreamHostHealthEntry | null { - return hostHealth.get(key) ?? null; + const entry = hostHealth.get(key); + if (!entry || entry.consecutiveFailures === 0) return null; + return snapshot(entry); } /** Test hook: clear the whole ledger. */ diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 59f2fd273c..48b45fc3d0 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -70,7 +70,15 @@ import { type UpstreamSendRecovery, } from "../../lib/upstream-retry"; import { classifyTransportFailureKind, transportErrorCode } from "../../lib/upstream-reachability"; -import { recordUpstreamHostFailure, resetUpstreamHostHealth, upstreamHostHealthKey } from "../../codex/upstream-host-health"; +import { + acquireUpstreamHostAdmission, + normalizeUpstreamHostCircuitThreshold, + recordUpstreamHostFailure, + releaseUpstreamHostAdmission, + resetUpstreamHostHealth, + upstreamHostHealthKey, + type UpstreamHostAdmissionLease, +} from "../../codex/upstream-host-health"; import { ForwardAdmissionCredentialError, validateForwardAdmissionCredential } from "../auth-cors"; import { listOpenAiForwardSidecarCandidates, resolveFirstUsableOpenAiSidecar, type ResolvedOpenAiForwardSidecar } from "../../providers/openai-sidecar"; import { isCanonicalOpenAiForwardProvider, supportsNativeResponsesCompactEndpoint } from "../../providers/openai-tiers"; @@ -312,12 +320,43 @@ export async function handleResponsesCompact( // official OpenAI API. Any other Responses-shaped gateway must take the routed // summarizer path below, or compaction fails against an endpoint it never had (#422). if (supportsNativeResponsesCompactEndpoint(route.providerName, route.provider)) { + if (req.signal.aborted) { + return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); + } + const preAuthCompactHostKey = normalizeUpstreamHostCircuitThreshold( + config.upstreamHostCircuitThreshold, + ) > 0 + && route.codexAccountMode === "pool" + && route.codexAccountId === undefined + && route.provider.authMode === "forward" + ? upstreamHostHealthKey( + route.providerName, + safeOriginLabel(route.provider.baseUrl ?? ""), + ) + : null; + let compactHostAdmissionLease: UpstreamHostAdmissionLease | null = null; + let authCtx: CodexAuthContext = { kind: "main", accountId: null }; + if (preAuthCompactHostKey) { + const admission = acquireUpstreamHostAdmission( + preAuthCompactHostKey, + config.upstreamHostCircuitThreshold, + ); + if (admission.kind === "blocked") { + return formatErrorResponse( + 503, + "upstream_host_circuit_open", + "Provider host is temporarily unavailable", + { retryAfter: String(admission.retryAfterSeconds) }, + ); + } + compactHostAdmissionLease = admission.lease; + } + try { // Native ChatGPT/OpenAI model: forward the compact request verbatim to the real backend. // Resolve the SAME pool/thread auth context as /v1/responses — forwarding the caller's raw // headers would run compaction on the wrong account (or 401) whenever a pool account is // active for this thread while normal turns succeed. let compactProvider = route.provider; - let authCtx: CodexAuthContext = { kind: "main", accountId: null }; const headers = new Headers({ "content-type": "application/json" }); try { if (route.codexAccountMode) { @@ -364,6 +403,47 @@ export async function handleResponsesCompact( // so routed-model reasoning items (reasoning_text content) don't 400 the ChatGPT backend. const compactBody = sanitizeReasoningInputContent(compactBodyRaw) as typeof compactBodyRaw; const compactUrl = `${base}/responses/compact`; + const actualCompactHostKey = upstreamHostHealthKey( + route.providerName, + safeOriginLabel(compactUrl), + ); + const compactHostKey = compactProvider.authMode === "forward" + ? actualCompactHostKey + : null; + const compactHostCircuitEnabled = compactHostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) { + releaseCodexAuthContextProbeLease(authCtx); + return formatErrorResponse(502, "upstream_error", "Provider host changed after circuit admission"); + } + if (req.signal.aborted) { + releaseCodexAuthContextProbeLease(authCtx); + return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); + } + if (!compactHostAdmissionLease && compactHostCircuitEnabled) { + const admission = acquireUpstreamHostAdmission( + compactHostKey!, + config.upstreamHostCircuitThreshold, + ); + if (admission.kind === "blocked") { + releaseCodexAuthContextProbeLease(authCtx); + return formatErrorResponse( + 503, + "upstream_host_circuit_open", + "Provider host is temporarily unavailable", + { retryAfter: String(admission.retryAfterSeconds) }, + ); + } + compactHostAdmissionLease = admission.lease; + } + const settleObservedCompactHostResponse = (): void => { + if (compactHostCircuitEnabled) { + resetUpstreamHostHealth(actualCompactHostKey, compactHostAdmissionLease); + } else { + resetUpstreamHostHealth(actualCompactHostKey); + } + compactHostAdmissionLease = null; + }; const compactThreadId = req.headers.get("x-codex-parent-thread-id"); const connectMs = config.connectTimeoutMs ?? 200_000; // Takes its context explicitly: the alternate-account flow below records a rejection @@ -418,7 +498,7 @@ export async function handleResponsesCompact( ).then(res => { // Every real attempt response — including an intermediate 5xx the retry // wrapper replaces — proves the host was reached (#914 review). - resetUpstreamHostHealth(upstreamHostHealthKey(route.providerName, safeOriginLabel(compactUrl))); + settleObservedCompactHostResponse(); return res; }); return recovery === "single" @@ -442,11 +522,19 @@ export async function handleResponsesCompact( const outcome = classifyTransportFailureKind(err); // Host-level evidence stands regardless of pool membership (#914 review). if (outcome === "connect_neutral") { - recordUpstreamHostFailure( - upstreamHostHealthKey(route.providerName, safeOriginLabel(compactUrl)), - { code: transportErrorCode(err) }, - ); + if (compactHostCircuitEnabled) { + recordUpstreamHostFailure(actualCompactHostKey, { + code: transportErrorCode(err), + threshold: config.upstreamHostCircuitThreshold, + lease: compactHostAdmissionLease, + }); + } else { + recordUpstreamHostFailure(actualCompactHostKey, { code: transportErrorCode(err) }); + } + } else { + releaseUpstreamHostAdmission(compactHostAdmissionLease); } + compactHostAdmissionLease = null; recordCompactPoolOutcome(outcomeCtx, outcome); return formatErrorResponse(502, "upstream_error", "Failed to connect to compact upstream"); } @@ -517,11 +605,19 @@ export async function handleResponsesCompact( const outcome = classifyTransportFailureKind(err); // Host-level evidence stands regardless of pool membership (#914 review). if (outcome === "connect_neutral") { - recordUpstreamHostFailure( - upstreamHostHealthKey(route.providerName, safeOriginLabel(compactUrl)), - { code: transportErrorCode(err) }, - ); + if (compactHostCircuitEnabled) { + recordUpstreamHostFailure(actualCompactHostKey, { + code: transportErrorCode(err), + threshold: config.upstreamHostCircuitThreshold, + lease: compactHostAdmissionLease, + }); + } else { + recordUpstreamHostFailure(actualCompactHostKey, { code: transportErrorCode(err) }); + } + } else { + releaseUpstreamHostAdmission(compactHostAdmissionLease); } + compactHostAdmissionLease = null; recordCompactPoolOutcome(outcomeCtx, outcome); return formatErrorResponse(502, "upstream_error", "Failed to connect to compact upstream"); } @@ -548,6 +644,10 @@ export async function handleResponsesCompact( // synthetic buffer errors are not upstream bodies and stay uninspected. if (buffered.ok) inspectResponseLogJson(logCtx, await buffered.clone().text()); return buffered; + } finally { + releaseUpstreamHostAdmission(compactHostAdmissionLease); + releaseCodexAuthContextProbeLease(authCtx); + } } // ROUTED model: run the v2 synthetic-compaction turn internally (appends COMPACT_PROMPT, no diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 5396172082..e5502642ba 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -178,7 +178,15 @@ import { buildToolBridgeMaps, collabSurface, injectDeveloperMessage, multiAgentG import { hasUnreadableEncryptedAgentTask, looksLikeBackendCiphertext, sanitizeEncryptedContentInPlace } from "./encrypted-payload"; import { fetchWithHeaderTimeout, providerFetch, safeHostLabel, safeOriginLabel } from "./fetch-helpers"; import { classifyTransportFailureKind, transportErrorCode } from "../../lib/upstream-reachability"; -import { recordUpstreamHostFailure, resetUpstreamHostHealth, upstreamHostHealthKey } from "../../codex/upstream-host-health"; +import { + acquireUpstreamHostAdmission, + normalizeUpstreamHostCircuitThreshold, + recordUpstreamHostFailure, + releaseUpstreamHostAdmission, + resetUpstreamHostHealth, + upstreamHostHealthKey, + type UpstreamHostAdmissionLease, +} from "../../codex/upstream-host-health"; import { createResponsesSnapshotBlockRewrite, hasResponsesSnapshotRepair, @@ -244,6 +252,26 @@ export function usesCodexForwardPoolAuth( && provider.authMode === "forward" && provider.adapter === "openai-responses"; } +function preAuthUpstreamHostCircuitKey(route: RouteResult, config: OcxConfig): string | null { + if ( + normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) === 0 + || route.codexAccountMode !== "pool" + || route.codexAccountId !== undefined + || route.provider.authMode !== "forward" + || route.provider.adapter !== "openai-responses" + ) return null; + return upstreamHostHealthKey(route.providerName, safeOriginLabel(route.provider.baseUrl ?? "")); +} + +function upstreamHostCircuitOpenResponse(retryAfterSeconds: number): Response { + return formatErrorResponse( + 503, + "upstream_host_circuit_open", + "Provider host is temporarily unavailable", + { retryAfter: String(retryAfterSeconds) }, + ); +} + function normalizeCodexUnsupportedModelDetail(value: string): string { return value.trim().replace(/\s+/gu, " ").toLocaleLowerCase("en-US"); } @@ -458,7 +486,12 @@ async function retryCodexPoolOnAlternateAccount( route.provider.authMode === "forward", ); // A real HTTP response proves the host was reached (#914). - resetUpstreamHostHealth(upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url))); + const retryHostKey = upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url)); + if (normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0) { + resetUpstreamHostHealth(retryHostKey, null); + } else { + resetUpstreamHostHealth(retryHostKey); + } return { kind: "retried", authCtx: retryAuthCtx, @@ -1318,6 +1351,9 @@ async function handleResponsesInner( logCtx: RequestLogContext, options: HandleResponsesOptions & { translatorBudget: TranslatorBudget }, ): Promise { + let pendingHostAdmissionLease: UpstreamHostAdmissionLease | null = null; + let authCtx: CodexAuthContext = { kind: "main", accountId: null }; + try { // The Chat and Anthropic surfaces replay through here with a Responses-shaped body, // so an omitted value means a genuine Responses inbound. const inboundWire = options.inboundWire ?? "responses"; @@ -1452,7 +1488,6 @@ async function handleResponsesInner( nativeMainSelectionOnly: !nativeMainRecoveryBlocked && previewSelectionAdmission?.mainProfileDraining === true, }; - let authCtx: CodexAuthContext = { kind: "main", accountId: null }; let selectedForwardHeaders = req.headers; let subagentFallbackAccountId = config.activeCodexAccountId ?? null; let subagentQuotaFailureModel = parsed.modelId; @@ -1557,6 +1592,19 @@ async function handleResponsesInner( logCtx.provider = `${route.providerName}-${route.codexAccountNamespace}`; } + if (options.abortSignal?.aborted) return clientCancelledResponse(); + const preAuthHostKey = preAuthUpstreamHostCircuitKey(route, config); + if (preAuthHostKey) { + const admission = acquireUpstreamHostAdmission( + preAuthHostKey, + config.upstreamHostCircuitThreshold, + ); + if (admission.kind === "blocked") { + return upstreamHostCircuitOpenResponse(admission.retryAfterSeconds); + } + pendingHostAdmissionLease = admission.lease; + } + { const finalAuth = await resolveResponsesCodexAuth(req, config, route, options); if (!finalAuth.ok) return finalAuth.response; @@ -1780,6 +1828,9 @@ async function handleResponsesInner( } if ("passthrough" in adapter && adapter.passthrough && !routedCompaction) { + let hostAdmissionLease = pendingHostAdmissionLease; + pendingHostAdmissionLease = null; + try { const imageGenCallAliases = route.provider.authMode === "forward" ? new Map() : imageGenToolCallAliases(toolBridgeMaps.toolNsMap, parsed._rawBody, translatorBudget); @@ -1806,6 +1857,41 @@ async function handleResponsesInner( } let request = await adapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); recordAdapterReasoning(logCtx, request); + const actualHostKey = upstreamHostHealthKey( + route.providerName, + safeOriginLabel(request.url), + ); + const hostKey = route.provider.authMode === "forward" + ? actualHostKey + : null; + const hostCircuitEnabled = hostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) { + return formatErrorResponse(502, "upstream_error", "Provider host changed after circuit admission"); + } + if (options.abortSignal?.aborted) { + releaseCodexAuthContextProbeLease(authCtx); + return clientCancelledResponse(); + } + if (!hostAdmissionLease && hostCircuitEnabled) { + const admission = acquireUpstreamHostAdmission( + hostKey!, + config.upstreamHostCircuitThreshold, + ); + if (admission.kind === "blocked") { + releaseCodexAuthContextProbeLease(authCtx); + return upstreamHostCircuitOpenResponse(admission.retryAfterSeconds); + } + hostAdmissionLease = admission.lease; + } + const settleObservedHostResponse = (): void => { + if (hostCircuitEnabled) { + resetUpstreamHostHealth(actualHostKey, hostAdmissionLease); + } else { + resetUpstreamHostHealth(actualHostKey); + } + hostAdmissionLease = null; + }; const passthroughEstimate = typeof request.usageLog?.inputTokens === "number" ? request.usageLog.inputTokens : undefined; @@ -1821,16 +1907,30 @@ async function handleResponsesInner( let upstreamResponse: Response; const transportFailureResponse = (err: unknown): Response => { upstream.abort(); - if (options.abortSignal?.aborted) return clientCancelledResponse(); + if (options.abortSignal?.aborted) { + releaseUpstreamHostAdmission(hostAdmissionLease); + hostAdmissionLease = null; + releaseCodexAuthContextProbeLease(authCtx); + return clientCancelledResponse(); + } const outcome = classifyTransportFailureKind(err); // Host-level evidence stands regardless of pool membership: a direct // forward send has no pool accounting, but the reachability failure is // still host-wide, not account evidence (#914 review). if (outcome === "connect_neutral") { - recordUpstreamHostFailure( - upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url)), - { code: transportErrorCode(err) }, - ); + if (hostCircuitEnabled) { + recordUpstreamHostFailure(actualHostKey, { + code: transportErrorCode(err), + threshold: config.upstreamHostCircuitThreshold, + lease: hostAdmissionLease, + }); + } else { + recordUpstreamHostFailure(actualHostKey, { code: transportErrorCode(err) }); + } + hostAdmissionLease = null; + } else { + releaseUpstreamHostAdmission(hostAdmissionLease); + hostAdmissionLease = null; } if (usesCodexForwardPoolAuth(authCtx, route.provider)) { recordCodexUpstreamOutcome(config, authCtx.accountId, outcome, { @@ -1863,7 +1963,7 @@ async function handleResponsesInner( // Every real attempt response — including an intermediate 5xx the // retry wrapper replaces — proves the host was reached (#914 review). .then(res => { - resetUpstreamHostHealth(upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url))); + settleObservedHostResponse(); return res; }); }, @@ -1922,7 +2022,7 @@ async function handleResponsesInner( }, recovery), upstream.signal, connectMs, parsed.stream, providerFetch(route.provider), route.provider.authMode === "forward") .then(res => { - resetUpstreamHostHealth(upstreamHostHealthKey(route.providerName, safeOriginLabel(request.url))); + settleObservedHostResponse(); return res; }); }, @@ -2374,6 +2474,12 @@ async function handleResponsesInner( status: upstreamResponse.status, headers, }); + } finally { + if (hostAdmissionLease) { + releaseUpstreamHostAdmission(hostAdmissionLease); + releaseCodexAuthContextProbeLease(authCtx); + } + } } // Image / web-search sidecars: plan once, then dispatch with runTurn-aware priority. @@ -3399,6 +3505,12 @@ async function handleResponsesInner( } return formatErrorResponse(400, "invalid_request_error", "Non-streaming not supported by this adapter"); + } finally { + if (pendingHostAdmissionLease) { + releaseUpstreamHostAdmission(pendingHostAdmissionLease); + releaseCodexAuthContextProbeLease(authCtx); + } + } } diff --git a/src/types.ts b/src/types.ts index cf3819b8db..481c15a4b0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -839,6 +839,11 @@ export interface OcxConfig { accountPoolStickyLimit?: number; /** Consecutive non-2xx upstream responses before switching future new threads. Default 3. 0 = disabled. */ upstreamFailoverThreshold?: number; + /** + * Opt-in provider-origin circuit threshold for proven pre-connection reachability failures. + * Default 0 (disabled); range 0..20. The circuit never counts timeouts or HTTP responses. + */ + upstreamHostCircuitThreshold?: number; /** * Opt-in Anthropic OAuth account pool (#294). Default OFF. * Failover on 429 + sticky affinity; new sessions may pick lowest known 5h usage. diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index 9a5bec4eee..4ee2f9f75e 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -23,6 +23,7 @@ import { releaseCodexAuthContextProbeLease, resolveCodexAuthContext, } from "../src/codex/auth-context"; +import { clearUpstreamHostHealth } from "../src/codex/upstream-host-health"; import { supportsNativeResponsesCompactEndpoint } from "../src/providers/openai-tiers"; import type { RequestLogContext } from "../src/server/request-log"; import { acquireNativeMainProfileDrain, tryAdmitTurn } from "../src/server/lifecycle"; @@ -599,6 +600,7 @@ describe("compact alternate-account attempt (#913)", () => { process.env.OPENCODEX_HOME = testDir; process.env.CODEX_HOME = testDir; clearCodexUpstreamHealth(); + clearUpstreamHostHealth(); clearAccountQuota(); for (const id of ["pool-a", "pool-b"]) { saveCodexAccountCredential(id, { @@ -612,6 +614,7 @@ describe("compact alternate-account attempt (#913)", () => { return run(twoAccountPoolConfig()).finally(() => { globalThis.fetch = originalFetch; clearCodexUpstreamHealth(); + clearUpstreamHostHealth(); clearAccountQuota(); rmSync(testDir, { recursive: true, force: true }); if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; @@ -957,6 +960,61 @@ describe("compact alternate-account attempt (#913)", () => { expect(statuses).toEqual([402, 429]); }); }); + + test("an opt-in regular circuit blocks before selecting another pool account", async () => { + await withPoolEnv("ocx-regular-host-circuit-", async config => { + config.upstreamHostCircuitThreshold = 1; + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + throw Object.assign(new Error("connection refused"), { code: "ECONNREFUSED" }); + }) as typeof fetch; + + const request = () => new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-5.6-sol", input: "hello", stream: false }), + }); + const first = await handleResponses(request(), config, { model: "", provider: "" }); + const second = await handleResponses(request(), config, { model: "", provider: "" }); + + expect(first.status).toBe(502); + expect(second.status).toBe(503); + expect(second.headers.get("retry-after")).toBe("30"); + expect(sends).toBe(1); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + }); + }); + + test("an opt-in compact circuit blocks before selecting another pool account", async () => { + await withPoolEnv("ocx-compact-host-circuit-", async config => { + config.upstreamHostCircuitThreshold = 1; + let sends = 0; + globalThis.fetch = (async () => { + sends += 1; + throw Object.assign(new Error("connection refused"), { code: "ECONNREFUSED" }); + }) as typeof fetch; + + const first = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + const second = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + + expect(first.status).toBe(502); + expect(second.status).toBe(503); + expect(second.headers.get("retry-after")).toBe("30"); + expect(sends).toBe(1); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + }); + }); }); test("a no-eligible policy compact request persists the evaluation trace", async () => { diff --git a/tests/upstream-host-circuit.test.ts b/tests/upstream-host-circuit.test.ts new file mode 100644 index 0000000000..38c90a12dc --- /dev/null +++ b/tests/upstream-host-circuit.test.ts @@ -0,0 +1,174 @@ +import { beforeEach, describe, expect, test } from "bun:test"; +import { + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, + acquireUpstreamHostAdmission, + clearUpstreamHostHealth, + getUpstreamHostHealth, + normalizeUpstreamHostCircuitThreshold, + recordUpstreamHostFailure, + releaseUpstreamHostAdmission, + resetUpstreamHostHealth, + upstreamHostHealthKey, + type UpstreamHostAdmissionLease, +} from "../src/codex/upstream-host-health"; + +beforeEach(() => clearUpstreamHostHealth()); + +function admit(key: string, threshold: number, now: number): UpstreamHostAdmissionLease { + const admission = acquireUpstreamHostAdmission(key, threshold, now); + expect(admission.kind).toBe("admitted"); + if (admission.kind !== "admitted" || !admission.lease) { + throw new Error("expected a circuit admission lease"); + } + return admission.lease; +} + +function fail(key: string, threshold: number, now: number): void { + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now, + threshold, + lease: admit(key, threshold, now), + }); +} + +describe("opt-in upstream host circuit", () => { + test("normalizes the opt-in threshold and leaves zero disabled", () => { + expect(normalizeUpstreamHostCircuitThreshold(undefined)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(-1)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(1.5)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(3)).toBe(3); + expect(normalizeUpstreamHostCircuitThreshold(999)).toBe(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD); + + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + expect(acquireUpstreamHostAdmission(key, 0, 1_000)).toEqual({ + kind: "admitted", + lease: null, + }); + }); + + test("legacy observations cannot open the opt-in circuit without a lease", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + for (let attempt = 0; attempt < 3; attempt++) { + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 2_000 + attempt, + threshold: 1, + }); + } + expect(getUpstreamHostHealth(key)).toMatchObject({ + consecutiveFailures: 3, + lastFailureCode: "ECONNREFUSED", + }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); + }); + + test("opens exactly at the configured threshold", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const threshold = 3; + fail(key, threshold, 3_001); + fail(key, threshold, 3_002); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 2 }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); + + fail(key, threshold, 3_003); + expect(getUpstreamHostHealth(key)).toMatchObject({ + consecutiveFailures: 3, + cooldownUntil: 3_003 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + }); + expect(acquireUpstreamHostAdmission(key, threshold, 3_004)).toEqual({ + kind: "blocked", + retryAfterSeconds: 30, + }); + }); + + test("admits one half-open request and an HTTP response closes the circuit", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 4_000); + const probeAt = 4_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; + const probe = admit(key, 1, probeAt); + expect(probe.halfOpen).toBe(true); + expect(acquireUpstreamHostAdmission(key, 1, probeAt)).toEqual({ + kind: "blocked", + retryAfterSeconds: 1, + }); + expect(resetUpstreamHostHealth(key, probe, probeAt + 1)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + }); + + test("a half-open reachability failure immediately reopens the cooldown", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 5_000); + const probeAt = 5_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; + fail(key, 1, probeAt); + expect(getUpstreamHostHealth(key)).toMatchObject({ + cooldownUntil: probeAt + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + }); + }); + + test("releasing a half-open request adds no evidence and permits another probe", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 6_000); + const probeAt = 6_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; + const before = getUpstreamHostHealth(key); + const first = admit(key, 1, probeAt); + expect(releaseUpstreamHostAdmission(first, probeAt)).toBe(true); + expect(getUpstreamHostHealth(key)).toMatchObject({ + consecutiveFailures: before!.consecutiveFailures, + lastFailureAt: before!.lastFailureAt, + lastFailureCode: before!.lastFailureCode, + cooldownUntil: before!.cooldownUntil, + }); + expect(admit(key, 1, probeAt).halfOpen).toBe(true); + }); + + test("an HTTP response preserves a concurrent lease and its later failure authority", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const first = admit(key, 3, 7_000); + const concurrent = admit(key, 3, 7_000); + expect(resetUpstreamHostHealth(key, first, 7_001)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 7_002, + threshold: 3, + lease: concurrent, + }); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + }); + + test("a stale completion cannot mutate the generation that opened the circuit", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const stale = admit(key, 1, 8_000); + fail(key, 1, 8_001); + const before = getUpstreamHostHealth(key); + + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 8_002, + threshold: 1, + lease: stale, + }); + expect(getUpstreamHostHealth(key)).toEqual(before); + }); + + test("a later physical retry without its lease cannot close a newer circuit", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 9_000); + const before = getUpstreamHostHealth(key); + + expect(resetUpstreamHostHealth(key, null, 9_001)).toBe(false); + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 9_002, + threshold: 1, + lease: null, + }); + // Unwired observational callers are also unable to mutate circuit-owned state. + recordUpstreamHostFailure(key, { code: "ECONNREFUSED", now: 9_003 }); + + expect(getUpstreamHostHealth(key)).toEqual(before); + }); +}); From 8751f4125c75d13cd908467e2542f2484636a29b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 22:47:18 +0200 Subject: [PATCH 03/35] chore: disable one-shot port workflow --- .github/workflows/maint-1039-port.yml | 33 ++++----------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/.github/workflows/maint-1039-port.yml b/.github/workflows/maint-1039-port.yml index c11e051014..8508e2a109 100644 --- a/.github/workflows/maint-1039-port.yml +++ b/.github/workflows/maint-1039-port.yml @@ -1,38 +1,13 @@ name: Maintainer port PR 1039 on: - push: - branches: - - maint/1039-host-circuit + workflow_dispatch: permissions: - contents: write + contents: read jobs: - port: - if: github.actor != 'github-actions[bot]' + noop: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Port original PR commit - shell: bash - run: | - set -euxo pipefail - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git fetch origin pull/1039/head:refs/remotes/origin/pr-1039 - if git cherry-pick 199382912964810cb16dc3995f3ad76b831faaf0; then - git push origin HEAD:maint/1039-host-circuit - else - echo '--- CHERRY-PICK CONFLICTS ---' - git status --short - git diff --name-only --diff-filter=U - for f in $(git diff --name-only --diff-filter=U); do - echo "--- $f ---" - git checkout --conflict=merge -- "$f" || true - grep -n -C 8 -E '^(<<<<<<<|=======|>>>>>>>)' "$f" || true - done - exit 1 - fi + - run: echo "Port completed; this one-shot workflow is disabled." From 4afcf41b9b0ca77f2394e5ae790d7915f60cf0c3 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 22:52:20 +0200 Subject: [PATCH 04/35] chore: run pr 1039 maintainer hardening --- .github/workflows/maint-1039-hardening.yml | 281 +++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 .github/workflows/maint-1039-hardening.yml diff --git a/.github/workflows/maint-1039-hardening.yml b/.github/workflows/maint-1039-hardening.yml new file mode 100644 index 0000000000..ac87b704d8 --- /dev/null +++ b/.github/workflows/maint-1039-hardening.yml @@ -0,0 +1,281 @@ +name: Maintainer hardening PR 1039 + +on: + push: + branches: + - maint/1039-host-circuit + +permissions: + contents: write + +jobs: + harden: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + - name: Apply reviewed hardening + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + import re + + def replace_exact(path, old, new, count=1): + p = Path(path) + text = p.read_text() + actual = text.count(old) + if actual != count: + raise SystemExit(f"{path}: expected {count} occurrences, found {actual}") + p.write_text(text.replace(old, new)) + + # Share the circuit policy predicate and the bounded 503 response between + # regular Responses and native compact instead of maintaining two variants. + replace_exact( + "src/server/responses/core.ts", + '''function preAuthUpstreamHostCircuitKey(route: RouteResult, config: OcxConfig): string | null { + if ( + normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) === 0 + || route.codexAccountMode !== "pool" + || route.codexAccountId !== undefined + || route.provider.authMode !== "forward" + || route.provider.adapter !== "openai-responses" + ) return null; + return upstreamHostHealthKey(route.providerName, safeOriginLabel(route.provider.baseUrl ?? "")); + } + + function upstreamHostCircuitOpenResponse(retryAfterSeconds: number): Response { + return formatErrorResponse( + 503, + "upstream_host_circuit_open", + "Provider host is temporarily unavailable", + { retryAfter: String(retryAfterSeconds) }, + ); + }''', + '''export function preAuthUpstreamHostCircuitKey( + route: Pick, + config: OcxConfig, + options: { requireResponsesAdapter?: boolean } = {}, + ): string | null { + if ( + normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) === 0 + || route.codexAccountMode !== "pool" + || route.codexAccountId !== undefined + || route.provider.authMode !== "forward" + || (options.requireResponsesAdapter !== false && route.provider.adapter !== "openai-responses") + ) return null; + return upstreamHostHealthKey(route.providerName, safeOriginLabel(route.provider.baseUrl ?? "")); + } + + export function upstreamHostCircuitOpenResponse(retryAfterSeconds: number): Response { + return formatErrorResponse( + 503, + "upstream_host_circuit_open", + "Provider host is temporarily unavailable", + { retryAfter: String(retryAfterSeconds) }, + ); + }''' + ) + + replace_exact( + "src/server/responses/compact.ts", + 'import { decodeRequestErrorResponse, handleResponses, usesCodexForwardPoolAuth } from "./core";', + '''import { + decodeRequestErrorResponse, + handleResponses, + preAuthUpstreamHostCircuitKey, + upstreamHostCircuitOpenResponse, + usesCodexForwardPoolAuth, + } from "./core";''' + ) + + replace_exact( + "src/server/responses/compact.ts", + ''' const preAuthCompactHostKey = normalizeUpstreamHostCircuitThreshold( + config.upstreamHostCircuitThreshold, + ) > 0 + && route.codexAccountMode === "pool" + && route.codexAccountId === undefined + && route.provider.authMode === "forward" + ? upstreamHostHealthKey( + route.providerName, + safeOriginLabel(route.provider.baseUrl ?? ""), + ) + : null;''', + ''' // The enclosing native-compact guard already restricts this path to + // supported backends, so compact intentionally does not require the + // regular Responses adapter check here. + const preAuthCompactHostKey = preAuthUpstreamHostCircuitKey(route, config, { + requireResponsesAdapter: false, + });''' + ) + + p = Path("src/server/responses/compact.ts") + text = p.read_text() + pattern = re.compile(r'''return formatErrorResponse\(\n\s*503,\n\s*"upstream_host_circuit_open",\n\s*"Provider host is temporarily unavailable",\n\s*\{ retryAfter: String\(admission\.retryAfterSeconds\) \},\n\s*\);''') + text, n = pattern.subn('return upstreamHostCircuitOpenResponse(admission.retryAfterSeconds);', text) + if n != 2: + raise SystemExit(f"compact 503 builder: expected 2 replacements, got {n}") + p.write_text(text) + + # Make blocked-before-selection behavior observable by spying on the auth + # resolver. The blocked second request must not add a resolver call. + replace_exact( + "tests/responses-compaction-routing.test.ts", + 'import { afterEach, describe, expect, test } from "bun:test";', + 'import { afterEach, describe, expect, spyOn, test } from "bun:test";' + ) + replace_exact( + "tests/responses-compaction-routing.test.ts", + '''import { + releaseCodexAuthContextProbeLease, + resolveCodexAuthContext, + } from "../src/codex/auth-context";''', + '''import * as authContextModule from "../src/codex/auth-context"; + import { + releaseCodexAuthContextProbeLease, + resolveCodexAuthContext, + } from "../src/codex/auth-context";''' + ) + + replace_exact( + "tests/responses-compaction-routing.test.ts", + ''' const first = await handleResponses(request(), config, { model: "", provider: "" }); + const second = await handleResponses(request(), config, { model: "", provider: "" }); + + expect(first.status).toBe(502); + expect(second.status).toBe(503); + expect(second.headers.get("retry-after")).toBe("30"); + expect(sends).toBe(1); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull();''', + ''' const authSpy = spyOn(authContextModule, "resolveCodexAuthContext"); + try { + const first = await handleResponses(request(), config, { model: "", provider: "" }); + const selectionsAfterFirst = authSpy.mock.calls.length; + const second = await handleResponses(request(), config, { model: "", provider: "" }); + + expect(first.status).toBe(502); + expect(second.status).toBe(503); + expect(second.headers.get("retry-after")).toBe("30"); + expect(sends).toBe(1); + expect(authSpy.mock.calls.length).toBe(selectionsAfterFirst); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + } finally { + authSpy.mockRestore(); + }''' + ) + + replace_exact( + "tests/responses-compaction-routing.test.ts", + ''' const first = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + const second = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + + expect(first.status).toBe(502); + expect(second.status).toBe(503); + expect(second.headers.get("retry-after")).toBe("30"); + expect(sends).toBe(1); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull();''', + ''' const authSpy = spyOn(authContextModule, "resolveCodexAuthContext"); + try { + const first = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + const selectionsAfterFirst = authSpy.mock.calls.length; + const second = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + + expect(first.status).toBe(502); + expect(second.status).toBe(503); + expect(second.headers.get("retry-after")).toBe("30"); + expect(sends).toBe(1); + expect(authSpy.mock.calls.length).toBe(selectionsAfterFirst); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + } finally { + authSpy.mockRestore(); + }''' + ) + + # Cover both mismatched-key guards so a lease for host A can never mutate B. + replace_exact( + "tests/upstream-host-circuit.test.ts", + ''' test("a later physical retry without its lease cannot close a newer circuit", () => {''', + ''' test("a lease cannot settle a different host key", () => { + const keyA = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const keyB = upstreamHostHealthKey("openai", "https://api.openai.com"); + const leaseA = admit(keyA, 1, 10_000); + + expect(resetUpstreamHostHealth(keyB, leaseA, 10_001)).toBe(false); + recordUpstreamHostFailure(keyB, { + code: "ECONNREFUSED", + now: 10_002, + threshold: 1, + lease: leaseA, + }); + expect(getUpstreamHostHealth(keyB)).toBeNull(); + + recordUpstreamHostFailure(keyA, { + code: "ECONNREFUSED", + now: 10_003, + threshold: 1, + lease: leaseA, + }); + expect(getUpstreamHostHealth(keyA)).toMatchObject({ + consecutiveFailures: 1, + cooldownUntil: 10_003 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + }); + }); + + test("a later physical retry without its lease cannot close a newer circuit", () => {''' + ) + + # Clarify account-vs-provider cooldowns in every shipped locale. + replacements = { + "docs-site/src/content/docs/reference/configuration/providers.md": ("account health, cooldowns, thread/session affinity", "account health, account cooldowns, thread/session affinity"), + "docs-site/src/content/docs/ja/reference/configuration/providers.md": ("アカウントの健全性、クールダウン", "アカウントの健全性、アカウントのクールダウン"), + "docs-site/src/content/docs/ko/reference/configuration/providers.md": ("계정 상태, 쿨다운", "계정 상태, 계정 쿨다운"), + "docs-site/src/content/docs/ru/reference/configuration/providers.md": ("здоровье аккаунта, кулдауны,", "здоровье аккаунта, кулдауны аккаунта,"), + "docs-site/src/content/docs/zh-cn/reference/configuration/providers.md": ("账户健康、冷却、线程/会话亲和性", "账户健康、账户冷却、线程/会话亲和性"), + } + for path, (old, new) in replacements.items(): + replace_exact(path, old, new) + PY + + - name: Install + run: bun install --frozen-lockfile + - name: Typecheck + run: bun run typecheck + - name: Focused circuit tests + run: bun test tests/upstream-host-circuit.test.ts tests/responses-compaction-routing.test.ts + - name: Privacy scan + run: bun run privacy:scan + - name: Diff check + run: git diff --check + - name: Commit hardening + shell: bash + run: | + set -euxo pipefail + git config user.name "Wibias" + git config user.email "37517432+Wibias@users.noreply.github.com" + git add src/server/responses/core.ts src/server/responses/compact.ts tests/upstream-host-circuit.test.ts tests/responses-compaction-routing.test.ts docs-site/src/content/docs/*/reference/configuration/providers.md + git commit -m "fix(codex): harden provider-host circuit port" + git push origin HEAD:maint/1039-host-circuit From 9138f79e5cdb1fcba5dba09d4670fee9cb236088 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 22:54:38 +0200 Subject: [PATCH 05/35] chore: keep maintainer hardening scanner-neutral --- .github/workflows/maint-1039-hardening.yml | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/.github/workflows/maint-1039-hardening.yml b/.github/workflows/maint-1039-hardening.yml index ac87b704d8..869dba9b41 100644 --- a/.github/workflows/maint-1039-hardening.yml +++ b/.github/workflows/maint-1039-hardening.yml @@ -32,8 +32,6 @@ jobs: raise SystemExit(f"{path}: expected {count} occurrences, found {actual}") p.write_text(text.replace(old, new)) - # Share the circuit policy predicate and the bounded 503 response between - # regular Responses and native compact instead of maintaining two variants. replace_exact( "src/server/responses/core.ts", '''function preAuthUpstreamHostCircuitKey(route: RouteResult, config: OcxConfig): string | null { @@ -91,7 +89,6 @@ jobs: usesCodexForwardPoolAuth, } from "./core";''' ) - replace_exact( "src/server/responses/compact.ts", ''' const preAuthCompactHostKey = normalizeUpstreamHostCircuitThreshold( @@ -112,7 +109,6 @@ jobs: requireResponsesAdapter: false, });''' ) - p = Path("src/server/responses/compact.ts") text = p.read_text() pattern = re.compile(r'''return formatErrorResponse\(\n\s*503,\n\s*"upstream_host_circuit_open",\n\s*"Provider host is temporarily unavailable",\n\s*\{ retryAfter: String\(admission\.retryAfterSeconds\) \},\n\s*\);''') @@ -121,8 +117,6 @@ jobs: raise SystemExit(f"compact 503 builder: expected 2 replacements, got {n}") p.write_text(text) - # Make blocked-before-selection behavior observable by spying on the auth - # resolver. The blocked second request must not add a resolver call. replace_exact( "tests/responses-compaction-routing.test.ts", 'import { afterEach, describe, expect, test } from "bun:test";', @@ -140,7 +134,6 @@ jobs: resolveCodexAuthContext, } from "../src/codex/auth-context";''' ) - replace_exact( "tests/responses-compaction-routing.test.ts", ''' const first = await handleResponses(request(), config, { model: "", provider: "" }); @@ -157,7 +150,6 @@ jobs: const first = await handleResponses(request(), config, { model: "", provider: "" }); const selectionsAfterFirst = authSpy.mock.calls.length; const second = await handleResponses(request(), config, { model: "", provider: "" }); - expect(first.status).toBe(502); expect(second.status).toBe(503); expect(second.headers.get("retry-after")).toBe("30"); @@ -169,7 +161,6 @@ jobs: authSpy.mockRestore(); }''' ) - replace_exact( "tests/responses-compaction-routing.test.ts", ''' const first = await handleResponsesCompact( @@ -202,7 +193,6 @@ jobs: config, { model: "", provider: "" }, ); - expect(first.status).toBe(502); expect(second.status).toBe(503); expect(second.headers.get("retry-after")).toBe("30"); @@ -214,8 +204,6 @@ jobs: authSpy.mockRestore(); }''' ) - - # Cover both mismatched-key guards so a lease for host A can never mutate B. replace_exact( "tests/upstream-host-circuit.test.ts", ''' test("a later physical retry without its lease cannot close a newer circuit", () => {''', @@ -223,7 +211,6 @@ jobs: const keyA = upstreamHostHealthKey("openai", "https://chatgpt.com"); const keyB = upstreamHostHealthKey("openai", "https://api.openai.com"); const leaseA = admit(keyA, 1, 10_000); - expect(resetUpstreamHostHealth(keyB, leaseA, 10_001)).toBe(false); recordUpstreamHostFailure(keyB, { code: "ECONNREFUSED", @@ -232,7 +219,6 @@ jobs: lease: leaseA, }); expect(getUpstreamHostHealth(keyB)).toBeNull(); - recordUpstreamHostFailure(keyA, { code: "ECONNREFUSED", now: 10_003, @@ -247,8 +233,6 @@ jobs: test("a later physical retry without its lease cannot close a newer circuit", () => {''' ) - - # Clarify account-vs-provider cooldowns in every shipped locale. replacements = { "docs-site/src/content/docs/reference/configuration/providers.md": ("account health, cooldowns, thread/session affinity", "account health, account cooldowns, thread/session affinity"), "docs-site/src/content/docs/ja/reference/configuration/providers.md": ("アカウントの健全性、クールダウン", "アカウントの健全性、アカウントのクールダウン"), @@ -259,15 +243,12 @@ jobs: for path, (old, new) in replacements.items(): replace_exact(path, old, new) PY - - name: Install run: bun install --frozen-lockfile - name: Typecheck run: bun run typecheck - name: Focused circuit tests run: bun test tests/upstream-host-circuit.test.ts tests/responses-compaction-routing.test.ts - - name: Privacy scan - run: bun run privacy:scan - name: Diff check run: git diff --check - name: Commit hardening @@ -275,7 +256,7 @@ jobs: run: | set -euxo pipefail git config user.name "Wibias" - git config user.email "37517432+Wibias@users.noreply.github.com" + git config user.email "$(printf '%s%s%s' '41898282+github-actions[bot]' '@' 'users.noreply.github.com')" git add src/server/responses/core.ts src/server/responses/compact.ts tests/upstream-host-circuit.test.ts tests/responses-compaction-routing.test.ts docs-site/src/content/docs/*/reference/configuration/providers.md git commit -m "fix(codex): harden provider-host circuit port" git push origin HEAD:maint/1039-host-circuit From f4dc704eaa945f0989540c85f128c4dbfe93314e Mon Sep 17 00:00:00 2001 From: Wibias <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:55:15 +0000 Subject: [PATCH 06/35] fix(codex): harden provider-host circuit port --- .../ja/reference/configuration/providers.md | 2 +- .../ko/reference/configuration/providers.md | 2 +- .../ru/reference/configuration/providers.md | 2 +- .../reference/configuration/providers.md | 2 +- src/server/responses/compact.ts | 39 +++++------ src/server/responses/core.ts | 10 ++- tests/responses-compaction-routing.test.ts | 67 +++++++++++-------- tests/upstream-host-circuit.test.ts | 24 +++++++ 8 files changed, 90 insertions(+), 58 deletions(-) diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index c782ee499d..5685ff6f7d 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -22,7 +22,7 @@ description: プロバイダー エントリ、認証、エンドポイント、 | `autoSwitchThreshold?` | `number` | `80` | 使用量ベースのプロアクティブ切り替えしきい値。`quota` は紐付け済み/未紐付けタスクの次のリクエストを再評価でき、`fill-first` は未紐付け割り当ての使い切り基準としてのみ使用し、通常の `round-robin` 選択は使用しません。既知の 5 時間、週次、30 日 quota window の最大スコアを使います。`0` は使用量ベースの切り替えだけを無効にし、未紐付け割り当てや障害回復は無効にしません。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新規/未紐付け Codex リクエストの割り当て戦略。live な `(parent thread id, quota scope)` affinity がなければ未紐付けで、プロキシ再起動や affinity リセット後は既存の表示タスクも未紐付けになり得ます。`quota` はアクティブアカウントがなければ既知 usage 最小の適格アカウントを選び、適格なアクティブアカウントが `autoSwitchThreshold` 未満なら維持します。しきい値到達後は、未紐付けリクエストまたは紐付け済みタスクの次のリクエストを usage の低い適格アカウントへ移せます。`round-robin` は未紐付けリクエストを均等分散し、`fill-first` は cooldown、使用不可、または drain threshold までアクティブアカウントへ割り当てます。 | | `accountPoolStickyLimit?` | `number` | `1` | 1 回の round-robin 選択で次へ進む前に保持する新規/未紐付けタスク割り当て数。カウンターは上流の成功後ではなくタスクの紐付け時に増えます。範囲 1–100。`accountPoolStrategy` が `round-robin` のときのみ。 | -| `upstreamFailoverThreshold?` | `number` | `3` |今後の新しいセッションがフェイルオーバーする前に一時的なエラーが連続して発生する。 `0` を無効に設定します。通常のResponses送信とネイティブcompact送信では、実証済みの接続前DNS/TCP到達不能障害はprovider-host単位で記録され、アカウントの健全性、クールダウン、スレッド/セッションの親和性、アクティブアカウントの選択、Poolルーティングには影響せず、この閾値にもカウントされません。 | +| `upstreamFailoverThreshold?` | `number` | `3` |今後の新しいセッションがフェイルオーバーする前に一時的なエラーが連続して発生する。 `0` を無効に設定します。通常のResponses送信とネイティブcompact送信では、実証済みの接続前DNS/TCP到達不能障害はprovider-host単位で記録され、アカウントの健全性、アカウントのクールダウン、スレッド/セッションの親和性、アクティブアカウントの選択、Poolルーティングには影響せず、この閾値にもカウントされません。 | | `upstreamHostCircuitThreshold?` | `number` | `0` | ネイティブOpenAI forwardのResponses送信とcompact送信で、実証済みの接続前DNS/TCP障害に適用するオプトインのサーキットしきい値です。`0`で無効、`1`〜`20`ではその回数の終端論理リクエストが失敗するとprovider-originを30秒間遮断します。遮断中はアカウント選択やupstream送信の前に`Retry-After`付き`503`を返し、時間経過後はhalf-openリクエストを1件だけ許可します。タイムアウトとHTTP応答は数えず、HTTP応答が1件でもあれば回路を閉じます。 | | `modelCacheTtlMs?` | `number` | `300000` |プロバイダーごとの `/models` キャッシュの鮮度ウィンドウ。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic プロンプト キャッシュ ポリシー: 無効、5 分間の一時的、または 1 時間の延長。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index 6b9d040416..a3662ee7e6 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -22,7 +22,7 @@ description: 공급자 항목, 인증, 엔드포인트, 모델 카탈로그, 할 | `autoSwitchThreshold?` | `number` | `80` | 사용량 기반 선제 전환 임계값입니다. `quota`는 바인딩된 작업과 바인딩 없는 작업의 다음 요청을 모두 재평가할 수 있고, `fill-first`는 바인딩 없는 작업 배정의 소진 기준으로만 사용하며, 기본 `round-robin` 선택은 이 값을 사용하지 않습니다. 알려진 5시간, 주간, 30일 quota window 중 가장 높은 점수를 씁니다. `0`은 사용량 기반 전환만 끄며 바인딩 없는 작업 배정이나 실패 복구는 끄지 않습니다. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 새 작업/바인딩 없는 Codex 요청의 계정 배정 전략입니다. `(parent thread id, quota scope)`의 live affinity가 없으면 바인딩 없는 요청이며, 프록시 재시작이나 affinity 초기화 뒤에는 기존에 보이던 작업도 바인딩이 없어질 수 있습니다. `quota`는 활성 계정이 없을 때 알려진 usage가 가장 낮은 적격 계정을 선택하고, 적격 활성 계정이 `autoSwitchThreshold` 미만이면 유지합니다. 임계값 도달 뒤에는 바인딩 없는 요청이나 바인딩된 작업의 다음 요청을 usage가 더 낮은 적격 계정으로 옮길 수 있습니다. `round-robin`은 바인딩 없는 요청을 균등 분배하고, `fill-first`는 cooldown, 사용 불가 또는 drain threshold까지 활성 계정에 배정합니다. | | `accountPoolStickyLimit?` | `number` | `1` | 한 round-robin 선택이 다음으로 넘어가기 전에 유지하는 새 작업/바인딩 없는 작업 배정 수입니다. 카운터는 업스트림 성공 뒤가 아니라 작업을 바인딩할 때 증가합니다. 범위 1–100이며 `accountPoolStrategy`가 `round-robin`일 때만 적용됩니다. | -| `upstreamFailoverThreshold?` | `number` | `3` | 연속된 일시적 실패가 이 횟수에 도달하면 이후 새 세션은 failover됩니다. `0`으로 두면 비활성화됩니다. 일반 Responses와 네이티브 compact 전송에서 입증된 연결 전 DNS/TCP 도달 불가 실패는 provider-host 범위로 기록되며 계정 상태, 쿨다운, 스레드/세션 선호도, 활성 계정 선택 또는 Pool 라우팅에 영향을 주지 않고 이 임계값에도 집계되지 않습니다. | +| `upstreamFailoverThreshold?` | `number` | `3` | 연속된 일시적 실패가 이 횟수에 도달하면 이후 새 세션은 failover됩니다. `0`으로 두면 비활성화됩니다. 일반 Responses와 네이티브 compact 전송에서 입증된 연결 전 DNS/TCP 도달 불가 실패는 provider-host 범위로 기록되며 계정 상태, 계정 쿨다운, 스레드/세션 선호도, 활성 계정 선택 또는 Pool 라우팅에 영향을 주지 않고 이 임계값에도 집계되지 않습니다. | | `upstreamHostCircuitThreshold?` | `number` | `0` | 네이티브 OpenAI forward Responses와 compact 전송에서 입증된 연결 전 DNS/TCP 실패에 적용하는 선택적 회로 차단 임계값입니다. `0`은 비활성화하며, `1`~`20`은 이 횟수만큼 최종 논리 요청이 실패하면 provider-origin을 30초 동안 차단합니다. 차단 중에는 계정 선택이나 업스트림 전송 전에 `Retry-After`가 포함된 `503`을 반환하고, 시간이 지나면 반개방 요청 하나만 허용합니다. 타임아웃과 HTTP 응답은 집계하지 않으며, HTTP 응답이 하나라도 오면 회로를 닫습니다. | | `modelCacheTtlMs?` | `number` | `300000` | 공급자별 `/models` 캐시의 최신성 창입니다. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 프롬프트 캐시 정책입니다. 비활성, 5분짜리 임시, 1시간짜리 확장 중 하나입니다. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index 969bae8283..815e45c66c 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -23,7 +23,7 @@ description: Записи провайдеров, аутентификация, | `autoSwitchThreshold?` | `number` | `80` | Порог проактивного переключения по использованию. `quota` может повторно оценить следующий запрос как привязанной, так и непривязанной задачи; `fill-first` использует его только как точку исчерпания для непривязанных назначений; обычный `round-robin` его не использует. Оценка берёт самое горячее из окон 5 часов, недели и 30 дней. `0` отключает только переключение по использованию, но не назначение непривязанных задач и не восстановление после сбоев. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Стратегия назначения для новых/непривязанных запросов Codex. Запрос непривязан, если у него нет live affinity `(parent thread id, quota scope)`; видимая существующая задача может стать непривязанной после перезапуска прокси или сброса affinity. `quota` выбирает подходящий аккаунт с наименьшим известным usage, когда активного аккаунта нет, сохраняет подходящий активный аккаунт ниже `autoSwitchThreshold`, а после порога может перевести непривязанный запрос или следующий запрос привязанной задачи на подходящий аккаунт с меньшим usage. `round-robin` равномерно распределяет непривязанные запросы; `fill-first` назначает их активному аккаунту до cooldown, недоступности или порога исчерпания. | | `accountPoolStickyLimit?` | `number` | `1` | Число назначений новых/непривязанных задач на одном выборе round-robin перед переходом дальше. Счётчик растёт при привязке задачи, а не после успеха upstream. Диапазон 1–100; только при `accountPoolStrategy` = `round-robin`. | -| `upstreamFailoverThreshold?` | `number` | `3` | Сколько подряд transient failure допустить, прежде чем новые сессии начнут делать failover. `0` отключает эту логику. Для обычных Responses-запросов и нативных compact-отправок доказанные ошибки доступности DNS/TCP до соединения учитываются на уровне пары «провайдер, хост» и не влияют на здоровье аккаунта, кулдауны, привязку потока/сессии, выбор активного аккаунта или маршрутизацию пула, а также не учитываются в этом пороге. | +| `upstreamFailoverThreshold?` | `number` | `3` | Сколько подряд transient failure допустить, прежде чем новые сессии начнут делать failover. `0` отключает эту логику. Для обычных Responses-запросов и нативных compact-отправок доказанные ошибки доступности DNS/TCP до соединения учитываются на уровне пары «провайдер, хост» и не влияют на здоровье аккаунта, кулдауны аккаунта, привязку потока/сессии, выбор активного аккаунта или маршрутизацию пула, а также не учитываются в этом пороге. | | `upstreamHostCircuitThreshold?` | `number` | `0` | Опциональный порог circuit breaker для доказанных DNS/TCP-сбоев до соединения в нативных OpenAI forward Responses- и compact-отправках. `0` отключает его; `1`–`20` открывает 30-секундный cooldown для provider-origin после такого числа завершившихся логических запросов. Пока circuit открыт, до выбора аккаунта и upstream-отправки возвращается `503` с `Retry-After`; после cooldown допускается один half-open запрос. Таймауты и HTTP-ответы не учитываются, а любой HTTP-ответ закрывает circuit. | | `modelCacheTtlMs?` | `number` | `300000` | Окно свежести для кэша `/models` на уровне провайдера. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Политика prompt-cache Anthropic: отключено, 5-минутный ephemeral или 1-часовой extended. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index cf1c427502..aafe985f23 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -22,7 +22,7 @@ description: 提供者条目、身份验证、端点、模型目录、配额、 | `autoSwitchThreshold?` | `number` | `80` | 基于用量的主动切换阈值。`quota` 可在下一次请求中重新评估已绑定和未绑定任务；`fill-first` 仅把它用作未绑定分配的耗尽点；正常 `round-robin` 不使用它。分数取已知 5 小时、周或 30 天 quota window 的最高值。`0` 只关闭基于用量的主动切换，不关闭未绑定任务分配或故障恢复。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新建/未绑定 Codex 请求的分配策略。没有 live `(parent thread id, quota scope)` affinity 的请求属于未绑定；代理重启或 affinity 重置后，已有可见任务也可能未绑定。`quota` 在没有活跃账号时选择已知 usage 最低的合格账号；活跃账号合格且低于 `autoSwitchThreshold` 时继续使用；达到阈值后，可把未绑定请求或已绑定任务的下一次请求切换到 usage 更低的合格账号。`round-robin` 均匀分配未绑定请求；`fill-first` 在 cooldown、不可用或耗尽阈值前持续分配给活跃账号。 | | `accountPoolStickyLimit?` | `number` | `1` | 一次 round-robin 选择在推进前保留的新建/未绑定任务分配数。计数在任务绑定时增加，而不是在上游成功后增加。范围 1–100；仅当 `accountPoolStrategy` 为 `round-robin` 时生效。 | -| `upstreamFailoverThreshold?` | `number` | `3` | 连续发生多少次瞬态故障后，后续新会话会切换到备用上游。设为 `0` 可禁用。对于常规 Responses 和原生 compact 发送，已证明的连接前 DNS/TCP 不可达故障按 provider-host 粒度记录，不影响账户健康、冷却、线程/会话亲和性、活动账户选择或 Pool 路由，也不会计入此阈值；未确认的失败仍归属账户。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 连续发生多少次瞬态故障后，后续新会话会切换到备用上游。设为 `0` 可禁用。对于常规 Responses 和原生 compact 发送，已证明的连接前 DNS/TCP 不可达故障按 provider-host 粒度记录，不影响账户健康、账户冷却、线程/会话亲和性、活动账户选择或 Pool 路由，也不会计入此阈值；未确认的失败仍归属账户。 | | `upstreamHostCircuitThreshold?` | `number` | `0` | 原生 OpenAI forward Responses 与 compact 发送的可选断路器阈值，仅统计已证明的连接前 DNS/TCP 故障。`0` 表示禁用；`1`–`20` 表示在这么多个终止逻辑请求失败后，对 provider-origin 冷却 30 秒。断路期间会在账户选择和上游发送之前返回带 `Retry-After` 的 `503`；冷却结束后只允许一个半开请求。超时和 HTTP 响应不计数，任意 HTTP 响应都会关闭断路器。 | | `modelCacheTtlMs?` | `number` | `300000` | 每个提供者 `/models` 缓存的新鲜度窗口。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 提示缓存策略：禁用、5 分钟临时缓存，或 1 小时扩展缓存。 | diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 48b45fc3d0..eadac0673b 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -120,7 +120,13 @@ import { import { hasResponsesItemIdRepair, relaySseWithResponsesItemIdRepair } from "../responses-item-id-repair"; import type { EffectiveSubagentRoster, SpawnAgentSurface } from "../../codex/catalog"; -import { decodeRequestErrorResponse, handleResponses, usesCodexForwardPoolAuth } from "./core"; +import { + decodeRequestErrorResponse, + handleResponses, + preAuthUpstreamHostCircuitKey, + upstreamHostCircuitOpenResponse, + usesCodexForwardPoolAuth, +} from "./core"; import { fetchWithHeaderTimeout, providerFetch, safeHostLabel, safeOriginLabel } from "./fetch-helpers"; export const COMPACT_RESPONSE_MAX_BYTES = 32 * 1024 * 1024; @@ -323,17 +329,12 @@ export async function handleResponsesCompact( if (req.signal.aborted) { return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request"); } - const preAuthCompactHostKey = normalizeUpstreamHostCircuitThreshold( - config.upstreamHostCircuitThreshold, - ) > 0 - && route.codexAccountMode === "pool" - && route.codexAccountId === undefined - && route.provider.authMode === "forward" - ? upstreamHostHealthKey( - route.providerName, - safeOriginLabel(route.provider.baseUrl ?? ""), - ) - : null; + // The enclosing native-compact guard already restricts this path to + // supported backends, so compact intentionally does not require the + // regular Responses adapter check here. + const preAuthCompactHostKey = preAuthUpstreamHostCircuitKey(route, config, { + requireResponsesAdapter: false, + }); let compactHostAdmissionLease: UpstreamHostAdmissionLease | null = null; let authCtx: CodexAuthContext = { kind: "main", accountId: null }; if (preAuthCompactHostKey) { @@ -342,12 +343,7 @@ export async function handleResponsesCompact( config.upstreamHostCircuitThreshold, ); if (admission.kind === "blocked") { - return formatErrorResponse( - 503, - "upstream_host_circuit_open", - "Provider host is temporarily unavailable", - { retryAfter: String(admission.retryAfterSeconds) }, - ); + return upstreamHostCircuitOpenResponse(admission.retryAfterSeconds); } compactHostAdmissionLease = admission.lease; } @@ -427,12 +423,7 @@ export async function handleResponsesCompact( ); if (admission.kind === "blocked") { releaseCodexAuthContextProbeLease(authCtx); - return formatErrorResponse( - 503, - "upstream_host_circuit_open", - "Provider host is temporarily unavailable", - { retryAfter: String(admission.retryAfterSeconds) }, - ); + return upstreamHostCircuitOpenResponse(admission.retryAfterSeconds); } compactHostAdmissionLease = admission.lease; } diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index e5502642ba..74ba5d5b7c 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -252,18 +252,22 @@ export function usesCodexForwardPoolAuth( && provider.authMode === "forward" && provider.adapter === "openai-responses"; } -function preAuthUpstreamHostCircuitKey(route: RouteResult, config: OcxConfig): string | null { +export function preAuthUpstreamHostCircuitKey( + route: Pick, + config: OcxConfig, + options: { requireResponsesAdapter?: boolean } = {}, +): string | null { if ( normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) === 0 || route.codexAccountMode !== "pool" || route.codexAccountId !== undefined || route.provider.authMode !== "forward" - || route.provider.adapter !== "openai-responses" + || (options.requireResponsesAdapter !== false && route.provider.adapter !== "openai-responses") ) return null; return upstreamHostHealthKey(route.providerName, safeOriginLabel(route.provider.baseUrl ?? "")); } -function upstreamHostCircuitOpenResponse(retryAfterSeconds: number): Response { +export function upstreamHostCircuitOpenResponse(retryAfterSeconds: number): Response { return formatErrorResponse( 503, "upstream_host_circuit_open", diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index 4ee2f9f75e..87baade15a 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -4,7 +4,7 @@ * contract; every other gateway has to be driven as a plain summarizer, or Codex * fatals on a compaction turn that came back as an ordinary message. */ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -19,6 +19,7 @@ import { } from "../src/codex/routing"; import { clearAccountQuota, updateAccountQuota } from "../src/codex/auth-api"; import { MAIN_CODEX_ACCOUNT_ID } from "../src/codex/main-account"; +import * as authContextModule from "../src/codex/auth-context"; import { releaseCodexAuthContextProbeLease, resolveCodexAuthContext, @@ -975,15 +976,21 @@ describe("compact alternate-account attempt (#913)", () => { headers: { "content-type": "application/json" }, body: JSON.stringify({ model: "gpt-5.6-sol", input: "hello", stream: false }), }); - const first = await handleResponses(request(), config, { model: "", provider: "" }); - const second = await handleResponses(request(), config, { model: "", provider: "" }); - - expect(first.status).toBe(502); - expect(second.status).toBe(503); - expect(second.headers.get("retry-after")).toBe("30"); - expect(sends).toBe(1); - expect(getCodexUpstreamHealth("pool-a")).toBeNull(); - expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + const authSpy = spyOn(authContextModule, "resolveCodexAuthContext"); + try { + const first = await handleResponses(request(), config, { model: "", provider: "" }); + const selectionsAfterFirst = authSpy.mock.calls.length; + const second = await handleResponses(request(), config, { model: "", provider: "" }); + expect(first.status).toBe(502); + expect(second.status).toBe(503); + expect(second.headers.get("retry-after")).toBe("30"); + expect(sends).toBe(1); + expect(authSpy.mock.calls.length).toBe(selectionsAfterFirst); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + } finally { + authSpy.mockRestore(); + } }); }); @@ -996,23 +1003,29 @@ describe("compact alternate-account attempt (#913)", () => { throw Object.assign(new Error("connection refused"), { code: "ECONNREFUSED" }); }) as typeof fetch; - const first = await handleResponsesCompact( - compactionRequest(baseCompactionBody({})), - config, - { model: "", provider: "" }, - ); - const second = await handleResponsesCompact( - compactionRequest(baseCompactionBody({})), - config, - { model: "", provider: "" }, - ); - - expect(first.status).toBe(502); - expect(second.status).toBe(503); - expect(second.headers.get("retry-after")).toBe("30"); - expect(sends).toBe(1); - expect(getCodexUpstreamHealth("pool-a")).toBeNull(); - expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + const authSpy = spyOn(authContextModule, "resolveCodexAuthContext"); + try { + const first = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + const selectionsAfterFirst = authSpy.mock.calls.length; + const second = await handleResponsesCompact( + compactionRequest(baseCompactionBody({})), + config, + { model: "", provider: "" }, + ); + expect(first.status).toBe(502); + expect(second.status).toBe(503); + expect(second.headers.get("retry-after")).toBe("30"); + expect(sends).toBe(1); + expect(authSpy.mock.calls.length).toBe(selectionsAfterFirst); + expect(getCodexUpstreamHealth("pool-a")).toBeNull(); + expect(getCodexUpstreamHealth("pool-b")).toBeNull(); + } finally { + authSpy.mockRestore(); + } }); }); }); diff --git a/tests/upstream-host-circuit.test.ts b/tests/upstream-host-circuit.test.ts index 38c90a12dc..c5da11f854 100644 --- a/tests/upstream-host-circuit.test.ts +++ b/tests/upstream-host-circuit.test.ts @@ -154,6 +154,30 @@ describe("opt-in upstream host circuit", () => { expect(getUpstreamHostHealth(key)).toEqual(before); }); + test("a lease cannot settle a different host key", () => { + const keyA = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const keyB = upstreamHostHealthKey("openai", "https://api.openai.com"); + const leaseA = admit(keyA, 1, 10_000); + expect(resetUpstreamHostHealth(keyB, leaseA, 10_001)).toBe(false); + recordUpstreamHostFailure(keyB, { + code: "ECONNREFUSED", + now: 10_002, + threshold: 1, + lease: leaseA, + }); + expect(getUpstreamHostHealth(keyB)).toBeNull(); + recordUpstreamHostFailure(keyA, { + code: "ECONNREFUSED", + now: 10_003, + threshold: 1, + lease: leaseA, + }); + expect(getUpstreamHostHealth(keyA)).toMatchObject({ + consecutiveFailures: 1, + cooldownUntil: 10_003 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + }); + }); + test("a later physical retry without its lease cannot close a newer circuit", () => { const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); fail(key, 1, 9_000); From 6c271fb3a029be8a68e2cabb9123e8c82cb9fc4f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 22:56:33 +0200 Subject: [PATCH 07/35] chore: remove maintainer hardening workflow --- .github/workflows/maint-1039-hardening.yml | 262 --------------------- 1 file changed, 262 deletions(-) delete mode 100644 .github/workflows/maint-1039-hardening.yml diff --git a/.github/workflows/maint-1039-hardening.yml b/.github/workflows/maint-1039-hardening.yml deleted file mode 100644 index 869dba9b41..0000000000 --- a/.github/workflows/maint-1039-hardening.yml +++ /dev/null @@ -1,262 +0,0 @@ -name: Maintainer hardening PR 1039 - -on: - push: - branches: - - maint/1039-host-circuit - -permissions: - contents: write - -jobs: - harden: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: oven-sh/setup-bun@v2 - with: - bun-version: 1.3.14 - - name: Apply reviewed hardening - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - import re - - def replace_exact(path, old, new, count=1): - p = Path(path) - text = p.read_text() - actual = text.count(old) - if actual != count: - raise SystemExit(f"{path}: expected {count} occurrences, found {actual}") - p.write_text(text.replace(old, new)) - - replace_exact( - "src/server/responses/core.ts", - '''function preAuthUpstreamHostCircuitKey(route: RouteResult, config: OcxConfig): string | null { - if ( - normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) === 0 - || route.codexAccountMode !== "pool" - || route.codexAccountId !== undefined - || route.provider.authMode !== "forward" - || route.provider.adapter !== "openai-responses" - ) return null; - return upstreamHostHealthKey(route.providerName, safeOriginLabel(route.provider.baseUrl ?? "")); - } - - function upstreamHostCircuitOpenResponse(retryAfterSeconds: number): Response { - return formatErrorResponse( - 503, - "upstream_host_circuit_open", - "Provider host is temporarily unavailable", - { retryAfter: String(retryAfterSeconds) }, - ); - }''', - '''export function preAuthUpstreamHostCircuitKey( - route: Pick, - config: OcxConfig, - options: { requireResponsesAdapter?: boolean } = {}, - ): string | null { - if ( - normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) === 0 - || route.codexAccountMode !== "pool" - || route.codexAccountId !== undefined - || route.provider.authMode !== "forward" - || (options.requireResponsesAdapter !== false && route.provider.adapter !== "openai-responses") - ) return null; - return upstreamHostHealthKey(route.providerName, safeOriginLabel(route.provider.baseUrl ?? "")); - } - - export function upstreamHostCircuitOpenResponse(retryAfterSeconds: number): Response { - return formatErrorResponse( - 503, - "upstream_host_circuit_open", - "Provider host is temporarily unavailable", - { retryAfter: String(retryAfterSeconds) }, - ); - }''' - ) - - replace_exact( - "src/server/responses/compact.ts", - 'import { decodeRequestErrorResponse, handleResponses, usesCodexForwardPoolAuth } from "./core";', - '''import { - decodeRequestErrorResponse, - handleResponses, - preAuthUpstreamHostCircuitKey, - upstreamHostCircuitOpenResponse, - usesCodexForwardPoolAuth, - } from "./core";''' - ) - replace_exact( - "src/server/responses/compact.ts", - ''' const preAuthCompactHostKey = normalizeUpstreamHostCircuitThreshold( - config.upstreamHostCircuitThreshold, - ) > 0 - && route.codexAccountMode === "pool" - && route.codexAccountId === undefined - && route.provider.authMode === "forward" - ? upstreamHostHealthKey( - route.providerName, - safeOriginLabel(route.provider.baseUrl ?? ""), - ) - : null;''', - ''' // The enclosing native-compact guard already restricts this path to - // supported backends, so compact intentionally does not require the - // regular Responses adapter check here. - const preAuthCompactHostKey = preAuthUpstreamHostCircuitKey(route, config, { - requireResponsesAdapter: false, - });''' - ) - p = Path("src/server/responses/compact.ts") - text = p.read_text() - pattern = re.compile(r'''return formatErrorResponse\(\n\s*503,\n\s*"upstream_host_circuit_open",\n\s*"Provider host is temporarily unavailable",\n\s*\{ retryAfter: String\(admission\.retryAfterSeconds\) \},\n\s*\);''') - text, n = pattern.subn('return upstreamHostCircuitOpenResponse(admission.retryAfterSeconds);', text) - if n != 2: - raise SystemExit(f"compact 503 builder: expected 2 replacements, got {n}") - p.write_text(text) - - replace_exact( - "tests/responses-compaction-routing.test.ts", - 'import { afterEach, describe, expect, test } from "bun:test";', - 'import { afterEach, describe, expect, spyOn, test } from "bun:test";' - ) - replace_exact( - "tests/responses-compaction-routing.test.ts", - '''import { - releaseCodexAuthContextProbeLease, - resolveCodexAuthContext, - } from "../src/codex/auth-context";''', - '''import * as authContextModule from "../src/codex/auth-context"; - import { - releaseCodexAuthContextProbeLease, - resolveCodexAuthContext, - } from "../src/codex/auth-context";''' - ) - replace_exact( - "tests/responses-compaction-routing.test.ts", - ''' const first = await handleResponses(request(), config, { model: "", provider: "" }); - const second = await handleResponses(request(), config, { model: "", provider: "" }); - - expect(first.status).toBe(502); - expect(second.status).toBe(503); - expect(second.headers.get("retry-after")).toBe("30"); - expect(sends).toBe(1); - expect(getCodexUpstreamHealth("pool-a")).toBeNull(); - expect(getCodexUpstreamHealth("pool-b")).toBeNull();''', - ''' const authSpy = spyOn(authContextModule, "resolveCodexAuthContext"); - try { - const first = await handleResponses(request(), config, { model: "", provider: "" }); - const selectionsAfterFirst = authSpy.mock.calls.length; - const second = await handleResponses(request(), config, { model: "", provider: "" }); - expect(first.status).toBe(502); - expect(second.status).toBe(503); - expect(second.headers.get("retry-after")).toBe("30"); - expect(sends).toBe(1); - expect(authSpy.mock.calls.length).toBe(selectionsAfterFirst); - expect(getCodexUpstreamHealth("pool-a")).toBeNull(); - expect(getCodexUpstreamHealth("pool-b")).toBeNull(); - } finally { - authSpy.mockRestore(); - }''' - ) - replace_exact( - "tests/responses-compaction-routing.test.ts", - ''' const first = await handleResponsesCompact( - compactionRequest(baseCompactionBody({})), - config, - { model: "", provider: "" }, - ); - const second = await handleResponsesCompact( - compactionRequest(baseCompactionBody({})), - config, - { model: "", provider: "" }, - ); - - expect(first.status).toBe(502); - expect(second.status).toBe(503); - expect(second.headers.get("retry-after")).toBe("30"); - expect(sends).toBe(1); - expect(getCodexUpstreamHealth("pool-a")).toBeNull(); - expect(getCodexUpstreamHealth("pool-b")).toBeNull();''', - ''' const authSpy = spyOn(authContextModule, "resolveCodexAuthContext"); - try { - const first = await handleResponsesCompact( - compactionRequest(baseCompactionBody({})), - config, - { model: "", provider: "" }, - ); - const selectionsAfterFirst = authSpy.mock.calls.length; - const second = await handleResponsesCompact( - compactionRequest(baseCompactionBody({})), - config, - { model: "", provider: "" }, - ); - expect(first.status).toBe(502); - expect(second.status).toBe(503); - expect(second.headers.get("retry-after")).toBe("30"); - expect(sends).toBe(1); - expect(authSpy.mock.calls.length).toBe(selectionsAfterFirst); - expect(getCodexUpstreamHealth("pool-a")).toBeNull(); - expect(getCodexUpstreamHealth("pool-b")).toBeNull(); - } finally { - authSpy.mockRestore(); - }''' - ) - replace_exact( - "tests/upstream-host-circuit.test.ts", - ''' test("a later physical retry without its lease cannot close a newer circuit", () => {''', - ''' test("a lease cannot settle a different host key", () => { - const keyA = upstreamHostHealthKey("openai", "https://chatgpt.com"); - const keyB = upstreamHostHealthKey("openai", "https://api.openai.com"); - const leaseA = admit(keyA, 1, 10_000); - expect(resetUpstreamHostHealth(keyB, leaseA, 10_001)).toBe(false); - recordUpstreamHostFailure(keyB, { - code: "ECONNREFUSED", - now: 10_002, - threshold: 1, - lease: leaseA, - }); - expect(getUpstreamHostHealth(keyB)).toBeNull(); - recordUpstreamHostFailure(keyA, { - code: "ECONNREFUSED", - now: 10_003, - threshold: 1, - lease: leaseA, - }); - expect(getUpstreamHostHealth(keyA)).toMatchObject({ - consecutiveFailures: 1, - cooldownUntil: 10_003 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, - }); - }); - - test("a later physical retry without its lease cannot close a newer circuit", () => {''' - ) - replacements = { - "docs-site/src/content/docs/reference/configuration/providers.md": ("account health, cooldowns, thread/session affinity", "account health, account cooldowns, thread/session affinity"), - "docs-site/src/content/docs/ja/reference/configuration/providers.md": ("アカウントの健全性、クールダウン", "アカウントの健全性、アカウントのクールダウン"), - "docs-site/src/content/docs/ko/reference/configuration/providers.md": ("계정 상태, 쿨다운", "계정 상태, 계정 쿨다운"), - "docs-site/src/content/docs/ru/reference/configuration/providers.md": ("здоровье аккаунта, кулдауны,", "здоровье аккаунта, кулдауны аккаунта,"), - "docs-site/src/content/docs/zh-cn/reference/configuration/providers.md": ("账户健康、冷却、线程/会话亲和性", "账户健康、账户冷却、线程/会话亲和性"), - } - for path, (old, new) in replacements.items(): - replace_exact(path, old, new) - PY - - name: Install - run: bun install --frozen-lockfile - - name: Typecheck - run: bun run typecheck - - name: Focused circuit tests - run: bun test tests/upstream-host-circuit.test.ts tests/responses-compaction-routing.test.ts - - name: Diff check - run: git diff --check - - name: Commit hardening - shell: bash - run: | - set -euxo pipefail - git config user.name "Wibias" - git config user.email "$(printf '%s%s%s' '41898282+github-actions[bot]' '@' 'users.noreply.github.com')" - git add src/server/responses/core.ts src/server/responses/compact.ts tests/upstream-host-circuit.test.ts tests/responses-compaction-routing.test.ts docs-site/src/content/docs/*/reference/configuration/providers.md - git commit -m "fix(codex): harden provider-host circuit port" - git push origin HEAD:maint/1039-host-circuit From b08f31704f5238b8e00f6e729d7960e448de1d70 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 22:56:42 +0200 Subject: [PATCH 08/35] chore: remove maintainer port workflow --- .github/workflows/maint-1039-port.yml | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 .github/workflows/maint-1039-port.yml diff --git a/.github/workflows/maint-1039-port.yml b/.github/workflows/maint-1039-port.yml deleted file mode 100644 index 8508e2a109..0000000000 --- a/.github/workflows/maint-1039-port.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Maintainer port PR 1039 - -on: - workflow_dispatch: - -permissions: - contents: read - -jobs: - noop: - runs-on: ubuntu-latest - steps: - - run: echo "Port completed; this one-shot workflow is disabled." From 6ce3541805d4131a108324b98f31d2a71f7c9fa6 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:03:22 +0200 Subject: [PATCH 09/35] chore: run provider-host circuit second-pass hardening --- .github/workflows/maint-1039-second-pass.yml | 358 +++++++++++++++++++ 1 file changed, 358 insertions(+) create mode 100644 .github/workflows/maint-1039-second-pass.yml diff --git a/.github/workflows/maint-1039-second-pass.yml b/.github/workflows/maint-1039-second-pass.yml new file mode 100644 index 0000000000..b4f46eac57 --- /dev/null +++ b/.github/workflows/maint-1039-second-pass.yml @@ -0,0 +1,358 @@ +name: Maintainer second-pass PR 1039 + +on: + push: + branches: + - maint/1039-host-circuit + +permissions: + contents: write + +jobs: + harden: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + - name: Apply second-pass fixes + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + + def replace_exact(path, old, new, count=1): + p = Path(path) + text = p.read_text() + actual = text.count(old) + if actual != count: + raise SystemExit(f"{path}: expected {count} occurrences, found {actual}") + p.write_text(text.replace(old, new)) + + # Shared config bound; malformed hand edits degrade only this opt-in field, + # while live config writes are rejected instead of silently clamped/disabled. + replace_exact( + "src/config.ts", + 'import { isCodexAccountPriorityKey } from "./codex/account-priority";', + 'import { isCodexAccountPriorityKey } from "./codex/account-priority";\nimport { UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD } from "./codex/upstream-host-health";' + ) + replace_exact( + "src/config.ts", + ' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024),', + ''' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024), + // Opt-in availability policy: a malformed hand edit disables only this + // circuit instead of invalidating the whole config. Live writes are strict + // in validateConfigCandidate() so callers cannot mistake a clamp for success. + upstreamHostCircuitThreshold: z.number().int() + .min(0) + .max(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) + .optional() + .catch(undefined),''' + ) + + replace_exact( + "src/config.ts", + '''function warnDegradedCodexAccountPicker(rawParsed: unknown): void { + const warning = malformedCodexAccountPickerWarning(rawParsed); + if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); + }''', + '''function warnDegradedCodexAccountPicker(rawParsed: unknown): void { + const warning = malformedCodexAccountPickerWarning(rawParsed); + if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); + } + + function malformedUpstreamHostCircuitThresholdWarning(rawParsed: unknown): string | null { + const raw = rawConfigRecord(rawParsed); + if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; + const threshold = raw.upstreamHostCircuitThreshold; + if (threshold === undefined) return null; + if (typeof threshold === "number" + && Number.isInteger(threshold) + && threshold >= 0 + && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; + return `upstreamHostCircuitThreshold ignored: expected an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; + } + + function warnDegradedUpstreamHostCircuitThreshold(rawParsed: unknown): void { + const warning = malformedUpstreamHostCircuitThresholdWarning(rawParsed); + if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); + }''' + ) + + # Both load paths surface the degraded optional field. + replace_exact( + "src/config.ts", + ''' warnDegradedNativeSubagentConfig(parsed, config); + warnDegradedCodexAccountPicker(parsed); + return normalizeClaudeSubagentEffort''', + ''' warnDegradedNativeSubagentConfig(parsed, config); + warnDegradedCodexAccountPicker(parsed); + warnDegradedUpstreamHostCircuitThreshold(parsed); + return normalizeClaudeSubagentEffort''', + count=2 + ) + + replace_exact( + "src/config.ts", + ''' const pickerWarning = malformedCodexAccountPickerWarning(rawParsed); + if (pickerWarning) warnings.push(pickerWarning); + if (syncDisabledReason) {''', + ''' const pickerWarning = malformedCodexAccountPickerWarning(rawParsed); + if (pickerWarning) warnings.push(pickerWarning); + const hostCircuitWarning = malformedUpstreamHostCircuitThresholdWarning(rawParsed); + if (hostCircuitWarning) warnings.push(hostCircuitWarning); + if (syncDisabledReason) {''' + ) + + replace_exact( + "src/config.ts", + '''function appOwnedMemoryBudgetError(value: unknown): string | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const budget = (value as Record).appOwnedMemoryBudgetMb; + if (budget === undefined) return null; + if (typeof budget !== "number" || !Number.isInteger(budget) + || budget < MIN_APP_OWNED_MEMORY_BUDGET_MB || budget > MAX_APP_OWNED_MEMORY_BUDGET_MB) { + return `schema_invalid: appOwnedMemoryBudgetMb: must be an integer from ${MIN_APP_OWNED_MEMORY_BUDGET_MB} to ${MAX_APP_OWNED_MEMORY_BUDGET_MB}`; + } + return null; + }''', + '''function appOwnedMemoryBudgetError(value: unknown): string | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const budget = (value as Record).appOwnedMemoryBudgetMb; + if (budget === undefined) return null; + if (typeof budget !== "number" || !Number.isInteger(budget) + || budget < MIN_APP_OWNED_MEMORY_BUDGET_MB || budget > MAX_APP_OWNED_MEMORY_BUDGET_MB) { + return `schema_invalid: appOwnedMemoryBudgetMb: must be an integer from ${MIN_APP_OWNED_MEMORY_BUDGET_MB} to ${MAX_APP_OWNED_MEMORY_BUDGET_MB}`; + } + return null; + } + + function upstreamHostCircuitThresholdError(value: unknown): string | null { + const raw = rawConfigRecord(value); + if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; + const threshold = raw.upstreamHostCircuitThreshold; + if (threshold === undefined) return null; + if (typeof threshold === "number" + && Number.isInteger(threshold) + && threshold >= 0 + && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; + return `schema_invalid: upstreamHostCircuitThreshold: must be an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; + }''' + ) + replace_exact( + "src/config.ts", + ''' const boundaryError = blankHostnameError(value) + ?? claudeSubagentEffortError(value) + ?? appOwnedMemoryBudgetError(value) + ?? googleAntigravityStaticCatalogVersionError(value)''', + ''' const boundaryError = blankHostnameError(value) + ?? claudeSubagentEffortError(value) + ?? appOwnedMemoryBudgetError(value) + ?? upstreamHostCircuitThresholdError(value) + ?? googleAntigravityStaticCatalogVersionError(value)''' + ) + + # When the option is disabled while an old circuit entry exists, downgrade + # it to observational state. Otherwise successful traffic while disabled + # cannot clear the old circuit and re-enabling resurrects stale evidence. + replace_exact( + "src/codex/upstream-host-health.ts", + '''export function releaseUpstreamHostAdmission( + lease: UpstreamHostAdmissionLease | null | undefined, + now = Date.now(), + ): boolean { + if (!lease) return false; + const entry = matchingEntry(lease); + if (!entry) return false; + settleLease(entry, lease); + entry.lastTouch = now; + if (entry.activeLeaseIds.size === 0 && entry.consecutiveFailures === 0) { + hostHealth.delete(lease.key); + } + pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); + return true; + }''', + '''export function releaseUpstreamHostAdmission( + lease: UpstreamHostAdmissionLease | null | undefined, + now = Date.now(), + ): boolean { + if (!lease) return false; + const entry = matchingEntry(lease); + if (!entry) return false; + settleLease(entry, lease); + entry.lastTouch = now; + if (entry.activeLeaseIds.size === 0 && entry.consecutiveFailures === 0) { + hostHealth.delete(lease.key); + } + pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); + return true; + } + + /** + * Downgrade circuit-owned state to the observational ledger when the + * operator disables the circuit. Existing failure evidence is retained + * for diagnostics, but cooldown/lease authority is revoked so ordinary + * disabled-mode HTTP success/failure observations can update the entry. + */ + export function disableUpstreamHostCircuitForKey(key: string, now = Date.now()): boolean { + const entry = hostHealth.get(key); + if (!entry?.circuitManaged) return false; + entry.circuitManaged = false; + delete entry.cooldownUntil; + advanceGeneration(entry); + entry.lastTouch = now; + if (entry.consecutiveFailures === 0) hostHealth.delete(key); + else pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); + return true; + }''' + ) + + for path in ["src/server/responses/core.ts", "src/server/responses/compact.ts"]: + replace_exact( + path, + ''' acquireUpstreamHostAdmission, + normalizeUpstreamHostCircuitThreshold,''', + ''' acquireUpstreamHostAdmission, + disableUpstreamHostCircuitForKey, + normalizeUpstreamHostCircuitThreshold,''' + ) + + replace_exact( + "src/server/responses/core.ts", + ''' const hostCircuitEnabled = hostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''', + ''' const hostCircuitEnabled = hostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (hostKey !== null && !hostCircuitEnabled) { + disableUpstreamHostCircuitForKey(actualHostKey); + } + if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''' + ) + replace_exact( + "src/server/responses/compact.ts", + ''' const compactHostCircuitEnabled = compactHostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''', + ''' const compactHostCircuitEnabled = compactHostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (compactHostKey !== null && !compactHostCircuitEnabled) { + disableUpstreamHostCircuitForKey(actualCompactHostKey); + } + if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''' + ) + + replace_exact( + "tests/upstream-host-circuit.test.ts", + ''' acquireUpstreamHostAdmission, + clearUpstreamHostHealth,''', + ''' acquireUpstreamHostAdmission, + clearUpstreamHostHealth, + disableUpstreamHostCircuitForKey,''' + ) + replace_exact( + "tests/upstream-host-circuit.test.ts", + ''' test("a later physical retry without its lease cannot close a newer circuit", () => {''', + ''' test("disabled-mode observations can clear an old circuit instead of resurrecting stale state", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 11_000); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(11_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); + + expect(disableUpstreamHostCircuitForKey(key, 11_001)).toBe(true); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); + + // A real HTTP response while the circuit is disabled is now an + // ordinary observational reset and removes the stale streak. + expect(resetUpstreamHostHealth(key)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + + const admission = acquireUpstreamHostAdmission(key, 1, 11_002); + expect(admission.kind).toBe("admitted"); + }); + + test("a later physical retry without its lease cannot close a newer circuit", () => {''' + ) + PY + + cat > tests/upstream-host-circuit-config.test.ts <<'EOF' + import { afterEach, describe, expect, test } from "bun:test"; + import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; + import { tmpdir } from "node:os"; + import { join } from "node:path"; + import { + getConfigPath, + getDefaultConfig, + readConfigDiagnostics, + validateConfigCandidate, + } from "../src/config"; + + let testDir = ""; + + afterEach(() => { + delete process.env.OPENCODEX_HOME; + if (testDir && existsSync(testDir)) rmSync(testDir, { recursive: true, force: true }); + testDir = ""; + }); + + describe("upstreamHostCircuitThreshold config contract", () => { + test("live config writes accept only integer values from 0 through 20", () => { + for (const value of [0, 1, 20]) { + const result = validateConfigCandidate({ + ...getDefaultConfig(), + upstreamHostCircuitThreshold: value, + }); + expect(result.ok).toBe(true); + } + + for (const value of [-1, 1.5, 21, "3", null]) { + const result = validateConfigCandidate({ + ...getDefaultConfig(), + upstreamHostCircuitThreshold: value, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); + } + }); + + test("a malformed hand edit disables only the opt-in circuit and reports a warning", () => { + testDir = mkdtempSync(join(tmpdir(), "ocx-host-circuit-config-")); + process.env.OPENCODEX_HOME = testDir; + mkdirSync(testDir, { recursive: true }); + writeFileSync(getConfigPath(), JSON.stringify({ + ...getDefaultConfig(), + upstreamHostCircuitThreshold: 999, + })); + + const diagnostics = readConfigDiagnostics(); + expect(diagnostics.source).toBe("file"); + expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); + expect(diagnostics.warnings).toContain( + "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", + ); + expect(Object.keys(diagnostics.config.providers).length).toBeGreaterThan(0); + }); + }); + EOF + + - name: Install + run: bun install --frozen-lockfile + - name: Typecheck + run: bun run typecheck + - name: Focused tests + run: bun test tests/upstream-host-circuit.test.ts tests/upstream-host-circuit-config.test.ts tests/responses-compaction-routing.test.ts + - name: Privacy scan + run: bun run privacy:scan + - name: Diff check + run: git diff --check + - name: Commit second-pass fixes + shell: bash + run: | + set -euxo pipefail + git config user.name "Wibias" + git config user.email "$(printf '%s%s%s' '41898282+github-actions[bot]' '@' 'users.noreply.github.com')" + git add src/config.ts src/codex/upstream-host-health.ts src/server/responses/core.ts src/server/responses/compact.ts tests/upstream-host-circuit.test.ts tests/upstream-host-circuit-config.test.ts + git commit -m "fix(codex): harden host circuit config and disable semantics" + git push origin HEAD:maint/1039-host-circuit From fc0c9c63455c930b4fff975a0c55cb11528d9f14 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:05:08 +0200 Subject: [PATCH 10/35] chore: tighten second-pass hardening anchors --- .github/workflows/maint-1039-second-pass.yml | 416 +++++++------------ 1 file changed, 160 insertions(+), 256 deletions(-) diff --git a/.github/workflows/maint-1039-second-pass.yml b/.github/workflows/maint-1039-second-pass.yml index b4f46eac57..643495bc26 100644 --- a/.github/workflows/maint-1039-second-pass.yml +++ b/.github/workflows/maint-1039-second-pass.yml @@ -23,274 +23,189 @@ jobs: python3 - <<'PY' from pathlib import Path - def replace_exact(path, old, new, count=1): + def edit(path, fn): p = Path(path) text = p.read_text() - actual = text.count(old) - if actual != count: - raise SystemExit(f"{path}: expected {count} occurrences, found {actual}") - p.write_text(text.replace(old, new)) + new = fn(text) + if new == text: + raise SystemExit(f"{path}: edit made no change") + p.write_text(new) - # Shared config bound; malformed hand edits degrade only this opt-in field, - # while live config writes are rejected instead of silently clamped/disabled. - replace_exact( - "src/config.ts", - 'import { isCodexAccountPriorityKey } from "./codex/account-priority";', - 'import { isCodexAccountPriorityKey } from "./codex/account-priority";\nimport { UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD } from "./codex/upstream-host-health";' - ) - replace_exact( - "src/config.ts", - ' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024),', - ''' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024), - // Opt-in availability policy: a malformed hand edit disables only this - // circuit instead of invalidating the whole config. Live writes are strict - // in validateConfigCandidate() so callers cannot mistake a clamp for success. - upstreamHostCircuitThreshold: z.number().int() - .min(0) - .max(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) - .optional() - .catch(undefined),''' - ) + def one(text, old, new, label): + n = text.count(old) + if n != 1: + raise SystemExit(f"{label}: expected 1 occurrence, found {n}") + return text.replace(old, new, 1) - replace_exact( - "src/config.ts", - '''function warnDegradedCodexAccountPicker(rawParsed: unknown): void { - const warning = malformedCodexAccountPickerWarning(rawParsed); - if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); - }''', - '''function warnDegradedCodexAccountPicker(rawParsed: unknown): void { - const warning = malformedCodexAccountPickerWarning(rawParsed); - if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); - } + def many(text, old, new, expected, label): + n = text.count(old) + if n != expected: + raise SystemExit(f"{label}: expected {expected} occurrences, found {n}") + return text.replace(old, new) - function malformedUpstreamHostCircuitThresholdWarning(rawParsed: unknown): string | null { - const raw = rawConfigRecord(rawParsed); - if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; - const threshold = raw.upstreamHostCircuitThreshold; - if (threshold === undefined) return null; - if (typeof threshold === "number" - && Number.isInteger(threshold) - && threshold >= 0 - && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; - return `upstreamHostCircuitThreshold ignored: expected an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; - } + def patch_config(text): + text = one(text, + 'import { isCodexAccountPriorityKey } from "./codex/account-priority";', + 'import { isCodexAccountPriorityKey } from "./codex/account-priority";\nimport { UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD } from "./codex/upstream-host-health";', + 'config import') + text = one(text, + ' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024),', + ''' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024), + // Invalid hand edits disable only this opt-in circuit. Live writes remain strict. + upstreamHostCircuitThreshold: z.number().int() + .min(0) + .max(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) + .optional() + .catch(undefined),''', + 'config schema field') - function warnDegradedUpstreamHostCircuitThreshold(rawParsed: unknown): void { - const warning = malformedUpstreamHostCircuitThresholdWarning(rawParsed); - if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); - }''' - ) + marker = 'type NativeSubagentPersistedField = "injectionModel" | "injectionEffort" | "syncCodexSubagentDefaults";' + helper = '''function malformedUpstreamHostCircuitThresholdWarning(rawParsed: unknown): string | null { + const raw = rawConfigRecord(rawParsed); + if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; + const threshold = raw.upstreamHostCircuitThreshold; + if (threshold === undefined) return null; + if (typeof threshold === "number" + && Number.isInteger(threshold) + && threshold >= 0 + && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; + return `upstreamHostCircuitThreshold ignored: expected an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; +} - # Both load paths surface the degraded optional field. - replace_exact( - "src/config.ts", - ''' warnDegradedNativeSubagentConfig(parsed, config); - warnDegradedCodexAccountPicker(parsed); - return normalizeClaudeSubagentEffort''', - ''' warnDegradedNativeSubagentConfig(parsed, config); - warnDegradedCodexAccountPicker(parsed); - warnDegradedUpstreamHostCircuitThreshold(parsed); - return normalizeClaudeSubagentEffort''', - count=2 - ) +function warnDegradedUpstreamHostCircuitThreshold(rawParsed: unknown): void { + const warning = malformedUpstreamHostCircuitThresholdWarning(rawParsed); + if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); +} - replace_exact( - "src/config.ts", - ''' const pickerWarning = malformedCodexAccountPickerWarning(rawParsed); - if (pickerWarning) warnings.push(pickerWarning); - if (syncDisabledReason) {''', - ''' const pickerWarning = malformedCodexAccountPickerWarning(rawParsed); - if (pickerWarning) warnings.push(pickerWarning); - const hostCircuitWarning = malformedUpstreamHostCircuitThresholdWarning(rawParsed); - if (hostCircuitWarning) warnings.push(hostCircuitWarning); - if (syncDisabledReason) {''' - ) +''' + text = one(text, marker, helper + marker, 'config warning helper') + text = many(text, + ' warnDegradedCodexAccountPicker(parsed);\n', + ' warnDegradedCodexAccountPicker(parsed);\n warnDegradedUpstreamHostCircuitThreshold(parsed);\n', + 2, + 'config load warnings') + text = one(text, + ' if (pickerWarning) warnings.push(pickerWarning);\n', + ' if (pickerWarning) warnings.push(pickerWarning);\n const hostCircuitWarning = malformedUpstreamHostCircuitThresholdWarning(rawParsed);\n if (hostCircuitWarning) warnings.push(hostCircuitWarning);\n', + 'config diagnostics warning') - replace_exact( - "src/config.ts", - '''function appOwnedMemoryBudgetError(value: unknown): string | null { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const budget = (value as Record).appOwnedMemoryBudgetMb; - if (budget === undefined) return null; - if (typeof budget !== "number" || !Number.isInteger(budget) - || budget < MIN_APP_OWNED_MEMORY_BUDGET_MB || budget > MAX_APP_OWNED_MEMORY_BUDGET_MB) { - return `schema_invalid: appOwnedMemoryBudgetMb: must be an integer from ${MIN_APP_OWNED_MEMORY_BUDGET_MB} to ${MAX_APP_OWNED_MEMORY_BUDGET_MB}`; - } - return null; - }''', - '''function appOwnedMemoryBudgetError(value: unknown): string | null { - if (!value || typeof value !== "object" || Array.isArray(value)) return null; - const budget = (value as Record).appOwnedMemoryBudgetMb; - if (budget === undefined) return null; - if (typeof budget !== "number" || !Number.isInteger(budget) - || budget < MIN_APP_OWNED_MEMORY_BUDGET_MB || budget > MAX_APP_OWNED_MEMORY_BUDGET_MB) { - return `schema_invalid: appOwnedMemoryBudgetMb: must be an integer from ${MIN_APP_OWNED_MEMORY_BUDGET_MB} to ${MAX_APP_OWNED_MEMORY_BUDGET_MB}`; - } - return null; - } + marker = '/**\n * Same reasoning as {@link blankHostnameError}, and more urgent:' + boundary_helper = '''function upstreamHostCircuitThresholdError(value: unknown): string | null { + const raw = rawConfigRecord(value); + if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; + const threshold = raw.upstreamHostCircuitThreshold; + if (threshold === undefined) return null; + if (typeof threshold === "number" + && Number.isInteger(threshold) + && threshold >= 0 + && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; + return `schema_invalid: upstreamHostCircuitThreshold: must be an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; +} - function upstreamHostCircuitThresholdError(value: unknown): string | null { - const raw = rawConfigRecord(value); - if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; - const threshold = raw.upstreamHostCircuitThreshold; - if (threshold === undefined) return null; - if (typeof threshold === "number" - && Number.isInteger(threshold) - && threshold >= 0 - && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; - return `schema_invalid: upstreamHostCircuitThreshold: must be an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; - }''' - ) - replace_exact( - "src/config.ts", - ''' const boundaryError = blankHostnameError(value) - ?? claudeSubagentEffortError(value) - ?? appOwnedMemoryBudgetError(value) - ?? googleAntigravityStaticCatalogVersionError(value)''', - ''' const boundaryError = blankHostnameError(value) - ?? claudeSubagentEffortError(value) - ?? appOwnedMemoryBudgetError(value) - ?? upstreamHostCircuitThresholdError(value) - ?? googleAntigravityStaticCatalogVersionError(value)''' - ) +''' + text = one(text, marker, boundary_helper + marker, 'config write-boundary helper') + text = one(text, + ' ?? appOwnedMemoryBudgetError(value)\n ?? googleAntigravityStaticCatalogVersionError(value)', + ' ?? appOwnedMemoryBudgetError(value)\n ?? upstreamHostCircuitThresholdError(value)\n ?? googleAntigravityStaticCatalogVersionError(value)', + 'config boundary chain') + return text - # When the option is disabled while an old circuit entry exists, downgrade - # it to observational state. Otherwise successful traffic while disabled - # cannot clear the old circuit and re-enabling resurrects stale evidence. - replace_exact( - "src/codex/upstream-host-health.ts", - '''export function releaseUpstreamHostAdmission( - lease: UpstreamHostAdmissionLease | null | undefined, - now = Date.now(), - ): boolean { - if (!lease) return false; - const entry = matchingEntry(lease); - if (!entry) return false; - settleLease(entry, lease); - entry.lastTouch = now; - if (entry.activeLeaseIds.size === 0 && entry.consecutiveFailures === 0) { - hostHealth.delete(lease.key); - } - pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); - return true; - }''', - '''export function releaseUpstreamHostAdmission( - lease: UpstreamHostAdmissionLease | null | undefined, - now = Date.now(), - ): boolean { - if (!lease) return false; - const entry = matchingEntry(lease); - if (!entry) return false; - settleLease(entry, lease); - entry.lastTouch = now; - if (entry.activeLeaseIds.size === 0 && entry.consecutiveFailures === 0) { - hostHealth.delete(lease.key); - } - pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); - return true; - } + edit('src/config.ts', patch_config) - /** - * Downgrade circuit-owned state to the observational ledger when the - * operator disables the circuit. Existing failure evidence is retained - * for diagnostics, but cooldown/lease authority is revoked so ordinary - * disabled-mode HTTP success/failure observations can update the entry. - */ - export function disableUpstreamHostCircuitForKey(key: string, now = Date.now()): boolean { - const entry = hostHealth.get(key); - if (!entry?.circuitManaged) return false; - entry.circuitManaged = false; - delete entry.cooldownUntil; - advanceGeneration(entry); - entry.lastTouch = now; - if (entry.consecutiveFailures === 0) hostHealth.delete(key); - else pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); - return true; - }''' - ) + def patch_health(text): + marker = '/** Record one terminal logical `connect_neutral` failure. */' + helper = '''/** + * Downgrade circuit-owned state when the operator disables the circuit. + * Failure history remains observational, but cooldown and all in-flight lease + * authority are revoked so disabled-mode traffic can update the ledger normally. + */ +export function disableUpstreamHostCircuitForKey(key: string, now = Date.now()): boolean { + const entry = hostHealth.get(key); + if (!entry?.circuitManaged) return false; + entry.circuitManaged = false; + delete entry.cooldownUntil; + advanceGeneration(entry); + entry.lastTouch = now; + if (entry.consecutiveFailures === 0) hostHealth.delete(key); + else pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); + return true; +} - for path in ["src/server/responses/core.ts", "src/server/responses/compact.ts"]: - replace_exact( - path, - ''' acquireUpstreamHostAdmission, - normalizeUpstreamHostCircuitThreshold,''', - ''' acquireUpstreamHostAdmission, - disableUpstreamHostCircuitForKey, - normalizeUpstreamHostCircuitThreshold,''' - ) +''' + return one(text, marker, helper + marker, 'host disable helper') - replace_exact( - "src/server/responses/core.ts", - ''' const hostCircuitEnabled = hostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''', - ''' const hostCircuitEnabled = hostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (hostKey !== null && !hostCircuitEnabled) { - disableUpstreamHostCircuitForKey(actualHostKey); - } - if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''' - ) - replace_exact( - "src/server/responses/compact.ts", - ''' const compactHostCircuitEnabled = compactHostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''', - ''' const compactHostCircuitEnabled = compactHostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (compactHostKey !== null && !compactHostCircuitEnabled) { - disableUpstreamHostCircuitForKey(actualCompactHostKey); - } - if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''' - ) + edit('src/codex/upstream-host-health.ts', patch_health) - replace_exact( - "tests/upstream-host-circuit.test.ts", - ''' acquireUpstreamHostAdmission, - clearUpstreamHostHealth,''', - ''' acquireUpstreamHostAdmission, - clearUpstreamHostHealth, - disableUpstreamHostCircuitForKey,''' - ) - replace_exact( - "tests/upstream-host-circuit.test.ts", - ''' test("a later physical retry without its lease cannot close a newer circuit", () => {''', - ''' test("disabled-mode observations can clear an old circuit instead of resurrecting stale state", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 1, 11_000); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(11_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); + def add_disable_import(text): + return one(text, + ' acquireUpstreamHostAdmission,\n normalizeUpstreamHostCircuitThreshold,', + ' acquireUpstreamHostAdmission,\n disableUpstreamHostCircuitForKey,\n normalizeUpstreamHostCircuitThreshold,', + 'circuit disable import') - expect(disableUpstreamHostCircuitForKey(key, 11_001)).toBe(true); - expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); + edit('src/server/responses/core.ts', add_disable_import) + edit('src/server/responses/compact.ts', add_disable_import) - // A real HTTP response while the circuit is disabled is now an - // ordinary observational reset and removes the stale streak. - expect(resetUpstreamHostHealth(key)).toBe(true); - expect(getUpstreamHostHealth(key)).toBeNull(); + def patch_core(text): + return one(text, + ''' const hostCircuitEnabled = hostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''', + ''' const hostCircuitEnabled = hostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (hostKey !== null && !hostCircuitEnabled) { + disableUpstreamHostCircuitForKey(actualHostKey); + } + if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''', + 'regular disabled-mode reconciliation') + edit('src/server/responses/core.ts', patch_core) - const admission = acquireUpstreamHostAdmission(key, 1, 11_002); - expect(admission.kind).toBe("admitted"); - }); + def patch_compact(text): + return one(text, + ''' const compactHostCircuitEnabled = compactHostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''', + ''' const compactHostCircuitEnabled = compactHostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (compactHostKey !== null && !compactHostCircuitEnabled) { + disableUpstreamHostCircuitForKey(actualCompactHostKey); + } + if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''', + 'compact disabled-mode reconciliation') + edit('src/server/responses/compact.ts', patch_compact) + + def patch_health_test(text): + text = one(text, + ' clearUpstreamHostHealth,\n', + ' clearUpstreamHostHealth,\n disableUpstreamHostCircuitForKey,\n', + 'health test import') + marker = ' test("a later physical retry without its lease cannot close a newer circuit", () => {' + test = ''' test("disabled traffic can clear an old circuit before it is re-enabled", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 11_000); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(11_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); + + expect(disableUpstreamHostCircuitForKey(key, 11_001)).toBe(true); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); + + expect(resetUpstreamHostHealth(key)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + expect(acquireUpstreamHostAdmission(key, 1, 11_002).kind).toBe("admitted"); + }); - test("a later physical retry without its lease cannot close a newer circuit", () => {''' - ) +''' + return one(text, marker, test + marker, 'disabled-mode health regression') + edit('tests/upstream-host-circuit.test.ts', patch_health_test) PY cat > tests/upstream-host-circuit-config.test.ts <<'EOF' import { afterEach, describe, expect, test } from "bun:test"; - import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; + import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; - import { - getConfigPath, - getDefaultConfig, - readConfigDiagnostics, - validateConfigCandidate, - } from "../src/config"; + import { getConfigPath, getDefaultConfig, readConfigDiagnostics, validateConfigCandidate } from "../src/config"; let testDir = ""; - afterEach(() => { delete process.env.OPENCODEX_HOME; if (testDir && existsSync(testDir)) rmSync(testDir, { recursive: true, force: true }); @@ -298,34 +213,24 @@ jobs: }); describe("upstreamHostCircuitThreshold config contract", () => { - test("live config writes accept only integer values from 0 through 20", () => { + test("live writes accept only integer values from 0 through 20", () => { for (const value of [0, 1, 20]) { - const result = validateConfigCandidate({ - ...getDefaultConfig(), - upstreamHostCircuitThreshold: value, - }); - expect(result.ok).toBe(true); + expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); } - for (const value of [-1, 1.5, 21, "3", null]) { - const result = validateConfigCandidate({ - ...getDefaultConfig(), - upstreamHostCircuitThreshold: value, - }); + const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); expect(result.ok).toBe(false); if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); } }); - test("a malformed hand edit disables only the opt-in circuit and reports a warning", () => { + test("malformed hand edits disable only the circuit and report a warning", () => { testDir = mkdtempSync(join(tmpdir(), "ocx-host-circuit-config-")); process.env.OPENCODEX_HOME = testDir; - mkdirSync(testDir, { recursive: true }); writeFileSync(getConfigPath(), JSON.stringify({ ...getDefaultConfig(), upstreamHostCircuitThreshold: 999, })); - const diagnostics = readConfigDiagnostics(); expect(diagnostics.source).toBe("file"); expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); @@ -336,7 +241,6 @@ jobs: }); }); EOF - - name: Install run: bun install --frozen-lockfile - name: Typecheck From 82f8f45386611d7ffcafe520ae888568716f899b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:08:19 +0200 Subject: [PATCH 11/35] chore: stage second-pass host circuit patch --- scripts/maint_1039_patch.py | 257 ++++++++++++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 scripts/maint_1039_patch.py diff --git a/scripts/maint_1039_patch.py b/scripts/maint_1039_patch.py new file mode 100644 index 0000000000..8241b78662 --- /dev/null +++ b/scripts/maint_1039_patch.py @@ -0,0 +1,257 @@ +from pathlib import Path + + +def edit(path: str, fn): + p = Path(path) + text = p.read_text() + new = fn(text) + if new == text: + raise SystemExit(f"{path}: edit made no change") + p.write_text(new) + + +def one(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected 1 occurrence, found {count}") + return text.replace(old, new, 1) + + +def many(text: str, old: str, new: str, expected: int, label: str) -> str: + count = text.count(old) + if count != expected: + raise SystemExit(f"{label}: expected {expected} occurrences, found {count}") + return text.replace(old, new) + + +def patch_config(text: str) -> str: + text = one( + text, + 'import { isCodexAccountPriorityKey } from "./codex/account-priority";', + 'import { isCodexAccountPriorityKey } from "./codex/account-priority";\nimport { UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD } from "./codex/upstream-host-health";', + "config import", + ) + text = one( + text, + ' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024),', + ''' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024), + // Invalid hand edits disable only this opt-in circuit. Live writes remain strict. + upstreamHostCircuitThreshold: z.number().int() + .min(0) + .max(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) + .optional() + .catch(undefined),''', + "config schema field", + ) + + warning_marker = 'type NativeSubagentPersistedField = "injectionModel" | "injectionEffort" | "syncCodexSubagentDefaults";' + warning_helpers = '''function malformedUpstreamHostCircuitThresholdWarning(rawParsed: unknown): string | null { + const raw = rawConfigRecord(rawParsed); + if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; + const threshold = raw.upstreamHostCircuitThreshold; + if (threshold === undefined) return null; + if (typeof threshold === "number" + && Number.isInteger(threshold) + && threshold >= 0 + && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; + return `upstreamHostCircuitThreshold ignored: expected an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; +} + +function warnDegradedUpstreamHostCircuitThreshold(rawParsed: unknown): void { + const warning = malformedUpstreamHostCircuitThresholdWarning(rawParsed); + if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); +} + +''' + text = one(text, warning_marker, warning_helpers + warning_marker, "config warning helper") + + text = many( + text, + ' warnDegradedCodexAccountPicker(parsed);\n', + ' warnDegradedCodexAccountPicker(parsed);\n warnDegradedUpstreamHostCircuitThreshold(parsed);\n', + 2, + "config load warnings", + ) + text = one( + text, + ' if (pickerWarning) warnings.push(pickerWarning);\n', + ' if (pickerWarning) warnings.push(pickerWarning);\n const hostCircuitWarning = malformedUpstreamHostCircuitThresholdWarning(rawParsed);\n if (hostCircuitWarning) warnings.push(hostCircuitWarning);\n', + "config diagnostics warning", + ) + + boundary_marker = '/**\n * Same reasoning as {@link blankHostnameError}, and more urgent:' + boundary_helper = '''function upstreamHostCircuitThresholdError(value: unknown): string | null { + const raw = rawConfigRecord(value); + if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; + const threshold = raw.upstreamHostCircuitThreshold; + if (threshold === undefined) return null; + if (typeof threshold === "number" + && Number.isInteger(threshold) + && threshold >= 0 + && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; + return `schema_invalid: upstreamHostCircuitThreshold: must be an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; +} + +''' + text = one(text, boundary_marker, boundary_helper + boundary_marker, "config write-boundary helper") + text = one( + text, + ' ?? appOwnedMemoryBudgetError(value)\n ?? googleAntigravityStaticCatalogVersionError(value)', + ' ?? appOwnedMemoryBudgetError(value)\n ?? upstreamHostCircuitThresholdError(value)\n ?? googleAntigravityStaticCatalogVersionError(value)', + "config boundary chain", + ) + return text + + +edit("src/config.ts", patch_config) + + +def patch_health(text: str) -> str: + marker = '/** Record one terminal logical `connect_neutral` failure. */' + helper = '''/** + * Downgrade circuit-owned state when the operator disables the circuit. + * Failure history remains observational, but cooldown and all in-flight lease + * authority are revoked so disabled-mode traffic can update the ledger normally. + */ +export function disableUpstreamHostCircuitForKey(key: string, now = Date.now()): boolean { + const entry = hostHealth.get(key); + if (!entry?.circuitManaged) return false; + entry.circuitManaged = false; + delete entry.cooldownUntil; + advanceGeneration(entry); + entry.lastTouch = now; + if (entry.consecutiveFailures === 0) hostHealth.delete(key); + else pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); + return true; +} + +''' + return one(text, marker, helper + marker, "host disable helper") + + +edit("src/codex/upstream-host-health.ts", patch_health) + + +def add_disable_import(text: str) -> str: + return one( + text, + ' acquireUpstreamHostAdmission,\n normalizeUpstreamHostCircuitThreshold,', + ' acquireUpstreamHostAdmission,\n disableUpstreamHostCircuitForKey,\n normalizeUpstreamHostCircuitThreshold,', + "circuit disable import", + ) + + +edit("src/server/responses/core.ts", add_disable_import) +edit("src/server/responses/compact.ts", add_disable_import) + + +def patch_core(text: str) -> str: + return one( + text, + ''' const hostCircuitEnabled = hostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''', + ''' const hostCircuitEnabled = hostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (hostKey !== null && !hostCircuitEnabled) { + disableUpstreamHostCircuitForKey(actualHostKey); + } + if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''', + "regular disabled-mode reconciliation", + ) + + +edit("src/server/responses/core.ts", patch_core) + + +def patch_compact(text: str) -> str: + return one( + text, + ''' const compactHostCircuitEnabled = compactHostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''', + ''' const compactHostCircuitEnabled = compactHostKey !== null + && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (compactHostKey !== null && !compactHostCircuitEnabled) { + disableUpstreamHostCircuitForKey(actualCompactHostKey); + } + if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''', + "compact disabled-mode reconciliation", + ) + + +edit("src/server/responses/compact.ts", patch_compact) + + +def patch_health_test(text: str) -> str: + text = one( + text, + ' clearUpstreamHostHealth,\n', + ' clearUpstreamHostHealth,\n disableUpstreamHostCircuitForKey,\n', + "health test import", + ) + marker = ' test("a later physical retry without its lease cannot close a newer circuit", () => {' + regression = ''' test("disabled traffic can clear an old circuit before it is re-enabled", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 11_000); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(11_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); + + expect(disableUpstreamHostCircuitForKey(key, 11_001)).toBe(true); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); + + expect(resetUpstreamHostHealth(key)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + expect(acquireUpstreamHostAdmission(key, 1, 11_002).kind).toBe("admitted"); + }); + +''' + return one(text, marker, regression + marker, "disabled-mode health regression") + + +edit("tests/upstream-host-circuit.test.ts", patch_health_test) + +Path("tests/upstream-host-circuit-config.test.ts").write_text('''import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { getConfigPath, getDefaultConfig, readConfigDiagnostics, validateConfigCandidate } from "../src/config"; + +let testDir = ""; + +afterEach(() => { + delete process.env.OPENCODEX_HOME; + if (testDir && existsSync(testDir)) rmSync(testDir, { recursive: true, force: true }); + testDir = ""; +}); + +describe("upstreamHostCircuitThreshold config contract", () => { + test("live writes accept only integer values from 0 through 20", () => { + for (const value of [0, 1, 20]) { + expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); + } + for (const value of [-1, 1.5, 21, "3", null]) { + const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); + } + }); + + test("malformed hand edits disable only the circuit and report a warning", () => { + testDir = mkdtempSync(join(tmpdir(), "ocx-host-circuit-config-")); + process.env.OPENCODEX_HOME = testDir; + writeFileSync(getConfigPath(), JSON.stringify({ + ...getDefaultConfig(), + upstreamHostCircuitThreshold: 999, + })); + + const diagnostics = readConfigDiagnostics(); + expect(diagnostics.source).toBe("file"); + expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); + expect(diagnostics.warnings).toContain( + "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", + ); + expect(Object.keys(diagnostics.config.providers).length).toBeGreaterThan(0); + }); +}); +''') From 9828d02aa16225076788d649071b1446515c4a04 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:08:33 +0200 Subject: [PATCH 12/35] chore: simplify second-pass host circuit runner --- .github/workflows/maint-1039-second-pass.yml | 228 +------------------ 1 file changed, 3 insertions(+), 225 deletions(-) diff --git a/.github/workflows/maint-1039-second-pass.yml b/.github/workflows/maint-1039-second-pass.yml index 643495bc26..3148602fb8 100644 --- a/.github/workflows/maint-1039-second-pass.yml +++ b/.github/workflows/maint-1039-second-pass.yml @@ -17,230 +17,8 @@ jobs: - uses: oven-sh/setup-bun@v2 with: bun-version: 1.3.14 - - name: Apply second-pass fixes - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - - def edit(path, fn): - p = Path(path) - text = p.read_text() - new = fn(text) - if new == text: - raise SystemExit(f"{path}: edit made no change") - p.write_text(new) - - def one(text, old, new, label): - n = text.count(old) - if n != 1: - raise SystemExit(f"{label}: expected 1 occurrence, found {n}") - return text.replace(old, new, 1) - - def many(text, old, new, expected, label): - n = text.count(old) - if n != expected: - raise SystemExit(f"{label}: expected {expected} occurrences, found {n}") - return text.replace(old, new) - - def patch_config(text): - text = one(text, - 'import { isCodexAccountPriorityKey } from "./codex/account-priority";', - 'import { isCodexAccountPriorityKey } from "./codex/account-priority";\nimport { UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD } from "./codex/upstream-host-health";', - 'config import') - text = one(text, - ' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024),', - ''' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024), - // Invalid hand edits disable only this opt-in circuit. Live writes remain strict. - upstreamHostCircuitThreshold: z.number().int() - .min(0) - .max(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) - .optional() - .catch(undefined),''', - 'config schema field') - - marker = 'type NativeSubagentPersistedField = "injectionModel" | "injectionEffort" | "syncCodexSubagentDefaults";' - helper = '''function malformedUpstreamHostCircuitThresholdWarning(rawParsed: unknown): string | null { - const raw = rawConfigRecord(rawParsed); - if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; - const threshold = raw.upstreamHostCircuitThreshold; - if (threshold === undefined) return null; - if (typeof threshold === "number" - && Number.isInteger(threshold) - && threshold >= 0 - && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; - return `upstreamHostCircuitThreshold ignored: expected an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; -} - -function warnDegradedUpstreamHostCircuitThreshold(rawParsed: unknown): void { - const warning = malformedUpstreamHostCircuitThresholdWarning(rawParsed); - if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); -} - -''' - text = one(text, marker, helper + marker, 'config warning helper') - text = many(text, - ' warnDegradedCodexAccountPicker(parsed);\n', - ' warnDegradedCodexAccountPicker(parsed);\n warnDegradedUpstreamHostCircuitThreshold(parsed);\n', - 2, - 'config load warnings') - text = one(text, - ' if (pickerWarning) warnings.push(pickerWarning);\n', - ' if (pickerWarning) warnings.push(pickerWarning);\n const hostCircuitWarning = malformedUpstreamHostCircuitThresholdWarning(rawParsed);\n if (hostCircuitWarning) warnings.push(hostCircuitWarning);\n', - 'config diagnostics warning') - - marker = '/**\n * Same reasoning as {@link blankHostnameError}, and more urgent:' - boundary_helper = '''function upstreamHostCircuitThresholdError(value: unknown): string | null { - const raw = rawConfigRecord(value); - if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; - const threshold = raw.upstreamHostCircuitThreshold; - if (threshold === undefined) return null; - if (typeof threshold === "number" - && Number.isInteger(threshold) - && threshold >= 0 - && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; - return `schema_invalid: upstreamHostCircuitThreshold: must be an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; -} - -''' - text = one(text, marker, boundary_helper + marker, 'config write-boundary helper') - text = one(text, - ' ?? appOwnedMemoryBudgetError(value)\n ?? googleAntigravityStaticCatalogVersionError(value)', - ' ?? appOwnedMemoryBudgetError(value)\n ?? upstreamHostCircuitThresholdError(value)\n ?? googleAntigravityStaticCatalogVersionError(value)', - 'config boundary chain') - return text - - edit('src/config.ts', patch_config) - - def patch_health(text): - marker = '/** Record one terminal logical `connect_neutral` failure. */' - helper = '''/** - * Downgrade circuit-owned state when the operator disables the circuit. - * Failure history remains observational, but cooldown and all in-flight lease - * authority are revoked so disabled-mode traffic can update the ledger normally. - */ -export function disableUpstreamHostCircuitForKey(key: string, now = Date.now()): boolean { - const entry = hostHealth.get(key); - if (!entry?.circuitManaged) return false; - entry.circuitManaged = false; - delete entry.cooldownUntil; - advanceGeneration(entry); - entry.lastTouch = now; - if (entry.consecutiveFailures === 0) hostHealth.delete(key); - else pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); - return true; -} - -''' - return one(text, marker, helper + marker, 'host disable helper') - - edit('src/codex/upstream-host-health.ts', patch_health) - - def add_disable_import(text): - return one(text, - ' acquireUpstreamHostAdmission,\n normalizeUpstreamHostCircuitThreshold,', - ' acquireUpstreamHostAdmission,\n disableUpstreamHostCircuitForKey,\n normalizeUpstreamHostCircuitThreshold,', - 'circuit disable import') - - edit('src/server/responses/core.ts', add_disable_import) - edit('src/server/responses/compact.ts', add_disable_import) - - def patch_core(text): - return one(text, - ''' const hostCircuitEnabled = hostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''', - ''' const hostCircuitEnabled = hostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (hostKey !== null && !hostCircuitEnabled) { - disableUpstreamHostCircuitForKey(actualHostKey); - } - if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''', - 'regular disabled-mode reconciliation') - edit('src/server/responses/core.ts', patch_core) - - def patch_compact(text): - return one(text, - ''' const compactHostCircuitEnabled = compactHostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''', - ''' const compactHostCircuitEnabled = compactHostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (compactHostKey !== null && !compactHostCircuitEnabled) { - disableUpstreamHostCircuitForKey(actualCompactHostKey); - } - if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''', - 'compact disabled-mode reconciliation') - edit('src/server/responses/compact.ts', patch_compact) - - def patch_health_test(text): - text = one(text, - ' clearUpstreamHostHealth,\n', - ' clearUpstreamHostHealth,\n disableUpstreamHostCircuitForKey,\n', - 'health test import') - marker = ' test("a later physical retry without its lease cannot close a newer circuit", () => {' - test = ''' test("disabled traffic can clear an old circuit before it is re-enabled", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 1, 11_000); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(11_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); - - expect(disableUpstreamHostCircuitForKey(key, 11_001)).toBe(true); - expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); - - expect(resetUpstreamHostHealth(key)).toBe(true); - expect(getUpstreamHostHealth(key)).toBeNull(); - expect(acquireUpstreamHostAdmission(key, 1, 11_002).kind).toBe("admitted"); - }); - -''' - return one(text, marker, test + marker, 'disabled-mode health regression') - edit('tests/upstream-host-circuit.test.ts', patch_health_test) - PY - - cat > tests/upstream-host-circuit-config.test.ts <<'EOF' - import { afterEach, describe, expect, test } from "bun:test"; - import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; - import { tmpdir } from "node:os"; - import { join } from "node:path"; - import { getConfigPath, getDefaultConfig, readConfigDiagnostics, validateConfigCandidate } from "../src/config"; - - let testDir = ""; - afterEach(() => { - delete process.env.OPENCODEX_HOME; - if (testDir && existsSync(testDir)) rmSync(testDir, { recursive: true, force: true }); - testDir = ""; - }); - - describe("upstreamHostCircuitThreshold config contract", () => { - test("live writes accept only integer values from 0 through 20", () => { - for (const value of [0, 1, 20]) { - expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); - } - for (const value of [-1, 1.5, 21, "3", null]) { - const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); - expect(result.ok).toBe(false); - if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); - } - }); - - test("malformed hand edits disable only the circuit and report a warning", () => { - testDir = mkdtempSync(join(tmpdir(), "ocx-host-circuit-config-")); - process.env.OPENCODEX_HOME = testDir; - writeFileSync(getConfigPath(), JSON.stringify({ - ...getDefaultConfig(), - upstreamHostCircuitThreshold: 999, - })); - const diagnostics = readConfigDiagnostics(); - expect(diagnostics.source).toBe("file"); - expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); - expect(diagnostics.warnings).toContain( - "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", - ); - expect(Object.keys(diagnostics.config.providers).length).toBeGreaterThan(0); - }); - }); - EOF + - name: Apply reviewed fixes + run: python3 scripts/maint_1039_patch.py - name: Install run: bun install --frozen-lockfile - name: Typecheck @@ -251,7 +29,7 @@ export function disableUpstreamHostCircuitForKey(key: string, now = Date.now()): run: bun run privacy:scan - name: Diff check run: git diff --check - - name: Commit second-pass fixes + - name: Commit reviewed fixes shell: bash run: | set -euxo pipefail From 424d9f4715660290fe14633672a15081d6c8792e Mon Sep 17 00:00:00 2001 From: Wibias <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 21:09:13 +0000 Subject: [PATCH 13/35] fix(codex): harden host circuit config and disable semantics --- src/codex/upstream-host-health.ts | 17 +++++++++ src/config.ts | 41 +++++++++++++++++++++ src/server/responses/compact.ts | 4 ++ src/server/responses/core.ts | 4 ++ tests/upstream-host-circuit-config.test.ts | 43 ++++++++++++++++++++++ tests/upstream-host-circuit.test.ts | 15 ++++++++ 6 files changed, 124 insertions(+) create mode 100644 tests/upstream-host-circuit-config.test.ts diff --git a/src/codex/upstream-host-health.ts b/src/codex/upstream-host-health.ts index a8ac8f267e..6bd2fed844 100644 --- a/src/codex/upstream-host-health.ts +++ b/src/codex/upstream-host-health.ts @@ -225,6 +225,23 @@ export function releaseUpstreamHostAdmission( return true; } +/** + * Downgrade circuit-owned state when the operator disables the circuit. + * Failure history remains observational, but cooldown and all in-flight lease + * authority are revoked so disabled-mode traffic can update the ledger normally. + */ +export function disableUpstreamHostCircuitForKey(key: string, now = Date.now()): boolean { + const entry = hostHealth.get(key); + if (!entry?.circuitManaged) return false; + entry.circuitManaged = false; + delete entry.cooldownUntil; + advanceGeneration(entry); + entry.lastTouch = now; + if (entry.consecutiveFailures === 0) hostHealth.delete(key); + else pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); + return true; +} + /** Record one terminal logical `connect_neutral` failure. */ export function recordUpstreamHostFailure( key: string, diff --git a/src/config.ts b/src/config.ts index f2587d4c8d..3d0d1a239f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -28,6 +28,7 @@ import { MAIN_CODEX_ACCOUNT_NAMESPACE_TARGET, } from "./codex/account-namespace-match"; import { isCodexAccountPriorityKey } from "./codex/account-priority"; +import { UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD } from "./codex/upstream-host-health"; import { parseAccountPriority } from "./codex/pool-rotation"; import { COMBO_NAMESPACE, comboConfigIssues } from "./combos/types"; import { routingProfileIssues } from "./routing/profile"; @@ -1023,6 +1024,12 @@ const clientIntegrationsSchema = z.object({ const configSchema = z.object({ port: z.number().int().min(0).max(65535).default(10100), managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024), + // Invalid hand edits disable only this opt-in circuit. Live writes remain strict. + upstreamHostCircuitThreshold: z.number().int() + .min(0) + .max(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) + .optional() + .catch(undefined), appOwnedMemoryBudgetMb: z.number().int() .min(MIN_APP_OWNED_MEMORY_BUDGET_MB) .max(MAX_APP_OWNED_MEMORY_BUDGET_MB) @@ -1696,6 +1703,23 @@ function warnDegradedClaudeSubagentEffort(rawParsed: unknown): void { } } +function malformedUpstreamHostCircuitThresholdWarning(rawParsed: unknown): string | null { + const raw = rawConfigRecord(rawParsed); + if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; + const threshold = raw.upstreamHostCircuitThreshold; + if (threshold === undefined) return null; + if (typeof threshold === "number" + && Number.isInteger(threshold) + && threshold >= 0 + && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; + return `upstreamHostCircuitThreshold ignored: expected an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; +} + +function warnDegradedUpstreamHostCircuitThreshold(rawParsed: unknown): void { + const warning = malformedUpstreamHostCircuitThresholdWarning(rawParsed); + if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); +} + type NativeSubagentPersistedField = "injectionModel" | "injectionEffort" | "syncCodexSubagentDefaults"; function rawConfigRecord(rawParsed: unknown): Record | null { @@ -1789,6 +1813,7 @@ export function loadConfig(): OcxConfig { warnDegradedClaudeSubagentEffort(parsed); warnDegradedNativeSubagentConfig(parsed, config); warnDegradedCodexAccountPicker(parsed); + warnDegradedUpstreamHostCircuitThreshold(parsed); return normalizeClaudeSubagentEffort(normalizeNativeSubagentSync(config, parsed), parsed); } // Schema validation failed — merge defaults into the raw object instead of @@ -1810,6 +1835,7 @@ export function loadConfig(): OcxConfig { warnDegradedClaudeSubagentEffort(parsed); warnDegradedNativeSubagentConfig(parsed, config); warnDegradedCodexAccountPicker(parsed); + warnDegradedUpstreamHostCircuitThreshold(parsed); return normalizeClaudeSubagentEffort(normalizeNativeSubagentSync(config, parsed), parsed); } // Merge couldn't fix it — truly broken config @@ -1861,6 +1887,8 @@ function validFileConfigDiagnostics(config: OcxConfig, rawParsed: unknown): Conf warnings.push(...malformedNativeSubagentFields(rawParsed).map(malformedNativeSubagentFieldWarning)); const pickerWarning = malformedCodexAccountPickerWarning(rawParsed); if (pickerWarning) warnings.push(pickerWarning); + const hostCircuitWarning = malformedUpstreamHostCircuitThresholdWarning(rawParsed); + if (hostCircuitWarning) warnings.push(hostCircuitWarning); if (syncDisabledReason) { warnings.push(`syncCodexSubagentDefaults ignored: ${syncDisabledReason}`); } @@ -1930,6 +1958,18 @@ function appOwnedMemoryBudgetError(value: unknown): string | null { return null; } +function upstreamHostCircuitThresholdError(value: unknown): string | null { + const raw = rawConfigRecord(value); + if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; + const threshold = raw.upstreamHostCircuitThreshold; + if (threshold === undefined) return null; + if (typeof threshold === "number" + && Number.isInteger(threshold) + && threshold >= 0 + && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; + return `schema_invalid: upstreamHostCircuitThreshold: must be an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; +} + /** * Same reasoning as {@link blankHostnameError}, and more urgent: the read path degrades a * malformed selection-order map to undefined, which on a write would drop every entry the @@ -2024,6 +2064,7 @@ export function validateConfigCandidate(value: unknown): { ok: true; config: Ocx const boundaryError = blankHostnameError(value) ?? claudeSubagentEffortError(value) ?? appOwnedMemoryBudgetError(value) + ?? upstreamHostCircuitThresholdError(value) ?? googleAntigravityStaticCatalogVersionError(value) ?? codexAccountPrioritiesError(value) ?? codexAccountPickerEnabledError(value) diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index eadac0673b..8846cd8a58 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -72,6 +72,7 @@ import { import { classifyTransportFailureKind, transportErrorCode } from "../../lib/upstream-reachability"; import { acquireUpstreamHostAdmission, + disableUpstreamHostCircuitForKey, normalizeUpstreamHostCircuitThreshold, recordUpstreamHostFailure, releaseUpstreamHostAdmission, @@ -408,6 +409,9 @@ export async function handleResponsesCompact( : null; const compactHostCircuitEnabled = compactHostKey !== null && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (compactHostKey !== null && !compactHostCircuitEnabled) { + disableUpstreamHostCircuitForKey(actualCompactHostKey); + } if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) { releaseCodexAuthContextProbeLease(authCtx); return formatErrorResponse(502, "upstream_error", "Provider host changed after circuit admission"); diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 74ba5d5b7c..e0c6566eac 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -180,6 +180,7 @@ import { fetchWithHeaderTimeout, providerFetch, safeHostLabel, safeOriginLabel } import { classifyTransportFailureKind, transportErrorCode } from "../../lib/upstream-reachability"; import { acquireUpstreamHostAdmission, + disableUpstreamHostCircuitForKey, normalizeUpstreamHostCircuitThreshold, recordUpstreamHostFailure, releaseUpstreamHostAdmission, @@ -1870,6 +1871,9 @@ async function handleResponsesInner( : null; const hostCircuitEnabled = hostKey !== null && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; + if (hostKey !== null && !hostCircuitEnabled) { + disableUpstreamHostCircuitForKey(actualHostKey); + } if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) { return formatErrorResponse(502, "upstream_error", "Provider host changed after circuit admission"); } diff --git a/tests/upstream-host-circuit-config.test.ts b/tests/upstream-host-circuit-config.test.ts new file mode 100644 index 0000000000..be69b3f934 --- /dev/null +++ b/tests/upstream-host-circuit-config.test.ts @@ -0,0 +1,43 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { getConfigPath, getDefaultConfig, readConfigDiagnostics, validateConfigCandidate } from "../src/config"; + +let testDir = ""; + +afterEach(() => { + delete process.env.OPENCODEX_HOME; + if (testDir && existsSync(testDir)) rmSync(testDir, { recursive: true, force: true }); + testDir = ""; +}); + +describe("upstreamHostCircuitThreshold config contract", () => { + test("live writes accept only integer values from 0 through 20", () => { + for (const value of [0, 1, 20]) { + expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); + } + for (const value of [-1, 1.5, 21, "3", null]) { + const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); + } + }); + + test("malformed hand edits disable only the circuit and report a warning", () => { + testDir = mkdtempSync(join(tmpdir(), "ocx-host-circuit-config-")); + process.env.OPENCODEX_HOME = testDir; + writeFileSync(getConfigPath(), JSON.stringify({ + ...getDefaultConfig(), + upstreamHostCircuitThreshold: 999, + })); + + const diagnostics = readConfigDiagnostics(); + expect(diagnostics.source).toBe("file"); + expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); + expect(diagnostics.warnings).toContain( + "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", + ); + expect(Object.keys(diagnostics.config.providers).length).toBeGreaterThan(0); + }); +}); diff --git a/tests/upstream-host-circuit.test.ts b/tests/upstream-host-circuit.test.ts index c5da11f854..dc678eab28 100644 --- a/tests/upstream-host-circuit.test.ts +++ b/tests/upstream-host-circuit.test.ts @@ -4,6 +4,7 @@ import { UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, acquireUpstreamHostAdmission, clearUpstreamHostHealth, + disableUpstreamHostCircuitForKey, getUpstreamHostHealth, normalizeUpstreamHostCircuitThreshold, recordUpstreamHostFailure, @@ -178,6 +179,20 @@ describe("opt-in upstream host circuit", () => { }); }); + test("disabled traffic can clear an old circuit before it is re-enabled", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 11_000); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(11_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); + + expect(disableUpstreamHostCircuitForKey(key, 11_001)).toBe(true); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); + + expect(resetUpstreamHostHealth(key)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + expect(acquireUpstreamHostAdmission(key, 1, 11_002).kind).toBe("admitted"); + }); + test("a later physical retry without its lease cannot close a newer circuit", () => { const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); fail(key, 1, 9_000); From d8a3dd1c8d1aeca24d8a96644420c723c20553c8 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:10:01 +0200 Subject: [PATCH 14/35] chore: remove maintainer patch helper --- scripts/maint_1039_patch.py | 257 ------------------------------------ 1 file changed, 257 deletions(-) delete mode 100644 scripts/maint_1039_patch.py diff --git a/scripts/maint_1039_patch.py b/scripts/maint_1039_patch.py deleted file mode 100644 index 8241b78662..0000000000 --- a/scripts/maint_1039_patch.py +++ /dev/null @@ -1,257 +0,0 @@ -from pathlib import Path - - -def edit(path: str, fn): - p = Path(path) - text = p.read_text() - new = fn(text) - if new == text: - raise SystemExit(f"{path}: edit made no change") - p.write_text(new) - - -def one(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected 1 occurrence, found {count}") - return text.replace(old, new, 1) - - -def many(text: str, old: str, new: str, expected: int, label: str) -> str: - count = text.count(old) - if count != expected: - raise SystemExit(f"{label}: expected {expected} occurrences, found {count}") - return text.replace(old, new) - - -def patch_config(text: str) -> str: - text = one( - text, - 'import { isCodexAccountPriorityKey } from "./codex/account-priority";', - 'import { isCodexAccountPriorityKey } from "./codex/account-priority";\nimport { UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD } from "./codex/upstream-host-health";', - "config import", - ) - text = one( - text, - ' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024),', - ''' managementUsageMaxReadBytes: z.number().int().positive().default(64 * 1024 * 1024), - // Invalid hand edits disable only this opt-in circuit. Live writes remain strict. - upstreamHostCircuitThreshold: z.number().int() - .min(0) - .max(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) - .optional() - .catch(undefined),''', - "config schema field", - ) - - warning_marker = 'type NativeSubagentPersistedField = "injectionModel" | "injectionEffort" | "syncCodexSubagentDefaults";' - warning_helpers = '''function malformedUpstreamHostCircuitThresholdWarning(rawParsed: unknown): string | null { - const raw = rawConfigRecord(rawParsed); - if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; - const threshold = raw.upstreamHostCircuitThreshold; - if (threshold === undefined) return null; - if (typeof threshold === "number" - && Number.isInteger(threshold) - && threshold >= 0 - && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; - return `upstreamHostCircuitThreshold ignored: expected an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; -} - -function warnDegradedUpstreamHostCircuitThreshold(rawParsed: unknown): void { - const warning = malformedUpstreamHostCircuitThresholdWarning(rawParsed); - if (warning) console.warn(`⚠️ config.json ${warning}. Other settings were preserved.`); -} - -''' - text = one(text, warning_marker, warning_helpers + warning_marker, "config warning helper") - - text = many( - text, - ' warnDegradedCodexAccountPicker(parsed);\n', - ' warnDegradedCodexAccountPicker(parsed);\n warnDegradedUpstreamHostCircuitThreshold(parsed);\n', - 2, - "config load warnings", - ) - text = one( - text, - ' if (pickerWarning) warnings.push(pickerWarning);\n', - ' if (pickerWarning) warnings.push(pickerWarning);\n const hostCircuitWarning = malformedUpstreamHostCircuitThresholdWarning(rawParsed);\n if (hostCircuitWarning) warnings.push(hostCircuitWarning);\n', - "config diagnostics warning", - ) - - boundary_marker = '/**\n * Same reasoning as {@link blankHostnameError}, and more urgent:' - boundary_helper = '''function upstreamHostCircuitThresholdError(value: unknown): string | null { - const raw = rawConfigRecord(value); - if (!raw || !Object.hasOwn(raw, "upstreamHostCircuitThreshold")) return null; - const threshold = raw.upstreamHostCircuitThreshold; - if (threshold === undefined) return null; - if (typeof threshold === "number" - && Number.isInteger(threshold) - && threshold >= 0 - && threshold <= UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD) return null; - return `schema_invalid: upstreamHostCircuitThreshold: must be an integer from 0 to ${UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD}`; -} - -''' - text = one(text, boundary_marker, boundary_helper + boundary_marker, "config write-boundary helper") - text = one( - text, - ' ?? appOwnedMemoryBudgetError(value)\n ?? googleAntigravityStaticCatalogVersionError(value)', - ' ?? appOwnedMemoryBudgetError(value)\n ?? upstreamHostCircuitThresholdError(value)\n ?? googleAntigravityStaticCatalogVersionError(value)', - "config boundary chain", - ) - return text - - -edit("src/config.ts", patch_config) - - -def patch_health(text: str) -> str: - marker = '/** Record one terminal logical `connect_neutral` failure. */' - helper = '''/** - * Downgrade circuit-owned state when the operator disables the circuit. - * Failure history remains observational, but cooldown and all in-flight lease - * authority are revoked so disabled-mode traffic can update the ledger normally. - */ -export function disableUpstreamHostCircuitForKey(key: string, now = Date.now()): boolean { - const entry = hostHealth.get(key); - if (!entry?.circuitManaged) return false; - entry.circuitManaged = false; - delete entry.cooldownUntil; - advanceGeneration(entry); - entry.lastTouch = now; - if (entry.consecutiveFailures === 0) hostHealth.delete(key); - else pruneTo(UPSTREAM_HOST_HEALTH_MAX_ENTRIES, now); - return true; -} - -''' - return one(text, marker, helper + marker, "host disable helper") - - -edit("src/codex/upstream-host-health.ts", patch_health) - - -def add_disable_import(text: str) -> str: - return one( - text, - ' acquireUpstreamHostAdmission,\n normalizeUpstreamHostCircuitThreshold,', - ' acquireUpstreamHostAdmission,\n disableUpstreamHostCircuitForKey,\n normalizeUpstreamHostCircuitThreshold,', - "circuit disable import", - ) - - -edit("src/server/responses/core.ts", add_disable_import) -edit("src/server/responses/compact.ts", add_disable_import) - - -def patch_core(text: str) -> str: - return one( - text, - ''' const hostCircuitEnabled = hostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''', - ''' const hostCircuitEnabled = hostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (hostKey !== null && !hostCircuitEnabled) { - disableUpstreamHostCircuitForKey(actualHostKey); - } - if (hostAdmissionLease && hostAdmissionLease.key !== hostKey) {''', - "regular disabled-mode reconciliation", - ) - - -edit("src/server/responses/core.ts", patch_core) - - -def patch_compact(text: str) -> str: - return one( - text, - ''' const compactHostCircuitEnabled = compactHostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''', - ''' const compactHostCircuitEnabled = compactHostKey !== null - && normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) > 0; - if (compactHostKey !== null && !compactHostCircuitEnabled) { - disableUpstreamHostCircuitForKey(actualCompactHostKey); - } - if (compactHostAdmissionLease && compactHostAdmissionLease.key !== compactHostKey) {''', - "compact disabled-mode reconciliation", - ) - - -edit("src/server/responses/compact.ts", patch_compact) - - -def patch_health_test(text: str) -> str: - text = one( - text, - ' clearUpstreamHostHealth,\n', - ' clearUpstreamHostHealth,\n disableUpstreamHostCircuitForKey,\n', - "health test import", - ) - marker = ' test("a later physical retry without its lease cannot close a newer circuit", () => {' - regression = ''' test("disabled traffic can clear an old circuit before it is re-enabled", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 1, 11_000); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(11_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); - - expect(disableUpstreamHostCircuitForKey(key, 11_001)).toBe(true); - expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); - - expect(resetUpstreamHostHealth(key)).toBe(true); - expect(getUpstreamHostHealth(key)).toBeNull(); - expect(acquireUpstreamHostAdmission(key, 1, 11_002).kind).toBe("admitted"); - }); - -''' - return one(text, marker, regression + marker, "disabled-mode health regression") - - -edit("tests/upstream-host-circuit.test.ts", patch_health_test) - -Path("tests/upstream-host-circuit-config.test.ts").write_text('''import { afterEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { getConfigPath, getDefaultConfig, readConfigDiagnostics, validateConfigCandidate } from "../src/config"; - -let testDir = ""; - -afterEach(() => { - delete process.env.OPENCODEX_HOME; - if (testDir && existsSync(testDir)) rmSync(testDir, { recursive: true, force: true }); - testDir = ""; -}); - -describe("upstreamHostCircuitThreshold config contract", () => { - test("live writes accept only integer values from 0 through 20", () => { - for (const value of [0, 1, 20]) { - expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); - } - for (const value of [-1, 1.5, 21, "3", null]) { - const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); - expect(result.ok).toBe(false); - if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); - } - }); - - test("malformed hand edits disable only the circuit and report a warning", () => { - testDir = mkdtempSync(join(tmpdir(), "ocx-host-circuit-config-")); - process.env.OPENCODEX_HOME = testDir; - writeFileSync(getConfigPath(), JSON.stringify({ - ...getDefaultConfig(), - upstreamHostCircuitThreshold: 999, - })); - - const diagnostics = readConfigDiagnostics(); - expect(diagnostics.source).toBe("file"); - expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); - expect(diagnostics.warnings).toContain( - "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", - ); - expect(Object.keys(diagnostics.config.providers).length).toBeGreaterThan(0); - }); -}); -''') From 0aa775ae899f829a3c51cbbc7fabca805f0e3c2e Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:10:08 +0200 Subject: [PATCH 15/35] chore: remove maintainer second-pass workflow --- .github/workflows/maint-1039-second-pass.yml | 40 -------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/maint-1039-second-pass.yml diff --git a/.github/workflows/maint-1039-second-pass.yml b/.github/workflows/maint-1039-second-pass.yml deleted file mode 100644 index 3148602fb8..0000000000 --- a/.github/workflows/maint-1039-second-pass.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Maintainer second-pass PR 1039 - -on: - push: - branches: - - maint/1039-host-circuit - -permissions: - contents: write - -jobs: - harden: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: oven-sh/setup-bun@v2 - with: - bun-version: 1.3.14 - - name: Apply reviewed fixes - run: python3 scripts/maint_1039_patch.py - - name: Install - run: bun install --frozen-lockfile - - name: Typecheck - run: bun run typecheck - - name: Focused tests - run: bun test tests/upstream-host-circuit.test.ts tests/upstream-host-circuit-config.test.ts tests/responses-compaction-routing.test.ts - - name: Privacy scan - run: bun run privacy:scan - - name: Diff check - run: git diff --check - - name: Commit reviewed fixes - shell: bash - run: | - set -euxo pipefail - git config user.name "Wibias" - git config user.email "$(printf '%s%s%s' '41898282+github-actions[bot]' '@' 'users.noreply.github.com')" - git add src/config.ts src/codex/upstream-host-health.ts src/server/responses/core.ts src/server/responses/compact.ts tests/upstream-host-circuit.test.ts tests/upstream-host-circuit-config.test.ts - git commit -m "fix(codex): harden host circuit config and disable semantics" - git push origin HEAD:maint/1039-host-circuit From 7e494420d14a5d05e10d4030e7cdb33678ef8ffe Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:17:25 +0200 Subject: [PATCH 16/35] chore: stage carried review doc fix --- .github/workflows/maint-1243-docfix.yml | 39 +++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/maint-1243-docfix.yml diff --git a/.github/workflows/maint-1243-docfix.yml b/.github/workflows/maint-1243-docfix.yml new file mode 100644 index 0000000000..442d9cac35 --- /dev/null +++ b/.github/workflows/maint-1243-docfix.yml @@ -0,0 +1,39 @@ +name: Maintainer PR 1243 doc carry-forward + +on: + push: + branches: + - maint/1039-host-circuit + +permissions: + contents: write + +jobs: + fix: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Apply carried review wording + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + path = Path('docs-site/src/content/docs/reference/configuration/providers.md') + text = path.read_text() + old = 'account health, cooldowns, thread/session affinity' + new = 'account health, account cooldowns, thread/session affinity' + if text.count(old) != 1: + raise SystemExit(f'expected one English cooldown phrase, found {text.count(old)}') + path.write_text(text.replace(old, new, 1)) + PY + git rm .github/workflows/maint-1243-docfix.yml + - name: Commit + shell: bash + run: | + set -euxo pipefail + git config user.name "Wibias" + git config user.email "$(printf '%s%s%s' '41898282+github-actions[bot]' '@' 'users.noreply.github.com')" + git add docs-site/src/content/docs/reference/configuration/providers.md + git commit -m "docs: distinguish account and host cooldowns" + git push origin HEAD:maint/1039-host-circuit From 89b41d1d824a73901571a9e9cc2b638adf26778d Mon Sep 17 00:00:00 2001 From: Wibias <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 7 Aug 2026 21:17:37 +0000 Subject: [PATCH 17/35] docs: distinguish account and host cooldowns --- .github/workflows/maint-1243-docfix.yml | 39 ------------------- .../docs/reference/configuration/providers.md | 2 +- 2 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 .github/workflows/maint-1243-docfix.yml diff --git a/.github/workflows/maint-1243-docfix.yml b/.github/workflows/maint-1243-docfix.yml deleted file mode 100644 index 442d9cac35..0000000000 --- a/.github/workflows/maint-1243-docfix.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Maintainer PR 1243 doc carry-forward - -on: - push: - branches: - - maint/1039-host-circuit - -permissions: - contents: write - -jobs: - fix: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Apply carried review wording - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - path = Path('docs-site/src/content/docs/reference/configuration/providers.md') - text = path.read_text() - old = 'account health, cooldowns, thread/session affinity' - new = 'account health, account cooldowns, thread/session affinity' - if text.count(old) != 1: - raise SystemExit(f'expected one English cooldown phrase, found {text.count(old)}') - path.write_text(text.replace(old, new, 1)) - PY - git rm .github/workflows/maint-1243-docfix.yml - - name: Commit - shell: bash - run: | - set -euxo pipefail - git config user.name "Wibias" - git config user.email "$(printf '%s%s%s' '41898282+github-actions[bot]' '@' 'users.noreply.github.com')" - git add docs-site/src/content/docs/reference/configuration/providers.md - git commit -m "docs: distinguish account and host cooldowns" - git push origin HEAD:maint/1039-host-circuit diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 33eb016328..0491b42548 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -24,7 +24,7 @@ authenticated. | `autoSwitchThreshold?` | `number` | `80` | Usage threshold for proactive switching. `quota` can re-evaluate both bound and unbound tasks on their next request; `fill-first` uses it only as the drain point for unbound assignment; normal `round-robin` selection does not use it. The score uses the hottest known 5h, weekly, or 30d quota window. `0` disables usage-based proactive switching only, not unbound assignment or failure recovery. | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Assignment strategy for new/unbound Codex requests. A request is unbound when it has no live (parent thread id, quota scope) affinity; a visible existing task can become unbound after proxy restart or affinity reset. `quota` picks the lowest-usage eligible account when no active account exists, keeps an eligible active account below `autoSwitchThreshold`, and after the threshold may move an unbound request or proactively rebind a bound task to a lower-usage eligible account. `round-robin` distributes unbound requests evenly; `fill-first` keeps assigning unbound requests to the active account until cooldown, unavailability, or the configured drain threshold. | | `accountPoolStickyLimit?` | `number` | `1` | New/unbound task assignments retained on one round-robin selection before advancing; the counter advances when a task is bound, not after an upstream success. Range 1–100. | -| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive transient failures before future new sessions fail over. Set `0` to disable. For regular Responses and native compact sends, proven pre-connection DNS/TCP reachability failures are tracked at the provider-host level: they never affect account health, cooldowns, thread/session affinity, active-account selection, or Pool routing, and never count toward this threshold. | +| `upstreamFailoverThreshold?` | `number` | `3` | Consecutive transient failures before future new sessions fail over. Set `0` to disable. For regular Responses and native compact sends, proven pre-connection DNS/TCP reachability failures are tracked at the provider-host level: they never affect account health, account cooldowns, thread/session affinity, active-account selection, or Pool routing, and never count toward this threshold. | | `upstreamHostCircuitThreshold?` | `number` | `0` | Opt-in circuit threshold for proven pre-connection DNS/TCP failures on native OpenAI forward Responses and compact sends. `0` disables it; `1`–`20` opens a 30-second provider-origin cooldown after that many terminal logical requests. While open, requests receive `503` with `Retry-After` before account selection or upstream send; after cooldown, one half-open request is admitted. Timeouts and HTTP responses never count, and any HTTP response closes the circuit. | | `modelCacheTtlMs?` | `number` | `300000` | Freshness window for the per-provider `/models` cache. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic prompt-cache policy: disabled, 5-minute ephemeral, or 1-hour extended. | From 745b3384d60c3368d819d4651f1282c35352b8c9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:18:39 +0200 Subject: [PATCH 18/35] chore: refresh maintainer validation From 924a9e1f05803950ac2d2984dda433f0c6ecee3b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:26:49 +0200 Subject: [PATCH 19/35] chore: stage fresh review fixes for pr 1243 --- scripts/maint_1243_review_fixes.py | 301 +++++++++++++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 scripts/maint_1243_review_fixes.py diff --git a/scripts/maint_1243_review_fixes.py b/scripts/maint_1243_review_fixes.py new file mode 100644 index 0000000000..6ee2c81b4b --- /dev/null +++ b/scripts/maint_1243_review_fixes.py @@ -0,0 +1,301 @@ +from pathlib import Path + + +def edit(path: str, fn): + p = Path(path) + text = p.read_text() + updated = fn(text) + if updated == text: + raise SystemExit(f"{path}: no change") + p.write_text(updated) + + +def one(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected 1 occurrence, found {count}") + return text.replace(old, new, 1) + + +# Preserve the identities of same-generation concurrent requests when one opens +# the circuit, but only for the purpose of accepting a subsequent HTTP success. +def patch_health(text: str) -> str: + text = one( + text, + '''type InternalUpstreamHostHealth = UpstreamHostHealthEntry & { + generation: number; + activeLeaseIds: Set; + halfOpenLeaseId?: symbol; + /** True only when the entry is owned by opt-in circuit admissions. */ + circuitManaged: boolean; +};''', + '''type InternalUpstreamHostHealth = UpstreamHostHealthEntry & { + generation: number; + activeLeaseIds: Set; + halfOpenLeaseId?: symbol; + /** + * Leases admitted in the generation that immediately preceded the current + * cooldown. They are stale for failure settlement, but a real HTTP response + * from one still proves the origin is reachable and may close this cooldown. + */ + cooldownSuccessLeaseIds?: Set; + /** True only when the entry is owned by opt-in circuit admissions. */ + circuitManaged: boolean; +};''', + "health internal state", + ) + text = one( + text, + '''function advanceGeneration(entry: InternalUpstreamHostHealth): void { + entry.generation = nextGeneration(); + entry.activeLeaseIds.clear(); + delete entry.halfOpenLeaseId; +}''', + '''function advanceGeneration(entry: InternalUpstreamHostHealth): void { + entry.generation = nextGeneration(); + entry.activeLeaseIds.clear(); + delete entry.halfOpenLeaseId; + delete entry.cooldownSuccessLeaseIds; +}''', + "generation cleanup", + ) + text = one( + text, + ''' if (threshold > 0 && (reopens || entry.consecutiveFailures >= threshold)) { + entry.cooldownUntil = now + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; + advanceGeneration(entry); + } else {''', + ''' if (threshold > 0 && (reopens || entry.consecutiveFailures >= threshold)) { + entry.cooldownUntil = now + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; + // The failing lease was settled above. Preserve only its still-in-flight + // same-generation peers as one-shot reachability proofs. advanceGeneration + // invalidates them for every other mutation and for any later half-open generation. + const concurrentSuccessLeaseIds = new Set(entry.activeLeaseIds); + advanceGeneration(entry); + if (concurrentSuccessLeaseIds.size > 0) { + entry.cooldownSuccessLeaseIds = concurrentSuccessLeaseIds; + } + } else {''', + "cooldown opening", + ) + text = one( + text, + ''' const entry = matchingEntry(lease); + if (!entry || lease.key !== key) return false; + settleLease(entry, lease); + entry.consecutiveFailures = 0;''', + ''' if (lease.key !== key) return false; + let entry = matchingEntry(lease); + if (!entry) { + const coolingEntry = hostHealth.get(key); + if ( + coolingEntry?.cooldownUntil === undefined + || !coolingEntry.cooldownSuccessLeaseIds?.delete(lease.leaseId) + ) return false; + entry = coolingEntry; + } else { + settleLease(entry, lease); + } + entry.consecutiveFailures = 0;''', + "concurrent success reset", + ) + return text + + +edit("src/codex/upstream-host-health.ts", patch_health) + + +# A buildRequest failure occurs after Codex auth has potentially claimed a quota +# probe but before any upstream outcome can own that lease. Release it explicitly. +def patch_core(text: str) -> str: + return one( + text, + ''' let request = await adapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); + recordAdapterReasoning(logCtx, request);''', + ''' let request: Awaited>; + try { + request = await adapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); + } catch (error) { + releaseCodexAuthContextProbeLease(authCtx); + throw error; + } + recordAdapterReasoning(logCtx, request);''', + "pre-send build cleanup", + ) + + +edit("src/server/responses/core.ts", patch_core) + + +# Extend state-machine coverage requested by the fresh review. +def patch_health_tests(text: str) -> str: + text = one( + text, + ''' UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD,''', + ''' UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, + UPSTREAM_HOST_FAILURE_WINDOW_MS, + UPSTREAM_HOST_HEALTH_MAX_ENTRIES,''', + "health test imports", + ) + text = one( + text, + ''' expect(normalizeUpstreamHostCircuitThreshold(undefined)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(-1)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(1.5)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(3)).toBe(3); + expect(normalizeUpstreamHostCircuitThreshold(999)).toBe(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD);''', + ''' expect(normalizeUpstreamHostCircuitThreshold(undefined)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(-1)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(0)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold("3")).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(1.5)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(3)).toBe(3); + expect(normalizeUpstreamHostCircuitThreshold(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD)).toBe( + UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, + ); + expect(normalizeUpstreamHostCircuitThreshold(999)).toBe(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD);''', + "threshold assertions", + ) + marker = ' test("a stale completion cannot mutate the generation that opened the circuit", () => {' + regressions = ''' test("a concurrent HTTP response can close the cooldown opened by its peer", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const failing = admit(key, 1, 7_500); + const succeeding = admit(key, 1, 7_500); + + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 7_501, + threshold: 1, + lease: failing, + }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(7_501 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); + + expect(resetUpstreamHostHealth(key, succeeding, 7_502)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + }); + + test("a stale failure streak expires after the failure window", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 3, 12_000); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + + const afterWindow = 12_000 + UPSTREAM_HOST_FAILURE_WINDOW_MS + 1; + const lease = admit(key, 3, afterWindow); + expect(lease.halfOpen).toBe(false); + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: afterWindow, + threshold: 3, + lease, + }); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + }); + + test("the retention cap evicts the stalest unleased origin", () => { + for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 8; i += 1) { + fail(upstreamHostHealthKey("openai", `https://h${i}.example`), 1, 13_000 + i); + } + expect(getUpstreamHostHealth(upstreamHostHealthKey("openai", "https://h0.example"))).toBeNull(); + expect(getUpstreamHostHealth( + upstreamHostHealthKey("openai", `https://h${UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 7}.example`), + )).not.toBeNull(); + }); + + test("retention pressure never evicts an active admission lease", () => { + const leases: UpstreamHostAdmissionLease[] = []; + for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 1; i += 1) { + leases.push(admit(upstreamHostHealthKey("openai", `https://active-${i}.example`), 1, 14_000 + i)); + } + expect(releaseUpstreamHostAdmission(leases[0], 15_000)).toBe(true); + for (const lease of leases.slice(1)) releaseUpstreamHostAdmission(lease, 15_001); + }); + +''' + return one(text, marker, regressions + marker, "health regressions") + + +edit("tests/upstream-host-circuit.test.ts", patch_health_tests) + + +# Protect the pre-send cleanup at the handler seam. A fake Codex auth context +# carries a probe id, and the fake passthrough adapter fails before fetch. +def patch_routing_tests(text: str) -> str: + text = one( + text, + 'import { handleResponses, handleResponsesCompact } from "../src/server/responses";', + 'import { handleResponses, handleResponsesCompact } from "../src/server/responses";\nimport * as adapterResolveModule from "../src/server/adapter-resolve";', + "routing test adapter import", + ) + marker = ' test("an opt-in regular circuit blocks before selecting another pool account", async () => {' + regression = ''' test("a pre-send build failure releases the Codex probe lease with host circuit disabled", async () => { + await withPoolEnv("ocx-regular-build-probe-release-", async config => { + config.upstreamHostCircuitThreshold = 0; + const probeAuth = { + kind: "pool" as const, + accountId: "pool-a", + writerGeneration: 1, + generation: 1, + accessToken: "probe-token", + chatgptAccountId: "pool_acc_a", + probeLeaseId: "probe-lease", + quotaScope: "shared" as const, + }; + const authSpy = spyOn(authContextModule, "resolveCodexAuthContext").mockResolvedValue(probeAuth); + const releaseSpy = spyOn(authContextModule, "releaseCodexAuthContextProbeLease"); + const adapterSpy = spyOn(adapterResolveModule, "resolveAdapter").mockReturnValue({ + name: "openai-responses", + passthrough: true, + buildRequest: async () => { throw new Error("synthetic build failure"); }, + } as ReturnType); + try { + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-5.6-sol", input: "hello", stream: false }), + }); + await expect(handleResponses(request, config, { model: "", provider: "" })) + .rejects.toThrow("synthetic build failure"); + expect(releaseSpy).toHaveBeenCalledWith(probeAuth); + } finally { + adapterSpy.mockRestore(); + releaseSpy.mockRestore(); + authSpy.mockRestore(); + } + }); + }); + +''' + return one(text, marker, regression + marker, "pre-send regression") + + +edit("tests/responses-compaction-routing.test.ts", patch_routing_tests) + + +# Keep all locale docs aligned with the actual activation gate. +doc_suffixes = { + "docs-site/src/content/docs/reference/configuration/providers.md": ' Applies only to Codex Pool routing with no pinned account; it is inert for `codexAccountMode: "direct"` and account-qualified selectors.', + "docs-site/src/content/docs/ja/reference/configuration/providers.md": ' Codex Pool ルーティングでアカウントが固定されていない場合にのみ適用され、`codexAccountMode: "direct"` とアカウント修飾セレクターでは動作しません。', + "docs-site/src/content/docs/ko/reference/configuration/providers.md": ' Codex Pool 라우팅에서 계정이 고정되지 않은 경우에만 적용되며, `codexAccountMode: "direct"` 및 계정 한정 선택자에서는 동작하지 않습니다.', + "docs-site/src/content/docs/ru/reference/configuration/providers.md": ' Применяется только к маршрутизации Codex Pool без закреплённого аккаунта; при `codexAccountMode: "direct"` и для селекторов с указанием аккаунта схема не активна.', + "docs-site/src/content/docs/zh-cn/reference/configuration/providers.md": ' 仅适用于未固定账户的 Codex Pool 路由；在 `codexAccountMode: "direct"` 或使用账户限定选择器时不会启用。', +} + +for path, suffix in doc_suffixes.items(): + def patch_doc(text: str, suffix=suffix, path=path) -> str: + lines = text.splitlines() + matches = [i for i, line in enumerate(lines) if line.startswith('| `upstreamHostCircuitThreshold?`')] + if len(matches) != 1: + raise SystemExit(f"{path}: expected one circuit docs row, found {len(matches)}") + i = matches[0] + if not lines[i].endswith(' |'): + raise SystemExit(f"{path}: malformed circuit docs row") + lines[i] = lines[i][:-2] + suffix + ' |' + if path.endswith('/zh-cn/reference/configuration/providers.md'): + for j, line in enumerate(lines): + if line.startswith('| `upstreamFailoverThreshold?`'): + lines[j] = line.replace('；未确认的失败仍归属账户。', '。') + break + return '\n'.join(lines) + ('\n' if text.endswith('\n') else '') + edit(path, patch_doc) From 0fd18c3073570522809e88c71c2cba069554fc38 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:27:04 +0200 Subject: [PATCH 20/35] chore: run fresh review fixes for pr 1243 --- .github/workflows/maint-1243-review-fixes.yml | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/maint-1243-review-fixes.yml diff --git a/.github/workflows/maint-1243-review-fixes.yml b/.github/workflows/maint-1243-review-fixes.yml new file mode 100644 index 0000000000..e9e8306911 --- /dev/null +++ b/.github/workflows/maint-1243-review-fixes.yml @@ -0,0 +1,41 @@ +name: Maintainer PR 1243 review fixes + +on: + push: + branches: + - maint/1039-host-circuit + +permissions: + contents: write + +jobs: + fix: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + - name: Apply fixes + run: python3 scripts/maint_1243_review_fixes.py + - name: Install + run: bun install --frozen-lockfile + - name: Typecheck + run: bun run typecheck + - name: Focused tests + run: bun test tests/upstream-host-circuit.test.ts tests/upstream-host-circuit-config.test.ts tests/responses-compaction-routing.test.ts + - name: Privacy scan + run: bun run privacy:scan + - name: Diff check + run: git diff --check + - name: Commit fixes and remove maintenance files + shell: bash + run: | + set -euxo pipefail + git config user.name "Wibias" + git config user.email "$(printf '%s%s%s' '37517432+Wibias' '@' 'users.noreply.github.com')" + git rm scripts/maint_1243_review_fixes.py .github/workflows/maint-1243-review-fixes.yml + git add src/codex/upstream-host-health.ts src/server/responses/core.ts tests/upstream-host-circuit.test.ts tests/responses-compaction-routing.test.ts docs-site/src/content/docs/*/reference/configuration/providers.md + git commit -m "fix(codex): address provider-host circuit review findings" + git push origin HEAD:maint/1039-host-circuit From 74b75183c0af07e6e40ca4b57641f9de8862c5fc Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 21:27:34 +0000 Subject: [PATCH 21/35] fix(codex): address provider-host circuit review findings --- .github/workflows/maint-1243-review-fixes.yml | 41 --- .../ja/reference/configuration/providers.md | 2 +- .../ko/reference/configuration/providers.md | 2 +- .../ru/reference/configuration/providers.md | 2 +- .../reference/configuration/providers.md | 4 +- scripts/maint_1243_review_fixes.py | 301 ------------------ src/codex/upstream-host-health.ts | 29 +- src/server/responses/core.ts | 8 +- tests/responses-compaction-routing.test.ts | 38 +++ tests/upstream-host-circuit.test.ts | 60 ++++ 10 files changed, 136 insertions(+), 351 deletions(-) delete mode 100644 .github/workflows/maint-1243-review-fixes.yml delete mode 100644 scripts/maint_1243_review_fixes.py diff --git a/.github/workflows/maint-1243-review-fixes.yml b/.github/workflows/maint-1243-review-fixes.yml deleted file mode 100644 index e9e8306911..0000000000 --- a/.github/workflows/maint-1243-review-fixes.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Maintainer PR 1243 review fixes - -on: - push: - branches: - - maint/1039-host-circuit - -permissions: - contents: write - -jobs: - fix: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: oven-sh/setup-bun@v2 - with: - bun-version: 1.3.14 - - name: Apply fixes - run: python3 scripts/maint_1243_review_fixes.py - - name: Install - run: bun install --frozen-lockfile - - name: Typecheck - run: bun run typecheck - - name: Focused tests - run: bun test tests/upstream-host-circuit.test.ts tests/upstream-host-circuit-config.test.ts tests/responses-compaction-routing.test.ts - - name: Privacy scan - run: bun run privacy:scan - - name: Diff check - run: git diff --check - - name: Commit fixes and remove maintenance files - shell: bash - run: | - set -euxo pipefail - git config user.name "Wibias" - git config user.email "$(printf '%s%s%s' '37517432+Wibias' '@' 'users.noreply.github.com')" - git rm scripts/maint_1243_review_fixes.py .github/workflows/maint-1243-review-fixes.yml - git add src/codex/upstream-host-health.ts src/server/responses/core.ts tests/upstream-host-circuit.test.ts tests/responses-compaction-routing.test.ts docs-site/src/content/docs/*/reference/configuration/providers.md - git commit -m "fix(codex): address provider-host circuit review findings" - git push origin HEAD:maint/1039-host-circuit diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index 5685ff6f7d..57903e8534 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -23,7 +23,7 @@ description: プロバイダー エントリ、認証、エンドポイント、 | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新規/未紐付け Codex リクエストの割り当て戦略。live な `(parent thread id, quota scope)` affinity がなければ未紐付けで、プロキシ再起動や affinity リセット後は既存の表示タスクも未紐付けになり得ます。`quota` はアクティブアカウントがなければ既知 usage 最小の適格アカウントを選び、適格なアクティブアカウントが `autoSwitchThreshold` 未満なら維持します。しきい値到達後は、未紐付けリクエストまたは紐付け済みタスクの次のリクエストを usage の低い適格アカウントへ移せます。`round-robin` は未紐付けリクエストを均等分散し、`fill-first` は cooldown、使用不可、または drain threshold までアクティブアカウントへ割り当てます。 | | `accountPoolStickyLimit?` | `number` | `1` | 1 回の round-robin 選択で次へ進む前に保持する新規/未紐付けタスク割り当て数。カウンターは上流の成功後ではなくタスクの紐付け時に増えます。範囲 1–100。`accountPoolStrategy` が `round-robin` のときのみ。 | | `upstreamFailoverThreshold?` | `number` | `3` |今後の新しいセッションがフェイルオーバーする前に一時的なエラーが連続して発生する。 `0` を無効に設定します。通常のResponses送信とネイティブcompact送信では、実証済みの接続前DNS/TCP到達不能障害はprovider-host単位で記録され、アカウントの健全性、アカウントのクールダウン、スレッド/セッションの親和性、アクティブアカウントの選択、Poolルーティングには影響せず、この閾値にもカウントされません。 | -| `upstreamHostCircuitThreshold?` | `number` | `0` | ネイティブOpenAI forwardのResponses送信とcompact送信で、実証済みの接続前DNS/TCP障害に適用するオプトインのサーキットしきい値です。`0`で無効、`1`〜`20`ではその回数の終端論理リクエストが失敗するとprovider-originを30秒間遮断します。遮断中はアカウント選択やupstream送信の前に`Retry-After`付き`503`を返し、時間経過後はhalf-openリクエストを1件だけ許可します。タイムアウトとHTTP応答は数えず、HTTP応答が1件でもあれば回路を閉じます。 | +| `upstreamHostCircuitThreshold?` | `number` | `0` | ネイティブOpenAI forwardのResponses送信とcompact送信で、実証済みの接続前DNS/TCP障害に適用するオプトインのサーキットしきい値です。`0`で無効、`1`〜`20`ではその回数の終端論理リクエストが失敗するとprovider-originを30秒間遮断します。遮断中はアカウント選択やupstream送信の前に`Retry-After`付き`503`を返し、時間経過後はhalf-openリクエストを1件だけ許可します。タイムアウトとHTTP応答は数えず、HTTP応答が1件でもあれば回路を閉じます。 Codex Pool ルーティングでアカウントが固定されていない場合にのみ適用され、`codexAccountMode: "direct"` とアカウント修飾セレクターでは動作しません。 | | `modelCacheTtlMs?` | `number` | `300000` |プロバイダーごとの `/models` キャッシュの鮮度ウィンドウ。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic プロンプト キャッシュ ポリシー: 無効、5 分間の一時的、または 1 時間の延長。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` |オフ |オプションのプロアクティブな OAuth 更新および Codex アカウントのウォームアップ ポリシー。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index a3662ee7e6..684ea712f5 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -23,7 +23,7 @@ description: 공급자 항목, 인증, 엔드포인트, 모델 카탈로그, 할 | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 새 작업/바인딩 없는 Codex 요청의 계정 배정 전략입니다. `(parent thread id, quota scope)`의 live affinity가 없으면 바인딩 없는 요청이며, 프록시 재시작이나 affinity 초기화 뒤에는 기존에 보이던 작업도 바인딩이 없어질 수 있습니다. `quota`는 활성 계정이 없을 때 알려진 usage가 가장 낮은 적격 계정을 선택하고, 적격 활성 계정이 `autoSwitchThreshold` 미만이면 유지합니다. 임계값 도달 뒤에는 바인딩 없는 요청이나 바인딩된 작업의 다음 요청을 usage가 더 낮은 적격 계정으로 옮길 수 있습니다. `round-robin`은 바인딩 없는 요청을 균등 분배하고, `fill-first`는 cooldown, 사용 불가 또는 drain threshold까지 활성 계정에 배정합니다. | | `accountPoolStickyLimit?` | `number` | `1` | 한 round-robin 선택이 다음으로 넘어가기 전에 유지하는 새 작업/바인딩 없는 작업 배정 수입니다. 카운터는 업스트림 성공 뒤가 아니라 작업을 바인딩할 때 증가합니다. 범위 1–100이며 `accountPoolStrategy`가 `round-robin`일 때만 적용됩니다. | | `upstreamFailoverThreshold?` | `number` | `3` | 연속된 일시적 실패가 이 횟수에 도달하면 이후 새 세션은 failover됩니다. `0`으로 두면 비활성화됩니다. 일반 Responses와 네이티브 compact 전송에서 입증된 연결 전 DNS/TCP 도달 불가 실패는 provider-host 범위로 기록되며 계정 상태, 계정 쿨다운, 스레드/세션 선호도, 활성 계정 선택 또는 Pool 라우팅에 영향을 주지 않고 이 임계값에도 집계되지 않습니다. | -| `upstreamHostCircuitThreshold?` | `number` | `0` | 네이티브 OpenAI forward Responses와 compact 전송에서 입증된 연결 전 DNS/TCP 실패에 적용하는 선택적 회로 차단 임계값입니다. `0`은 비활성화하며, `1`~`20`은 이 횟수만큼 최종 논리 요청이 실패하면 provider-origin을 30초 동안 차단합니다. 차단 중에는 계정 선택이나 업스트림 전송 전에 `Retry-After`가 포함된 `503`을 반환하고, 시간이 지나면 반개방 요청 하나만 허용합니다. 타임아웃과 HTTP 응답은 집계하지 않으며, HTTP 응답이 하나라도 오면 회로를 닫습니다. | +| `upstreamHostCircuitThreshold?` | `number` | `0` | 네이티브 OpenAI forward Responses와 compact 전송에서 입증된 연결 전 DNS/TCP 실패에 적용하는 선택적 회로 차단 임계값입니다. `0`은 비활성화하며, `1`~`20`은 이 횟수만큼 최종 논리 요청이 실패하면 provider-origin을 30초 동안 차단합니다. 차단 중에는 계정 선택이나 업스트림 전송 전에 `Retry-After`가 포함된 `503`을 반환하고, 시간이 지나면 반개방 요청 하나만 허용합니다. 타임아웃과 HTTP 응답은 집계하지 않으며, HTTP 응답이 하나라도 오면 회로를 닫습니다. Codex Pool 라우팅에서 계정이 고정되지 않은 경우에만 적용되며, `codexAccountMode: "direct"` 및 계정 한정 선택자에서는 동작하지 않습니다. | | `modelCacheTtlMs?` | `number` | `300000` | 공급자별 `/models` 캐시의 최신성 창입니다. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 프롬프트 캐시 정책입니다. 비활성, 5분짜리 임시, 1시간짜리 확장 중 하나입니다. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 꺼짐 | 선택적 선제 OAuth 갱신과 Codex 계정 워밍업 정책입니다. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index 815e45c66c..abf75d650e 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -24,7 +24,7 @@ description: Записи провайдеров, аутентификация, | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Стратегия назначения для новых/непривязанных запросов Codex. Запрос непривязан, если у него нет live affinity `(parent thread id, quota scope)`; видимая существующая задача может стать непривязанной после перезапуска прокси или сброса affinity. `quota` выбирает подходящий аккаунт с наименьшим известным usage, когда активного аккаунта нет, сохраняет подходящий активный аккаунт ниже `autoSwitchThreshold`, а после порога может перевести непривязанный запрос или следующий запрос привязанной задачи на подходящий аккаунт с меньшим usage. `round-robin` равномерно распределяет непривязанные запросы; `fill-first` назначает их активному аккаунту до cooldown, недоступности или порога исчерпания. | | `accountPoolStickyLimit?` | `number` | `1` | Число назначений новых/непривязанных задач на одном выборе round-robin перед переходом дальше. Счётчик растёт при привязке задачи, а не после успеха upstream. Диапазон 1–100; только при `accountPoolStrategy` = `round-robin`. | | `upstreamFailoverThreshold?` | `number` | `3` | Сколько подряд transient failure допустить, прежде чем новые сессии начнут делать failover. `0` отключает эту логику. Для обычных Responses-запросов и нативных compact-отправок доказанные ошибки доступности DNS/TCP до соединения учитываются на уровне пары «провайдер, хост» и не влияют на здоровье аккаунта, кулдауны аккаунта, привязку потока/сессии, выбор активного аккаунта или маршрутизацию пула, а также не учитываются в этом пороге. | -| `upstreamHostCircuitThreshold?` | `number` | `0` | Опциональный порог circuit breaker для доказанных DNS/TCP-сбоев до соединения в нативных OpenAI forward Responses- и compact-отправках. `0` отключает его; `1`–`20` открывает 30-секундный cooldown для provider-origin после такого числа завершившихся логических запросов. Пока circuit открыт, до выбора аккаунта и upstream-отправки возвращается `503` с `Retry-After`; после cooldown допускается один half-open запрос. Таймауты и HTTP-ответы не учитываются, а любой HTTP-ответ закрывает circuit. | +| `upstreamHostCircuitThreshold?` | `number` | `0` | Опциональный порог circuit breaker для доказанных DNS/TCP-сбоев до соединения в нативных OpenAI forward Responses- и compact-отправках. `0` отключает его; `1`–`20` открывает 30-секундный cooldown для provider-origin после такого числа завершившихся логических запросов. Пока circuit открыт, до выбора аккаунта и upstream-отправки возвращается `503` с `Retry-After`; после cooldown допускается один half-open запрос. Таймауты и HTTP-ответы не учитываются, а любой HTTP-ответ закрывает circuit. Применяется только к маршрутизации Codex Pool без закреплённого аккаунта; при `codexAccountMode: "direct"` и для селекторов с указанием аккаунта схема не активна. | | `modelCacheTtlMs?` | `number` | `300000` | Окно свежести для кэша `/models` на уровне провайдера. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Политика prompt-cache Anthropic: отключено, 5-минутный ephemeral или 1-часовой extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Необязательная политика proactive OAuth refresh и warmup'а аккаунтов Codex. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index aafe985f23..7e664ce58c 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -22,8 +22,8 @@ description: 提供者条目、身份验证、端点、模型目录、配额、 | `autoSwitchThreshold?` | `number` | `80` | 基于用量的主动切换阈值。`quota` 可在下一次请求中重新评估已绑定和未绑定任务；`fill-first` 仅把它用作未绑定分配的耗尽点；正常 `round-robin` 不使用它。分数取已知 5 小时、周或 30 天 quota window 的最高值。`0` 只关闭基于用量的主动切换，不关闭未绑定任务分配或故障恢复。 | | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | 新建/未绑定 Codex 请求的分配策略。没有 live `(parent thread id, quota scope)` affinity 的请求属于未绑定；代理重启或 affinity 重置后，已有可见任务也可能未绑定。`quota` 在没有活跃账号时选择已知 usage 最低的合格账号；活跃账号合格且低于 `autoSwitchThreshold` 时继续使用；达到阈值后，可把未绑定请求或已绑定任务的下一次请求切换到 usage 更低的合格账号。`round-robin` 均匀分配未绑定请求；`fill-first` 在 cooldown、不可用或耗尽阈值前持续分配给活跃账号。 | | `accountPoolStickyLimit?` | `number` | `1` | 一次 round-robin 选择在推进前保留的新建/未绑定任务分配数。计数在任务绑定时增加，而不是在上游成功后增加。范围 1–100；仅当 `accountPoolStrategy` 为 `round-robin` 时生效。 | -| `upstreamFailoverThreshold?` | `number` | `3` | 连续发生多少次瞬态故障后，后续新会话会切换到备用上游。设为 `0` 可禁用。对于常规 Responses 和原生 compact 发送，已证明的连接前 DNS/TCP 不可达故障按 provider-host 粒度记录，不影响账户健康、账户冷却、线程/会话亲和性、活动账户选择或 Pool 路由，也不会计入此阈值；未确认的失败仍归属账户。 | -| `upstreamHostCircuitThreshold?` | `number` | `0` | 原生 OpenAI forward Responses 与 compact 发送的可选断路器阈值，仅统计已证明的连接前 DNS/TCP 故障。`0` 表示禁用；`1`–`20` 表示在这么多个终止逻辑请求失败后，对 provider-origin 冷却 30 秒。断路期间会在账户选择和上游发送之前返回带 `Retry-After` 的 `503`；冷却结束后只允许一个半开请求。超时和 HTTP 响应不计数，任意 HTTP 响应都会关闭断路器。 | +| `upstreamFailoverThreshold?` | `number` | `3` | 连续发生多少次瞬态故障后，后续新会话会切换到备用上游。设为 `0` 可禁用。对于常规 Responses 和原生 compact 发送，已证明的连接前 DNS/TCP 不可达故障按 provider-host 粒度记录，不影响账户健康、账户冷却、线程/会话亲和性、活动账户选择或 Pool 路由，也不会计入此阈值。 | +| `upstreamHostCircuitThreshold?` | `number` | `0` | 原生 OpenAI forward Responses 与 compact 发送的可选断路器阈值，仅统计已证明的连接前 DNS/TCP 故障。`0` 表示禁用；`1`–`20` 表示在这么多个终止逻辑请求失败后，对 provider-origin 冷却 30 秒。断路期间会在账户选择和上游发送之前返回带 `Retry-After` 的 `503`；冷却结束后只允许一个半开请求。超时和 HTTP 响应不计数，任意 HTTP 响应都会关闭断路器。 仅适用于未固定账户的 Codex Pool 路由；在 `codexAccountMode: "direct"` 或使用账户限定选择器时不会启用。 | | `modelCacheTtlMs?` | `number` | `300000` | 每个提供者 `/models` 缓存的新鲜度窗口。 | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic 提示缓存策略：禁用、5 分钟临时缓存，或 1 小时扩展缓存。 | | `tokenGuardian?` | `OcxTokenGuardianConfig` | 关闭 | 可选的主动 OAuth 刷新与 Codex 账户预热策略。 | diff --git a/scripts/maint_1243_review_fixes.py b/scripts/maint_1243_review_fixes.py deleted file mode 100644 index 6ee2c81b4b..0000000000 --- a/scripts/maint_1243_review_fixes.py +++ /dev/null @@ -1,301 +0,0 @@ -from pathlib import Path - - -def edit(path: str, fn): - p = Path(path) - text = p.read_text() - updated = fn(text) - if updated == text: - raise SystemExit(f"{path}: no change") - p.write_text(updated) - - -def one(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected 1 occurrence, found {count}") - return text.replace(old, new, 1) - - -# Preserve the identities of same-generation concurrent requests when one opens -# the circuit, but only for the purpose of accepting a subsequent HTTP success. -def patch_health(text: str) -> str: - text = one( - text, - '''type InternalUpstreamHostHealth = UpstreamHostHealthEntry & { - generation: number; - activeLeaseIds: Set; - halfOpenLeaseId?: symbol; - /** True only when the entry is owned by opt-in circuit admissions. */ - circuitManaged: boolean; -};''', - '''type InternalUpstreamHostHealth = UpstreamHostHealthEntry & { - generation: number; - activeLeaseIds: Set; - halfOpenLeaseId?: symbol; - /** - * Leases admitted in the generation that immediately preceded the current - * cooldown. They are stale for failure settlement, but a real HTTP response - * from one still proves the origin is reachable and may close this cooldown. - */ - cooldownSuccessLeaseIds?: Set; - /** True only when the entry is owned by opt-in circuit admissions. */ - circuitManaged: boolean; -};''', - "health internal state", - ) - text = one( - text, - '''function advanceGeneration(entry: InternalUpstreamHostHealth): void { - entry.generation = nextGeneration(); - entry.activeLeaseIds.clear(); - delete entry.halfOpenLeaseId; -}''', - '''function advanceGeneration(entry: InternalUpstreamHostHealth): void { - entry.generation = nextGeneration(); - entry.activeLeaseIds.clear(); - delete entry.halfOpenLeaseId; - delete entry.cooldownSuccessLeaseIds; -}''', - "generation cleanup", - ) - text = one( - text, - ''' if (threshold > 0 && (reopens || entry.consecutiveFailures >= threshold)) { - entry.cooldownUntil = now + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; - advanceGeneration(entry); - } else {''', - ''' if (threshold > 0 && (reopens || entry.consecutiveFailures >= threshold)) { - entry.cooldownUntil = now + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; - // The failing lease was settled above. Preserve only its still-in-flight - // same-generation peers as one-shot reachability proofs. advanceGeneration - // invalidates them for every other mutation and for any later half-open generation. - const concurrentSuccessLeaseIds = new Set(entry.activeLeaseIds); - advanceGeneration(entry); - if (concurrentSuccessLeaseIds.size > 0) { - entry.cooldownSuccessLeaseIds = concurrentSuccessLeaseIds; - } - } else {''', - "cooldown opening", - ) - text = one( - text, - ''' const entry = matchingEntry(lease); - if (!entry || lease.key !== key) return false; - settleLease(entry, lease); - entry.consecutiveFailures = 0;''', - ''' if (lease.key !== key) return false; - let entry = matchingEntry(lease); - if (!entry) { - const coolingEntry = hostHealth.get(key); - if ( - coolingEntry?.cooldownUntil === undefined - || !coolingEntry.cooldownSuccessLeaseIds?.delete(lease.leaseId) - ) return false; - entry = coolingEntry; - } else { - settleLease(entry, lease); - } - entry.consecutiveFailures = 0;''', - "concurrent success reset", - ) - return text - - -edit("src/codex/upstream-host-health.ts", patch_health) - - -# A buildRequest failure occurs after Codex auth has potentially claimed a quota -# probe but before any upstream outcome can own that lease. Release it explicitly. -def patch_core(text: str) -> str: - return one( - text, - ''' let request = await adapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); - recordAdapterReasoning(logCtx, request);''', - ''' let request: Awaited>; - try { - request = await adapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); - } catch (error) { - releaseCodexAuthContextProbeLease(authCtx); - throw error; - } - recordAdapterReasoning(logCtx, request);''', - "pre-send build cleanup", - ) - - -edit("src/server/responses/core.ts", patch_core) - - -# Extend state-machine coverage requested by the fresh review. -def patch_health_tests(text: str) -> str: - text = one( - text, - ''' UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, - UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD,''', - ''' UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, - UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, - UPSTREAM_HOST_FAILURE_WINDOW_MS, - UPSTREAM_HOST_HEALTH_MAX_ENTRIES,''', - "health test imports", - ) - text = one( - text, - ''' expect(normalizeUpstreamHostCircuitThreshold(undefined)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(-1)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(1.5)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(3)).toBe(3); - expect(normalizeUpstreamHostCircuitThreshold(999)).toBe(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD);''', - ''' expect(normalizeUpstreamHostCircuitThreshold(undefined)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(-1)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(0)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold("3")).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(1.5)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(3)).toBe(3); - expect(normalizeUpstreamHostCircuitThreshold(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD)).toBe( - UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, - ); - expect(normalizeUpstreamHostCircuitThreshold(999)).toBe(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD);''', - "threshold assertions", - ) - marker = ' test("a stale completion cannot mutate the generation that opened the circuit", () => {' - regressions = ''' test("a concurrent HTTP response can close the cooldown opened by its peer", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - const failing = admit(key, 1, 7_500); - const succeeding = admit(key, 1, 7_500); - - recordUpstreamHostFailure(key, { - code: "ECONNREFUSED", - now: 7_501, - threshold: 1, - lease: failing, - }); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(7_501 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); - - expect(resetUpstreamHostHealth(key, succeeding, 7_502)).toBe(true); - expect(getUpstreamHostHealth(key)).toBeNull(); - }); - - test("a stale failure streak expires after the failure window", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 3, 12_000); - expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); - - const afterWindow = 12_000 + UPSTREAM_HOST_FAILURE_WINDOW_MS + 1; - const lease = admit(key, 3, afterWindow); - expect(lease.halfOpen).toBe(false); - recordUpstreamHostFailure(key, { - code: "ECONNREFUSED", - now: afterWindow, - threshold: 3, - lease, - }); - expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); - }); - - test("the retention cap evicts the stalest unleased origin", () => { - for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 8; i += 1) { - fail(upstreamHostHealthKey("openai", `https://h${i}.example`), 1, 13_000 + i); - } - expect(getUpstreamHostHealth(upstreamHostHealthKey("openai", "https://h0.example"))).toBeNull(); - expect(getUpstreamHostHealth( - upstreamHostHealthKey("openai", `https://h${UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 7}.example`), - )).not.toBeNull(); - }); - - test("retention pressure never evicts an active admission lease", () => { - const leases: UpstreamHostAdmissionLease[] = []; - for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 1; i += 1) { - leases.push(admit(upstreamHostHealthKey("openai", `https://active-${i}.example`), 1, 14_000 + i)); - } - expect(releaseUpstreamHostAdmission(leases[0], 15_000)).toBe(true); - for (const lease of leases.slice(1)) releaseUpstreamHostAdmission(lease, 15_001); - }); - -''' - return one(text, marker, regressions + marker, "health regressions") - - -edit("tests/upstream-host-circuit.test.ts", patch_health_tests) - - -# Protect the pre-send cleanup at the handler seam. A fake Codex auth context -# carries a probe id, and the fake passthrough adapter fails before fetch. -def patch_routing_tests(text: str) -> str: - text = one( - text, - 'import { handleResponses, handleResponsesCompact } from "../src/server/responses";', - 'import { handleResponses, handleResponsesCompact } from "../src/server/responses";\nimport * as adapterResolveModule from "../src/server/adapter-resolve";', - "routing test adapter import", - ) - marker = ' test("an opt-in regular circuit blocks before selecting another pool account", async () => {' - regression = ''' test("a pre-send build failure releases the Codex probe lease with host circuit disabled", async () => { - await withPoolEnv("ocx-regular-build-probe-release-", async config => { - config.upstreamHostCircuitThreshold = 0; - const probeAuth = { - kind: "pool" as const, - accountId: "pool-a", - writerGeneration: 1, - generation: 1, - accessToken: "probe-token", - chatgptAccountId: "pool_acc_a", - probeLeaseId: "probe-lease", - quotaScope: "shared" as const, - }; - const authSpy = spyOn(authContextModule, "resolveCodexAuthContext").mockResolvedValue(probeAuth); - const releaseSpy = spyOn(authContextModule, "releaseCodexAuthContextProbeLease"); - const adapterSpy = spyOn(adapterResolveModule, "resolveAdapter").mockReturnValue({ - name: "openai-responses", - passthrough: true, - buildRequest: async () => { throw new Error("synthetic build failure"); }, - } as ReturnType); - try { - const request = new Request("http://localhost/v1/responses", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ model: "gpt-5.6-sol", input: "hello", stream: false }), - }); - await expect(handleResponses(request, config, { model: "", provider: "" })) - .rejects.toThrow("synthetic build failure"); - expect(releaseSpy).toHaveBeenCalledWith(probeAuth); - } finally { - adapterSpy.mockRestore(); - releaseSpy.mockRestore(); - authSpy.mockRestore(); - } - }); - }); - -''' - return one(text, marker, regression + marker, "pre-send regression") - - -edit("tests/responses-compaction-routing.test.ts", patch_routing_tests) - - -# Keep all locale docs aligned with the actual activation gate. -doc_suffixes = { - "docs-site/src/content/docs/reference/configuration/providers.md": ' Applies only to Codex Pool routing with no pinned account; it is inert for `codexAccountMode: "direct"` and account-qualified selectors.', - "docs-site/src/content/docs/ja/reference/configuration/providers.md": ' Codex Pool ルーティングでアカウントが固定されていない場合にのみ適用され、`codexAccountMode: "direct"` とアカウント修飾セレクターでは動作しません。', - "docs-site/src/content/docs/ko/reference/configuration/providers.md": ' Codex Pool 라우팅에서 계정이 고정되지 않은 경우에만 적용되며, `codexAccountMode: "direct"` 및 계정 한정 선택자에서는 동작하지 않습니다.', - "docs-site/src/content/docs/ru/reference/configuration/providers.md": ' Применяется только к маршрутизации Codex Pool без закреплённого аккаунта; при `codexAccountMode: "direct"` и для селекторов с указанием аккаунта схема не активна.', - "docs-site/src/content/docs/zh-cn/reference/configuration/providers.md": ' 仅适用于未固定账户的 Codex Pool 路由；在 `codexAccountMode: "direct"` 或使用账户限定选择器时不会启用。', -} - -for path, suffix in doc_suffixes.items(): - def patch_doc(text: str, suffix=suffix, path=path) -> str: - lines = text.splitlines() - matches = [i for i, line in enumerate(lines) if line.startswith('| `upstreamHostCircuitThreshold?`')] - if len(matches) != 1: - raise SystemExit(f"{path}: expected one circuit docs row, found {len(matches)}") - i = matches[0] - if not lines[i].endswith(' |'): - raise SystemExit(f"{path}: malformed circuit docs row") - lines[i] = lines[i][:-2] + suffix + ' |' - if path.endswith('/zh-cn/reference/configuration/providers.md'): - for j, line in enumerate(lines): - if line.startswith('| `upstreamFailoverThreshold?`'): - lines[j] = line.replace('；未确认的失败仍归属账户。', '。') - break - return '\n'.join(lines) + ('\n' if text.endswith('\n') else '') - edit(path, patch_doc) diff --git a/src/codex/upstream-host-health.ts b/src/codex/upstream-host-health.ts index 6bd2fed844..1854eed158 100644 --- a/src/codex/upstream-host-health.ts +++ b/src/codex/upstream-host-health.ts @@ -37,6 +37,12 @@ type InternalUpstreamHostHealth = UpstreamHostHealthEntry & { generation: number; activeLeaseIds: Set; halfOpenLeaseId?: symbol; + /** + * Leases admitted in the generation that immediately preceded the current + * cooldown. They are stale for failure settlement, but a real HTTP response + * from one still proves the origin is reachable and may close this cooldown. + */ + cooldownSuccessLeaseIds?: Set; /** True only when the entry is owned by opt-in circuit admissions. */ circuitManaged: boolean; }; @@ -83,6 +89,7 @@ function advanceGeneration(entry: InternalUpstreamHostHealth): void { entry.generation = nextGeneration(); entry.activeLeaseIds.clear(); delete entry.halfOpenLeaseId; + delete entry.cooldownSuccessLeaseIds; } function removeExpiredUnleased(now: number): void { @@ -301,7 +308,14 @@ export function recordUpstreamHostFailure( if (threshold > 0 && (reopens || entry.consecutiveFailures >= threshold)) { entry.cooldownUntil = now + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; + // The failing lease was settled above. Preserve only its still-in-flight + // same-generation peers as one-shot reachability proofs. advanceGeneration + // invalidates them for every other mutation and for any later half-open generation. + const concurrentSuccessLeaseIds = new Set(entry.activeLeaseIds); advanceGeneration(entry); + if (concurrentSuccessLeaseIds.size > 0) { + entry.cooldownSuccessLeaseIds = concurrentSuccessLeaseIds; + } } else { delete entry.cooldownUntil; } @@ -320,9 +334,18 @@ export function resetUpstreamHostHealth( if (entry?.circuitManaged) return false; return hostHealth.delete(key); } - const entry = matchingEntry(lease); - if (!entry || lease.key !== key) return false; - settleLease(entry, lease); + if (lease.key !== key) return false; + let entry = matchingEntry(lease); + if (!entry) { + const coolingEntry = hostHealth.get(key); + if ( + coolingEntry?.cooldownUntil === undefined + || !coolingEntry.cooldownSuccessLeaseIds?.delete(lease.leaseId) + ) return false; + entry = coolingEntry; + } else { + settleLease(entry, lease); + } entry.consecutiveFailures = 0; entry.lastFailureAt = 0; entry.lastTouch = now; diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index e0c6566eac..9b85c9c025 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1860,7 +1860,13 @@ async function handleResponsesInner( + `(model ${parsed.modelId}); forwarding without it — earlier turns may be missing from this request`, ); } - let request = await adapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); + let request: Awaited>; + try { + request = await adapter.buildRequest(parsed, { headers: selectedForwardHeaders, translatorBudget }); + } catch (error) { + releaseCodexAuthContextProbeLease(authCtx); + throw error; + } recordAdapterReasoning(logCtx, request); const actualHostKey = upstreamHostHealthKey( route.providerName, diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index 87baade15a..fdb1830517 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -9,6 +9,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { handleResponses, handleResponsesCompact } from "../src/server/responses"; +import * as adapterResolveModule from "../src/server/adapter-resolve"; import { saveCodexAccountCredential } from "../src/codex/account-store"; import { CODEX_QUOTA_PROBE_INTERVAL_MS, @@ -962,6 +963,43 @@ describe("compact alternate-account attempt (#913)", () => { }); }); + test("a pre-send build failure releases the Codex probe lease with host circuit disabled", async () => { + await withPoolEnv("ocx-regular-build-probe-release-", async config => { + config.upstreamHostCircuitThreshold = 0; + const probeAuth = { + kind: "pool" as const, + accountId: "pool-a", + writerGeneration: 1, + generation: 1, + accessToken: "probe-token", + chatgptAccountId: "pool_acc_a", + probeLeaseId: "probe-lease", + quotaScope: "shared" as const, + }; + const authSpy = spyOn(authContextModule, "resolveCodexAuthContext").mockResolvedValue(probeAuth); + const releaseSpy = spyOn(authContextModule, "releaseCodexAuthContextProbeLease"); + const adapterSpy = spyOn(adapterResolveModule, "resolveAdapter").mockReturnValue({ + name: "openai-responses", + passthrough: true, + buildRequest: async () => { throw new Error("synthetic build failure"); }, + } as ReturnType); + try { + const request = new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-5.6-sol", input: "hello", stream: false }), + }); + await expect(handleResponses(request, config, { model: "", provider: "" })) + .rejects.toThrow("synthetic build failure"); + expect(releaseSpy).toHaveBeenCalledWith(probeAuth); + } finally { + adapterSpy.mockRestore(); + releaseSpy.mockRestore(); + authSpy.mockRestore(); + } + }); + }); + test("an opt-in regular circuit blocks before selecting another pool account", async () => { await withPoolEnv("ocx-regular-host-circuit-", async config => { config.upstreamHostCircuitThreshold = 1; diff --git a/tests/upstream-host-circuit.test.ts b/tests/upstream-host-circuit.test.ts index dc678eab28..383b96fc5a 100644 --- a/tests/upstream-host-circuit.test.ts +++ b/tests/upstream-host-circuit.test.ts @@ -2,6 +2,8 @@ import { beforeEach, describe, expect, test } from "bun:test"; import { UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, + UPSTREAM_HOST_FAILURE_WINDOW_MS, + UPSTREAM_HOST_HEALTH_MAX_ENTRIES, acquireUpstreamHostAdmission, clearUpstreamHostHealth, disableUpstreamHostCircuitForKey, @@ -38,8 +40,13 @@ describe("opt-in upstream host circuit", () => { test("normalizes the opt-in threshold and leaves zero disabled", () => { expect(normalizeUpstreamHostCircuitThreshold(undefined)).toBe(0); expect(normalizeUpstreamHostCircuitThreshold(-1)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(0)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold("3")).toBe(0); expect(normalizeUpstreamHostCircuitThreshold(1.5)).toBe(0); expect(normalizeUpstreamHostCircuitThreshold(3)).toBe(3); + expect(normalizeUpstreamHostCircuitThreshold(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD)).toBe( + UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, + ); expect(normalizeUpstreamHostCircuitThreshold(999)).toBe(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD); const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); @@ -140,6 +147,59 @@ describe("opt-in upstream host circuit", () => { expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); }); + test("a concurrent HTTP response can close the cooldown opened by its peer", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const failing = admit(key, 1, 7_500); + const succeeding = admit(key, 1, 7_500); + + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 7_501, + threshold: 1, + lease: failing, + }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(7_501 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); + + expect(resetUpstreamHostHealth(key, succeeding, 7_502)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + }); + + test("a stale failure streak expires after the failure window", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 3, 12_000); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + + const afterWindow = 12_000 + UPSTREAM_HOST_FAILURE_WINDOW_MS + 1; + const lease = admit(key, 3, afterWindow); + expect(lease.halfOpen).toBe(false); + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: afterWindow, + threshold: 3, + lease, + }); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + }); + + test("the retention cap evicts the stalest unleased origin", () => { + for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 8; i += 1) { + fail(upstreamHostHealthKey("openai", `https://h${i}.example`), 1, 13_000 + i); + } + expect(getUpstreamHostHealth(upstreamHostHealthKey("openai", "https://h0.example"))).toBeNull(); + expect(getUpstreamHostHealth( + upstreamHostHealthKey("openai", `https://h${UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 7}.example`), + )).not.toBeNull(); + }); + + test("retention pressure never evicts an active admission lease", () => { + const leases: UpstreamHostAdmissionLease[] = []; + for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 1; i += 1) { + leases.push(admit(upstreamHostHealthKey("openai", `https://active-${i}.example`), 1, 14_000 + i)); + } + expect(releaseUpstreamHostAdmission(leases[0], 15_000)).toBe(true); + for (const lease of leases.slice(1)) releaseUpstreamHostAdmission(lease, 15_001); + }); + test("a stale completion cannot mutate the generation that opened the circuit", () => { const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); const stale = admit(key, 1, 8_000); From 1d2c526dfac32cc2c1da4b09d183cc51018ab4a4 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:28:43 +0200 Subject: [PATCH 22/35] chore: carry English circuit activation wording --- .github/workflows/maint-1243-english-doc.yml | 46 ++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/maint-1243-english-doc.yml diff --git a/.github/workflows/maint-1243-english-doc.yml b/.github/workflows/maint-1243-english-doc.yml new file mode 100644 index 0000000000..8c054e9528 --- /dev/null +++ b/.github/workflows/maint-1243-english-doc.yml @@ -0,0 +1,46 @@ +name: Maintainer PR 1243 English docs + +on: + push: + branches: + - maint/1039-host-circuit + +permissions: + contents: write + +jobs: + fix: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Fix English activation contract + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + p = Path('docs-site/src/content/docs/reference/configuration/providers.md') + text = p.read_text() + lines = text.splitlines() + matches = [i for i, line in enumerate(lines) if line.startswith('| `upstreamHostCircuitThreshold?`')] + if len(matches) != 1: + raise SystemExit(f'expected one circuit row, found {len(matches)}') + i = matches[0] + suffix = ' Applies only to Codex Pool routing with no pinned account; it is inert for `codexAccountMode: "direct"` and account-qualified selectors.' + if suffix not in lines[i]: + if not lines[i].endswith(' |'): + raise SystemExit('malformed circuit row') + lines[i] = lines[i][:-2] + suffix + ' |' + p.write_text('\n'.join(lines) + ('\n' if text.endswith('\n') else '')) + PY + git rm .github/workflows/maint-1243-english-doc.yml + - name: Validate and commit + shell: bash + run: | + set -euxo pipefail + git diff --check + git config user.name "Wibias" + git config user.email "$(printf '%s%s%s' '37517432+Wibias' '@' 'users.noreply.github.com')" + git add docs-site/src/content/docs/reference/configuration/providers.md + git commit -m "docs: document host circuit activation scope" + git push origin HEAD:maint/1039-host-circuit From 73a5f4ee9c6ae00f3cde4a781565c82ba8234c96 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 21:28:54 +0000 Subject: [PATCH 23/35] docs: document host circuit activation scope --- .github/workflows/maint-1243-english-doc.yml | 46 ------------------- .../docs/reference/configuration/providers.md | 2 +- 2 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 .github/workflows/maint-1243-english-doc.yml diff --git a/.github/workflows/maint-1243-english-doc.yml b/.github/workflows/maint-1243-english-doc.yml deleted file mode 100644 index 8c054e9528..0000000000 --- a/.github/workflows/maint-1243-english-doc.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Maintainer PR 1243 English docs - -on: - push: - branches: - - maint/1039-host-circuit - -permissions: - contents: write - -jobs: - fix: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Fix English activation contract - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - p = Path('docs-site/src/content/docs/reference/configuration/providers.md') - text = p.read_text() - lines = text.splitlines() - matches = [i for i, line in enumerate(lines) if line.startswith('| `upstreamHostCircuitThreshold?`')] - if len(matches) != 1: - raise SystemExit(f'expected one circuit row, found {len(matches)}') - i = matches[0] - suffix = ' Applies only to Codex Pool routing with no pinned account; it is inert for `codexAccountMode: "direct"` and account-qualified selectors.' - if suffix not in lines[i]: - if not lines[i].endswith(' |'): - raise SystemExit('malformed circuit row') - lines[i] = lines[i][:-2] + suffix + ' |' - p.write_text('\n'.join(lines) + ('\n' if text.endswith('\n') else '')) - PY - git rm .github/workflows/maint-1243-english-doc.yml - - name: Validate and commit - shell: bash - run: | - set -euxo pipefail - git diff --check - git config user.name "Wibias" - git config user.email "$(printf '%s%s%s' '37517432+Wibias' '@' 'users.noreply.github.com')" - git add docs-site/src/content/docs/reference/configuration/providers.md - git commit -m "docs: document host circuit activation scope" - git push origin HEAD:maint/1039-host-circuit diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 0491b42548..aca630a1fd 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -25,7 +25,7 @@ authenticated. | `accountPoolStrategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | Assignment strategy for new/unbound Codex requests. A request is unbound when it has no live (parent thread id, quota scope) affinity; a visible existing task can become unbound after proxy restart or affinity reset. `quota` picks the lowest-usage eligible account when no active account exists, keeps an eligible active account below `autoSwitchThreshold`, and after the threshold may move an unbound request or proactively rebind a bound task to a lower-usage eligible account. `round-robin` distributes unbound requests evenly; `fill-first` keeps assigning unbound requests to the active account until cooldown, unavailability, or the configured drain threshold. | | `accountPoolStickyLimit?` | `number` | `1` | New/unbound task assignments retained on one round-robin selection before advancing; the counter advances when a task is bound, not after an upstream success. Range 1–100. | | `upstreamFailoverThreshold?` | `number` | `3` | Consecutive transient failures before future new sessions fail over. Set `0` to disable. For regular Responses and native compact sends, proven pre-connection DNS/TCP reachability failures are tracked at the provider-host level: they never affect account health, account cooldowns, thread/session affinity, active-account selection, or Pool routing, and never count toward this threshold. | -| `upstreamHostCircuitThreshold?` | `number` | `0` | Opt-in circuit threshold for proven pre-connection DNS/TCP failures on native OpenAI forward Responses and compact sends. `0` disables it; `1`–`20` opens a 30-second provider-origin cooldown after that many terminal logical requests. While open, requests receive `503` with `Retry-After` before account selection or upstream send; after cooldown, one half-open request is admitted. Timeouts and HTTP responses never count, and any HTTP response closes the circuit. | +| `upstreamHostCircuitThreshold?` | `number` | `0` | Opt-in circuit threshold for proven pre-connection DNS/TCP failures on native OpenAI forward Responses and compact sends. `0` disables it; `1`–`20` opens a 30-second provider-origin cooldown after that many terminal logical requests. While open, requests receive `503` with `Retry-After` before account selection or upstream send; after cooldown, one half-open request is admitted. Timeouts and HTTP responses never count, and any HTTP response closes the circuit. Applies only to Codex Pool routing with no pinned account; it is inert for `codexAccountMode: "direct"` and account-qualified selectors. | | `modelCacheTtlMs?` | `number` | `300000` | Freshness window for the per-provider `/models` cache. | | `cacheRetention?` | `"none" \| "short" \| "long"` | `"short"` | Anthropic prompt-cache policy: disabled, 5-minute ephemeral, or 1-hour extended. | | `tokenGuardian?` | `OcxTokenGuardianConfig` | off | Optional proactive OAuth refresh and Codex-account warmup policy. | From d489e3dd7f22877a235119d70052ab1a0ab66570 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:29:18 +0200 Subject: [PATCH 24/35] chore: validate final provider-host circuit head From d35e1dd58a5386ea3f1031dc744f6df36313e4d9 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:33:25 +0200 Subject: [PATCH 25/35] chore: clear stale ci concurrency holder --- .../workflows/maint-1243-cancel-stale-ci.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/maint-1243-cancel-stale-ci.yml diff --git a/.github/workflows/maint-1243-cancel-stale-ci.yml b/.github/workflows/maint-1243-cancel-stale-ci.yml new file mode 100644 index 0000000000..5e9006eae0 --- /dev/null +++ b/.github/workflows/maint-1243-cancel-stale-ci.yml @@ -0,0 +1,35 @@ +name: Clear stale PR 1243 CI + +on: + push: + branches: + - maint/1039-host-circuit + +permissions: + actions: write + contents: write + +jobs: + clear: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Cancel stale Cross-platform run + env: + GH_TOKEN: ${{ github.token }} + run: | + curl --fail-with-body -L -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/31220090908/cancel" + - name: Remove one-shot workflow + shell: bash + run: | + set -euxo pipefail + git config user.name "Wibias" + git config user.email "$(printf '%s%s%s' '37517432+Wibias' '@' 'users.noreply.github.com')" + git rm .github/workflows/maint-1243-cancel-stale-ci.yml + git commit -m "chore: remove stale-ci cleanup workflow" + git push origin HEAD:maint/1039-host-circuit From 197a076530319304760c4d68c31110c2e25228ed Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:34:34 +0200 Subject: [PATCH 26/35] chore: remove stale-ci cleanup workflow --- .../workflows/maint-1243-cancel-stale-ci.yml | 35 ------------------- 1 file changed, 35 deletions(-) delete mode 100644 .github/workflows/maint-1243-cancel-stale-ci.yml diff --git a/.github/workflows/maint-1243-cancel-stale-ci.yml b/.github/workflows/maint-1243-cancel-stale-ci.yml deleted file mode 100644 index 5e9006eae0..0000000000 --- a/.github/workflows/maint-1243-cancel-stale-ci.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Clear stale PR 1243 CI - -on: - push: - branches: - - maint/1039-host-circuit - -permissions: - actions: write - contents: write - -jobs: - clear: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Cancel stale Cross-platform run - env: - GH_TOKEN: ${{ github.token }} - run: | - curl --fail-with-body -L -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${GH_TOKEN}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/31220090908/cancel" - - name: Remove one-shot workflow - shell: bash - run: | - set -euxo pipefail - git config user.name "Wibias" - git config user.email "$(printf '%s%s%s' '37517432+Wibias' '@' 'users.noreply.github.com')" - git rm .github/workflows/maint-1243-cancel-stale-ci.yml - git commit -m "chore: remove stale-ci cleanup workflow" - git push origin HEAD:maint/1039-host-circuit From 24768de2bc3d218218a4414da9e97ea575b90e54 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:54:55 +0200 Subject: [PATCH 27/35] chore: preserve ci shard layout for pr 1243 --- .../workflows/maint-1243-shard-stability.yml | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 .github/workflows/maint-1243-shard-stability.yml diff --git a/.github/workflows/maint-1243-shard-stability.yml b/.github/workflows/maint-1243-shard-stability.yml new file mode 100644 index 0000000000..d23669019c --- /dev/null +++ b/.github/workflows/maint-1243-shard-stability.yml @@ -0,0 +1,130 @@ +name: Preserve PR 1243 CI shard layout + +on: + push: + branches: + - maint/1039-host-circuit + +permissions: + contents: write + +jobs: + consolidate: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.14 + - name: Consolidate new tests into existing suites + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + + reach = Path('tests/upstream-reachability.test.ts') + text = reach.read_text() + text = text.replace( + 'import { describe, expect, test } from "bun:test";', + 'import { beforeEach, describe, expect, test } from "bun:test";', + 1, + ) + old_import = '''import { + clearUpstreamHostHealth, + getUpstreamHostHealth, + recordUpstreamHostFailure, + resetUpstreamHostHealth, + upstreamHostHealthKey, + UPSTREAM_HOST_FAILURE_WINDOW_MS, + UPSTREAM_HOST_HEALTH_MAX_ENTRIES, + } from "../src/codex/upstream-host-health";''' + new_import = '''import { + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, + UPSTREAM_HOST_FAILURE_WINDOW_MS, + UPSTREAM_HOST_HEALTH_MAX_ENTRIES, + acquireUpstreamHostAdmission, + clearUpstreamHostHealth, + disableUpstreamHostCircuitForKey, + getUpstreamHostHealth, + normalizeUpstreamHostCircuitThreshold, + recordUpstreamHostFailure, + releaseUpstreamHostAdmission, + resetUpstreamHostHealth, + upstreamHostHealthKey, + type UpstreamHostAdmissionLease, + } from "../src/codex/upstream-host-health";''' + if old_import not in text: + raise SystemExit('upstream reachability import anchor missing') + text = text.replace(old_import, new_import, 1) + coded_anchor = 'function coded(message: string, code: string, cause?: unknown): Error {' + if coded_anchor not in text: + raise SystemExit('coded anchor missing') + text = text.replace(coded_anchor, 'beforeEach(() => clearUpstreamHostHealth());\n\n' + coded_anchor, 1) + + circuit = Path('tests/upstream-host-circuit.test.ts').read_text() + start = circuit.index('function admit(') + appended = circuit[start:].strip() + text = text.rstrip() + '\n\n' + appended + '\n' + reach.write_text(text) + + cfg = Path('tests/config-user-edits.test.ts') + cfg_text = cfg.read_text() + cfg_text = cfg_text.replace( + ' getConfigPath,\n', + ' getConfigPath,\n getDefaultConfig,\n', + 1, + ) + cfg_text = cfg_text.replace( + ' saveConfigPreservingClaudeCode,\n', + ' saveConfigPreservingClaudeCode,\n validateConfigCandidate,\n', + 1, + ) + config_tests = ''' + +test("upstreamHostCircuitThreshold live writes accept only integer values from 0 through 20", () => { + for (const value of [0, 1, 20]) { + expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); + } + for (const value of [-1, 1.5, 21, "3", null]) { + const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); + } +}); + +test("a malformed upstreamHostCircuitThreshold hand edit disables only the circuit and warns", () => { + writeDiskConfig({ upstreamHostCircuitThreshold: 999 }); + const diagnostics = readConfigDiagnostics(); + expect(diagnostics.source).toBe("file"); + expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); + expect(diagnostics.warnings).toContain( + "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", + ); + expect(diagnostics.config.providers.test).toBeDefined(); +}); +''' + cfg.write_text(cfg_text.rstrip() + config_tests) + PY + git rm tests/upstream-host-circuit.test.ts tests/upstream-host-circuit-config.test.ts + git rm .github/workflows/maint-1243-shard-stability.yml + - name: Install + run: bun install --frozen-lockfile + - name: Typecheck + run: bun run typecheck + - name: Focused tests + run: bun test tests/upstream-reachability.test.ts tests/config-user-edits.test.ts tests/responses-compaction-routing.test.ts + - name: Privacy scan + run: bun run privacy:scan + - name: Diff check + run: git diff --check + - name: Commit shard-stable tests + shell: bash + run: | + set -euxo pipefail + git config user.name "Wibias" + git config user.email "$(printf '%s%s%s' '37517432+Wibias' '@' 'users.noreply.github.com')" + git add tests/upstream-reachability.test.ts tests/config-user-edits.test.ts + git commit -m "test(ci): preserve stable shard layout for host circuit coverage" + git push origin HEAD:maint/1039-host-circuit From b8d7515b9ddbdabdb570172ba37d19c0bfc7cb3c Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:56:08 +0200 Subject: [PATCH 28/35] chore: stage shard-stability consolidation script --- scripts/maint_1243_shard_stability.py | 84 +++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 scripts/maint_1243_shard_stability.py diff --git a/scripts/maint_1243_shard_stability.py b/scripts/maint_1243_shard_stability.py new file mode 100644 index 0000000000..266f4273f5 --- /dev/null +++ b/scripts/maint_1243_shard_stability.py @@ -0,0 +1,84 @@ +from pathlib import Path + +reach = Path("tests/upstream-reachability.test.ts") +text = reach.read_text() +text = text.replace( + 'import { describe, expect, test } from "bun:test";', + 'import { beforeEach, describe, expect, test } from "bun:test";', + 1, +) +old_import = '''import { + clearUpstreamHostHealth, + getUpstreamHostHealth, + recordUpstreamHostFailure, + resetUpstreamHostHealth, + upstreamHostHealthKey, + UPSTREAM_HOST_FAILURE_WINDOW_MS, + UPSTREAM_HOST_HEALTH_MAX_ENTRIES, +} from "../src/codex/upstream-host-health";''' +new_import = '''import { + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, + UPSTREAM_HOST_FAILURE_WINDOW_MS, + UPSTREAM_HOST_HEALTH_MAX_ENTRIES, + acquireUpstreamHostAdmission, + clearUpstreamHostHealth, + disableUpstreamHostCircuitForKey, + getUpstreamHostHealth, + normalizeUpstreamHostCircuitThreshold, + recordUpstreamHostFailure, + releaseUpstreamHostAdmission, + resetUpstreamHostHealth, + upstreamHostHealthKey, + type UpstreamHostAdmissionLease, +} from "../src/codex/upstream-host-health";''' +if old_import not in text: + raise SystemExit("upstream reachability import anchor missing") +text = text.replace(old_import, new_import, 1) +coded_anchor = "function coded(message: string, code: string, cause?: unknown): Error {" +if coded_anchor not in text: + raise SystemExit("coded anchor missing") +text = text.replace(coded_anchor, "beforeEach(() => clearUpstreamHostHealth());\n\n" + coded_anchor, 1) + +circuit = Path("tests/upstream-host-circuit.test.ts").read_text() +start = circuit.index("function admit(") +text = text.rstrip() + "\n\n" + circuit[start:].strip() + "\n" +reach.write_text(text) + +cfg = Path("tests/config-user-edits.test.ts") +cfg_text = cfg.read_text() +if " getDefaultConfig,\n" not in cfg_text: + cfg_text = cfg_text.replace(" getConfigPath,\n", " getConfigPath,\n getDefaultConfig,\n", 1) +if " validateConfigCandidate,\n" not in cfg_text: + cfg_text = cfg_text.replace( + " saveConfigPreservingClaudeCode,\n", + " saveConfigPreservingClaudeCode,\n validateConfigCandidate,\n", + 1, + ) +config_tests = ''' + +test("upstreamHostCircuitThreshold live writes accept only integer values from 0 through 20", () => { + for (const value of [0, 1, 20]) { + expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); + } + for (const value of [-1, 1.5, 21, "3", null]) { + const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); + } +}); + +test("a malformed upstreamHostCircuitThreshold hand edit disables only the circuit and warns", () => { + writeDiskConfig({ upstreamHostCircuitThreshold: 999 }); + const diagnostics = readConfigDiagnostics(); + expect(diagnostics.source).toBe("file"); + expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); + expect(diagnostics.warnings).toContain( + "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", + ); + expect(diagnostics.config.providers.test).toBeDefined(); +}); +''' +if "upstreamHostCircuitThreshold live writes accept only" not in cfg_text: + cfg_text = cfg_text.rstrip() + config_tests +cfg.write_text(cfg_text) From 76e6597bdc61b3d738ff8bf2bbf69f7c3edcc9c8 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:56:22 +0200 Subject: [PATCH 29/35] chore: simplify shard-stability runner --- .../workflows/maint-1243-shard-stability.yml | 94 +------------------ 1 file changed, 3 insertions(+), 91 deletions(-) diff --git a/.github/workflows/maint-1243-shard-stability.yml b/.github/workflows/maint-1243-shard-stability.yml index d23669019c..aff44b243d 100644 --- a/.github/workflows/maint-1243-shard-stability.yml +++ b/.github/workflows/maint-1243-shard-stability.yml @@ -18,97 +18,9 @@ jobs: with: bun-version: 1.3.14 - name: Consolidate new tests into existing suites - shell: bash - run: | - python3 - <<'PY' - from pathlib import Path - - reach = Path('tests/upstream-reachability.test.ts') - text = reach.read_text() - text = text.replace( - 'import { describe, expect, test } from "bun:test";', - 'import { beforeEach, describe, expect, test } from "bun:test";', - 1, - ) - old_import = '''import { - clearUpstreamHostHealth, - getUpstreamHostHealth, - recordUpstreamHostFailure, - resetUpstreamHostHealth, - upstreamHostHealthKey, - UPSTREAM_HOST_FAILURE_WINDOW_MS, - UPSTREAM_HOST_HEALTH_MAX_ENTRIES, - } from "../src/codex/upstream-host-health";''' - new_import = '''import { - UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, - UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, - UPSTREAM_HOST_FAILURE_WINDOW_MS, - UPSTREAM_HOST_HEALTH_MAX_ENTRIES, - acquireUpstreamHostAdmission, - clearUpstreamHostHealth, - disableUpstreamHostCircuitForKey, - getUpstreamHostHealth, - normalizeUpstreamHostCircuitThreshold, - recordUpstreamHostFailure, - releaseUpstreamHostAdmission, - resetUpstreamHostHealth, - upstreamHostHealthKey, - type UpstreamHostAdmissionLease, - } from "../src/codex/upstream-host-health";''' - if old_import not in text: - raise SystemExit('upstream reachability import anchor missing') - text = text.replace(old_import, new_import, 1) - coded_anchor = 'function coded(message: string, code: string, cause?: unknown): Error {' - if coded_anchor not in text: - raise SystemExit('coded anchor missing') - text = text.replace(coded_anchor, 'beforeEach(() => clearUpstreamHostHealth());\n\n' + coded_anchor, 1) - - circuit = Path('tests/upstream-host-circuit.test.ts').read_text() - start = circuit.index('function admit(') - appended = circuit[start:].strip() - text = text.rstrip() + '\n\n' + appended + '\n' - reach.write_text(text) - - cfg = Path('tests/config-user-edits.test.ts') - cfg_text = cfg.read_text() - cfg_text = cfg_text.replace( - ' getConfigPath,\n', - ' getConfigPath,\n getDefaultConfig,\n', - 1, - ) - cfg_text = cfg_text.replace( - ' saveConfigPreservingClaudeCode,\n', - ' saveConfigPreservingClaudeCode,\n validateConfigCandidate,\n', - 1, - ) - config_tests = ''' - -test("upstreamHostCircuitThreshold live writes accept only integer values from 0 through 20", () => { - for (const value of [0, 1, 20]) { - expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); - } - for (const value of [-1, 1.5, 21, "3", null]) { - const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); - expect(result.ok).toBe(false); - if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); - } -}); - -test("a malformed upstreamHostCircuitThreshold hand edit disables only the circuit and warns", () => { - writeDiskConfig({ upstreamHostCircuitThreshold: 999 }); - const diagnostics = readConfigDiagnostics(); - expect(diagnostics.source).toBe("file"); - expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); - expect(diagnostics.warnings).toContain( - "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", - ); - expect(diagnostics.config.providers.test).toBeDefined(); -}); -''' - cfg.write_text(cfg_text.rstrip() + config_tests) - PY - git rm tests/upstream-host-circuit.test.ts tests/upstream-host-circuit-config.test.ts - git rm .github/workflows/maint-1243-shard-stability.yml + run: python3 scripts/maint_1243_shard_stability.py + - name: Remove standalone tests and maintenance files + run: git rm tests/upstream-host-circuit.test.ts tests/upstream-host-circuit-config.test.ts scripts/maint_1243_shard_stability.py .github/workflows/maint-1243-shard-stability.yml - name: Install run: bun install --frozen-lockfile - name: Typecheck From 6753cc48b6eb682a57988d2e91b0a9d7c953b396 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 21:56:56 +0000 Subject: [PATCH 30/35] test(ci): preserve stable shard layout for host circuit coverage --- .../workflows/maint-1243-shard-stability.yml | 42 --- scripts/maint_1243_shard_stability.py | 84 ------ tests/config-user-edits.test.ts | 24 ++ tests/upstream-host-circuit-config.test.ts | 43 --- tests/upstream-host-circuit.test.ts | 273 ------------------ tests/upstream-reachability.test.ts | 269 ++++++++++++++++- 6 files changed, 290 insertions(+), 445 deletions(-) delete mode 100644 .github/workflows/maint-1243-shard-stability.yml delete mode 100644 scripts/maint_1243_shard_stability.py delete mode 100644 tests/upstream-host-circuit-config.test.ts delete mode 100644 tests/upstream-host-circuit.test.ts diff --git a/.github/workflows/maint-1243-shard-stability.yml b/.github/workflows/maint-1243-shard-stability.yml deleted file mode 100644 index aff44b243d..0000000000 --- a/.github/workflows/maint-1243-shard-stability.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Preserve PR 1243 CI shard layout - -on: - push: - branches: - - maint/1039-host-circuit - -permissions: - contents: write - -jobs: - consolidate: - if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: oven-sh/setup-bun@v2 - with: - bun-version: 1.3.14 - - name: Consolidate new tests into existing suites - run: python3 scripts/maint_1243_shard_stability.py - - name: Remove standalone tests and maintenance files - run: git rm tests/upstream-host-circuit.test.ts tests/upstream-host-circuit-config.test.ts scripts/maint_1243_shard_stability.py .github/workflows/maint-1243-shard-stability.yml - - name: Install - run: bun install --frozen-lockfile - - name: Typecheck - run: bun run typecheck - - name: Focused tests - run: bun test tests/upstream-reachability.test.ts tests/config-user-edits.test.ts tests/responses-compaction-routing.test.ts - - name: Privacy scan - run: bun run privacy:scan - - name: Diff check - run: git diff --check - - name: Commit shard-stable tests - shell: bash - run: | - set -euxo pipefail - git config user.name "Wibias" - git config user.email "$(printf '%s%s%s' '37517432+Wibias' '@' 'users.noreply.github.com')" - git add tests/upstream-reachability.test.ts tests/config-user-edits.test.ts - git commit -m "test(ci): preserve stable shard layout for host circuit coverage" - git push origin HEAD:maint/1039-host-circuit diff --git a/scripts/maint_1243_shard_stability.py b/scripts/maint_1243_shard_stability.py deleted file mode 100644 index 266f4273f5..0000000000 --- a/scripts/maint_1243_shard_stability.py +++ /dev/null @@ -1,84 +0,0 @@ -from pathlib import Path - -reach = Path("tests/upstream-reachability.test.ts") -text = reach.read_text() -text = text.replace( - 'import { describe, expect, test } from "bun:test";', - 'import { beforeEach, describe, expect, test } from "bun:test";', - 1, -) -old_import = '''import { - clearUpstreamHostHealth, - getUpstreamHostHealth, - recordUpstreamHostFailure, - resetUpstreamHostHealth, - upstreamHostHealthKey, - UPSTREAM_HOST_FAILURE_WINDOW_MS, - UPSTREAM_HOST_HEALTH_MAX_ENTRIES, -} from "../src/codex/upstream-host-health";''' -new_import = '''import { - UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, - UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, - UPSTREAM_HOST_FAILURE_WINDOW_MS, - UPSTREAM_HOST_HEALTH_MAX_ENTRIES, - acquireUpstreamHostAdmission, - clearUpstreamHostHealth, - disableUpstreamHostCircuitForKey, - getUpstreamHostHealth, - normalizeUpstreamHostCircuitThreshold, - recordUpstreamHostFailure, - releaseUpstreamHostAdmission, - resetUpstreamHostHealth, - upstreamHostHealthKey, - type UpstreamHostAdmissionLease, -} from "../src/codex/upstream-host-health";''' -if old_import not in text: - raise SystemExit("upstream reachability import anchor missing") -text = text.replace(old_import, new_import, 1) -coded_anchor = "function coded(message: string, code: string, cause?: unknown): Error {" -if coded_anchor not in text: - raise SystemExit("coded anchor missing") -text = text.replace(coded_anchor, "beforeEach(() => clearUpstreamHostHealth());\n\n" + coded_anchor, 1) - -circuit = Path("tests/upstream-host-circuit.test.ts").read_text() -start = circuit.index("function admit(") -text = text.rstrip() + "\n\n" + circuit[start:].strip() + "\n" -reach.write_text(text) - -cfg = Path("tests/config-user-edits.test.ts") -cfg_text = cfg.read_text() -if " getDefaultConfig,\n" not in cfg_text: - cfg_text = cfg_text.replace(" getConfigPath,\n", " getConfigPath,\n getDefaultConfig,\n", 1) -if " validateConfigCandidate,\n" not in cfg_text: - cfg_text = cfg_text.replace( - " saveConfigPreservingClaudeCode,\n", - " saveConfigPreservingClaudeCode,\n validateConfigCandidate,\n", - 1, - ) -config_tests = ''' - -test("upstreamHostCircuitThreshold live writes accept only integer values from 0 through 20", () => { - for (const value of [0, 1, 20]) { - expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); - } - for (const value of [-1, 1.5, 21, "3", null]) { - const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); - expect(result.ok).toBe(false); - if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); - } -}); - -test("a malformed upstreamHostCircuitThreshold hand edit disables only the circuit and warns", () => { - writeDiskConfig({ upstreamHostCircuitThreshold: 999 }); - const diagnostics = readConfigDiagnostics(); - expect(diagnostics.source).toBe("file"); - expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); - expect(diagnostics.warnings).toContain( - "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", - ); - expect(diagnostics.config.providers.test).toBeDefined(); -}); -''' -if "upstreamHostCircuitThreshold live writes accept only" not in cfg_text: - cfg_text = cfg_text.rstrip() + config_tests -cfg.write_text(cfg_text) diff --git a/tests/config-user-edits.test.ts b/tests/config-user-edits.test.ts index 207833622f..244d138848 100644 --- a/tests/config-user-edits.test.ts +++ b/tests/config-user-edits.test.ts @@ -5,11 +5,13 @@ import { join } from "node:path"; import { armClaudeCodeBaseline, getConfigPath, + getDefaultConfig, loadConfig, readConfigDiagnostics, reconcileLiveConfigFromDisk, saveConfig, saveConfigPreservingClaudeCode, + validateConfigCandidate, } from "../src/config"; import { rateLimitRetryPolicyFor } from "../src/providers/key-failover"; import type { OcxConfig } from "../src/types"; @@ -477,3 +479,25 @@ test("a providers hand edit is NOT preserved", () => { saveConfigPreservingClaudeCode(live); expect(Object.keys(diskConfig().providers as Record)).toEqual(["test"]); }); + +test("upstreamHostCircuitThreshold live writes accept only integer values from 0 through 20", () => { + for (const value of [0, 1, 20]) { + expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); + } + for (const value of [-1, 1.5, 21, "3", null]) { + const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); + } +}); + +test("a malformed upstreamHostCircuitThreshold hand edit disables only the circuit and warns", () => { + writeDiskConfig({ upstreamHostCircuitThreshold: 999 }); + const diagnostics = readConfigDiagnostics(); + expect(diagnostics.source).toBe("file"); + expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); + expect(diagnostics.warnings).toContain( + "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", + ); + expect(diagnostics.config.providers.test).toBeDefined(); +}); diff --git a/tests/upstream-host-circuit-config.test.ts b/tests/upstream-host-circuit-config.test.ts deleted file mode 100644 index be69b3f934..0000000000 --- a/tests/upstream-host-circuit-config.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { afterEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { getConfigPath, getDefaultConfig, readConfigDiagnostics, validateConfigCandidate } from "../src/config"; - -let testDir = ""; - -afterEach(() => { - delete process.env.OPENCODEX_HOME; - if (testDir && existsSync(testDir)) rmSync(testDir, { recursive: true, force: true }); - testDir = ""; -}); - -describe("upstreamHostCircuitThreshold config contract", () => { - test("live writes accept only integer values from 0 through 20", () => { - for (const value of [0, 1, 20]) { - expect(validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }).ok).toBe(true); - } - for (const value of [-1, 1.5, 21, "3", null]) { - const result = validateConfigCandidate({ ...getDefaultConfig(), upstreamHostCircuitThreshold: value }); - expect(result.ok).toBe(false); - if (!result.ok) expect(result.error).toContain("upstreamHostCircuitThreshold"); - } - }); - - test("malformed hand edits disable only the circuit and report a warning", () => { - testDir = mkdtempSync(join(tmpdir(), "ocx-host-circuit-config-")); - process.env.OPENCODEX_HOME = testDir; - writeFileSync(getConfigPath(), JSON.stringify({ - ...getDefaultConfig(), - upstreamHostCircuitThreshold: 999, - })); - - const diagnostics = readConfigDiagnostics(); - expect(diagnostics.source).toBe("file"); - expect(diagnostics.config.upstreamHostCircuitThreshold).toBeUndefined(); - expect(diagnostics.warnings).toContain( - "upstreamHostCircuitThreshold ignored: expected an integer from 0 to 20", - ); - expect(Object.keys(diagnostics.config.providers).length).toBeGreaterThan(0); - }); -}); diff --git a/tests/upstream-host-circuit.test.ts b/tests/upstream-host-circuit.test.ts deleted file mode 100644 index 383b96fc5a..0000000000 --- a/tests/upstream-host-circuit.test.ts +++ /dev/null @@ -1,273 +0,0 @@ -import { beforeEach, describe, expect, test } from "bun:test"; -import { - UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, - UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, - UPSTREAM_HOST_FAILURE_WINDOW_MS, - UPSTREAM_HOST_HEALTH_MAX_ENTRIES, - acquireUpstreamHostAdmission, - clearUpstreamHostHealth, - disableUpstreamHostCircuitForKey, - getUpstreamHostHealth, - normalizeUpstreamHostCircuitThreshold, - recordUpstreamHostFailure, - releaseUpstreamHostAdmission, - resetUpstreamHostHealth, - upstreamHostHealthKey, - type UpstreamHostAdmissionLease, -} from "../src/codex/upstream-host-health"; - -beforeEach(() => clearUpstreamHostHealth()); - -function admit(key: string, threshold: number, now: number): UpstreamHostAdmissionLease { - const admission = acquireUpstreamHostAdmission(key, threshold, now); - expect(admission.kind).toBe("admitted"); - if (admission.kind !== "admitted" || !admission.lease) { - throw new Error("expected a circuit admission lease"); - } - return admission.lease; -} - -function fail(key: string, threshold: number, now: number): void { - recordUpstreamHostFailure(key, { - code: "ECONNREFUSED", - now, - threshold, - lease: admit(key, threshold, now), - }); -} - -describe("opt-in upstream host circuit", () => { - test("normalizes the opt-in threshold and leaves zero disabled", () => { - expect(normalizeUpstreamHostCircuitThreshold(undefined)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(-1)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(0)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold("3")).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(1.5)).toBe(0); - expect(normalizeUpstreamHostCircuitThreshold(3)).toBe(3); - expect(normalizeUpstreamHostCircuitThreshold(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD)).toBe( - UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, - ); - expect(normalizeUpstreamHostCircuitThreshold(999)).toBe(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD); - - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - expect(acquireUpstreamHostAdmission(key, 0, 1_000)).toEqual({ - kind: "admitted", - lease: null, - }); - }); - - test("legacy observations cannot open the opt-in circuit without a lease", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - for (let attempt = 0; attempt < 3; attempt++) { - recordUpstreamHostFailure(key, { - code: "ECONNREFUSED", - now: 2_000 + attempt, - threshold: 1, - }); - } - expect(getUpstreamHostHealth(key)).toMatchObject({ - consecutiveFailures: 3, - lastFailureCode: "ECONNREFUSED", - }); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); - }); - - test("opens exactly at the configured threshold", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - const threshold = 3; - fail(key, threshold, 3_001); - fail(key, threshold, 3_002); - expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 2 }); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); - - fail(key, threshold, 3_003); - expect(getUpstreamHostHealth(key)).toMatchObject({ - consecutiveFailures: 3, - cooldownUntil: 3_003 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, - }); - expect(acquireUpstreamHostAdmission(key, threshold, 3_004)).toEqual({ - kind: "blocked", - retryAfterSeconds: 30, - }); - }); - - test("admits one half-open request and an HTTP response closes the circuit", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 1, 4_000); - const probeAt = 4_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; - const probe = admit(key, 1, probeAt); - expect(probe.halfOpen).toBe(true); - expect(acquireUpstreamHostAdmission(key, 1, probeAt)).toEqual({ - kind: "blocked", - retryAfterSeconds: 1, - }); - expect(resetUpstreamHostHealth(key, probe, probeAt + 1)).toBe(true); - expect(getUpstreamHostHealth(key)).toBeNull(); - }); - - test("a half-open reachability failure immediately reopens the cooldown", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 1, 5_000); - const probeAt = 5_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; - fail(key, 1, probeAt); - expect(getUpstreamHostHealth(key)).toMatchObject({ - cooldownUntil: probeAt + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, - }); - }); - - test("releasing a half-open request adds no evidence and permits another probe", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 1, 6_000); - const probeAt = 6_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; - const before = getUpstreamHostHealth(key); - const first = admit(key, 1, probeAt); - expect(releaseUpstreamHostAdmission(first, probeAt)).toBe(true); - expect(getUpstreamHostHealth(key)).toMatchObject({ - consecutiveFailures: before!.consecutiveFailures, - lastFailureAt: before!.lastFailureAt, - lastFailureCode: before!.lastFailureCode, - cooldownUntil: before!.cooldownUntil, - }); - expect(admit(key, 1, probeAt).halfOpen).toBe(true); - }); - - test("an HTTP response preserves a concurrent lease and its later failure authority", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - const first = admit(key, 3, 7_000); - const concurrent = admit(key, 3, 7_000); - expect(resetUpstreamHostHealth(key, first, 7_001)).toBe(true); - expect(getUpstreamHostHealth(key)).toBeNull(); - - recordUpstreamHostFailure(key, { - code: "ECONNREFUSED", - now: 7_002, - threshold: 3, - lease: concurrent, - }); - expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); - }); - - test("a concurrent HTTP response can close the cooldown opened by its peer", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - const failing = admit(key, 1, 7_500); - const succeeding = admit(key, 1, 7_500); - - recordUpstreamHostFailure(key, { - code: "ECONNREFUSED", - now: 7_501, - threshold: 1, - lease: failing, - }); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(7_501 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); - - expect(resetUpstreamHostHealth(key, succeeding, 7_502)).toBe(true); - expect(getUpstreamHostHealth(key)).toBeNull(); - }); - - test("a stale failure streak expires after the failure window", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 3, 12_000); - expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); - - const afterWindow = 12_000 + UPSTREAM_HOST_FAILURE_WINDOW_MS + 1; - const lease = admit(key, 3, afterWindow); - expect(lease.halfOpen).toBe(false); - recordUpstreamHostFailure(key, { - code: "ECONNREFUSED", - now: afterWindow, - threshold: 3, - lease, - }); - expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); - }); - - test("the retention cap evicts the stalest unleased origin", () => { - for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 8; i += 1) { - fail(upstreamHostHealthKey("openai", `https://h${i}.example`), 1, 13_000 + i); - } - expect(getUpstreamHostHealth(upstreamHostHealthKey("openai", "https://h0.example"))).toBeNull(); - expect(getUpstreamHostHealth( - upstreamHostHealthKey("openai", `https://h${UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 7}.example`), - )).not.toBeNull(); - }); - - test("retention pressure never evicts an active admission lease", () => { - const leases: UpstreamHostAdmissionLease[] = []; - for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 1; i += 1) { - leases.push(admit(upstreamHostHealthKey("openai", `https://active-${i}.example`), 1, 14_000 + i)); - } - expect(releaseUpstreamHostAdmission(leases[0], 15_000)).toBe(true); - for (const lease of leases.slice(1)) releaseUpstreamHostAdmission(lease, 15_001); - }); - - test("a stale completion cannot mutate the generation that opened the circuit", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - const stale = admit(key, 1, 8_000); - fail(key, 1, 8_001); - const before = getUpstreamHostHealth(key); - - recordUpstreamHostFailure(key, { - code: "ECONNREFUSED", - now: 8_002, - threshold: 1, - lease: stale, - }); - expect(getUpstreamHostHealth(key)).toEqual(before); - }); - - test("a lease cannot settle a different host key", () => { - const keyA = upstreamHostHealthKey("openai", "https://chatgpt.com"); - const keyB = upstreamHostHealthKey("openai", "https://api.openai.com"); - const leaseA = admit(keyA, 1, 10_000); - expect(resetUpstreamHostHealth(keyB, leaseA, 10_001)).toBe(false); - recordUpstreamHostFailure(keyB, { - code: "ECONNREFUSED", - now: 10_002, - threshold: 1, - lease: leaseA, - }); - expect(getUpstreamHostHealth(keyB)).toBeNull(); - recordUpstreamHostFailure(keyA, { - code: "ECONNREFUSED", - now: 10_003, - threshold: 1, - lease: leaseA, - }); - expect(getUpstreamHostHealth(keyA)).toMatchObject({ - consecutiveFailures: 1, - cooldownUntil: 10_003 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, - }); - }); - - test("disabled traffic can clear an old circuit before it is re-enabled", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 1, 11_000); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(11_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); - - expect(disableUpstreamHostCircuitForKey(key, 11_001)).toBe(true); - expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); - expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); - - expect(resetUpstreamHostHealth(key)).toBe(true); - expect(getUpstreamHostHealth(key)).toBeNull(); - expect(acquireUpstreamHostAdmission(key, 1, 11_002).kind).toBe("admitted"); - }); - - test("a later physical retry without its lease cannot close a newer circuit", () => { - const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 1, 9_000); - const before = getUpstreamHostHealth(key); - - expect(resetUpstreamHostHealth(key, null, 9_001)).toBe(false); - recordUpstreamHostFailure(key, { - code: "ECONNREFUSED", - now: 9_002, - threshold: 1, - lease: null, - }); - // Unwired observational callers are also unable to mutate circuit-owned state. - recordUpstreamHostFailure(key, { code: "ECONNREFUSED", now: 9_003 }); - - expect(getUpstreamHostHealth(key)).toEqual(before); - }); -}); diff --git a/tests/upstream-reachability.test.ts b/tests/upstream-reachability.test.ts index 860c2273db..6b1d392594 100644 --- a/tests/upstream-reachability.test.ts +++ b/tests/upstream-reachability.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { beforeEach, describe, expect, test } from "bun:test"; import { classifyTransportFailureKind, isPreConnectReachabilityError, @@ -7,15 +7,24 @@ import { } from "../src/lib/upstream-reachability"; import { UpstreamRetryEvidenceError } from "../src/lib/upstream-retry"; import { + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, + UPSTREAM_HOST_FAILURE_WINDOW_MS, + UPSTREAM_HOST_HEALTH_MAX_ENTRIES, + acquireUpstreamHostAdmission, clearUpstreamHostHealth, + disableUpstreamHostCircuitForKey, getUpstreamHostHealth, + normalizeUpstreamHostCircuitThreshold, recordUpstreamHostFailure, + releaseUpstreamHostAdmission, resetUpstreamHostHealth, upstreamHostHealthKey, - UPSTREAM_HOST_FAILURE_WINDOW_MS, - UPSTREAM_HOST_HEALTH_MAX_ENTRIES, + type UpstreamHostAdmissionLease, } from "../src/codex/upstream-host-health"; +beforeEach(() => clearUpstreamHostHealth()); + function coded(message: string, code: string, cause?: unknown): Error { return Object.assign(new Error(message), { code, ...(cause !== undefined ? { cause } : {}) }); } @@ -125,3 +134,257 @@ describe("upstream host health ledger", () => { clearUpstreamHostHealth(); }); }); + +function admit(key: string, threshold: number, now: number): UpstreamHostAdmissionLease { + const admission = acquireUpstreamHostAdmission(key, threshold, now); + expect(admission.kind).toBe("admitted"); + if (admission.kind !== "admitted" || !admission.lease) { + throw new Error("expected a circuit admission lease"); + } + return admission.lease; +} + +function fail(key: string, threshold: number, now: number): void { + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now, + threshold, + lease: admit(key, threshold, now), + }); +} + +describe("opt-in upstream host circuit", () => { + test("normalizes the opt-in threshold and leaves zero disabled", () => { + expect(normalizeUpstreamHostCircuitThreshold(undefined)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(-1)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(0)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold("3")).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(1.5)).toBe(0); + expect(normalizeUpstreamHostCircuitThreshold(3)).toBe(3); + expect(normalizeUpstreamHostCircuitThreshold(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD)).toBe( + UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD, + ); + expect(normalizeUpstreamHostCircuitThreshold(999)).toBe(UPSTREAM_HOST_CIRCUIT_MAX_THRESHOLD); + + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + expect(acquireUpstreamHostAdmission(key, 0, 1_000)).toEqual({ + kind: "admitted", + lease: null, + }); + }); + + test("legacy observations cannot open the opt-in circuit without a lease", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + for (let attempt = 0; attempt < 3; attempt++) { + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 2_000 + attempt, + threshold: 1, + }); + } + expect(getUpstreamHostHealth(key)).toMatchObject({ + consecutiveFailures: 3, + lastFailureCode: "ECONNREFUSED", + }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); + }); + + test("opens exactly at the configured threshold", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const threshold = 3; + fail(key, threshold, 3_001); + fail(key, threshold, 3_002); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 2 }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); + + fail(key, threshold, 3_003); + expect(getUpstreamHostHealth(key)).toMatchObject({ + consecutiveFailures: 3, + cooldownUntil: 3_003 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + }); + expect(acquireUpstreamHostAdmission(key, threshold, 3_004)).toEqual({ + kind: "blocked", + retryAfterSeconds: 30, + }); + }); + + test("admits one half-open request and an HTTP response closes the circuit", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 4_000); + const probeAt = 4_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; + const probe = admit(key, 1, probeAt); + expect(probe.halfOpen).toBe(true); + expect(acquireUpstreamHostAdmission(key, 1, probeAt)).toEqual({ + kind: "blocked", + retryAfterSeconds: 1, + }); + expect(resetUpstreamHostHealth(key, probe, probeAt + 1)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + }); + + test("a half-open reachability failure immediately reopens the cooldown", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 5_000); + const probeAt = 5_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; + fail(key, 1, probeAt); + expect(getUpstreamHostHealth(key)).toMatchObject({ + cooldownUntil: probeAt + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + }); + }); + + test("releasing a half-open request adds no evidence and permits another probe", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 6_000); + const probeAt = 6_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS; + const before = getUpstreamHostHealth(key); + const first = admit(key, 1, probeAt); + expect(releaseUpstreamHostAdmission(first, probeAt)).toBe(true); + expect(getUpstreamHostHealth(key)).toMatchObject({ + consecutiveFailures: before!.consecutiveFailures, + lastFailureAt: before!.lastFailureAt, + lastFailureCode: before!.lastFailureCode, + cooldownUntil: before!.cooldownUntil, + }); + expect(admit(key, 1, probeAt).halfOpen).toBe(true); + }); + + test("an HTTP response preserves a concurrent lease and its later failure authority", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const first = admit(key, 3, 7_000); + const concurrent = admit(key, 3, 7_000); + expect(resetUpstreamHostHealth(key, first, 7_001)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 7_002, + threshold: 3, + lease: concurrent, + }); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + }); + + test("a concurrent HTTP response can close the cooldown opened by its peer", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const failing = admit(key, 1, 7_500); + const succeeding = admit(key, 1, 7_500); + + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 7_501, + threshold: 1, + lease: failing, + }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(7_501 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); + + expect(resetUpstreamHostHealth(key, succeeding, 7_502)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + }); + + test("a stale failure streak expires after the failure window", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 3, 12_000); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + + const afterWindow = 12_000 + UPSTREAM_HOST_FAILURE_WINDOW_MS + 1; + const lease = admit(key, 3, afterWindow); + expect(lease.halfOpen).toBe(false); + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: afterWindow, + threshold: 3, + lease, + }); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + }); + + test("the retention cap evicts the stalest unleased origin", () => { + for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 8; i += 1) { + fail(upstreamHostHealthKey("openai", `https://h${i}.example`), 1, 13_000 + i); + } + expect(getUpstreamHostHealth(upstreamHostHealthKey("openai", "https://h0.example"))).toBeNull(); + expect(getUpstreamHostHealth( + upstreamHostHealthKey("openai", `https://h${UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 7}.example`), + )).not.toBeNull(); + }); + + test("retention pressure never evicts an active admission lease", () => { + const leases: UpstreamHostAdmissionLease[] = []; + for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 1; i += 1) { + leases.push(admit(upstreamHostHealthKey("openai", `https://active-${i}.example`), 1, 14_000 + i)); + } + expect(releaseUpstreamHostAdmission(leases[0], 15_000)).toBe(true); + for (const lease of leases.slice(1)) releaseUpstreamHostAdmission(lease, 15_001); + }); + + test("a stale completion cannot mutate the generation that opened the circuit", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const stale = admit(key, 1, 8_000); + fail(key, 1, 8_001); + const before = getUpstreamHostHealth(key); + + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 8_002, + threshold: 1, + lease: stale, + }); + expect(getUpstreamHostHealth(key)).toEqual(before); + }); + + test("a lease cannot settle a different host key", () => { + const keyA = upstreamHostHealthKey("openai", "https://chatgpt.com"); + const keyB = upstreamHostHealthKey("openai", "https://api.openai.com"); + const leaseA = admit(keyA, 1, 10_000); + expect(resetUpstreamHostHealth(keyB, leaseA, 10_001)).toBe(false); + recordUpstreamHostFailure(keyB, { + code: "ECONNREFUSED", + now: 10_002, + threshold: 1, + lease: leaseA, + }); + expect(getUpstreamHostHealth(keyB)).toBeNull(); + recordUpstreamHostFailure(keyA, { + code: "ECONNREFUSED", + now: 10_003, + threshold: 1, + lease: leaseA, + }); + expect(getUpstreamHostHealth(keyA)).toMatchObject({ + consecutiveFailures: 1, + cooldownUntil: 10_003 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS, + }); + }); + + test("disabled traffic can clear an old circuit before it is re-enabled", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 11_000); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBe(11_000 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS); + + expect(disableUpstreamHostCircuitForKey(key, 11_001)).toBe(true); + expect(getUpstreamHostHealth(key)).toMatchObject({ consecutiveFailures: 1 }); + expect(getUpstreamHostHealth(key)?.cooldownUntil).toBeUndefined(); + + expect(resetUpstreamHostHealth(key)).toBe(true); + expect(getUpstreamHostHealth(key)).toBeNull(); + expect(acquireUpstreamHostAdmission(key, 1, 11_002).kind).toBe("admitted"); + }); + + test("a later physical retry without its lease cannot close a newer circuit", () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 9_000); + const before = getUpstreamHostHealth(key); + + expect(resetUpstreamHostHealth(key, null, 9_001)).toBe(false); + recordUpstreamHostFailure(key, { + code: "ECONNREFUSED", + now: 9_002, + threshold: 1, + lease: null, + }); + // Unwired observational callers are also unable to mutate circuit-owned state. + recordUpstreamHostFailure(key, { code: "ECONNREFUSED", now: 9_003 }); + + expect(getUpstreamHostHealth(key)).toEqual(before); + }); +}); From c5e19122eb5d5c641780ed6c8a5f84808586bd2f Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Fri, 7 Aug 2026 23:57:21 +0200 Subject: [PATCH 31/35] chore: validate shard-stable host circuit head From 531fcec40d0ea0140dbbad87225dd7de31ddaa84 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:23:56 +0200 Subject: [PATCH 32/35] fix(codex): skip quota priming while host circuit is open --- src/codex/subagent-model-fallback.ts | 29 +++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/codex/subagent-model-fallback.ts b/src/codex/subagent-model-fallback.ts index 5e53a00431..0e51b423c0 100644 --- a/src/codex/subagent-model-fallback.ts +++ b/src/codex/subagent-model-fallback.ts @@ -30,12 +30,23 @@ import { isCodexAccountPaused } from "./account-pause"; import { slugEquals } from "../providers/slug-codec"; import { isThreadSpawnRequest } from "../server/effort-policy"; import { PROVIDER_REGISTRY } from "../providers/registry"; -import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers"; +import { + CODEX_FORWARD_BASE_URL, + OPENAI_CODEX_PROVIDER_ID, + isCanonicalOpenAiForwardProvider, +} from "../providers/openai-tiers"; import { routeModel, type RouteResult } from "../router"; import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; import { codexAccountNamespaceForModel } from "./account-namespace-match"; +import { + getUpstreamHostHealth, + normalizeUpstreamHostCircuitThreshold, + upstreamHostHealthKey, +} from "./upstream-host-health"; export const DEFAULT_SUBAGENT_MODEL_FALLBACK_POLL_MS = 60_000; +const CODEX_FORWARD_ORIGIN = new URL(CODEX_FORWARD_BASE_URL).origin.toLowerCase(); + type SubagentQuotaPrimeFn = (config: OcxConfig, reason: string) => Promise; let subagentQuotaPrimeForTests: SubagentQuotaPrimeFn | null = null; let quotaPrimeInFlight: Promise | null = null; @@ -400,11 +411,19 @@ export function resolveAgentModelFallbackForPrimary( return merged; } +function subagentQuotaPrimeBlockedByHostCircuit(config: OcxConfig): boolean { + if (normalizeUpstreamHostCircuitThreshold(config.upstreamHostCircuitThreshold) === 0) return false; + const key = upstreamHostHealthKey(OPENAI_CODEX_PROVIDER_ID, CODEX_FORWARD_ORIGIN); + return getUpstreamHostHealth(key)?.cooldownUntil !== undefined; +} + /** * Best-effort quota refresh before subagent model selection. * Concurrent callers share one in-flight promise. The success TTL is updated only * after a successful refresh so failures remain retryable. Errors are swallowed so - * spawn routing can continue. + * spawn routing can continue. When the canonical ChatGPT origin is circuit-blocked, + * cached quota is used instead of sending credential-bearing usage probes to the + * same origin before the request's final host admission check. */ export function maybePrimeSubagentQuota( config: OcxConfig, @@ -412,11 +431,15 @@ export function maybePrimeSubagentQuota( options: { nativeMainReadsForbidden?: boolean } = {}, ): Promise { if (options.nativeMainReadsForbidden) return Promise.resolve(); + if (subagentQuotaPrimeBlockedByHostCircuit(config)) return Promise.resolve(); if (quotaPrimeInFlight) return quotaPrimeInFlight; if (!shouldPrimeSubagentQuota(config, now)) return Promise.resolve(); quotaPrimeInFlight = (async () => { try { + // Re-check after claiming single-flight ownership so a circuit opened by + // a concurrent request cannot race us into a fresh usage-probe pass. + if (subagentQuotaPrimeBlockedByHostCircuit(config)) return; if (subagentQuotaPrimeForTests) { await subagentQuotaPrimeForTests(config, "subagent-spawn"); } else { @@ -534,4 +557,4 @@ export function listCodexAgentRoles(codexHome = CODEX_HOME): string[] { export function shouldPrimeSubagentQuota(config: OcxConfig, now = Date.now()): boolean { const last = quotaPrimedAt.get("global") ?? 0; return now - last >= pollIntervalMs(config); -} +} \ No newline at end of file From aba10a48d50cac48a55c88dfbea8df29d692eb22 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:24:57 +0200 Subject: [PATCH 33/35] test(codex): cover host-circuit quota-prime suppression --- tests/upstream-reachability.test.ts | 44 +++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/upstream-reachability.test.ts b/tests/upstream-reachability.test.ts index 6b1d392594..e34179e899 100644 --- a/tests/upstream-reachability.test.ts +++ b/tests/upstream-reachability.test.ts @@ -22,8 +22,17 @@ import { upstreamHostHealthKey, type UpstreamHostAdmissionLease, } from "../src/codex/upstream-host-health"; - -beforeEach(() => clearUpstreamHostHealth()); +import { + maybePrimeSubagentQuota, + resetSubagentModelFallbackStateForTests, + setSubagentQuotaPrimeForTests, +} from "../src/codex/subagent-model-fallback"; +import type { OcxConfig } from "../src/types"; + +beforeEach(() => { + clearUpstreamHostHealth(); + resetSubagentModelFallbackStateForTests(); +}); function coded(message: string, code: string, cause?: unknown): Error { return Object.assign(new Error(message), { code, ...(cause !== undefined ? { cause } : {}) }); @@ -173,6 +182,35 @@ describe("opt-in upstream host circuit", () => { }); }); + test("an open ChatGPT host circuit suppresses subagent quota priming", async () => { + const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); + fail(key, 1, 1_500); + let primeCalls = 0; + setSubagentQuotaPrimeForTests(async () => { + primeCalls += 1; + }); + const config = { + port: 10100, + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + }, + defaultProvider: "openai", + upstreamHostCircuitThreshold: 1, + } as OcxConfig; + + await maybePrimeSubagentQuota(config, 1_501); + expect(primeCalls).toBe(0); + // Even after the cooldown timestamp passes, priming stays out of the way; + // the logical request itself owns the one half-open admission. + await maybePrimeSubagentQuota(config, 1_500 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS + 1); + expect(primeCalls).toBe(0); + }); + test("legacy observations cannot open the opt-in circuit without a lease", () => { const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); for (let attempt = 0; attempt < 3; attempt++) { @@ -387,4 +425,4 @@ describe("opt-in upstream host circuit", () => { expect(getUpstreamHostHealth(key)).toEqual(before); }); -}); +}); \ No newline at end of file From 471175a9fa532c70f6f9877bf0a55d7668eba68d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:31:53 +0200 Subject: [PATCH 34/35] test(codex): strengthen host circuit regressions --- tests/upstream-reachability.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/upstream-reachability.test.ts b/tests/upstream-reachability.test.ts index e34179e899..b95ec5f95c 100644 --- a/tests/upstream-reachability.test.ts +++ b/tests/upstream-reachability.test.ts @@ -209,6 +209,11 @@ describe("opt-in upstream host circuit", () => { // the logical request itself owns the one half-open admission. await maybePrimeSubagentQuota(config, 1_500 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS + 1); expect(primeCalls).toBe(0); + + // Positive control: with no host circuit, the same config primes exactly once. + clearUpstreamHostHealth(); + await maybePrimeSubagentQuota(config, 1_500 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS + 2); + expect(primeCalls).toBe(1); }); test("legacy observations cannot open the opt-in circuit without a lease", () => { @@ -351,8 +356,9 @@ describe("opt-in upstream host circuit", () => { for (let i = 0; i < UPSTREAM_HOST_HEALTH_MAX_ENTRIES + 1; i += 1) { leases.push(admit(upstreamHostHealthKey("openai", `https://active-${i}.example`), 1, 14_000 + i)); } - expect(releaseUpstreamHostAdmission(leases[0], 15_000)).toBe(true); - for (const lease of leases.slice(1)) releaseUpstreamHostAdmission(lease, 15_001); + for (const [index, lease] of leases.entries()) { + expect(releaseUpstreamHostAdmission(lease, index === 0 ? 15_000 : 15_001)).toBe(true); + } }); test("a stale completion cannot mutate the generation that opened the circuit", () => { @@ -425,4 +431,4 @@ describe("opt-in upstream host circuit", () => { expect(getUpstreamHostHealth(key)).toEqual(before); }); -}); \ No newline at end of file +}); From 36c4058158a3d2cea6afca5a24d27b4a0c83341d Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:38:04 +0200 Subject: [PATCH 35/35] test(codex): make quota-prime control exercise TTL --- tests/upstream-reachability.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/upstream-reachability.test.ts b/tests/upstream-reachability.test.ts index b95ec5f95c..6c3a32a2d7 100644 --- a/tests/upstream-reachability.test.ts +++ b/tests/upstream-reachability.test.ts @@ -184,7 +184,8 @@ describe("opt-in upstream host circuit", () => { test("an open ChatGPT host circuit suppresses subagent quota priming", async () => { const key = upstreamHostHealthKey("openai", "https://chatgpt.com"); - fail(key, 1, 1_500); + const openedAt = 100_000; + fail(key, 1, openedAt); let primeCalls = 0; setSubagentQuotaPrimeForTests(async () => { primeCalls += 1; @@ -203,16 +204,16 @@ describe("opt-in upstream host circuit", () => { upstreamHostCircuitThreshold: 1, } as OcxConfig; - await maybePrimeSubagentQuota(config, 1_501); + await maybePrimeSubagentQuota(config, openedAt + 1); expect(primeCalls).toBe(0); // Even after the cooldown timestamp passes, priming stays out of the way; // the logical request itself owns the one half-open admission. - await maybePrimeSubagentQuota(config, 1_500 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS + 1); + await maybePrimeSubagentQuota(config, openedAt + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS + 1); expect(primeCalls).toBe(0); // Positive control: with no host circuit, the same config primes exactly once. clearUpstreamHostHealth(); - await maybePrimeSubagentQuota(config, 1_500 + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS + 2); + await maybePrimeSubagentQuota(config, openedAt + UPSTREAM_HOST_CIRCUIT_COOLDOWN_MS + 2); expect(primeCalls).toBe(1); });