Self-heal codex quota walls via CODEX_GATEWAY_POOL rotation

docs/ARCHITECTURE.md

@@ -139,6 +139,13 @@ Codex uses whatever auth you've configured. `CODEX_PROVIDER` (`-c model_provider
 (`-c model=…`) let you point it at a specific gateway or model. There's no API key in stupify itself;
 credentials are Codex's concern.
 
+If your Codex rides a pool of interchangeable gateway accounts (e.g. exe.dev `llm` integrations, each fronting
+a ChatGPT plan), `CODEX_GATEWAY_POOL` (ordered comma-separated hostnames) lets the sweep self-heal a quota
+wall: when a review dies rate-limited, it rewrites the gateway hostname in `~/.codex/config.toml` to the next
+pool entry — Codex re-reads the file each sweep, so the next sweep runs on the fresh account. No probing (the
+real failure is the signal) and at most one step per `CODEX_ROTATE_COOLDOWN_MIN` (default 10), so a fully
+drained pool cycles calmly until a weekly reset rescues it. Unset = off.
+
 ## Why curated, not inferred
 
 An earlier experiment auto-extracted a "good code" corpus from the repo. It reliably praised the exact slop it

src/review-sweep.test.ts

@@ -28,6 +28,8 @@ const cfg = (): Config => ({
   codexModel: '',
   githubStatus: true,
   githubStatusContext: 'stupify/review',
+  gatewayPool: '',
+  rotateCooldownMs: 600_000,
 })
 
 const pr = (number: number, sha: string): Pr => ({
@@ -295,3 +297,47 @@ test('the prefix is large enough to be cache-eligible (well past the ~1024-token
   const approxTokens = Math.round(bytes / 4) // ~4 chars/token, the standard rough estimate
   expect(approxTokens).toBeGreaterThan(1024)
 })
+
+// Gateway rotation: on a quota wall the sweep advances ~/.codex/config.toml to the next CODEX_GATEWAY_POOL
+// entry — the same env contract bunion/earshot rotate on. Real files in a temp dir, no mocks.
+import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { rotateGateway } from './review-sweep'
+
+const POOL = 'llm.int.exe.xyz,llm-3.int.exe.xyz,llm-4.int.exe.xyz'
+const TOML = 'model_provider = "exe-llm"\n[model_providers.exe-llm]\nbase_url = "https://llm.int.exe.xyz/v1"\n'
+
+const rotateSetup = (pool = POOL, toml = TOML) => {
+  const dir = mkdtempSync(join(tmpdir(), 'rotate-'))
+  mkdirSync(join(dir, 'state'))
+  const configPath = join(dir, 'config.toml')
+  writeFileSync(configPath, toml)
+  const c: Config = { ...cfg(), stateDir: join(dir, 'state'), gatewayPool: pool, rotateCooldownMs: 600_000 }
+  return { c, configPath }
+}
+
+test('rotateGateway advances the ring, stamps the cooldown, and leaves the rest of the config alone', () => {
+  const { c, configPath } = rotateSetup()
+  expect(rotateGateway(c, configPath, 1000)).toEqual({ rotated: true, from: 'llm.int.exe.xyz', to: 'llm-3.int.exe.xyz' })
+  const toml = readFileSync(configPath, 'utf8')
+  expect(toml).toContain('base_url = "https://llm-3.int.exe.xyz/v1"')
+  expect(toml).toContain('model_provider = "exe-llm"')
+  // the ring wraps from the last entry back to the first (cooldown elapsed)
+  writeFileSync(configPath, TOML.replace('llm.int.exe.xyz', 'llm-4.int.exe.xyz'))
+  expect(rotateGateway(c, configPath, 700_000)).toEqual({ rotated: true, from: 'llm-4.int.exe.xyz', to: 'llm.int.exe.xyz' })
+})
+
+test('rotateGateway cooldown suppresses back-to-back rotations, then re-arms', () => {
+  const { c, configPath } = rotateSetup()
+  expect(rotateGateway(c, configPath, 1000).rotated).toBe(true)
+  expect(rotateGateway(c, configPath, 500_000)).toEqual({ rotated: false, why: 'rotated recently — cooling down' })
+  expect(rotateGateway(c, configPath, 700_000).rotated).toBe(true)
+})
+
+test('rotateGateway is off without a pool, and never touches a config outside the pool', () => {
+  const off = rotateSetup('')
+  expect(rotateGateway(off.c, off.configPath, 1000).rotated).toBe(false)
+  expect(readFileSync(off.configPath, 'utf8')).toBe(TOML)
+  const foreign = rotateSetup(POOL, TOML.replace('llm.int.exe.xyz', 'llm-2.int.exe.xyz'))
+  expect(rotateGateway(foreign.c, foreign.configPath, 1000)).toEqual({ rotated: false, why: 'no pool gateway in codex config' })
+})

src/review-sweep.ts

@@ -19,6 +19,7 @@
  * `flock` dependency. Every knob lives in config.env next to this file (read fresh each run). Run: `bun review-sweep.ts`.
  */
 import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
+import { homedir } from 'node:os'
 import { dirname, join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import {
@@ -59,6 +60,8 @@ export interface Config {
   codexModel: string // optional `-c model=...`; empty = codex's default model
   githubStatus: boolean // post GitHub commit statuses (`stupify/review`) for PR-head workflow visibility
   githubStatusContext: string
+  gatewayPool: string // CODEX_GATEWAY_POOL: ordered comma-separated gateway hostnames codex may rotate through; empty = rotation off
+  rotateCooldownMs: number // min gap between gateway rotations, so a fully-drained pool cycles calmly instead of thrashing
 }
 
 function loadConfig(): Config {
@@ -119,6 +122,8 @@ function loadConfig(): Config {
     codexModel: pick('CODEX_MODEL', ''),
     githubStatus: bool('GITHUB_STATUS', true, false), // default visible in GitHub; typo disables instead of surprise-posting
     githubStatusContext: pick('GITHUB_STATUS_CONTEXT', 'stupify/review').trim() || 'stupify/review',
+    gatewayPool: pick('CODEX_GATEWAY_POOL', ''),
+    rotateCooldownMs: int('CODEX_ROTATE_COOLDOWN_MIN', 10, 0) * 60_000,
   }
 }
 
@@ -957,6 +962,32 @@ function reviewOne(cfg: Config, ref: string, post: boolean): void {
   console.log(`posted to ${slug}#${number} ✅ (${findings.length} inline)`)
 }
 
+/** Self-heal the wall isRateLimited just hit: advance `base_url` in ~/.codex/config.toml to the next gateway in
+ *  CODEX_GATEWAY_POOL (an ordered ring of interchangeable ChatGPT-account gateways — the same env contract
+ *  bunion and earshot rotate on via @bevyl-ai/agent-tools; inlined here to keep this engine dependency-free).
+ *  Codex re-reads the file each sweep, so the next sweep lands on the fresh account — no probing, no restarts.
+ *  A dead account fails fast and advances the ring again, so the pool converges on whichever has quota. The
+ *  cooldown keeps a fully-drained pool cycling calmly, one step per wall. Unset pool = rotation off. */
+export function rotateGateway(cfg: Config, configPath?: string, now = Date.now()): { rotated: true; from: string; to: string } | { rotated: false; why: string } {
+  const pool = cfg.gatewayPool.split(',').map((h) => h.trim()).filter(Boolean)
+  if (pool.length < 2) return { rotated: false, why: pool.length === 0 ? 'CODEX_GATEWAY_POOL unset — rotation off' : 'pool has a single entry' }
+  const path = configPath ?? join(process.env.CODEX_HOME ?? join(homedir(), '.codex'), 'config.toml')
+  if (!existsSync(path)) return { rotated: false, why: `no codex config at ${path}` }
+  const config = readFileSync(path, 'utf8')
+  const i = pool.findIndex((host) => config.includes(host)) // never touches hosts outside the pool, so other providers are safe
+  if (i === -1) return { rotated: false, why: 'no pool gateway in codex config' }
+  const stampPath = join(cfg.stateDir, 'gateway-rotated-at')
+  if (existsSync(stampPath)) {
+    const last = Number(readFileSync(stampPath, 'utf8'))
+    if (Number.isFinite(last) && now - last < cfg.rotateCooldownMs) return { rotated: false, why: 'rotated recently — cooling down' }
+  }
+  const from = pool[i] as string
+  const to = pool[(i + 1) % pool.length] as string
+  writeFileSync(path, config.replaceAll(from, to))
+  writeFileSync(stampPath, String(now))
+  return { rotated: true, from, to }
+}
+
 /** codex prints `tokens used` then the count on the next line — read the last such pair. */
 function parseTokens(out: string): number | null {
   const lines = out.split('\n')
@@ -1129,6 +1160,8 @@ function main(): void {
     setCommitStatus(cfg, commitStatuses, pr, 'pending', `stupify is reviewing ${lines} diff lines`)
     const used = reviewPr(cfg, pr, prior.memory, diff, firstReview, prior.openThreadIds, prior.dismissed)
     if (used === 'limit') {
+      const r = rotateGateway(cfg) // self-heal: next sweep runs on the next CODEX_GATEWAY_POOL account (no-op if unset)
+      if (r.rotated) log(`codex gateway rotated: ${r.from} → ${r.to}`)
       log('codex plan is rate-limited — ending this sweep early (the rest would fail the same way); retries next sweep')
       setStatusPr(cfg, status, pr, 'failed', 'codex plan is rate-limited; ending sweep early', lines)
       setStatusStage(cfg, status, 'blocked', 'codex plan is rate-limited')