diff --git a/webapps/console/lib/api.ts b/webapps/console/lib/api.ts index d15e48aef..873f86303 100644 --- a/webapps/console/lib/api.ts +++ b/webapps/console/lib/api.ts @@ -8,7 +8,7 @@ import { assertDefined, checkHash, checkRawToken, getErrorMessage, requireDefine import { getServerSession, Session } from "next-auth"; import { nextAuthConfig } from "./nextauth.config"; import { inferTokenTypeFromId, SessionUser } from "./schema"; -import { db } from "./server/db"; +import { db, isDatabaseReachable } from "./server/db"; import { isMaintenanceActive } from "./server/maintenance"; import { prepareZodObjectForDeserialization, safeParseWithDate } from "./zod"; import { ApiError } from "./shared/errors"; @@ -345,6 +345,27 @@ export async function getUser( return session ? getUserFromSession(session) : undefined; } +/** + * On the request error path, tells a database outage apart from an ordinary + * failure by actively probing the DB. When it's unreachable, responds with a + * clear `database_unavailable` 503 (fail-closed, with Retry-After) and returns + * true; otherwise leaves the response untouched and returns false so the caller + * falls back to its normal handling. Centralized here so every route — not just + * rate-limited ones — surfaces the same message when Postgres is down. + */ +async function respondIfDatabaseDown(req: NextApiRequest, res: NextApiResponse, e: unknown): Promise { + if (await isDatabaseReachable()) { + return false; + } + log.atError().withCause(e).log(`database is unreachable for ${req.method} ${req.url}; request rejected`); + res.setHeader("Retry-After", "5"); + res.status(503).json({ + error: "database_unavailable", + message: "Database is unavailable; request rejected.", + }); + return true; +} + export function nextJsApiHandler(api: Api): NextApiHandler { const handleRequest = async (req: NextApiRequest, res: NextApiResponse) => { const method = req.method as HttpMethodType; @@ -423,6 +444,14 @@ export function nextJsApiHandler(api: Api): NextApiHandler { // Fail closed: the limiter store is unavailable, so we reject rather than // silently letting everyone through. A future per-route failOpen flag can // carve exceptions when we need them. + // + // The Postgres-backed limiter runs an INSERT on every request, so a DB + // outage tends to surface here first. Probe the DB and, when it's the + // real cause, report that instead of blaming the rate limiter — which + // would point developers at the wrong subsystem. + if (await respondIfDatabaseDown(req, res, e)) { + return; + } log.atWarn().withCause(e).log(`rate limiter unavailable for ${req.method} ${req.url}`); res.setHeader("Retry-After", "5"); res.status(503).json({ @@ -524,6 +553,12 @@ export function nextJsApiHandler(api: Api): NextApiHandler { } res.status(status).send(errorBody); } else { + // A dead database bubbles up here as a raw Prisma error (not an ApiError). + // Probe and, if it's really down, return the unified database_unavailable + // 503 so every route reports the same clear cause. + if (await respondIfDatabaseDown(req, res, e)) { + return; + } log.atError().withCause(e).log(`Request for ${req.method} ${req.url} failed`); res .status(500) diff --git a/webapps/console/lib/server/db.ts b/webapps/console/lib/server/db.ts index f3a108435..b990cce08 100644 --- a/webapps/console/lib/server/db.ts +++ b/webapps/console/lib/server/db.ts @@ -77,6 +77,36 @@ export const db = { pgHelper: () => pgHelper, } as const; +/** + * Actively probes the database with a trivial `SELECT 1`. Prisma exposes no + * "am I connected?" flag — connections are lazy and pooled — so the only + * reliable signal is to run a cheap query and see whether it succeeds. Used on + * the request error path to tell a genuine outage apart from an ordinary query + * failure, without having to pattern-match on Prisma error codes. + * + * Bounded by `timeoutMs` so an unreachable server can't stall the response + * while the driver waits out its own (much longer) connect timeout. + */ +export async function isDatabaseReachable(timeoutMs = 1500): Promise { + const ping = db.prisma().$queryRaw`SELECT 1`; + // If the timeout wins the race the query promise still settles later; swallow + // its rejection so it can't surface as an unhandled rejection. + Promise.resolve(ping).catch(() => {}); + try { + await Promise.race([ + ping, + new Promise((_, reject) => { + const t = setTimeout(() => reject(new Error("db ping timed out")), timeoutMs); + // Don't let the probe timer keep the event loop alive on its own. + (t as { unref?: () => void }).unref?.(); + }), + ]); + return true; + } catch { + return false; + } +} + export type DatabaseConnection = typeof db; export type PrismaSSLMode = "disable" | "prefer" | "require" | "no-verify";