From 8fd336427e4fe9408bc7fedc235b363c71afe0a4 Mon Sep 17 00:00:00 2001 From: Vladimir Klimontovich Date: Thu, 23 Jul 2026 15:36:50 +0300 Subject: [PATCH 1/2] fix(console): distinguish DB-down from rate-limiter-unavailable in 503 error --- webapps/console/lib/api.ts | 19 +++++++++++++++++-- webapps/console/lib/server/db.ts | 32 +++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/webapps/console/lib/api.ts b/webapps/console/lib/api.ts index d15e48aef..3128eb178 100644 --- a/webapps/console/lib/api.ts +++ b/webapps/console/lib/api.ts @@ -8,7 +8,7 @@ import { assertDefined, checkHash, checkRawToken, getErrorMessage, requireDefine import { getServerSession, Session } from "next-auth"; import { nextAuthConfig } from "./nextauth.config"; import { inferTokenTypeFromId, SessionUser } from "./schema"; -import { db } from "./server/db"; +import { db, getDatabaseErrorCode, isDatabaseConnectivityError } from "./server/db"; import { isMaintenanceActive } from "./server/maintenance"; import { prepareZodObjectForDeserialization, safeParseWithDate } from "./zod"; import { ApiError } from "./shared/errors"; @@ -423,8 +423,23 @@ export function nextJsApiHandler(api: Api): NextApiHandler { // Fail closed: the limiter store is unavailable, so we reject rather than // silently letting everyone through. A future per-route failOpen flag can // carve exceptions when we need them. - log.atWarn().withCause(e).log(`rate limiter unavailable for ${req.method} ${req.url}`); res.setHeader("Retry-After", "5"); + if (isDatabaseConnectivityError(e)) { + // The Postgres-backed limiter runs an INSERT on every request, so a DB + // outage surfaces here first. Report the real cause instead of blaming + // the rate limiter, which points developers at the wrong subsystem. + const code = getDatabaseErrorCode(e); + log + .atError() + .withCause(e) + .log(`database is unreachable${code ? ` (${code})` : ""} for ${req.method} ${req.url}; request rejected`); + res.status(503).json({ + error: "database_unavailable", + message: "Database is unavailable; request rejected.", + }); + return; + } + log.atWarn().withCause(e).log(`rate limiter unavailable for ${req.method} ${req.url}`); res.status(503).json({ error: "rate_limit_unavailable", message: "Rate limiter store is unavailable; request rejected.", diff --git a/webapps/console/lib/server/db.ts b/webapps/console/lib/server/db.ts index f3a108435..1a95cc902 100644 --- a/webapps/console/lib/server/db.ts +++ b/webapps/console/lib/server/db.ts @@ -1,4 +1,4 @@ -import { PrismaClient } from "@prisma/client"; +import { Prisma, PrismaClient } from "@prisma/client"; import { Pool, PoolClient } from "pg"; import Cursor from "pg-cursor"; import { getSingleton, namedParameters, newError, requireDefined, stopwatch, hideSensitiveInfo } from "juava"; @@ -71,6 +71,36 @@ const pgHelper: PgHelper = { }, }; +// Prisma error codes that indicate the database itself is unreachable or the +// connection dropped, as opposed to a query/logic error. +// P1001: can't reach database server, P1002: connection timed out, +// P1008: operation timed out, P1017: server closed the connection. +const DB_CONNECTIVITY_CODES = new Set(["P1001", "P1002", "P1008", "P1017"]); + +/** + * Returns true when the error signals that the database is down/unreachable + * (connection could not be established or was lost) rather than a query error. + */ +export function isDatabaseConnectivityError(e: unknown): boolean { + if (e instanceof Prisma.PrismaClientInitializationError) { + return true; + } + if (e instanceof Prisma.PrismaClientKnownRequestError) { + return DB_CONNECTIVITY_CODES.has(e.code); + } + return false; +} + +export function getDatabaseErrorCode(e: unknown): string | undefined { + if (e instanceof Prisma.PrismaClientInitializationError) { + return e.errorCode; + } + if (e instanceof Prisma.PrismaClientKnownRequestError) { + return e.code; + } + return undefined; +} + export const db = { prisma: getSingleton("prisma", createPrisma), pgPool: getSingleton("pg", createPg), From 7fff40aa6ee942be31e69b8dc49e2e8e5c03f20e Mon Sep 17 00:00:00 2001 From: Vladimir Klimontovich Date: Thu, 23 Jul 2026 19:48:22 +0300 Subject: [PATCH 2/2] refactor(console): probe the DB instead of matching Prisma error codes Replace the hardcoded P1xxx code list with an active `SELECT 1` probe (`isDatabaseReachable`). Prisma exposes no "am I connected?" flag, so probing is the only reliable signal and there is no error-code list to keep in sync. Move DB-down handling into the shared top-level catch so every route, not just rate-limited ones, returns `database_unavailable` when Postgres is down. The rate-limiter catch reuses the same probe. Fail-closed behavior is unchanged. --- webapps/console/lib/api.ts | 50 ++++++++++++++++++-------- webapps/console/lib/server/db.ts | 60 ++++++++++++++++---------------- 2 files changed, 65 insertions(+), 45 deletions(-) diff --git a/webapps/console/lib/api.ts b/webapps/console/lib/api.ts index 3128eb178..873f86303 100644 --- a/webapps/console/lib/api.ts +++ b/webapps/console/lib/api.ts @@ -8,7 +8,7 @@ import { assertDefined, checkHash, checkRawToken, getErrorMessage, requireDefine import { getServerSession, Session } from "next-auth"; import { nextAuthConfig } from "./nextauth.config"; import { inferTokenTypeFromId, SessionUser } from "./schema"; -import { db, getDatabaseErrorCode, isDatabaseConnectivityError } from "./server/db"; +import { db, isDatabaseReachable } from "./server/db"; import { isMaintenanceActive } from "./server/maintenance"; import { prepareZodObjectForDeserialization, safeParseWithDate } from "./zod"; import { ApiError } from "./shared/errors"; @@ -345,6 +345,27 @@ export async function getUser( return session ? getUserFromSession(session) : undefined; } +/** + * On the request error path, tells a database outage apart from an ordinary + * failure by actively probing the DB. When it's unreachable, responds with a + * clear `database_unavailable` 503 (fail-closed, with Retry-After) and returns + * true; otherwise leaves the response untouched and returns false so the caller + * falls back to its normal handling. Centralized here so every route — not just + * rate-limited ones — surfaces the same message when Postgres is down. + */ +async function respondIfDatabaseDown(req: NextApiRequest, res: NextApiResponse, e: unknown): Promise { + if (await isDatabaseReachable()) { + return false; + } + log.atError().withCause(e).log(`database is unreachable for ${req.method} ${req.url}; request rejected`); + res.setHeader("Retry-After", "5"); + res.status(503).json({ + error: "database_unavailable", + message: "Database is unavailable; request rejected.", + }); + return true; +} + export function nextJsApiHandler(api: Api): NextApiHandler { const handleRequest = async (req: NextApiRequest, res: NextApiResponse) => { const method = req.method as HttpMethodType; @@ -423,23 +444,16 @@ export function nextJsApiHandler(api: Api): NextApiHandler { // Fail closed: the limiter store is unavailable, so we reject rather than // silently letting everyone through. A future per-route failOpen flag can // carve exceptions when we need them. - res.setHeader("Retry-After", "5"); - if (isDatabaseConnectivityError(e)) { - // The Postgres-backed limiter runs an INSERT on every request, so a DB - // outage surfaces here first. Report the real cause instead of blaming - // the rate limiter, which points developers at the wrong subsystem. - const code = getDatabaseErrorCode(e); - log - .atError() - .withCause(e) - .log(`database is unreachable${code ? ` (${code})` : ""} for ${req.method} ${req.url}; request rejected`); - res.status(503).json({ - error: "database_unavailable", - message: "Database is unavailable; request rejected.", - }); + // + // The Postgres-backed limiter runs an INSERT on every request, so a DB + // outage tends to surface here first. Probe the DB and, when it's the + // real cause, report that instead of blaming the rate limiter — which + // would point developers at the wrong subsystem. + if (await respondIfDatabaseDown(req, res, e)) { return; } log.atWarn().withCause(e).log(`rate limiter unavailable for ${req.method} ${req.url}`); + res.setHeader("Retry-After", "5"); res.status(503).json({ error: "rate_limit_unavailable", message: "Rate limiter store is unavailable; request rejected.", @@ -539,6 +553,12 @@ export function nextJsApiHandler(api: Api): NextApiHandler { } res.status(status).send(errorBody); } else { + // A dead database bubbles up here as a raw Prisma error (not an ApiError). + // Probe and, if it's really down, return the unified database_unavailable + // 503 so every route reports the same clear cause. + if (await respondIfDatabaseDown(req, res, e)) { + return; + } log.atError().withCause(e).log(`Request for ${req.method} ${req.url} failed`); res .status(500) diff --git a/webapps/console/lib/server/db.ts b/webapps/console/lib/server/db.ts index 1a95cc902..b990cce08 100644 --- a/webapps/console/lib/server/db.ts +++ b/webapps/console/lib/server/db.ts @@ -1,4 +1,4 @@ -import { Prisma, PrismaClient } from "@prisma/client"; +import { PrismaClient } from "@prisma/client"; import { Pool, PoolClient } from "pg"; import Cursor from "pg-cursor"; import { getSingleton, namedParameters, newError, requireDefined, stopwatch, hideSensitiveInfo } from "juava"; @@ -71,42 +71,42 @@ const pgHelper: PgHelper = { }, }; -// Prisma error codes that indicate the database itself is unreachable or the -// connection dropped, as opposed to a query/logic error. -// P1001: can't reach database server, P1002: connection timed out, -// P1008: operation timed out, P1017: server closed the connection. -const DB_CONNECTIVITY_CODES = new Set(["P1001", "P1002", "P1008", "P1017"]); +export const db = { + prisma: getSingleton("prisma", createPrisma), + pgPool: getSingleton("pg", createPg), + pgHelper: () => pgHelper, +} as const; /** - * Returns true when the error signals that the database is down/unreachable - * (connection could not be established or was lost) rather than a query error. + * Actively probes the database with a trivial `SELECT 1`. Prisma exposes no + * "am I connected?" flag — connections are lazy and pooled — so the only + * reliable signal is to run a cheap query and see whether it succeeds. Used on + * the request error path to tell a genuine outage apart from an ordinary query + * failure, without having to pattern-match on Prisma error codes. + * + * Bounded by `timeoutMs` so an unreachable server can't stall the response + * while the driver waits out its own (much longer) connect timeout. */ -export function isDatabaseConnectivityError(e: unknown): boolean { - if (e instanceof Prisma.PrismaClientInitializationError) { +export async function isDatabaseReachable(timeoutMs = 1500): Promise { + const ping = db.prisma().$queryRaw`SELECT 1`; + // If the timeout wins the race the query promise still settles later; swallow + // its rejection so it can't surface as an unhandled rejection. + Promise.resolve(ping).catch(() => {}); + try { + await Promise.race([ + ping, + new Promise((_, reject) => { + const t = setTimeout(() => reject(new Error("db ping timed out")), timeoutMs); + // Don't let the probe timer keep the event loop alive on its own. + (t as { unref?: () => void }).unref?.(); + }), + ]); return true; + } catch { + return false; } - if (e instanceof Prisma.PrismaClientKnownRequestError) { - return DB_CONNECTIVITY_CODES.has(e.code); - } - return false; -} - -export function getDatabaseErrorCode(e: unknown): string | undefined { - if (e instanceof Prisma.PrismaClientInitializationError) { - return e.errorCode; - } - if (e instanceof Prisma.PrismaClientKnownRequestError) { - return e.code; - } - return undefined; } -export const db = { - prisma: getSingleton("prisma", createPrisma), - pgPool: getSingleton("pg", createPg), - pgHelper: () => pgHelper, -} as const; - export type DatabaseConnection = typeof db; export type PrismaSSLMode = "disable" | "prefer" | "require" | "no-verify";