From 546e63529f3a88967b8a5fa748052831cfc89846 Mon Sep 17 00:00:00 2001
From: Atul Gupta <iamatulmgupta@gmail.com>
Date: Sun, 29 Mar 2026 18:42:59 -0700
Subject: [PATCH] fix: add retry logic with fast timeouts for MongoDB startup

The API crashes immediately if MongoDB isn't reachable during startup
(ServerSelectionTimeoutError). In Kubernetes, pods can start before
MongoDB is fully available.

Changes:
- Retry loop: 10 attempts with 3s delay between retries
- Reduced timeouts: serverSelection=5s, connect=5s, socket=10s
  (down from 30s/20s/20s defaults) so each retry fails fast
- Each attempt now takes ~8s instead of ~33s
- Structured logging for retry visibility

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 api/app/database.py | 50 ++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 45 insertions(+), 5 deletions(-)

diff --git a/api/app/database.py b/api/app/database.py
index 8505025..cc72d14 100644
--- a/api/app/database.py
+++ b/api/app/database.py
@@ -1,22 +1,62 @@
 """MongoDB connection management using Beanie ODM."""
 
+import asyncio
+
+import structlog
 from beanie import init_beanie
 from pymongo import AsyncMongoClient
 
 from app.config import settings
 
+logger = structlog.get_logger()
+
 client: AsyncMongoClient | None = None
 
+MAX_RETRIES = 10
+RETRY_DELAY = 3  # seconds
+
 
 async def init_db() -> None:
-    """Initialize MongoDB connection and Beanie ODM."""
+    """Initialize MongoDB connection and Beanie ODM with retry logic."""
     global client
-    client = AsyncMongoClient(settings.mongo_uri)
-    db = client[settings.mongo_db]
-
     from app.models import ALL_MODELS
 
-    await init_beanie(database=db, document_models=ALL_MODELS)
+    for attempt in range(1, MAX_RETRIES + 1):
+        try:
+            client = AsyncMongoClient(
+                settings.mongo_uri,
+                serverSelectionTimeoutMS=5000,
+                connectTimeoutMS=5000,
+                socketTimeoutMS=10000,
+            )
+            db = client[settings.mongo_db]
+            await init_beanie(database=db, document_models=ALL_MODELS)
+            if attempt > 1:
+                logger.info("Database connected after retry", attempt=attempt)
+            return
+        except Exception as exc:
+            if attempt == MAX_RETRIES:
+                logger.error(
+                    "Database connection failed after all retries",
+                    attempts=MAX_RETRIES,
+                    error=str(exc),
+                )
+                raise
+            logger.warning(
+                "Database not ready, retrying...",
+                attempt=attempt,
+                max_retries=MAX_RETRIES,
+                delay=RETRY_DELAY,
+                error=str(exc),
+            )
+            # Close failed client before retrying
+            if client:
+                try:
+                    await client.close()
+                except Exception:
+                    pass
+                client = None
+            await asyncio.sleep(RETRY_DELAY)
 
 
 async def close_db() -> None: