feat: add API health checks and monitoring - PR-7

src/auth.py

@@ -0,0 +1,11 @@
+from functools import wraps
+from flask import request, jsonify
+
+def require_api_key(f):
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        api_key = request.headers.get('X-API-Key')
+        if api_key != 'your-secret-key':  # Replace with actual key or env var
+            return jsonify({'error': 'API key required'}), 401
+        return f(*args, **kwargs)
+    return decorated_function

src/health_endpoints.py

@@ -0,0 +1,83 @@
+from flask import Blueprint, jsonify, request
+from health_monitor import health_monitor
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Create health endpoints blueprint
+health_bp = Blueprint('health', __name__, url_prefix='/api/health')
+
+@health_bp.route('/', methods=['GET'])
+def health_check():
+    """Basic health check endpoint."""
+    try:
+        summary = health_monitor.get_health_summary()
+        status_code = 200 if summary["status"] in ["healthy", "warning"] else 503
+        return jsonify(summary), status_code
+    except Exception as e:
+        logger.error(f"Health check failed: {e}")
+        return jsonify({
+            "status": "error",
+            "message": "Health check failed"
+        }), 500
+
+@health_bp.route('/detailed', methods=['GET'])
+def detailed_health():
+    """Detailed health check with system metrics."""
+    try:
+        health_data = health_monitor.get_system_health()
+        status_code = 200 if health_data["status"] in ["healthy", "warning"] else 503
+        return jsonify(health_data), status_code
+    except Exception as e:
+        logger.error(f"Detailed health check failed: {e}")
+        return jsonify({
+            "status": "error",
+            "message": "Detailed health check failed"
+        }), 500
+
+@health_bp.route('/ready', methods=['GET'])
+def readiness_check():
+    """Kubernetes readiness probe endpoint."""
+    try:
+        health_data = health_monitor.get_system_health()
+        if health_data["status"] in ["healthy", "warning"]:
+            return jsonify({"ready": True}), 200
+        else:
+            return jsonify({"ready": False, "reason": health_data["status"]}), 503
+    except Exception as e:
+        logger.error(f"Readiness check failed: {e}")
+        return jsonify({"ready": False, "reason": "error"}), 503
+
+@health_bp.route('/live', methods=['GET'])
+def liveness_check():
+    """Kubernetes liveness probe endpoint."""
+    try:
+        # Simple liveness check - just verify the service is responding
+        return jsonify({"alive": True}), 200
+    except Exception as e:
+        logger.error(f"Liveness check failed: {e}")
+        return jsonify({"alive": False}), 500
+
+@health_bp.route('/metrics', methods=['GET'])
+def health_metrics():
+    """Health metrics endpoint for monitoring systems."""
+    try:
+        health_data = health_monitor.get_system_health()
+        metrics = {
+            "api_requests_total": health_data["process"]["request_count"],
+            "api_errors_total": health_data["process"]["error_count"],
+            "api_error_rate_percent": health_data["process"]["error_rate"],
+            "system_cpu_percent": health_data["system"]["cpu_percent"],
+            "system_memory_percent": health_data["system"]["memory_percent"],
+            "system_disk_percent": health_data["system"]["disk_percent"],
+            "uptime_seconds": health_data["uptime_hours"] * 3600
+        }
+        return jsonify(metrics), 200
+    except Exception as e:
+        logger.error(f"Metrics collection failed: {e}")
+        return jsonify({"error": "Metrics collection failed"}), 500
+
+def register_health_endpoints(app):
+    """Register health endpoints with the Flask app."""
+    app.register_blueprint(health_bp)
+    logger.info("Health endpoints registered: /api/health/*")

src/health_monitor.py

@@ -0,0 +1,90 @@
+import time
+import psutil
+from datetime import datetime
+from typing import Dict, Any, Optional
+import logging
+
+logger = logging.getLogger(__name__)
+
+class HealthMonitor:
+    """Health monitoring system for API endpoints and system resources."""
+
+    def __init__(self):
+        self.start_time = time.time()
+        self.request_count = 0
+        self.error_count = 0
+        self.last_health_check = None
+
+    def get_system_health(self) -> Dict[str, Any]:
+        """Get comprehensive system health metrics."""
+        try:
+            # System resource usage
+            cpu_percent = psutil.cpu_percent(interval=1)
+            memory = psutil.virtual_memory()
+            disk = psutil.disk_usage('/')
+
+            # Process information
+            process = psutil.Process()
+            process_memory = process.memory_info().rss / 1024 / 1024  # MB
+
+            # Uptime calculation
+            uptime_seconds = time.time() - self.start_time
+            uptime_hours = uptime_seconds / 3600
+
+            health_data = {
+                "status": "healthy",
+                "timestamp": datetime.utcnow().isoformat(),
+                "uptime_hours": round(uptime_hours, 2),
+                "system": {
+                    "cpu_percent": cpu_percent,
+                    "memory_percent": memory.percent,
+                    "memory_available_gb": round(memory.available / 1024**3, 2),
+                    "disk_percent": disk.percent,
+                    "disk_free_gb": round(disk.free / 1024**3, 2)
+                },
+                "process": {
+                    "memory_mb": round(process_memory, 2),
+                    "request_count": self.request_count,
+                    "error_count": self.error_count,
+                    "error_rate": round(self.error_count / max(self.request_count, 1) * 100, 2)
+                },
+                "last_health_check": self.last_health_check
+            }
+
+            # Determine overall health status
+            if cpu_percent > 90 or memory.percent > 90 or disk.percent > 90:
+                health_data["status"] = "warning"
+            if cpu_percent > 95 or memory.percent > 95 or disk.percent > 95:
+                health_data["status"] = "critical"
+            if self.error_count > 0 and self.error_count / max(self.request_count, 1) > 0.1:
+                health_data["status"] = "degraded"
+
+            self.last_health_check = health_data["timestamp"]
+            return health_data
+
+        except Exception as e:
+            logger.error(f"Health check failed: {e}")
+            return {
+                "status": "error",
+                "timestamp": datetime.utcnow().isoformat(),
+                "error": str(e)
+            }
+
+    def record_request(self, success: bool = True):
+        """Record a request for health monitoring."""
+        self.request_count += 1
+        if not success:
+            self.error_count += 1
+
+    def get_health_summary(self) -> Dict[str, Any]:
+        """Get a simplified health summary for quick checks."""
+        health = self.get_system_health()
+        return {
+            "status": health["status"],
+            "uptime_hours": health["uptime_hours"],
+            "request_count": health["process"]["request_count"],
+            "error_rate": health["process"]["error_rate"]
+        }
+
+# Global health monitor instance
+health_monitor = HealthMonitor()

src/rate_limiter.py

@@ -0,0 +1,21 @@
+from collections import defaultdict
+from datetime import datetime, timedelta
+from flask import abort, current_app
+
+# Simple rate limiter using memory (use Redis for production)
+rate_limit = defaultdict(list)
+
+def rate_limit(max_requests=100, window_minutes=1):
+    def decorator(f):
+        @wraps(f)
+        def decorated_function(*args, **kwargs):
+            client_ip = request.remote_addr
+            now = datetime.utcnow()
+            window_start = now - timedelta(minutes=window_minutes)
+            rate_limit[client_ip] = [req_time for req_time in rate_limit[client_ip] if req_time > window_start]
+            if len(rate_limit[client_ip]) >= max_requests:
+                abort(429, description="Rate limit exceeded")
+            rate_limit[client_ip].append(now)
+            return f(*args, **kwargs)
+        return decorated_function
+    return decorator

src/security/auth.py

@@ -0,0 +1,45 @@
+from fastapi import Depends, HTTPException, status
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from jose import JWTError, jwt
+from passlib.context import CryptContext
+from datetime import datetime, timedelta
+from typing import Optional
+
+# Security settings
+SECRET_KEY = "your-secret-key"  # Should be loaded from config
+ALGORITHM = "HS256"
+ACCESS_TOKEN_EXPIRE_MINUTES = 30
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+security = HTTPBearer()
+
+def verify_password(plain_password, hashed_password):
+    return pwd_context.verify(plain_password, hashed_password)
+
+def get_password_hash(password):
+    return pwd_context.hash(password)
+
+def create_access_token(data: dict, expires_delta: Optional[timedelta] = None):
+    to_encode = data.copy()
+    if expires_delta:
+        expire = datetime.utcnow() + expires_delta
+    else:
+        expire = datetime.utcnow() + timedelta(minutes=15)
+    to_encode.update({"exp": expire})
+    encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
+    return encoded_jwt
+
+async def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    credentials_exception = HTTPException(
+        status_code=status.HTTP_401_UNAUTHORIZED,
+        detail="Could not validate credentials",
+        headers={"WWW-Authenticate": "Bearer"},
+    )
+    try:
+        payload = jwt.decode(credentials.credentials, SECRET_KEY, algorithms=[ALGORITHM])
+        username: str = payload.get("sub")
+        if username is None:
+            raise credentials_exception
+    except JWTError:
+        raise credentials_exception
+    return username

src/security/rate_limiter.py

@@ -0,0 +1,31 @@
+from collections import defaultdict
+from datetime import datetime, timedelta
+from typing import Optional
+import time
+
+class RateLimiter:
+    def __init__(self, max_requests: int = 100, window_seconds: int = 3600):
+        self.max_requests = max_requests
+        self.window_seconds = window_seconds
+        self.requests = defaultdict(list)
+
+    def is_allowed(self, identifier: str) -> bool:
+        now = time.time()
+        window_start = now - self.window_seconds
+        self.requests[identifier] = [
+            timestamp for timestamp in self.requests[identifier]
+            if timestamp > window_start
+        ]
+        if len(self.requests[identifier]) < self.max_requests:
+            self.requests[identifier].append(now)
+            return True
+        return False
+
+    def get_remaining_requests(self, identifier: str) -> int:
+        now = time.time()
+        window_start = now - self.window_seconds
+        self.requests[identifier] = [
+            timestamp for timestamp in self.requests[identifier]
+            if timestamp > window_start
+        ]
+        return max(0, self.max_requests - len(self.requests[identifier]))

src/unified_api_server.py

@@ -0,0 +1,20 @@
+from flask import Flask, jsonify
+from flask_cors import CORS
+from auth import require_api_key
+from rate_limiter import rate_limit
+
+app = Flask(__name__)
+CORS(app)  # Enable CORS for all routes
+
+@app.route('/api/health')
+def health():
+    return jsonify({'status': 'healthy'})
+
+@app.route('/api/protected', methods=['POST'])
+@require_api_key
+@rate_limit(max_requests=10, window_minutes=1)
+def protected():
+    return jsonify({'message': 'Protected endpoint'})
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0', port=5000)