From 31804b9a08ace16bed801c68e75b963eb228c81c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 01:52:49 +0000 Subject: [PATCH 1/4] fix(security): sanitize all HTTP 500 responses to prevent info disclosure (supersedes #801) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multiple FastAPI handlers returned internal exception text to clients via `HTTPException(status_code=500, detail=str(e))` (or f-strings embedding `{e}`), leaking stack-adjacent messages, backend API errors, and database/Looker errors. This is an information-disclosure vector (CWE-209). #801 sanitized ~24 handlers but left three live endpoints leaking: `generate_video_pack` and `generate_blueprint` (v1/router.py) and the mounted `reporting_routes.py` dashboard endpoint — the exact gaps Copilot flagged on that PR. A tree-wide scan surfaced 13 further leaks in cloud_ai_routes.py, cloud_api_endpoints.py, and real_api_endpoints.py that #801 never touched. Changes: - Replace the dynamic `detail` in every 500 response across the backend with a static "Internal server error"; the full exception is now logged server-side (`logger.error(..., exc_info=True)`) so diagnostics are preserved. - Add reporting_routes.py a module logger (previously none). - Add tests/unit/test_500_info_disclosure.py: a hermetic source-scan guard that fails if any backend 500 response uses a dynamic `detail`, closing the test-coverage gap (existing exception-path tests asserted only status code, so they passed while the body leaked). Includes a self-check that the scanner detects a synthetic leak and ignores 4xx responses. 4xx responses (which echo client-supplied validation input) are intentionally left unchanged. Verified: all touched files compile; ruff findings are identical to the base branch (lint-neutral); guard test passes. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_018MF2xHBKyQBQtpdXt6bVRx --- .../backend/api/reporting_routes.py | 7 +- .../backend/api/v1/router.py | 44 ++++----- .../backend/cloud_ai_routes.py | 10 +- .../backend/cloud_api_endpoints.py | 9 +- src/youtube_extension/backend/main.py | 8 +- .../backend/real_api_endpoints.py | 15 ++- tests/unit/test_500_info_disclosure.py | 94 +++++++++++++++++++ 7 files changed, 147 insertions(+), 40 deletions(-) create mode 100644 tests/unit/test_500_info_disclosure.py diff --git a/src/youtube_extension/backend/api/reporting_routes.py b/src/youtube_extension/backend/api/reporting_routes.py index 041b1be06..dba800a07 100644 --- a/src/youtube_extension/backend/api/reporting_routes.py +++ b/src/youtube_extension/backend/api/reporting_routes.py @@ -1,9 +1,13 @@ +import logging + from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel from typing import Optional from src.integration.looker_embedded import LookerEmbeddedService +logger = logging.getLogger(__name__) + router = APIRouter(prefix="/api/v1/reporting", tags=["Reporting & Dashboards"]) class DashboardEmbedRequest(BaseModel): @@ -36,7 +40,8 @@ async def generate_dashboard_url( ) return DashboardEmbedResponse(embed_url=url) except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to generate dashboard embed URL: {str(e)}") + logger.error(f"Unhandled error in generate_dashboard_url: {e}", exc_info=True) + raise HTTPException(status_code=500, detail="Internal server error") @router.get("/dashboards") async def list_available_dashboards(tenant_id: str = Query(...)): diff --git a/src/youtube_extension/backend/api/v1/router.py b/src/youtube_extension/backend/api/v1/router.py index 7d33ffb63..275eadc63 100644 --- a/src/youtube_extension/backend/api/v1/router.py +++ b/src/youtube_extension/backend/api/v1/router.py @@ -249,7 +249,7 @@ async def health_check_v1( return HealthResponse(**health_status) except Exception as e: logger.error(f"Health check failed: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.get( @@ -278,7 +278,7 @@ async def detailed_health_check_v1( } except Exception as e: logger.error(f"Detailed health check failed: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") # Capabilities / Model availability @@ -657,7 +657,7 @@ async def chat_v1( except Exception as e: logger.error(f"Error in chat endpoint: {e}", exc_info=True) - raise HTTPException(status_code=500, detail=str(e)) from e + raise HTTPException(status_code=500, detail="Internal server error") from e # Video Processing Endpoints @@ -716,7 +716,7 @@ async def process_video_v1( {"url": request.video_url, "error": str(e)}, request.video_url, ) - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.post( @@ -762,7 +762,7 @@ async def process_video_markdown_v1( except Exception as e: logger.error(f"Error in markdown processing: {e}") health_service.increment_metric("error_total") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.post( @@ -799,7 +799,7 @@ async def video_to_software_v1( except Exception as e: logger.error(f"Video-to-software processing failed: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") # Cache Management Endpoints @@ -823,7 +823,7 @@ async def get_cache_stats_v1(cache_service: CacheService = Depends(get_cache_ser return CacheStats(**stats) except Exception as e: logger.error(f"Error getting cache stats: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.get( @@ -852,7 +852,7 @@ async def get_cached_video_v1( raise except Exception as e: logger.error(f"Error retrieving cached video: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.delete( @@ -875,7 +875,7 @@ async def clear_video_cache_v1( } except Exception as e: logger.error(f"Error clearing video cache: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.delete( @@ -895,7 +895,7 @@ async def clear_all_cache_v1(cache_service: CacheService = Depends(get_cache_ser } except Exception as e: logger.error(f"Error clearing all cache: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") # Data Endpoints @@ -933,7 +933,7 @@ async def list_videos_v1( except Exception as e: logger.error(f"Error listing videos: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.get( @@ -957,7 +957,7 @@ async def get_video_detail_v1( raise except Exception as e: logger.error(f"Error getting video detail: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.get( @@ -973,7 +973,7 @@ async def get_learning_log_v1(data_service: DataService = Depends(get_data_servi return learning_log except Exception as e: logger.error(f"Error getting learning log: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.post( @@ -1024,7 +1024,7 @@ async def get_actions_by_video_v1(video_id: str): return actions except Exception as e: logger.error(f"Error retrieving actions for {video_id}: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.put( @@ -1072,7 +1072,7 @@ async def update_action_v1(action_id: str, payload: dict[str, Any]): return {"success": bool(success)} except Exception as e: logger.error(f"Error updating action {action_id}: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") # Feedback Endpoints @@ -1119,7 +1119,7 @@ async def submit_feedback_v1( except Exception as e: logger.error(f"Error saving feedback: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") # Metrics Endpoints @@ -1139,7 +1139,7 @@ async def get_metrics_v1( return JSONResponse(content="\n".join(metrics_lines), media_type="text/plain") except Exception as e: logger.error(f"Metrics endpoint failed: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") # Frontend performance ingestion endpoints @@ -1159,7 +1159,7 @@ async def ingest_performance_alert_v1(payload: dict[str, Any]): return {"status": "ok", "recorded": metric_name} except Exception as e: logger.error(f"Failed to ingest performance alert: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.post("/performance/report", summary="Ingest frontend performance report") @@ -1177,7 +1177,7 @@ async def ingest_performance_report_v1(report: dict[str, Any]): return {"status": "ok", "metrics_recorded": len(metrics)} except Exception as e: logger.error(f"Failed to ingest performance report: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") # ============================================================ @@ -1710,7 +1710,7 @@ async def get_or_create_videopack(request: VideoPackRequest): return ApiResponse.success(pack.model_dump()) except Exception as e: logger.error(f"Failed to create VideoPack: {e}") - raise HTTPException(status_code=500, detail=f"VideoPack generation failed: {e}") + raise HTTPException(status_code=500, detail="Internal server error") @router.post( @@ -1759,7 +1759,7 @@ async def generate_blueprint(request: BlueprintRequest): return ApiResponse.success(blueprint) except Exception as e: logger.error(f"Failed to generate blueprint: {e}") - raise HTTPException(status_code=500, detail=f"Blueprint generation failed: {e}") + raise HTTPException(status_code=500, detail="Internal server error") @router.post( @@ -1785,7 +1785,7 @@ async def generate_project_code(request: GenerateCodeRequest): return ApiResponse.success(result) except Exception as e: logger.error(f"Code generation failed: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @router.get( diff --git a/src/youtube_extension/backend/cloud_ai_routes.py b/src/youtube_extension/backend/cloud_ai_routes.py index a1242a418..0172e1617 100644 --- a/src/youtube_extension/backend/cloud_ai_routes.py +++ b/src/youtube_extension/backend/cloud_ai_routes.py @@ -229,7 +229,7 @@ async def get_provider_status(): except Exception as e: logger.error(f"Failed to get provider status: {e}") - raise HTTPException(status_code=500, detail=f"Failed to get provider status: {str(e)}") + raise HTTPException(status_code=500, detail="Internal server error") @router.post("/analyze/video", response_model=VideoAnalysisResponse) @@ -263,12 +263,12 @@ async def analyze_video(request: VideoAnalysisRequest): raise HTTPException(status_code=429, detail=f"Rate limit exceeded: {str(e)}") except ConfigurationError as e: logger.error(f"Configuration error: {e}") - raise HTTPException(status_code=500, detail=f"Configuration error: {str(e)}") + raise HTTPException(status_code=500, detail="Internal server error") except HTTPException: raise except Exception as e: logger.error(f"Unexpected error during video analysis: {e}") - raise HTTPException(status_code=500, detail=f"Analysis failed: {str(e)}") + raise HTTPException(status_code=500, detail="Internal server error") @router.post("/analyze/batch") @@ -301,7 +301,7 @@ async def analyze_batch_videos(request: BatchAnalysisRequest, background_tasks: raise except Exception as e: logger.error(f"Failed to start batch analysis: {e}") - raise HTTPException(status_code=500, detail=f"Batch analysis failed: {str(e)}") + raise HTTPException(status_code=500, detail="Internal server error") @router.post("/analyze/multi-provider", response_model=list[VideoAnalysisResponse]) @@ -325,7 +325,7 @@ async def analyze_video_multi_provider(request: VideoAnalysisRequest): except Exception as e: logger.error(f"Multi-provider analysis failed: {e}") - raise HTTPException(status_code=500, detail=f"Multi-provider analysis failed: {str(e)}") + raise HTTPException(status_code=500, detail="Internal server error") @router.get("/analysis-types") diff --git a/src/youtube_extension/backend/cloud_api_endpoints.py b/src/youtube_extension/backend/cloud_api_endpoints.py index e38655f99..165bc76c9 100644 --- a/src/youtube_extension/backend/cloud_api_endpoints.py +++ b/src/youtube_extension/backend/cloud_api_endpoints.py @@ -260,9 +260,10 @@ async def batch_process_videos_cloud(request: BatchCloudProcessingRequest): except HTTPException: raise except Exception as e: + logger.error(f"Unhandled error in batch_process_videos_cloud: {e}", exc_info=True) raise HTTPException( status_code=500, - detail=f"Batch processing failed: {str(e)}" + detail="Internal server error" ) @router.get("/api/v3/videos/{video_id}/status", response_model=VideoStatusResponse) @@ -293,9 +294,10 @@ async def get_video_status(video_id: str): except HTTPException: raise except Exception as e: + logger.error(f"Unhandled error in get_video_status: {e}", exc_info=True) raise HTTPException( status_code=500, - detail=f"Error retrieving status: {str(e)}" + detail="Internal server error" ) @router.get("/api/v3/videos/{video_id}/result") @@ -330,9 +332,10 @@ async def get_video_result(video_id: str): except HTTPException: raise except Exception as e: + logger.error(f"Unhandled error in get_video_result: {e}", exc_info=True) raise HTTPException( status_code=500, - detail=f"Error retrieving result: {str(e)}" + detail="Internal server error" ) @router.get("/api/v3/queue/stats") diff --git a/src/youtube_extension/backend/main.py b/src/youtube_extension/backend/main.py index 0e52a9341..e0ffa5c3e 100644 --- a/src/youtube_extension/backend/main.py +++ b/src/youtube_extension/backend/main.py @@ -206,7 +206,7 @@ async def legacy_chat(request: dict): except Exception as e: logger.error(f"Legacy chat endpoint error: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @app.post("/api/process-video-markdown") @@ -240,7 +240,7 @@ async def legacy_process_video_markdown(request: dict): raise except Exception as e: logger.error(f"Legacy markdown processing error: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") @app.post("/api/process-video") @@ -297,7 +297,7 @@ async def legacy_process_video(request: dict): except Exception as e: logger.error(f"Legacy video processing error: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") # Other legacy endpoints with redirects @@ -368,7 +368,7 @@ async def system_info(): except Exception as e: logger.error(f"System info error: {e}") - raise HTTPException(status_code=500, detail=str(e)) + raise HTTPException(status_code=500, detail="Internal server error") # Enhanced OpenAPI schema generation diff --git a/src/youtube_extension/backend/real_api_endpoints.py b/src/youtube_extension/backend/real_api_endpoints.py index 03fd9e4e6..1b0433cfc 100644 --- a/src/youtube_extension/backend/real_api_endpoints.py +++ b/src/youtube_extension/backend/real_api_endpoints.py @@ -150,9 +150,10 @@ async def validate_video_url(request: VideoValidationRequest): } except Exception as e: + logger.error(f"Unhandled error in validate_video_url: {e}", exc_info=True) raise HTTPException( status_code=500, - detail=f"Validation failed: {str(e)}" + detail="Internal server error" ) @app.post("/api/v2/batch-process") @@ -177,9 +178,10 @@ async def batch_process_videos(request: BatchProcessingRequest): return result except Exception as e: + logger.error(f"Unhandled error in batch_process_videos: {e}", exc_info=True) raise HTTPException( status_code=500, - detail=f"Batch processing failed: {str(e)}" + detail="Internal server error" ) @app.get("/api/v2/videos/list") @@ -252,9 +254,10 @@ async def get_video_analysis(video_id: str): except HTTPException: raise except Exception as e: + logger.error(f"Unhandled error in get_video_analysis: {e}", exc_info=True) raise HTTPException( status_code=500, - detail=f"Error retrieving video analysis: {str(e)}" + detail="Internal server error" ) @app.get("/api/v2/cost-dashboard") @@ -384,9 +387,10 @@ async def clear_processing_cache(): } except Exception as e: + logger.error(f"Unhandled error in clear_processing_cache: {e}", exc_info=True) raise HTTPException( status_code=500, - detail=f"Failed to clear cache: {str(e)}" + detail="Internal server error" ) @app.post("/api/v2/search-videos") @@ -432,9 +436,10 @@ async def search_youtube_videos( } except Exception as e: + logger.error(f"Unhandled error in search_youtube_videos: {e}", exc_info=True) raise HTTPException( status_code=500, - detail=f"Search failed: {str(e)}" + detail="Internal server error" ) logger.info("🚀 Real API endpoints setup complete") diff --git a/tests/unit/test_500_info_disclosure.py b/tests/unit/test_500_info_disclosure.py new file mode 100644 index 000000000..d561c422e --- /dev/null +++ b/tests/unit/test_500_info_disclosure.py @@ -0,0 +1,94 @@ +"""Regression guard against information disclosure in HTTP 500 responses. + +Context: several FastAPI handlers historically raised +``HTTPException(status_code=500, detail=str(e))`` (or an f-string embedding the +exception), leaking internal exception text — stack-adjacent messages, backend +API errors, database errors — to clients. See PR #801, which sanitized most but +not all handlers. + +This test encodes the invariant directly on the source: a 500 response must use +a *static* ``detail`` string, never one derived from the caught exception. It is +hermetic (pure source scan, no app import / no pydantic) so it runs anywhere and +catches new leaks in any backend route, not just the ones fixed today. + +It deliberately does not constrain 4xx responses: those echo client-supplied +validation errors, which are not internal-disclosure vectors. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +_BACKEND = Path(__file__).resolve().parents[2] / "src" / "youtube_extension" / "backend" + +# Match a single `raise HTTPException(...)` call, capturing its argument list, +# tolerant of the call spanning multiple lines. +_HTTP_EXC = re.compile(r"HTTPException\((?P.*?)\)", re.DOTALL) + +# A `detail=` argument whose value is derived from a variable/exception: +# detail=str(e) detail=str(exc) +# detail=f"... {e} ..." detail=f"... {str(e)} ..." +_DYNAMIC_DETAIL = re.compile( + r"""detail\s*=\s*(?: + str\( # detail=str(...) + | f["'][^"']*\{ # detail=f"...{...}..." + )""", + re.VERBOSE, +) + + +def _backend_python_files() -> list[Path]: + return sorted(_BACKEND.rglob("*.py")) + + +def _iter_500_dynamic_detail(text: str): + """Yield (line_no, snippet) for each 500 HTTPException with a dynamic detail.""" + for m in _HTTP_EXC.finditer(text): + args = m.group("args") + if "status_code=500" not in args.replace(" ", "").replace( + "status_code =", "status_code=" + ): + # normalize minor spacing; only care about 500 responses + if "status_code=500" not in re.sub(r"\s+", "", args): + continue + if _DYNAMIC_DETAIL.search(args): + line_no = text.count("\n", 0, m.start()) + 1 + yield line_no, " ".join(args.split())[:120] + + +def test_no_dynamic_detail_in_500_responses() -> None: + offenders: list[str] = [] + for path in _backend_python_files(): + text = path.read_text(encoding="utf-8") + for line_no, snippet in _iter_500_dynamic_detail(text): + rel = path.relative_to(_BACKEND.parents[2]) + offenders.append(f"{rel}:{line_no}: HTTPException({snippet})") + + assert not offenders, ( + "HTTP 500 responses must use a static `detail` string (e.g. " + '"Internal server error") and never leak the caught exception. ' + "Log the full error server-side instead. Offending sites:\n " + + "\n ".join(offenders) + ) + + +def test_guard_detects_a_synthetic_leak() -> None: + """Sanity check: the scanner actually flags a dynamic 500 detail.""" + leaky = "raise HTTPException(status_code=500, detail=str(e))" + safe = 'raise HTTPException(status_code=500, detail="Internal server error")' + client_err = "raise HTTPException(status_code=400, detail=str(exc))" + + assert list(_iter_500_dynamic_detail(leaky)), "scanner missed a real 500 leak" + assert not list( + _iter_500_dynamic_detail(safe) + ), "scanner false-positived a static detail" + assert not list( + _iter_500_dynamic_detail(client_err) + ), "scanner must ignore 4xx responses" + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(pytest.main([__file__, "-v"])) From 448fe6822a3c037a7fb06ba1e692a08d6121a491 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 02:18:41 +0000 Subject: [PATCH 2/4] fix(security): close dict/variable 500-detail leaks and harden the guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The source-scan guard added for the 500 info-disclosure work only matched inline detail=str(e) / detail=f"...{e}...". It silently passed while three handlers still leaked internal state through detail shapes it did not model: - cloud_api_endpoints.py process_video_cloud -> detail={... "message": error_msg ...} - cloud_api_endpoints.py process_video_task -> detail=error_msg (bare variable) - real_api_endpoints.py process_video_real_api -> detail={... "message": error_msg ...} Fix all three to a static "Internal server error" (the exception is logged server-side via logger.error(..., exc_info=True)), and rewrite the guard to flag any 500 detail= that is not an inline static string literal — a leading '{' (dict) or identifier char (f-string, str(), or a bare variable) now fails the scan. Self-checks extended to cover the dict and variable shapes. test_error_response_includes_video_url asserted the old leaking body; it is replaced by test_error_response_is_sanitized, which asserts the 500 body is exactly "Internal server error" and contains neither the exception nor the caller-supplied video_url. Strict superset of the #807 approach; the router.py/reporting_routes.py sinks #804 targeted are already clean on main and covered by the tree-wide guard. --- .../backend/cloud_api_endpoints.py | 19 +++------ .../backend/real_api_endpoints.py | 14 ++----- tests/unit/test_500_info_disclosure.py | 41 ++++++++++++------- tests/unit/test_real_api_endpoints.py | 11 +++-- 4 files changed, 43 insertions(+), 42 deletions(-) diff --git a/src/youtube_extension/backend/cloud_api_endpoints.py b/src/youtube_extension/backend/cloud_api_endpoints.py index 165bc76c9..613aa6f2f 100644 --- a/src/youtube_extension/backend/cloud_api_endpoints.py +++ b/src/youtube_extension/backend/cloud_api_endpoints.py @@ -142,18 +142,10 @@ async def process_video_cloud( ) except Exception as e: - error_msg = f"Cloud processing failed: {str(e)}" - logger.error(error_msg) + logger.error(f"Cloud processing failed: {e}", exc_info=True) - raise HTTPException( - status_code=500, - detail={ - "error": "cloud_processing_failed", - "message": error_msg, - "video_url": request.video_url, - "timestamp": datetime.now(timezone.utc).isoformat() - } - ) + # detail must be a static string — the exception is logged above only. + raise HTTPException(status_code=500, detail="Internal server error") @router.post("/api/v3/process-video-task") async def process_video_task_handler( @@ -216,7 +208,7 @@ async def process_video_task_handler( except Exception as e: error_msg = f"Task processing failed: {str(e)}" - logger.error(error_msg) + logger.error(error_msg, exc_info=True) # Update state with error try: @@ -229,7 +221,8 @@ async def process_video_task_handler( except Exception as state_error: logger.error(f"Failed to update error state: {state_error}") - raise HTTPException(status_code=500, detail=error_msg) + # detail must be a static string — error_msg (with the exception) is logged above only. + raise HTTPException(status_code=500, detail="Internal server error") @router.post("/api/v3/batch-process") async def batch_process_videos_cloud(request: BatchCloudProcessingRequest): diff --git a/src/youtube_extension/backend/real_api_endpoints.py b/src/youtube_extension/backend/real_api_endpoints.py index 1b0433cfc..b886bf192 100644 --- a/src/youtube_extension/backend/real_api_endpoints.py +++ b/src/youtube_extension/backend/real_api_endpoints.py @@ -118,18 +118,10 @@ async def process_video_real_api(request: VideoProcessingRequest, background_tas return response except Exception as e: - error_msg = f"Real API processing failed: {str(e)}" - logger.error(error_msg) + logger.error(f"Real API processing failed: {e}", exc_info=True) - raise HTTPException( - status_code=500, - detail={ - "error": "video_processing_failed", - "message": error_msg, - "video_url": request.video_url, - "timestamp": datetime.now(timezone.utc).isoformat() - } - ) + # detail must be a static string — the exception is logged above only. + raise HTTPException(status_code=500, detail="Internal server error") @app.post("/api/v2/validate-video") async def validate_video_url(request: VideoValidationRequest): diff --git a/tests/unit/test_500_info_disclosure.py b/tests/unit/test_500_info_disclosure.py index d561c422e..d67f2bddc 100644 --- a/tests/unit/test_500_info_disclosure.py +++ b/tests/unit/test_500_info_disclosure.py @@ -28,16 +28,17 @@ # tolerant of the call spanning multiple lines. _HTTP_EXC = re.compile(r"HTTPException\((?P.*?)\)", re.DOTALL) -# A `detail=` argument whose value is derived from a variable/exception: -# detail=str(e) detail=str(exc) -# detail=f"... {e} ..." detail=f"... {str(e)} ..." -_DYNAMIC_DETAIL = re.compile( - r"""detail\s*=\s*(?: - str\( # detail=str(...) - | f["'][^"']*\{ # detail=f"...{...}..." - )""", - re.VERBOSE, -) +# A 500 `detail=` is safe only when it is an inline *static string literal* +# (``detail="Internal server error"``). Anything else can carry internal state to +# the client and is flagged: +# detail=str(e) detail=f"... {e} ..." (inline dynamic string) +# detail=error_msg (a variable — may hold f"...{e}...") +# detail={...} (a dict whose values embed the exception) +# The value after ``detail=`` is dynamic unless its first non-space character +# opens a plain string literal (``"`` or ``'``). A leading ``{`` (dict) or any +# identifier char — ``f`` of an f-string, ``s`` of ``str(``, or a bare variable +# name — means it is not a static literal. +_DYNAMIC_DETAIL = re.compile(r"""detail\s*=\s*(?:\{|[A-Za-z_])""") def _backend_python_files() -> list[Path]: @@ -76,15 +77,25 @@ def test_no_dynamic_detail_in_500_responses() -> None: def test_guard_detects_a_synthetic_leak() -> None: - """Sanity check: the scanner actually flags a dynamic 500 detail.""" - leaky = "raise HTTPException(status_code=500, detail=str(e))" + """Sanity check: the scanner flags every dynamic-detail shape, not just str(e).""" + # Each of these leaks internal state and must be flagged. + leaks = [ + "raise HTTPException(status_code=500, detail=str(e))", + 'raise HTTPException(status_code=500, detail=f"failed: {e}")', + "raise HTTPException(status_code=500, detail=error_msg)", # bare variable + 'raise HTTPException(status_code=500, detail={"message": error_msg})', # dict + ] + for leak in leaks: + assert list(_iter_500_dynamic_detail(leak)), f"scanner missed a real 500 leak: {leak}" + + # A static string literal is the only safe form. safe = 'raise HTTPException(status_code=500, detail="Internal server error")' - client_err = "raise HTTPException(status_code=400, detail=str(exc))" - - assert list(_iter_500_dynamic_detail(leaky)), "scanner missed a real 500 leak" assert not list( _iter_500_dynamic_detail(safe) ), "scanner false-positived a static detail" + + # 4xx responses echo client-supplied input and are intentionally out of scope. + client_err = "raise HTTPException(status_code=400, detail=str(exc))" assert not list( _iter_500_dynamic_detail(client_err) ), "scanner must ignore 4xx responses" diff --git a/tests/unit/test_real_api_endpoints.py b/tests/unit/test_real_api_endpoints.py index 4eb7109c6..b9f14406a 100644 --- a/tests/unit/test_real_api_endpoints.py +++ b/tests/unit/test_real_api_endpoints.py @@ -344,14 +344,19 @@ def test_returns_500_when_processor_raises(self, client, mock_processor): ) assert response.status_code == 500 - def test_error_response_includes_video_url(self, client, mock_processor): + def test_error_response_is_sanitized(self, client, mock_processor): + # A 500 must not leak internal state (CWE-209): the response body must be a + # static message, never the caught exception or the caller-supplied video_url. mock_processor.process_video = AsyncMock(side_effect=RuntimeError("crash")) response = client.post( "/api/v2/process-video", json={"video_url": "https://youtube.com/watch?v=auJzb1D-fag"}, ) - detail = response.json()["detail"] - assert "auJzb1D-fag" in str(detail) + assert response.status_code == 500 + detail = str(response.json()["detail"]) + assert detail == "Internal server error" + assert "crash" not in detail + assert "auJzb1D-fag" not in detail def test_missing_video_url_returns_422(self, client): response = client.post("/api/v2/process-video", json={}) From 93d4847c0df09bcdd8b0ffb3520a3d8d780645f1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 02:30:03 +0000 Subject: [PATCH 3/4] =?UTF-8?q?fix(security):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20global=20handler=20leak,=20persisted-error=20leak,?= =?UTF-8?q?=20400=E2=86=92500=20swallow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round of fixes from the CodeRabbit full review and the Vercel VADE bot on #815: - main.py global_exception_handler leaked str(exc) and the exception class name (error_type) in its 500 JSONResponse body (VADE finding). Return a static body; the full exception is already logged with exc_info=True. - cloud_api_endpoints.process_video_task_handler persisted error_msg (containing str(e)) as the Firestore 'error_message', which get_video_status / get_video_result return verbatim to clients — exfiltrating the exception despite the sanitized 500. Persist a generic message; keep the detail in logs only. - real_api_endpoints batch_process_videos and search_youtube_videos caught their own HTTPException(400) validation errors in the broad 'except Exception' and turned them into 500s. Re-raise HTTPException first to preserve the 400. - cloud_ai_routes: add exc_info=True to the five 500-handler logger.error calls so the traceback is retained server-side. - Guard: exception handlers are a second 500 sink the HTTPException scan didn't model. Add test_no_disclosure_in_500_exception_handlers — it flags any @app.exception_handler that returns 500 while placing str(exc)/str(e) or __class__.__name__ in the response body (docstrings/comments/log lines excluded). --- .../backend/cloud_ai_routes.py | 10 +- .../backend/cloud_api_endpoints.py | 4 +- src/youtube_extension/backend/main.py | 13 ++- .../backend/real_api_endpoints.py | 6 + tests/unit/test_500_info_disclosure.py | 105 ++++++++++++++++++ 5 files changed, 127 insertions(+), 11 deletions(-) diff --git a/src/youtube_extension/backend/cloud_ai_routes.py b/src/youtube_extension/backend/cloud_ai_routes.py index 0172e1617..a470cabe6 100644 --- a/src/youtube_extension/backend/cloud_ai_routes.py +++ b/src/youtube_extension/backend/cloud_ai_routes.py @@ -228,7 +228,7 @@ async def get_provider_status(): ) except Exception as e: - logger.error(f"Failed to get provider status: {e}") + logger.error(f"Failed to get provider status: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Internal server error") @@ -262,12 +262,12 @@ async def analyze_video(request: VideoAnalysisRequest): logger.warning(f"Rate limit exceeded: {e}") raise HTTPException(status_code=429, detail=f"Rate limit exceeded: {str(e)}") except ConfigurationError as e: - logger.error(f"Configuration error: {e}") + logger.error(f"Configuration error: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Internal server error") except HTTPException: raise except Exception as e: - logger.error(f"Unexpected error during video analysis: {e}") + logger.error(f"Unexpected error during video analysis: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Internal server error") @@ -300,7 +300,7 @@ async def analyze_batch_videos(request: BatchAnalysisRequest, background_tasks: except HTTPException: raise except Exception as e: - logger.error(f"Failed to start batch analysis: {e}") + logger.error(f"Failed to start batch analysis: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Internal server error") @@ -324,7 +324,7 @@ async def analyze_video_multi_provider(request: VideoAnalysisRequest): return formatted_results except Exception as e: - logger.error(f"Multi-provider analysis failed: {e}") + logger.error(f"Multi-provider analysis failed: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Internal server error") diff --git a/src/youtube_extension/backend/cloud_api_endpoints.py b/src/youtube_extension/backend/cloud_api_endpoints.py index 613aa6f2f..fc7dedee0 100644 --- a/src/youtube_extension/backend/cloud_api_endpoints.py +++ b/src/youtube_extension/backend/cloud_api_endpoints.py @@ -213,10 +213,12 @@ async def process_video_task_handler( # Update state with error try: firestore_service = await get_firestore_service() + # error_message is returned to clients by the status/result endpoints, + # so it must stay generic — the full error_msg is in the logs above only. await firestore_service.update_state( payload.video_id, status='failed', - error_message=error_msg + error_message="Internal server error" ) except Exception as state_error: logger.error(f"Failed to update error state: {state_error}") diff --git a/src/youtube_extension/backend/main.py b/src/youtube_extension/backend/main.py index ff1234740..c7dabe921 100644 --- a/src/youtube_extension/backend/main.py +++ b/src/youtube_extension/backend/main.py @@ -444,21 +444,24 @@ async def value_error_handler(request, exc): @app.exception_handler(Exception) async def global_exception_handler(request, exc): - """Global exception handler with enhanced error details""" + """Global handler for unhandled exceptions. + + The full exception — type, message, and traceback — is logged server-side + only. The client receives a static body: neither the exception message + (``str(exc)``) nor its class name may be disclosed, as both leak internal + state to the caller (CWE-209 information disclosure). + """ logger.error(f"Unhandled exception: {exc}", exc_info=True) error_detail = { "error": "Internal server error", - "detail": str(exc), + "detail": "Internal server error", "timestamp": datetime.now().isoformat(), "path": str(request.url) if hasattr(request, "url") else "unknown", "version": "2.0.0", "architecture": "service-oriented", } - if hasattr(exc, "__class__"): - error_detail["error_type"] = exc.__class__.__name__ - return JSONResponse(status_code=500, content=error_detail) diff --git a/src/youtube_extension/backend/real_api_endpoints.py b/src/youtube_extension/backend/real_api_endpoints.py index b886bf192..65c8b9fe6 100644 --- a/src/youtube_extension/backend/real_api_endpoints.py +++ b/src/youtube_extension/backend/real_api_endpoints.py @@ -169,6 +169,9 @@ async def batch_process_videos(request: BatchProcessingRequest): return result + except HTTPException: + # Preserve intentional client errors (e.g. the 400 above). + raise except Exception as e: logger.error(f"Unhandled error in batch_process_videos: {e}", exc_info=True) raise HTTPException( @@ -427,6 +430,9 @@ async def search_youtube_videos( "timestamp": datetime.now(timezone.utc).isoformat() } + except HTTPException: + # Preserve intentional client errors (e.g. the 400 above). + raise except Exception as e: logger.error(f"Unhandled error in search_youtube_videos: {e}", exc_info=True) raise HTTPException( diff --git a/tests/unit/test_500_info_disclosure.py b/tests/unit/test_500_info_disclosure.py index d67f2bddc..c90f15407 100644 --- a/tests/unit/test_500_info_disclosure.py +++ b/tests/unit/test_500_info_disclosure.py @@ -60,6 +60,62 @@ def _iter_500_dynamic_detail(text: str): yield line_no, " ".join(args.split())[:120] +# FastAPI exception handlers are a second 500 sink the HTTPException scan above +# does not model: they build a response body directly (dict / JSONResponse) rather +# than raising. A handler must not place the exception message (`str(exc)`) or its +# class name (`exc.__class__.__name__`) into that body — both leak internal state. +# Logging the exception server-side is fine; those tokens appear only in response +# construction, never in a `logger.`/`log`/`raise`/comment line, so we exclude +# those lines to avoid false positives. +_HANDLER_DECORATOR = re.compile(r"^\s*@\w+\.exception_handler\(", re.MULTILINE) +_HANDLER_DISCLOSURE = re.compile(r"str\(\s*(?:exc|e)\s*\)|__class__\.__name__") +# Triple-quoted docstrings, so prose that *mentions* str(exc) is not mistaken for code. +_TRIPLE_STR = re.compile(r'"""(?:.|\n)*?"""|\'\'\'(?:.|\n)*?\'\'\'') + + +def _strip_docstrings(body: str) -> str: + """Blank triple-quoted blocks, preserving line count so line numbers stay aligned.""" + return _TRIPLE_STR.sub(lambda m: "\n" * m.group(0).count("\n"), body) + + +def _exception_handler_bodies(text: str): + """Yield (start_line, body_text) for each @.exception_handler function.""" + lines = text.splitlines(keepends=True) + for m in _HANDLER_DECORATOR.finditer(text): + start_line = text.count("\n", 0, m.start()) + # Find the `def`/`async def` line that the decorator applies to. + i = start_line + while i < len(lines) and not lines[i].lstrip().startswith(("def ", "async def ")): + i += 1 + if i >= len(lines): + continue + def_indent = len(lines[i]) - len(lines[i].lstrip()) + j = i + 1 + body: list[str] = [] + while j < len(lines): + line = lines[j] + stripped = line.strip() + if stripped and (len(line) - len(line.lstrip())) <= def_indent: + break # dedented back to <= the def's level: end of function + body.append(line) + j += 1 + yield i + 1, "".join(body) + + +def _iter_handler_disclosures(text: str): + """Yield (line_no, snippet) for exception-handler bodies that leak exc into the response.""" + for start_line, body in _exception_handler_bodies(text): + if "status_code=500" not in re.sub(r"\s+", "", body): + continue + for offset, line in enumerate(_strip_docstrings(body).splitlines()): + code = line.split("#", 1)[0] # ignore inline comments + bare = line.strip() + if bare.startswith(("logger", "log", "self.logger", "raise")): + continue + if _HANDLER_DISCLOSURE.search(code): + yield start_line + offset, bare[:120] + + def test_no_dynamic_detail_in_500_responses() -> None: offenders: list[str] = [] for path in _backend_python_files(): @@ -76,6 +132,55 @@ def test_no_dynamic_detail_in_500_responses() -> None: ) +def test_no_disclosure_in_500_exception_handlers() -> None: + offenders: list[str] = [] + for path in _backend_python_files(): + text = path.read_text(encoding="utf-8") + for line_no, snippet in _iter_handler_disclosures(text): + rel = path.relative_to(_BACKEND.parents[2]) + offenders.append(f"{rel}:{line_no}: {snippet}") + + assert not offenders, ( + "A FastAPI exception handler that returns HTTP 500 must not place the " + "exception message (`str(exc)`) or its class name (`__class__.__name__`) " + "into the response body — log it server-side instead. Offending sites:\n " + + "\n ".join(offenders) + ) + + +def test_guard_detects_a_synthetic_handler_leak() -> None: + """The exception-handler scanner flags exc message/class-name disclosure in a 500 body.""" + leaky = ( + "@app.exception_handler(Exception)\n" + "async def h(request, exc):\n" + ' logger.error(f"boom: {exc}", exc_info=True)\n' + ' body = {"detail": str(exc), "error_type": exc.__class__.__name__}\n' + " return JSONResponse(status_code=500, content=body)\n" + ) + assert list(_iter_handler_disclosures(leaky)), "handler scanner missed a real leak" + + safe = ( + "@app.exception_handler(Exception)\n" + "async def h(request, exc):\n" + ' logger.error(f"boom: {exc}", exc_info=True)\n' + ' body = {"detail": "Internal server error"}\n' + " return JSONResponse(status_code=500, content=body)\n" + ) + assert not list( + _iter_handler_disclosures(safe) + ), "handler scanner false-positived a sanitized 500 handler" + + # A 4xx handler may echo the exception (client-supplied validation input). + client_err = ( + "@app.exception_handler(ValueError)\n" + "async def h(request, exc):\n" + ' return JSONResponse(status_code=400, content={"detail": str(exc)})\n' + ) + assert not list( + _iter_handler_disclosures(client_err) + ), "handler scanner must ignore 4xx handlers" + + def test_guard_detects_a_synthetic_leak() -> None: """Sanity check: the scanner flags every dynamic-detail shape, not just str(e).""" # Each of these leaks internal state and must be flagged. From 53fa3fcf44eb12d4a5551e85ba98e400b7fbd077 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 02:44:01 +0000 Subject: [PATCH 4/4] fix(api): preserve client 4xx in multi-provider route; align 500-handler test - analyze_video_multi_provider caught HTTPException(400) from parse_analysis_types in its broad 'except Exception' and re-raised it as a generic 500, masking a client error (flagged in review). Add 'except HTTPException: raise' to mirror analyze_video / analyze_batch_videos, plus a regression test asserting an invalid analysis type returns 400. - Update the stale global-exception-handler test: it asserted the 500 body leaks 'error_type' (the exception class name), which the info-disclosure hardening deliberately removes (CWE-209). Assert the secure contract instead: neither the exception type nor its message appears in the response. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_014KW6wTcE4r8Uk11QpXBhVq --- src/youtube_extension/backend/cloud_ai_routes.py | 4 ++++ tests/unit/test_backend_main.py | 9 +++++++-- tests/unit/test_cloud_routes.py | 10 ++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/youtube_extension/backend/cloud_ai_routes.py b/src/youtube_extension/backend/cloud_ai_routes.py index a470cabe6..6008cdf10 100644 --- a/src/youtube_extension/backend/cloud_ai_routes.py +++ b/src/youtube_extension/backend/cloud_ai_routes.py @@ -323,6 +323,10 @@ async def analyze_video_multi_provider(request: VideoAnalysisRequest): return formatted_results + except HTTPException: + # Preserve client-facing status codes (e.g. 400 from parse_analysis_types); + # do not let the broad handler below convert them into a generic 500. + raise except Exception as e: logger.error(f"Multi-provider analysis failed: {e}", exc_info=True) raise HTTPException(status_code=500, detail="Internal server error") diff --git a/tests/unit/test_backend_main.py b/tests/unit/test_backend_main.py index a73dcf792..f89a9df2b 100644 --- a/tests/unit/test_backend_main.py +++ b/tests/unit/test_backend_main.py @@ -590,14 +590,19 @@ async def test_global_exception_handler_returns_500(self): response = await main_module.global_exception_handler(mock_req, RuntimeError("crash")) assert response.status_code == 500 - async def test_global_exception_handler_includes_error_type(self): + async def test_global_exception_handler_omits_internal_details(self): + """The 500 body must not leak the exception class name or message (CWE-209).""" import json as _json mock_req = MagicMock() mock_req.url = "http://test/api" response = await main_module.global_exception_handler(mock_req, RuntimeError("crash")) body = _json.loads(response.body) - assert body["error_type"] == "RuntimeError" + # The exception type must not be disclosed to the client... + assert "error_type" not in body + # ...nor may the exception message appear anywhere in the response. + assert "crash" not in _json.dumps(body) + assert "RuntimeError" not in _json.dumps(body) async def test_global_exception_handler_includes_version(self): import json as _json diff --git a/tests/unit/test_cloud_routes.py b/tests/unit/test_cloud_routes.py index 4c415a3ed..ab7fa8870 100644 --- a/tests/unit/test_cloud_routes.py +++ b/tests/unit/test_cloud_routes.py @@ -485,6 +485,16 @@ def test_analyze_video_multi_provider_error(self): }) assert response.status_code == 500 + def test_analyze_video_multi_provider_invalid_analysis_type(self): + """A bad analysis type is a client error: the 400 from parse_analysis_types + must be preserved, not swallowed into a 500 by the broad exception handler.""" + client = TestClient(_make_cloud_ai_app()) + response = client.post("/api/v1/cloud-ai/analyze/multi-provider", json={ + "video_url": "https://www.youtube.com/watch?v=auJzb1D-fag", + "analysis_types": ["invalid_type"], + }) + assert response.status_code == 400 + # =========================================================================== # Tests for cloud_api_endpoints.py