From a2857230443063ac3bd538dc9482cef3be15ea96 Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Thu, 12 Mar 2026 18:10:37 +0000 Subject: [PATCH 1/4] feat: updating celery worker --- app/api/v1/routes/settings.py | 482 +++--- app/core/license.py | 282 ++-- app/dependencies.py | 230 ++- app/workers/celery_app.py | 1309 +---------------- app/workers/config.py | 37 + app/workers/tasks/__init__.py | 27 + app/workers/tasks/helpers/__init__.py | 17 + app/workers/tasks/helpers/audio_evaluation.py | 101 ++ app/workers/tasks/helpers/constants.py | 17 + app/workers/tasks/helpers/llm_evaluation.py | 318 ++++ app/workers/tasks/helpers/score_utils.py | 100 ++ app/workers/tasks/process_evaluation.py | 31 + app/workers/tasks/process_evaluator_result.py | 281 ++++ app/workers/tasks/run_evaluator.py | 143 ++ app/workers/tasks/tts_comparison.py | 458 ++++++ app/workers/tasks/tts_report.py | 91 ++ frontend/src/lib/api.ts | 23 +- .../pages/enterprise/EnterpriseUpgrade.tsx | 142 +- frontend/src/store/licenseStore.ts | 92 +- 19 files changed, 2288 insertions(+), 1893 deletions(-) create mode 100644 app/workers/config.py create mode 100644 app/workers/tasks/__init__.py create mode 100644 app/workers/tasks/helpers/__init__.py create mode 100644 app/workers/tasks/helpers/audio_evaluation.py create mode 100644 app/workers/tasks/helpers/constants.py create mode 100644 app/workers/tasks/helpers/llm_evaluation.py create mode 100644 app/workers/tasks/helpers/score_utils.py create mode 100644 app/workers/tasks/process_evaluation.py create mode 100644 app/workers/tasks/process_evaluator_result.py create mode 100644 app/workers/tasks/run_evaluator.py create mode 100644 app/workers/tasks/tts_comparison.py create mode 100644 app/workers/tasks/tts_report.py diff --git a/app/api/v1/routes/settings.py b/app/api/v1/routes/settings.py index a3d672a7..7f664a92 100644 --- a/app/api/v1/routes/settings.py +++ b/app/api/v1/routes/settings.py @@ -1,238 +1,244 @@ -""" -Settings API Routes -Manage API keys for authenticated users -""" -from fastapi import APIRouter, Depends, HTTPException, status -from sqlalchemy.orm import Session -from typing import List, Optional -from uuid import UUID -from datetime import datetime, timezone -import secrets -from pydantic import BaseModel - -from app.dependencies import get_db, get_api_key, get_organization_id -from app.models.database import APIKey, User, Organization -from app.models.schemas import MessageResponse -from app.api.v1.routes.profile import get_current_user -from app.core.license import get_license_info, is_feature_enabled, ENTERPRISE_FEATURES - - -class APIKeyCreateRequest(BaseModel): - name: Optional[str] = None - -router = APIRouter(prefix="/settings", tags=["Settings"]) - -# Maximum number of API keys per user -MAX_API_KEYS_PER_USER = 5 - - -@router.get("/license-info") -def license_info(organization_id: UUID = Depends(get_organization_id)): - """ - Return the current enterprise license status and enabled features. - When the license is scoped to an org_id, only returns features - that match the requesting organization. - """ - data = get_license_info() - all_licensed = data.get("features", []) - enabled_for_org = [f for f in all_licensed if is_feature_enabled(f, organization_id)] - return { - "is_enterprise": bool(enabled_for_org), - "enabled_features": enabled_for_org, - "all_enterprise_features": ENTERPRISE_FEATURES, - "organization": data.get("org"), - } - - -def mask_api_key(key: str) -> str: - """Mask API key for display (show first 8 and last 4 characters).""" - if len(key) <= 12: - return "*" * len(key) - return f"{key[:8]}...{key[-4:]}" - - -@router.get("/api-keys") -def list_api_keys( - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db) -): - """ - List all API keys for the current user. - Returns masked keys for security. - """ - # Get all API keys for this user - api_keys = db.query(APIKey).filter( - APIKey.user_id == current_user.id, - APIKey.is_active == True - ).order_by(APIKey.created_at.desc()).all() - - # Return masked keys - result = [] - for key in api_keys: - result.append({ - "id": str(key.id), - "key": mask_api_key(key.key), - "name": key.name, - "is_active": key.is_active, - "created_at": key.created_at.isoformat() if key.created_at else None, - "last_used": key.last_used.isoformat() if key.last_used else None, - }) - - return result - - -@router.post("/api-keys") -def create_api_key( - request: APIKeyCreateRequest, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db) -): - """ - Create a new API key for the current user. - Maximum 5 keys per user. - Returns the full key (only shown once). - """ - # Check current key count - key_count = db.query(APIKey).filter( - APIKey.user_id == current_user.id, - APIKey.is_active == True - ).count() - - if key_count >= MAX_API_KEYS_PER_USER: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Maximum of {MAX_API_KEYS_PER_USER} API keys allowed per user. Please delete an existing key first." - ) - - # Get user's organization (from first API key or organization membership) - from app.models.database import OrganizationMember - org_member = db.query(OrganizationMember).filter( - OrganizationMember.user_id == current_user.id - ).first() - - if not org_member: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="User is not associated with any organization" - ) - - organization_id = org_member.organization_id - - # Generate secure random API key - api_key = secrets.token_urlsafe(32) - - # Create API key record - db_key = APIKey( - key=api_key, - name=request.name, - organization_id=organization_id, - user_id=current_user.id, - is_active=True - ) - db.add(db_key) - db.commit() - db.refresh(db_key) - - # Return full key (only time it's shown) - return { - "id": str(db_key.id), - "key": db_key.key, # Full key shown only once - "name": db_key.name, - "is_active": db_key.is_active, - "created_at": db_key.created_at.isoformat() if db_key.created_at else None, - "last_used": None, - "message": "Save this API key securely. You won't be able to see it again." - } - - -@router.delete("/api-keys/{key_id}") -def delete_api_key( - key_id: UUID, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db) -): - """ - Delete (deactivate) an API key. - Only the owner can delete their own keys. - """ - api_key = db.query(APIKey).filter( - APIKey.id == key_id, - APIKey.user_id == current_user.id - ).first() - - if not api_key: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="API key not found or you don't have permission to delete it" - ) - - # Deactivate instead of deleting (soft delete) - api_key.is_active = False - db.commit() - - return MessageResponse(message="API key deleted successfully") - - -@router.post("/api-keys/{key_id}/regenerate") -def regenerate_api_key( - key_id: UUID, - current_user: User = Depends(get_current_user), - db: Session = Depends(get_db) -): - """ - Regenerate an API key. - Creates a new key and deactivates the old one. - Returns the new full key (only shown once). - """ - old_key = db.query(APIKey).filter( - APIKey.id == key_id, - APIKey.user_id == current_user.id - ).first() - - if not old_key: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="API key not found or you don't have permission to regenerate it" - ) - - # Check if we're at the limit (accounting for the key we're about to deactivate) - key_count = db.query(APIKey).filter( - APIKey.user_id == current_user.id, - APIKey.is_active == True - ).count() - - if key_count >= MAX_API_KEYS_PER_USER: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Maximum of {MAX_API_KEYS_PER_USER} API keys allowed per user. Please delete an existing key first." - ) - - # Generate new secure random API key - new_api_key = secrets.token_urlsafe(32) - - # Deactivate old key - old_key.is_active = False - - # Create new API key with same organization and user - new_db_key = APIKey( - key=new_api_key, - name=old_key.name, - organization_id=old_key.organization_id, - user_id=old_key.user_id, - is_active=True - ) - db.add(new_db_key) - db.commit() - db.refresh(new_db_key) - - # Return new full key (only time it's shown) - return { - "id": str(new_db_key.id), - "key": new_db_key.key, # Full key shown only once - "name": new_db_key.name, - "is_active": new_db_key.is_active, - "created_at": new_db_key.created_at.isoformat() if new_db_key.created_at else None, - "last_used": None, - "message": "Save this API key securely. You won't be able to see it again." - } - +""" +Settings API Routes +Manage API keys for authenticated users +""" +from fastapi import APIRouter, Depends, HTTPException, status +from sqlalchemy.orm import Session +from typing import List, Optional +from uuid import UUID +from datetime import datetime, timezone +import secrets +from pydantic import BaseModel + +from app.dependencies import get_db, get_api_key, get_organization_id +from app.models.database import APIKey, User, Organization +from app.models.schemas import MessageResponse +from app.api.v1.routes.profile import get_current_user +from app.core.license import ( + get_feature_catalog, + get_license_info, + is_feature_enabled, + ENTERPRISE_FEATURES, +) + + +class APIKeyCreateRequest(BaseModel): + name: Optional[str] = None + +router = APIRouter(prefix="/settings", tags=["Settings"]) + +# Maximum number of API keys per user +MAX_API_KEYS_PER_USER = 5 + + +@router.get("/license-info") +def license_info(organization_id: UUID = Depends(get_organization_id)): + """ + Return the current enterprise license status and enabled features. + When the license is scoped to an org_id, only returns features + that match the requesting organization. + """ + data = get_license_info() + all_licensed = data.get("features", []) if isinstance(data.get("features"), list) else [] + enabled_for_org = [f for f in all_licensed if is_feature_enabled(f, organization_id)] + return { + "is_enterprise": bool(enabled_for_org), + "enabled_features": enabled_for_org, + "all_enterprise_features": ENTERPRISE_FEATURES, + "feature_catalog": get_feature_catalog(), + "organization": data.get("org_id"), + } + + +def mask_api_key(key: str) -> str: + """Mask API key for display (show first 8 and last 4 characters).""" + if len(key) <= 12: + return "*" * len(key) + return f"{key[:8]}...{key[-4:]}" + + +@router.get("/api-keys") +def list_api_keys( + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """ + List all API keys for the current user. + Returns masked keys for security. + """ + # Get all API keys for this user + api_keys = db.query(APIKey).filter( + APIKey.user_id == current_user.id, + APIKey.is_active == True + ).order_by(APIKey.created_at.desc()).all() + + # Return masked keys + result = [] + for key in api_keys: + result.append({ + "id": str(key.id), + "key": mask_api_key(key.key), + "name": key.name, + "is_active": key.is_active, + "created_at": key.created_at.isoformat() if key.created_at else None, + "last_used": key.last_used.isoformat() if key.last_used else None, + }) + + return result + + +@router.post("/api-keys") +def create_api_key( + request: APIKeyCreateRequest, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """ + Create a new API key for the current user. + Maximum 5 keys per user. + Returns the full key (only shown once). + """ + # Check current key count + key_count = db.query(APIKey).filter( + APIKey.user_id == current_user.id, + APIKey.is_active == True + ).count() + + if key_count >= MAX_API_KEYS_PER_USER: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Maximum of {MAX_API_KEYS_PER_USER} API keys allowed per user. Please delete an existing key first." + ) + + # Get user's organization (from first API key or organization membership) + from app.models.database import OrganizationMember + org_member = db.query(OrganizationMember).filter( + OrganizationMember.user_id == current_user.id + ).first() + + if not org_member: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="User is not associated with any organization" + ) + + organization_id = org_member.organization_id + + # Generate secure random API key + api_key = secrets.token_urlsafe(32) + + # Create API key record + db_key = APIKey( + key=api_key, + name=request.name, + organization_id=organization_id, + user_id=current_user.id, + is_active=True + ) + db.add(db_key) + db.commit() + db.refresh(db_key) + + # Return full key (only time it's shown) + return { + "id": str(db_key.id), + "key": db_key.key, # Full key shown only once + "name": db_key.name, + "is_active": db_key.is_active, + "created_at": db_key.created_at.isoformat() if db_key.created_at else None, + "last_used": None, + "message": "Save this API key securely. You won't be able to see it again." + } + + +@router.delete("/api-keys/{key_id}") +def delete_api_key( + key_id: UUID, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """ + Delete (deactivate) an API key. + Only the owner can delete their own keys. + """ + api_key = db.query(APIKey).filter( + APIKey.id == key_id, + APIKey.user_id == current_user.id + ).first() + + if not api_key: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="API key not found or you don't have permission to delete it" + ) + + # Deactivate instead of deleting (soft delete) + api_key.is_active = False + db.commit() + + return MessageResponse(message="API key deleted successfully") + + +@router.post("/api-keys/{key_id}/regenerate") +def regenerate_api_key( + key_id: UUID, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """ + Regenerate an API key. + Creates a new key and deactivates the old one. + Returns the new full key (only shown once). + """ + old_key = db.query(APIKey).filter( + APIKey.id == key_id, + APIKey.user_id == current_user.id + ).first() + + if not old_key: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="API key not found or you don't have permission to regenerate it" + ) + + # Check if we're at the limit (accounting for the key we're about to deactivate) + key_count = db.query(APIKey).filter( + APIKey.user_id == current_user.id, + APIKey.is_active == True + ).count() + + if key_count >= MAX_API_KEYS_PER_USER: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Maximum of {MAX_API_KEYS_PER_USER} API keys allowed per user. Please delete an existing key first." + ) + + # Generate new secure random API key + new_api_key = secrets.token_urlsafe(32) + + # Deactivate old key + old_key.is_active = False + + # Create new API key with same organization and user + new_db_key = APIKey( + key=new_api_key, + name=old_key.name, + organization_id=old_key.organization_id, + user_id=old_key.user_id, + is_active=True + ) + db.add(new_db_key) + db.commit() + db.refresh(new_db_key) + + # Return new full key (only time it's shown) + return { + "id": str(new_db_key.id), + "key": new_db_key.key, # Full key shown only once + "name": new_db_key.name, + "is_active": new_db_key.is_active, + "created_at": new_db_key.created_at.isoformat() if new_db_key.created_at else None, + "last_used": None, + "message": "Save this API key securely. You won't be able to see it again." + } + diff --git a/app/core/license.py b/app/core/license.py index b160365e..d6ab1364 100644 --- a/app/core/license.py +++ b/app/core/license.py @@ -1,132 +1,150 @@ -""" -Enterprise license validation for EfficientAI. - -License keys are JWT tokens signed with RS256 (asymmetric RSA). -The private key is held by the EfficientAI team. -The public key below is used to verify licenses — it cannot be used to forge them. - -Customers set the license via the EFFICIENTAI_LICENSE env var, .env, or config.yml. - -JWT payload: - { - "features": ["voice_playground", ...], - "org": "customer-org-name", - "org_id": "uuid-or-null", # optional — restricts to a specific org - "exp": - } - -Behaviour: - - org_id omitted/null → license applies to the entire deployment (self-hosted) - - org_id set → license only applies to that organization (multi-tenant) -""" - -import os -from typing import Dict, Any, List, Optional -from uuid import UUID -from loguru import logger - -_license_cache: Dict[str, Any] | None = None - -ENTERPRISE_FEATURES = [ - "voice_playground", -] - -# RSA public key used to verify enterprise license JWTs. -# The corresponding private key is kept offline by the EfficientAI team. -# Even though this key is visible in the source, it can only VERIFY — not sign — tokens. -EFFICIENTAI_LICENSE_PUBLIC_KEY = """-----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAl6UAH0skICj4UytmqzKJ -jUGj6AFEmOT+NirCEp5nNnqQV7tIPr6GidNju0IWH8/q9QJww18To9PU++BliLi4 -3Tjy4fk6EgqbLIP/3ed9SMV2ChiS65QCt8nAhybJbspEjN5ViQy0Vfv9ZlVuR7Bs -nVE9nKi743y9RM6cDhEvKGlMEnHl+5EfG65rzDBbuX3F/U7QbGSi77i5SSxeW+Ac -KcRD+/5rpwuIK6C0BJgzR+zh73vPdfLxa3t3H2u2AzrFmYWxeTYjHZojVi7lp9u+ -/O0Ic4T0FliF/6XblXxMmsZ+af1PQfkZtSPHeHLRNAJR7BkeMtS09u0UbJk6MU2/ -UwIDAQAB ------END PUBLIC KEY-----""" - - -def _get_license_token() -> str | None: - """Resolve the license token from config.yml, .env, or environment variable.""" - try: - from app.config import settings - if settings.EFFICIENTAI_LICENSE: - return settings.EFFICIENTAI_LICENSE - except Exception: - pass - return os.getenv("EFFICIENTAI_LICENSE") - - -def _decode_license() -> Dict[str, Any]: - """Decode and validate the license JWT using RS256. Returns the payload or empty dict.""" - token = _get_license_token() - if not token: - return {} - - try: - from jose import jwt as jose_jwt, JWTError, ExpiredSignatureError - - payload = jose_jwt.decode( - token, - EFFICIENTAI_LICENSE_PUBLIC_KEY, - algorithms=["RS256"], - options={"verify_exp": True}, - ) - logger.info( - "EfficientAI Enterprise license validated — " - f"org={payload.get('org', 'unknown')}, " - f"org_id={payload.get('org_id', 'all')}, " - f"features={payload.get('features', [])}" - ) - return payload - - except ExpiredSignatureError: - logger.warning("EfficientAI Enterprise license has expired") - return {} - except (JWTError, Exception) as e: - logger.warning(f"Invalid EfficientAI Enterprise license: {e}") - return {} - - -def get_license_info() -> Dict[str, Any]: - """Return cached license payload, decoding on first call.""" - global _license_cache - if _license_cache is None: - _license_cache = _decode_license() - return _license_cache - - -def get_enabled_features() -> List[str]: - """Return the list of enterprise features enabled by the current license.""" - return get_license_info().get("features", []) - - -def get_licensed_org_id() -> Optional[str]: - """Return the org_id the license is scoped to, or None for deployment-wide.""" - return get_license_info().get("org_id") - - -def is_feature_enabled(feature: str, organization_id: Optional[UUID] = None) -> bool: - """ - Check whether an enterprise feature is enabled. - - If the license contains an org_id, the requesting organization must match. - If org_id is absent from the license, the feature is enabled deployment-wide. - """ - info = get_license_info() - if feature not in info.get("features", []): - return False - - licensed_org = info.get("org_id") - if licensed_org is None: - return True - - if organization_id is None: - return True - - return str(organization_id) == str(licensed_org) - - -def reset_license_cache() -> None: - """Force re-evaluation of the license (useful after env change in tests).""" - global _license_cache - _license_cache = None +""" +Enterprise license validation for EfficientAI. + +License keys are JWT tokens signed with RS256 (asymmetric RSA). +The private key is held by the EfficientAI team. +The public key below is used to verify licenses — it cannot be used to forge them. + +Customers set the license via the EFFICIENTAI_LICENSE env var, .env, or config.yml. + +JWT payload: + { + "features": ["voice_playground", ...], + "org": "customer-org-name", + "org_id": "uuid-or-null", # optional — restricts to a specific org + "exp": + } + +Behaviour: + - org_id omitted/null → license applies to the entire deployment (self-hosted) + - org_id set → license only applies to that organization (multi-tenant) +""" + +import os +from typing import Dict, Any, List, Optional +from uuid import UUID +from loguru import logger + +_license_cache: Dict[str, Any] | None = None + +FEATURE_CATALOG: Dict[str, Dict[str, str]] = { + "voice_playground": { + "title": "Voice Playground", + "description": "A/B test TTS providers with blind tests and quality analytics.", + "category": "playground", + }, +} + +# Backward-compatible export used by existing API response shape. +ENTERPRISE_FEATURES = list(FEATURE_CATALOG.keys()) + +# RSA public key used to verify enterprise license JWTs. +# The corresponding private key is kept offline by the EfficientAI team. +# Even though this key is visible in the source, it can only VERIFY — not sign — tokens. +EFFICIENTAI_LICENSE_PUBLIC_KEY = """-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAl6UAH0skICj4UytmqzKJ +jUGj6AFEmOT+NirCEp5nNnqQV7tIPr6GidNju0IWH8/q9QJww18To9PU++BliLi4 +3Tjy4fk6EgqbLIP/3ed9SMV2ChiS65QCt8nAhybJbspEjN5ViQy0Vfv9ZlVuR7Bs +nVE9nKi743y9RM6cDhEvKGlMEnHl+5EfG65rzDBbuX3F/U7QbGSi77i5SSxeW+Ac +KcRD+/5rpwuIK6C0BJgzR+zh73vPdfLxa3t3H2u2AzrFmYWxeTYjHZojVi7lp9u+ +/O0Ic4T0FliF/6XblXxMmsZ+af1PQfkZtSPHeHLRNAJR7BkeMtS09u0UbJk6MU2/ +UwIDAQAB +-----END PUBLIC KEY-----""" + + +def _get_license_token() -> str | None: + """Resolve the license token from config.yml, .env, or environment variable.""" + try: + from app.config import settings + if settings.EFFICIENTAI_LICENSE: + return settings.EFFICIENTAI_LICENSE + except Exception: + pass + return os.getenv("EFFICIENTAI_LICENSE") + + +def _decode_license() -> Dict[str, Any]: + """Decode and validate the license JWT using RS256. Returns the payload or empty dict.""" + token = _get_license_token() + if not token: + return {} + + try: + from jose import jwt as jose_jwt, JWTError, ExpiredSignatureError + + payload = jose_jwt.decode( + token, + EFFICIENTAI_LICENSE_PUBLIC_KEY, + algorithms=["RS256"], + options={"verify_exp": True}, + ) + logger.info( + "EfficientAI Enterprise license validated — " + f"org={payload.get('org', 'unknown')}, " + f"org_id={payload.get('org_id', 'all')}, " + f"features={payload.get('features', [])}" + ) + return payload + + except ExpiredSignatureError: + logger.warning("EfficientAI Enterprise license has expired") + return {} + except (JWTError, Exception) as e: + logger.warning(f"Invalid EfficientAI Enterprise license: {e}") + return {} + + +def get_license_info() -> Dict[str, Any]: + """Return cached license payload, decoding on first call.""" + global _license_cache + if _license_cache is None: + _license_cache = _decode_license() + return _license_cache + + +def get_enabled_features() -> List[str]: + """Return the list of enterprise features enabled by the current license.""" + licensed_features = get_license_info().get("features", []) + if not isinstance(licensed_features, list): + return [] + # Keep only known feature IDs to avoid accidental entitlement typos. + return [f for f in licensed_features if f in FEATURE_CATALOG] + + +def get_feature_catalog() -> Dict[str, Dict[str, str]]: + """Return metadata for all enterprise features.""" + # Return shallow copies to avoid callers mutating global state. + return {feature: meta.copy() for feature, meta in FEATURE_CATALOG.items()} + + +def get_licensed_org_id() -> Optional[str]: + """Return the org_id the license is scoped to, or None for deployment-wide.""" + return get_license_info().get("org_id") + + +def is_feature_enabled(feature: str, organization_id: Optional[UUID] = None) -> bool: + """ + Check whether an enterprise feature is enabled. + + If the license contains an org_id, the requesting organization must match. + If org_id is absent from the license, the feature is enabled deployment-wide. + """ + info = get_license_info() + if feature not in get_enabled_features(): + return False + + licensed_org = info.get("org_id") + if licensed_org is None: + return True + + # For org-scoped licenses, we require a concrete requesting organization. + if organization_id is None: + return False + + return str(organization_id) == str(licensed_org) + + +def reset_license_cache() -> None: + """Force re-evaluation of the license (useful after env change in tests).""" + global _license_cache + _license_cache = None diff --git a/app/dependencies.py b/app/dependencies.py index fc0ada75..6e7a6a40 100644 --- a/app/dependencies.py +++ b/app/dependencies.py @@ -1,119 +1,111 @@ -"""Common dependencies for FastAPI routes.""" - -from fastapi import Header, HTTPException, Depends -from typing import Optional, Tuple -from sqlalchemy.orm import Session -from uuid import UUID -from app.database import get_db -from app.core.security import verify_api_key, get_api_key_organization_id -from app.core.exceptions import InvalidAPIKeyError -from app.core.license import is_feature_enabled - - -def get_api_key( - x_api_key: Optional[str] = Header(None, alias="X-API-Key"), - x_efficientai_api_key: Optional[str] = Header( - None, alias="X-EFFICIENTAI-API-KEY" - ), -) -> str: - """ - Extract and validate API key from request headers. - - Args: - x_api_key: API key from X-API-Key header (legacy/SDK usage) - x_efficientai_api_key: API key from X-EFFICIENTAI-API-KEY header (webhooks) - - Returns: - Validated API key - - Raises: - HTTPException: If API key is missing or invalid - """ - api_key = x_api_key or x_efficientai_api_key - - if not api_key: - raise HTTPException(status_code=401, detail="API key is required") - - db = next(get_db()) - try: - verify_api_key(api_key, db) - return api_key - except InvalidAPIKeyError as e: - raise HTTPException(status_code=401, detail=str(e)) - finally: - db.close() - - -def get_organization_id( - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -) -> UUID: - """ - Get organization ID from validated API key. - - Args: - api_key: Validated API key from get_api_key dependency - db: Database session - - Returns: - Organization ID - - Raises: - HTTPException: If organization not found - """ - organization_id = get_api_key_organization_id(api_key, db) - if not organization_id: - raise HTTPException(status_code=500, detail="Organization not found for API key") - return organization_id - - -def get_db_session() -> Session: - """ - Get database session. - - Yields: - Database session - """ - return next(get_db()) - - -def require_enterprise_feature(feature: str): - """ - FastAPI dependency factory that gates a route behind an enterprise feature. - - When the license contains an org_id, the requesting organization must match. - When org_id is absent from the license, the feature is enabled deployment-wide. - - Usage: - router = APIRouter( - dependencies=[Depends(require_enterprise_feature("voice_playground"))] - ) - """ - def _check( - x_api_key: Optional[str] = Header(None, alias="X-API-Key"), - x_efficientai_api_key: Optional[str] = Header(None, alias="X-EFFICIENTAI-API-KEY"), - db: Session = Depends(get_db), - ): - organization_id = None - api_key = x_api_key or x_efficientai_api_key - if api_key: - try: - organization_id = get_api_key_organization_id(api_key, db) - except Exception: - pass - - if not is_feature_enabled(feature, organization_id): - raise HTTPException( - status_code=403, - detail={ - "error": "enterprise_feature_required", - "feature": feature, - "message": ( - f"'{feature}' is an EfficientAI Enterprise feature. " - "Please set EFFICIENTAI_LICENSE in your environment to unlock it. " - "Contact sales@efficientai.com to get an enterprise license key." - ), - }, - ) - return _check - +"""Common dependencies for FastAPI routes.""" + +from fastapi import Header, HTTPException, Depends +from typing import Optional +from sqlalchemy.orm import Session +from uuid import UUID +from app.database import get_db +from app.core.security import verify_api_key, get_api_key_organization_id +from app.core.exceptions import InvalidAPIKeyError +from app.core.license import is_feature_enabled + + +def get_api_key( + x_api_key: Optional[str] = Header(None, alias="X-API-Key"), + x_efficientai_api_key: Optional[str] = Header( + None, alias="X-EFFICIENTAI-API-KEY" + ), +) -> str: + """ + Extract and validate API key from request headers. + + Args: + x_api_key: API key from X-API-Key header (legacy/SDK usage) + x_efficientai_api_key: API key from X-EFFICIENTAI-API-KEY header (webhooks) + + Returns: + Validated API key + + Raises: + HTTPException: If API key is missing or invalid + """ + api_key = x_api_key or x_efficientai_api_key + + if not api_key: + raise HTTPException(status_code=401, detail="API key is required") + + db = next(get_db()) + try: + verify_api_key(api_key, db) + return api_key + except InvalidAPIKeyError as e: + raise HTTPException(status_code=401, detail=str(e)) + finally: + db.close() + + +def get_organization_id( + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +) -> UUID: + """ + Get organization ID from validated API key. + + Args: + api_key: Validated API key from get_api_key dependency + db: Database session + + Returns: + Organization ID + + Raises: + HTTPException: If organization not found + """ + organization_id = get_api_key_organization_id(api_key, db) + if not organization_id: + raise HTTPException(status_code=500, detail="Organization not found for API key") + return organization_id + + +def get_db_session() -> Session: + """ + Get database session. + + Yields: + Database session + """ + return next(get_db()) + + +def require_enterprise_feature(feature: str): + """ + FastAPI dependency factory that gates a route behind an enterprise feature. + + When the license contains an org_id, the requesting organization must match. + When org_id is absent from the license, the feature is enabled deployment-wide. + + Usage: + router = APIRouter( + dependencies=[Depends(require_enterprise_feature("voice_playground"))] + ) + """ + + def _check( + organization_id: UUID = Depends(get_organization_id), + ): + if not is_feature_enabled(feature, organization_id): + raise HTTPException( + status_code=403, + detail={ + "error": "enterprise_feature_required", + "feature": feature, + "message": ( + f"'{feature}' is an EfficientAI Enterprise feature. " + "Please set EFFICIENTAI_LICENSE in your environment to unlock it. " + "Contact sales@efficientai.com to get an enterprise license key." + ), + }, + ) + + return _check + diff --git a/app/workers/celery_app.py b/app/workers/celery_app.py index 5222204c..44705353 100644 --- a/app/workers/celery_app.py +++ b/app/workers/celery_app.py @@ -1,1290 +1,29 @@ -"""Celery application configuration and task definitions.""" +"""Celery application - compatibility entrypoint. -import time -from datetime import datetime -from pathlib import Path -from celery import Celery -from app.config import settings, load_config_from_file -from app.database import SessionLocal -from app.services.evaluation_service import evaluation_service -from uuid import UUID -from loguru import logger +This module preserves backward compatibility for: +- Imports: from app.workers.celery_app import process_evaluator_result_task, etc. +- Worker command: celery -A app.workers.celery_app worker -# Metrics that should never be evaluated. -REMOVED_EVALUATION_METRIC_NAMES = {"clarity and empathy"} - -# Load config.yml if it exists (before using settings) -# This ensures the Celery worker has the same configuration as the main app -config_path = Path("config.yml") -if config_path.exists(): - try: - load_config_from_file(str(config_path)) - logger.info(f"✅ Celery worker loaded configuration from {config_path}") - except Exception as e: - logger.warning(f"⚠️ Celery worker: Could not load config.yml: {e}") - -# Create Celery app -celery_app = Celery( - "efficientai", - broker=settings.CELERY_BROKER_URL, - backend=settings.CELERY_RESULT_BACKEND, -) - -# Celery configuration -celery_app.conf.update( - task_serializer="json", - accept_content=["json"], - result_serializer="json", - timezone="UTC", - enable_utc=True, - task_track_started=True, - task_time_limit=30 * 60, # 30 minutes - task_soft_time_limit=25 * 60, # 25 minutes -) - - -@celery_app.task(name="process_evaluation", bind=True, max_retries=3) -def process_evaluation_task(self, evaluation_id: str): - """ - Celery task to process an evaluation. - - Args: - self: Task instance - evaluation_id: Evaluation ID as string - - Returns: - Dictionary with evaluation results - """ - db = SessionLocal() - try: - eval_id = UUID(evaluation_id) - result = evaluation_service.process_evaluation(eval_id, db) - return result - except Exception as exc: - # Retry on failure - raise self.retry(exc=exc, countdown=60) - finally: - db.close() - - -@celery_app.task(name="process_evaluator_result", bind=True, max_retries=3) -def process_evaluator_result_task(self, result_id: str): - """ - Celery task to process an evaluator result: transcribe audio and evaluate metrics. - - Workflow: - 1. QUEUED -> Job is created and queued - 2. TRANSCRIBING -> Audio is being transcribed - 3. EVALUATING -> Transcription is being evaluated against metrics - 4. COMPLETED -> All processing is complete - 5. FAILED -> An error occurred - - Args: - self: Task instance - result_id: EvaluatorResult ID as string - - Returns: - Dictionary with processing results - """ - db = SessionLocal() - task_start_time = time.time() - - try: - from app.models.database import ( - EvaluatorResult, EvaluatorResultStatus, Metric, - Evaluator, Agent, Persona, Scenario, ModelProvider, AIProvider - ) - from app.services.transcription_service import transcription_service - from app.services.llm_service import llm_service - from app.core.encryption import decrypt_api_key - import json - import re - - # Helper function to compare provider values (handles string vs enum) - def provider_matches(db_provider, target_enum): - """Compare provider field (could be string or enum) with target enum.""" - if db_provider is None: - return False - if isinstance(db_provider, str): - return db_provider.lower() == target_enum.value.lower() - if hasattr(db_provider, 'value'): - return db_provider.value.lower() == target_enum.value.lower() - return db_provider == target_enum - - result_uuid = UUID(result_id) - result = db.query(EvaluatorResult).filter(EvaluatorResult.id == result_uuid).first() - - if not result: - logger.error(f"[EvaluatorResult {result_id}] Job not found in database") - return {"error": "Evaluator result not found"} - - logger.info(f"[EvaluatorResult {result.result_id}] Starting processing task") - - # Update Celery task ID - result.celery_task_id = self.request.id - db.commit() - - # Check if transcript is already available (from provider call_data, or from a previous run) - has_existing_transcript = bool(result.transcription) - - try: - if not result.audio_s3_key and not has_existing_transcript: - raise ValueError("No audio S3 key or existing transcript found") - - # Evaluator is optional - only load if evaluator_id is present - evaluator = None - if result.evaluator_id: - evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() - if not evaluator: - logger.warning(f"[EvaluatorResult {result.result_id}] Evaluator {result.evaluator_id} not found, continuing without evaluator") - - agent = None - if result.agent_id: - agent = db.query(Agent).filter(Agent.id == result.agent_id).first() - - persona = None - if result.persona_id: - persona = db.query(Persona).filter(Persona.id == result.persona_id).first() - - scenario = None - if result.scenario_id: - scenario = db.query(Scenario).filter(Scenario.id == result.scenario_id).first() - - is_custom_evaluator = evaluator and bool(evaluator.custom_prompt) - - if not is_custom_evaluator and not agent: - raise ValueError("Agent not found and no custom prompt available") - # Check if organization has configured AI providers (needed for evaluation even if skipping transcription) - ai_providers = db.query(AIProvider).filter( - AIProvider.organization_id == result.organization_id, - AIProvider.is_active == True - ).all() - - if has_existing_transcript: - transcription = result.transcription - speaker_segments = result.speaker_segments or [] - transcription_time = 0.0 - else: - result.status = EvaluatorResultStatus.TRANSCRIBING.value - db.commit() - - stt_provider = ModelProvider.OPENAI - stt_model = "whisper-1" - - openai_provider = next((p for p in ai_providers if provider_matches(p.provider, ModelProvider.OPENAI)), None) - if not openai_provider: - logger.warning(f"[EvaluatorResult {result.result_id}] No OpenAI provider found, using default whisper-1") - - transcription_start_time = time.time() - - transcription_result = transcription_service.transcribe( - audio_file_key=result.audio_s3_key, - stt_provider=stt_provider, - stt_model=stt_model, - organization_id=result.organization_id, - db=db, - language=None, # Auto-detect - enable_speaker_diarization=True - ) - - transcription_time = time.time() - transcription_start_time - transcription = transcription_result.get("transcript", "") - speaker_segments = transcription_result.get("speaker_segments", []) - - result.transcription = transcription - result.speaker_segments = speaker_segments if speaker_segments else None - db.commit() - - evaluation_time = None - - enabled_metrics = db.query(Metric).filter( - Metric.organization_id == result.organization_id, - Metric.enabled == True - ).all() - enabled_metrics = [ - m for m in enabled_metrics - if (m.name or "").strip().lower() not in REMOVED_EVALUATION_METRIC_NAMES - ] - - # Step 5: Evaluate against enabled metrics using LLM - metric_scores = {} - - # Audio-dependent metrics require actual audio signal analysis - # (Parselmouth, ML models). They cannot be evaluated by the LLM - # from a transcript. When audio IS available they are run through - # the audio analysis pipeline; otherwise they are skipped. - AUDIO_ONLY_METRIC_NAMES = { - "pitch variance", "jitter", "shimmer", "hnr", - "mos score", "emotion category", "emotion confidence", - "valence", "arousal", "speaker consistency", "prosody score", - } - - has_audio = bool(result.audio_s3_key) - llm_metrics = [] - audio_metrics = [] - for m in enabled_metrics: - if m.name.lower() in AUDIO_ONLY_METRIC_NAMES: - if has_audio: - audio_metrics.append(m) - else: - m_type = m.metric_type.value if hasattr(m.metric_type, 'value') else m.metric_type - metric_scores[str(m.id)] = { - "value": None, - "type": m_type.lower() if isinstance(m_type, str) else m_type, - "metric_name": m.name, - "skipped": "audio_required", - } - else: - llm_metrics.append(m) - - # --- Run audio analysis for audio-dependent metrics ---------------- - if audio_metrics and has_audio: - try: - import tempfile as _tempfile - import os as _os - from app.services.s3_service import s3_service - from app.services.voice_quality_service import calculate_audio_metrics - from app.services.qualitative_voice_service import qualitative_voice_service - - logger.info(f"[EvaluatorResult {result.result_id}] Running audio analysis on {len(audio_metrics)} metrics") - audio_bytes = s3_service.download_file_by_key(result.audio_s3_key) - if audio_bytes: - tmp_fd, tmp_path = _tempfile.mkstemp(suffix=".mp3") - _os.close(tmp_fd) - try: - with open(tmp_path, "wb") as _f: - _f.write(audio_bytes) - - audio_metric_names = [m.name for m in audio_metrics] - parselmouth_names = [n for n in audio_metric_names if n.lower() in {"pitch variance", "jitter", "shimmer", "hnr"}] - qualitative_names = [n for n in audio_metric_names if n not in parselmouth_names] - - raw_results: dict = {} - if parselmouth_names: - raw_results.update(calculate_audio_metrics(tmp_path, parselmouth_names, is_url=False)) - if qualitative_names: - raw_results.update(qualitative_voice_service.calculate_metrics(tmp_path, qualitative_names, is_url=False)) - - for m in audio_metrics: - m_type = m.metric_type.value if hasattr(m.metric_type, 'value') else m.metric_type - metric_scores[str(m.id)] = { - "value": raw_results.get(m.name), - "type": m_type.lower() if isinstance(m_type, str) else m_type, - "metric_name": m.name, - } - - logger.info(f"[EvaluatorResult {result.result_id}] Audio analysis complete: {list(raw_results.keys())}") - finally: - if _os.path.exists(tmp_path): - _os.unlink(tmp_path) - else: - logger.warning(f"[EvaluatorResult {result.result_id}] Could not download audio from S3") - for m in audio_metrics: - m_type = m.metric_type.value if hasattr(m.metric_type, 'value') else m.metric_type - metric_scores[str(m.id)] = { - "value": None, - "type": m_type.lower() if isinstance(m_type, str) else m_type, - "metric_name": m.name, - "error": "audio_download_failed", - } - except Exception as audio_err: - logger.error(f"[EvaluatorResult {result.result_id}] Audio analysis failed: {audio_err}", exc_info=True) - for m in audio_metrics: - m_type = m.metric_type.value if hasattr(m.metric_type, 'value') else m.metric_type - metric_scores[str(m.id)] = { - "value": None, - "type": m_type.lower() if isinstance(m_type, str) else m_type, - "metric_name": m.name, - "error": str(audio_err), - } - - if llm_metrics and transcription: - result.status = EvaluatorResultStatus.EVALUATING.value - db.commit() - # Build metric key mapping for later use - metric_key_map = {} # metric_key -> metric object - for metric in llm_metrics: - metric_key = metric.name.lower().replace(" ", "_") - metric_key_map[metric_key] = metric - metric_key_map[metric.name.lower()] = metric - - if is_custom_evaluator: - evaluation_prompt = f"""You are evaluating a conversation transcript against the agent's system prompt. You MUST evaluate ONLY the specific metrics listed below and use the EXACT metric keys provided. - -## Agent System Prompt -The following is the system prompt / instructions that the agent was configured with. Use this to understand the agent's goals, rules, and expected behavior when evaluating the conversation. - -{evaluator.custom_prompt} - -## Conversation Transcript -{transcription} - -## Metrics to Evaluate (use EXACT keys below) -""" - else: - call_type_val = (agent.call_type.value if hasattr(agent.call_type, 'value') else agent.call_type) if agent and agent.call_type else 'conversations' - language_val = (persona.language.value if hasattr(persona.language, 'value') else persona.language) if persona and persona.language else 'N/A' - agent_objective = agent.description if agent and agent.description else f"The agent's objective is to handle {call_type_val}." - scenario_context = scenario.description if scenario and scenario.description else "" - scenario_goals = scenario.required_info if scenario and scenario.required_info else {} - - evaluation_prompt = f"""You are evaluating a conversation transcript. You MUST evaluate ONLY the specific metrics listed below and use the EXACT metric keys provided. - -## Agent Information -- Name: {agent.name if agent else 'Unknown'} -- Objective/Purpose: {agent_objective} -- Call Type: {call_type_val if agent and agent.call_type else 'N/A'} -- Language: {language_val} - -## Scenario Information -- Name: {scenario.name if scenario else 'Unknown'} -- Description: {scenario_context} -- Required Information: {json.dumps(scenario_goals) if scenario_goals else 'N/A'} - -## Conversation Transcript -{transcription} - -## Metrics to Evaluate (use EXACT keys below) +Task implementations live in app/workers/tasks/*.py +Celery app creation lives in app/workers/config.py """ - - # Add metric descriptions to prompt with exact keys - for metric in llm_metrics: - metric_key = metric.name.lower().replace(" ", "_") - metric_desc = metric.description or f"Evaluate {metric.name}" - m_type = metric.metric_type.value if hasattr(metric.metric_type, 'value') else metric.metric_type - if m_type == "rating": - evaluation_prompt += f'\n- "{metric_key}" (rating 0.0-1.0): {metric_desc}' - elif m_type == "boolean": - evaluation_prompt += f'\n- "{metric_key}" (true/false): {metric_desc}' - elif m_type == "number": - evaluation_prompt += f'\n- "{metric_key}" (numeric value): {metric_desc}' - - evaluation_prompt += f""" - -## REQUIRED Response Format -You MUST respond with ONLY a JSON object using the EXACT metric keys listed above. No other keys allowed. - -Example format: -{{ -""" - for metric in llm_metrics: - metric_key = metric.name.lower().replace(" ", "_") - m_type = metric.metric_type.value if hasattr(metric.metric_type, 'value') else metric.metric_type - if m_type == "rating": - evaluation_prompt += f' "{metric_key}": 0.75,\n' - elif m_type == "boolean": - evaluation_prompt += f' "{metric_key}": true,\n' - elif m_type == "number": - evaluation_prompt += f' "{metric_key}": 5,\n' - - evaluation_prompt += """} - -CRITICAL RULES: -1. Use the EXACT metric keys shown above - copy them character-for-character -2. Each value must be a SINGLE NUMBER (not an object with score/comments) -3. Do NOT wrap in "metrics" or any other object -4. Do NOT add comments or explanations -5. Return ONLY the JSON object, nothing else""" - - # Call LLM service for evaluation - try: - # Determine LLM provider and model from evaluator config, falling back to defaults - evaluator_llm_provider = getattr(evaluator, 'llm_provider', None) if evaluator else None - evaluator_llm_model = getattr(evaluator, 'llm_model', None) if evaluator else None - - if evaluator_llm_provider and evaluator_llm_model: - if isinstance(evaluator_llm_provider, str): - llm_provider = ModelProvider(evaluator_llm_provider.lower()) - else: - llm_provider = evaluator_llm_provider - llm_model = evaluator_llm_model - else: - llm_provider = ModelProvider.OPENAI - llm_model = "gpt-4o" - - chosen_provider = next((p for p in ai_providers if provider_matches(p.provider, llm_provider)), None) - if not chosen_provider: - logger.warning(f"[EvaluatorResult {result.result_id}] Provider {llm_provider.value} not configured, evaluation may fail") - - # Build the list of exact metric keys for system message - exact_keys = [metric.name.lower().replace(" ", "_") for metric in llm_metrics] - - messages = [ - {"role": "system", "content": f"""You are an expert conversation evaluator. You MUST follow these rules STRICTLY: - -1. Return ONLY valid JSON - no markdown, no explanations, no comments -2. Use ONLY these exact metric keys (copy-paste them exactly): {json.dumps(exact_keys)} -3. Each value must be a single number (0.0-1.0 for ratings, 0 or 1 for boolean) - NO nested objects, NO comments -4. Do NOT rename, abbreviate, or modify the metric keys in any way - -Example of CORRECT format: -{{"follow_instructions": 0.8, "clarity_and_empathy": 0.7}} - -Example of WRONG format (DO NOT do this): -{{"metrics": {{"Clarity": {{"score": 7}}}}}}"""}, - {"role": "user", "content": evaluation_prompt} - ] - - evaluation_start_time = time.time() - - llm_result = llm_service.generate_response( - messages=messages, - llm_provider=llm_provider, - llm_model=llm_model, - organization_id=result.organization_id, - db=db, - temperature=0.3, # Lower temperature for more consistent evaluations - max_tokens=2000 - ) - - evaluation_time = time.time() - evaluation_start_time - - response_text = llm_result["text"].strip() - - # Try to extract JSON from response (handle cases where LLM adds markdown formatting) - if response_text.startswith("```json"): - response_text = response_text.replace("```json", "").replace("```", "").strip() - elif response_text.startswith("```"): - response_text = response_text.replace("```", "").strip() - - try: - evaluation_data = json.loads(response_text) - except json.JSONDecodeError as e: - logger.warning(f"[EvaluatorResult {result.result_id}] JSON parsing failed, attempting regex extraction") - json_match = re.search(r'\{[\s\S]*\}', response_text) - if json_match: - try: - evaluation_data = json.loads(json_match.group()) - except json.JSONDecodeError: - raise ValueError("Could not parse extracted JSON") - else: - raise ValueError("Could not parse LLM response as JSON") - - if "metrics" in evaluation_data and isinstance(evaluation_data["metrics"], dict): - evaluation_data = evaluation_data["metrics"] - - # Helper function to extract score from various formats - def extract_score(value): - """Extract numeric/boolean score from various response formats.""" - if value is None: - return None - # Direct value - if isinstance(value, (int, float, bool)): - return value - # Nested object with 'score' field - if isinstance(value, dict): - if 'score' in value: - return value['score'] - if 'value' in value: - return value['value'] - if 'rating' in value: - return value['rating'] - # String that might be a number - if isinstance(value, str): - try: - return float(value) - except ValueError: - if value.lower() in ('true', 'yes'): - return True - if value.lower() in ('false', 'no'): - return False - return None - - # Helper function to find matching key in response (case-insensitive, fuzzy) - def find_matching_key(target_key, response_keys): - """Find a matching key in the response, with fuzzy matching.""" - target_lower = target_key.lower().replace(" ", "_").replace("-", "_") - target_words = set(target_lower.replace("_", " ").split()) - - # Exact match (case-insensitive) - for key in response_keys: - if key.lower().replace(" ", "_").replace("-", "_") == target_lower: - return key - - # Partial match (key contains target or target contains key) - for key in response_keys: - key_lower = key.lower().replace(" ", "_").replace("-", "_") - if target_lower in key_lower or key_lower in target_lower: - return key - - # Word overlap match - best_match = None - best_overlap = 0 - for key in response_keys: - key_words = set(key.lower().replace("_", " ").replace("-", " ").split()) - overlap = len(target_words & key_words) - if overlap > best_overlap: - best_overlap = overlap - best_match = key - - if best_overlap >= 1: # At least one word matches - return best_match - - return None - - response_keys = list(evaluation_data.keys()) - - for metric in llm_metrics: - metric_key = metric.name.lower().replace(" ", "_") - m_type = metric.metric_type.value if hasattr(metric.metric_type, 'value') else metric.metric_type - - # Try exact key first - raw_score = evaluation_data.get(metric_key) - - # If not found, try fuzzy matching - if raw_score is None: - matched_key = find_matching_key(metric.name, response_keys) - if matched_key: - raw_score = evaluation_data.get(matched_key) - - # Extract score from various formats - score = extract_score(raw_score) - - # Validate and convert score based on metric type - if m_type == "rating": - if score is not None: - try: - score = float(score) - # If score is 0-10 range, normalize to 0-1 - if score > 1.0: - score = score / 10.0 - # Clamp to 0.0-1.0 range - score = max(0.0, min(1.0, score)) - except (ValueError, TypeError): - score = None - elif m_type == "boolean": - if score is not None: - if isinstance(score, bool): - pass # Already boolean - elif isinstance(score, (int, float)): - score = score > 0.5 if score <= 1 else score > 5 # Handle 0-1 or 0-10 ranges - else: - score = bool(score) - elif m_type == "number": - if score is not None: - try: - score = float(score) - except (ValueError, TypeError): - score = None - - metric_scores[str(metric.id)] = { - "value": score, - "type": m_type.lower() if isinstance(m_type, str) else m_type, - "metric_name": metric.name - } - - except Exception as e: - # Use str() to avoid format issues with curly braces in error messages - error_msg = str(e).replace("{", "{{").replace("}", "}}") - logger.error(f"[EvaluatorResult {result.result_id}] ✗ LLM evaluation failed: {error_msg}", exc_info=True) - # If LLM evaluation fails, mark metrics as None but don't fail the whole task - for metric in llm_metrics: - m_type = metric.metric_type.value if hasattr(metric.metric_type, 'value') else metric.metric_type - metric_scores[str(metric.id)] = { - "value": None, - "type": m_type.lower() if isinstance(m_type, str) else m_type, - "metric_name": metric.name, - "error": str(e) - } - else: - if not llm_metrics: - logger.warning(f"[EvaluatorResult {result.result_id}] No LLM-evaluable metrics found (audio-only metrics were skipped), skipping evaluation") - if not transcription: - logger.warning(f"[EvaluatorResult {result.result_id}] No transcription available, skipping evaluation") - - # Update status to COMPLETED - result.metric_scores = metric_scores - result.status = EvaluatorResultStatus.COMPLETED.value - db.commit() - - total_time = time.time() - task_start_time - logger.info(f"[EvaluatorResult {result.result_id}] Completed in {total_time:.2f}s, {len(metric_scores)} metrics evaluated") - - return { - "result_id": result_id, - "status": "completed", - "transcription": transcription, - "metrics_evaluated": len(metric_scores), - "processing_time": total_time, - "transcription_time": transcription_time if 'transcription_time' in locals() else None, - "evaluation_time": evaluation_time if 'evaluation_time' in locals() else None - } - - except Exception as e: - # Mark as failed - logger.error(f"[EvaluatorResult {result.result_id}] Processing failed: {e}", exc_info=True) - result.status = EvaluatorResultStatus.FAILED.value - result.error_message = str(e) - db.commit() - raise - - except Exception as exc: - # Retry on failure - raise self.retry(exc=exc, countdown=60) - finally: - db.close() - - -@celery_app.task(name="run_evaluator", bind=True, max_retries=3) -def run_evaluator_task(self, evaluator_id: str, evaluator_result_id: str): - """ - Celery task to run an evaluator: bridge test agent to Voice AI agent and record conversation. - - Args: - self: Task instance - evaluator_id: Evaluator ID as string - evaluator_result_id: Pre-created EvaluatorResult ID as string - - Returns: - Dictionary with execution results - """ - db = SessionLocal() - task_start_time = time.time() - - try: - from app.models.database import ( - Evaluator, EvaluatorResult, EvaluatorResultStatus, - Agent, Persona, Scenario - ) - from app.services.test_agent_bridge_service import test_agent_bridge_service - import asyncio - - evaluator_uuid = UUID(evaluator_id) - result_uuid = UUID(evaluator_result_id) - - evaluator = db.query(Evaluator).filter(Evaluator.id == evaluator_uuid).first() - if not evaluator: - logger.error(f"[RunEvaluator {evaluator_id}] Evaluator not found") - return {"error": "Evaluator not found"} - - result = db.query(EvaluatorResult).filter(EvaluatorResult.id == result_uuid).first() - if not result: - logger.error(f"[RunEvaluator {evaluator_id}] EvaluatorResult not found") - return {"error": "EvaluatorResult not found"} - - agent = db.query(Agent).filter(Agent.id == evaluator.agent_id).first() - if not agent: - logger.error(f"[RunEvaluator {evaluator_id}] Agent not found") - result.status = EvaluatorResultStatus.FAILED.value - result.error_message = "Agent not found" - db.commit() - return {"error": "Agent not found"} - - logger.info(f"[RunEvaluator {evaluator.evaluator_id}] Starting task (Result: {result.result_id})") - - result.celery_task_id = self.request.id - if result.status != EvaluatorResultStatus.QUEUED.value: - logger.warning(f"[RunEvaluator {evaluator.evaluator_id}] Status was {result.status}, expected QUEUED") - db.commit() - - has_voice_bundle = agent.voice_bundle_id is not None - has_voice_ai_integration = agent.voice_ai_integration_id is not None and agent.voice_ai_agent_id is not None - - if has_voice_bundle and has_voice_ai_integration: - - try: - result.status = EvaluatorResultStatus.CALL_INITIATING.value - result.call_event = "task_started" - db.commit() - - loop = asyncio.get_event_loop() - if loop.is_closed(): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - bridge_result = loop.run_until_complete( - test_agent_bridge_service.bridge_test_agent_to_voice_agent( - evaluator_id=evaluator_uuid, - evaluator_result_id=result_uuid, - organization_id=evaluator.organization_id, - db=db, - ) - ) - - # Don't reset status - the bridge service should have updated it - # Refresh the result to get the latest status from the bridge service - db.refresh(result) - - # Only update error_message if it was set to clear any temporary call info - if result.error_message and result.error_message.startswith("call_id:"): - # Keep the call info for now, it will be cleared when call ends - pass - - db.commit() - - return { - "evaluator_id": evaluator_id, - "result_id": evaluator_result_id, - "status": "initiated", - "bridge_result": bridge_result, - } - - except Exception as bridge_error: - logger.error(f"[RunEvaluator {evaluator.evaluator_id}] Bridge service error: {bridge_error}", exc_info=True) - result.status = EvaluatorResultStatus.FAILED.value - result.error_message = str(bridge_error) - result.call_event = "bridge_error" - db.commit() - raise - - elif has_voice_bundle: - result.status = EvaluatorResultStatus.FAILED.value - result.error_message = "Standard voice agent flow not yet implemented for evaluator runs" - db.commit() - return {"error": "Standard flow not implemented"} - - else: - logger.error(f"[RunEvaluator {evaluator.evaluator_id}] Agent missing required configuration") - result.status = EvaluatorResultStatus.FAILED.value - result.error_message = f"Agent missing required configuration: voice_bundle={has_voice_bundle}, voice_ai_integration={has_voice_ai_integration}" - result.call_event = "configuration_error" - db.commit() - return {"error": "Agent does not have required configuration for bridging"} - - except Exception as exc: - logger.error(f"[RunEvaluator {evaluator_id}] Task failed: {exc}", exc_info=True) - # Update result status - try: - result = db.query(EvaluatorResult).filter(EvaluatorResult.id == UUID(evaluator_result_id)).first() - if result: - result.status = EvaluatorResultStatus.FAILED.value - result.error_message = str(exc) - db.commit() - except: - pass - # Retry on failure - raise self.retry(exc=exc, countdown=60) - finally: - db.close() - - -# ====================================================================== -# TTS Comparison Tasks (Voice Playground) -# ====================================================================== - - -@celery_app.task(name="generate_tts_comparison", bind=True, max_retries=1) -def generate_tts_comparison_task(self, comparison_id: str): - """ - Generate TTS audio for every sample in a comparison, upload to S3, - then dispatch evaluation. - """ - from app.models.database import ( - TTSComparison, TTSSample, - TTSComparisonStatus, TTSSampleStatus, ModelProvider, - ) - from app.services.tts_service import tts_service, get_audio_file_extension - - db = SessionLocal() - try: - comp = db.query(TTSComparison).filter(TTSComparison.id == UUID(comparison_id)).first() - if not comp: - logger.error(f"[TTS Generate] Comparison {comparison_id} not found") - return {"error": "not found"} - - comp.status = TTSComparisonStatus.GENERATING.value - db.commit() - - samples = ( - db.query(TTSSample) - .filter(TTSSample.comparison_id == comp.id) - .order_by(TTSSample.sample_index) - .all() - ) - - voice_configs_a = {v["id"]: v for v in (comp.voices_a or []) if isinstance(v, dict)} - voice_configs_b = {v["id"]: v for v in (comp.voices_b or []) if isinstance(v, dict)} - - def _resolve_voice_meta(sample_obj): - """Match sample to correct side's voice config (A or B).""" - if sample_obj.side == "A": - return voice_configs_a.get(sample_obj.voice_id) or {} - if sample_obj.side == "B": - return voice_configs_b.get(sample_obj.voice_id) or {} - # Fallback for legacy samples without a side column - is_side_a = ( - sample_obj.provider == comp.provider_a and sample_obj.model == comp.model_a - ) - is_side_b = ( - sample_obj.provider == comp.provider_b and sample_obj.model == comp.model_b - ) - if is_side_a and not is_side_b: - return voice_configs_a.get(sample_obj.voice_id) or {} - if is_side_b and not is_side_a: - return voice_configs_b.get(sample_obj.voice_id) or {} - return voice_configs_a.get(sample_obj.voice_id) or voice_configs_b.get(sample_obj.voice_id) or {} - - failed_count = 0 - for sample in samples: - try: - sample.status = TTSSampleStatus.GENERATING.value - db.commit() - - voice_meta = _resolve_voice_meta(sample) - tts_config = {} - sample_rate_hz = voice_meta.get("sample_rate_hz") - if sample_rate_hz: - tts_config["sample_rate_hz"] = int(sample_rate_hz) - language_code = voice_meta.get("language_code") - if language_code: - tts_config["language_code"] = language_code - - provider_enum = ModelProvider(sample.provider) - logger.info( - f"[TTS Generate] Sample {sample.id} – " - f"provider={sample.provider} voice={sample.voice_id} " - f"sample_rate_hz={sample_rate_hz} config={tts_config}" - ) - audio_bytes, latency_ms, ttfb_ms = tts_service.synthesize_timed( - text=sample.text, - tts_provider=provider_enum, - tts_model=sample.model, - organization_id=comp.organization_id, - db=db, - voice=sample.voice_id, - config=tts_config or None, - ) - - from app.services.s3_service import s3_service - - audio_ext = get_audio_file_extension(sample.provider, int(sample_rate_hz) if sample_rate_hz else None) - s3_key = s3_service.upload_file_by_key( - file_content=audio_bytes, - key=f"{s3_service.prefix}organizations/{comp.organization_id}/voicePlayground/{comp.id}/{sample.id}.{audio_ext}", - ) - - if audio_ext == "wav" and len(audio_bytes) > 44: - import struct as _struct - sr = _struct.unpack_from(' 0 else None - else: - duration_est = len(audio_bytes) / (128000 / 8) if audio_bytes else None - - sample.audio_s3_key = s3_key - sample.latency_ms = round(latency_ms, 1) - sample.ttfb_ms = round(ttfb_ms, 1) - sample.duration_seconds = round(duration_est, 2) if duration_est else None - sample.status = TTSSampleStatus.COMPLETED.value - db.commit() - - logger.info( - f"[TTS Generate] Sample {sample.id} done – " - f"{sample.provider}/{sample.voice_name} ttfb={ttfb_ms:.0f}ms total={latency_ms:.0f}ms" - ) - - except Exception as e: - logger.error(f"[TTS Generate] Sample {sample.id} failed: {e}") - sample.status = TTSSampleStatus.FAILED.value - sample.error_message = str(e)[:500] - db.commit() - failed_count += 1 - - total = len(samples) - if failed_count == total: - comp.status = TTSComparisonStatus.FAILED.value - comp.error_message = "All samples failed to generate" - db.commit() - return {"error": "all failed"} - # Dispatch evaluation - comp.status = TTSComparisonStatus.EVALUATING.value - db.commit() - evaluate_tts_comparison_task.delay(comparison_id) - - return {"generated": total - failed_count, "failed": failed_count} - - except Exception as exc: - logger.error(f"[TTS Generate] Task failed: {exc}", exc_info=True) - try: - comp = db.query(TTSComparison).filter(TTSComparison.id == UUID(comparison_id)).first() - if comp: - comp.status = TTSComparisonStatus.FAILED.value - comp.error_message = str(exc)[:500] - db.commit() - except Exception: - pass - raise self.retry(exc=exc, countdown=30) - finally: - db.close() - - -def _compute_wer_cer(ground_truth: str, predicted: str): - """Compute raw and normalized WER/CER between reference and ASR text. - - Normalized scores reduce false penalties on numeric/currency phrasing - differences (for example "$1,234.56" vs "one thousand two hundred..."). - """ - import re - import string - try: - from jiwer import wer, cer - except ImportError: - logger.warning("[TTS Eval] jiwer not installed – skipping WER/CER") - return { - "raw_wer": None, - "raw_cer": None, - "normalized_wer": None, - "normalized_cer": None, - } - - number_words = { - "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", - "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", - "seventeen", "eighteen", "nineteen", "twenty", "thirty", "forty", "fifty", - "sixty", "seventy", "eighty", "ninety", "hundred", "thousand", "million", - "billion", "trillion", "point", "and", - } - currency_words = { - "dollar", "dollars", "usd", "cent", "cents", "rupee", "rupees", "inr", - "euro", "euros", "eur", "pound", "pounds", "gbp", - } - - def _is_numeric_token(token: str) -> bool: - return bool(re.fullmatch(r"\d+(?:\.\d+)?", token)) - - def _normalize_base(text: str) -> str: - punct_table = str.maketrans("", "", string.punctuation) - return text.lower().translate(punct_table).strip() - - def _normalize_entities(text: str) -> str: - tokens = _normalize_base(text).split() - normalized_tokens = [] - i = 0 - while i < len(tokens): - token = tokens[i] - is_entity_token = ( - _is_numeric_token(token) - or token in number_words - or token in currency_words - ) - if not is_entity_token: - normalized_tokens.append(token) - i += 1 - continue - - j = i - has_currency = token in currency_words - while j < len(tokens): - t = tokens[j] - if _is_numeric_token(t) or t in number_words or t in currency_words: - if t in currency_words: - has_currency = True - j += 1 - continue - break - - normalized_tokens.append("" if has_currency else "") - i = j - - return " ".join(normalized_tokens) - ref = _normalize_base(ground_truth) - hyp = _normalize_base(predicted) - - if not ref: - return { - "raw_wer": None, - "raw_cer": None, - "normalized_wer": None, - "normalized_cer": None, - } - - try: - raw_wer = round(wer(ref, hyp), 4) - raw_cer = round(cer(ref, hyp), 4) - - norm_ref = _normalize_entities(ground_truth) - norm_hyp = _normalize_entities(predicted) - normalized_wer = round(wer(norm_ref, norm_hyp), 4) if norm_ref else None - normalized_cer = round(cer(norm_ref, norm_hyp), 4) if norm_ref else None - - return { - "raw_wer": raw_wer, - "raw_cer": raw_cer, - "normalized_wer": normalized_wer, - "normalized_cer": normalized_cer, - } - except Exception as e: - logger.warning(f"[TTS Eval] WER/CER calculation error: {e}") - return { - "raw_wer": None, - "raw_cer": None, - "normalized_wer": None, - "normalized_cer": None, - } - - -# Singleton for the NeMo ASR model (loaded once per worker process) -_nemo_asr_model = None - - -def _get_nemo_asr_model(): - """Lazy-load NVIDIA NeMo Conformer CTC model for hallucination detection. - - Requires: pip install efficientai[nemo-asr] - Returns the model instance, or None if NeMo is not installed. - """ - global _nemo_asr_model - - if _nemo_asr_model is not None: - return _nemo_asr_model - - try: - import nemo.collections.asr as nemo_asr - logger.info("[TTS Eval] Loading NeMo ASR model (stt_en_conformer_ctc_large)...") - _nemo_asr_model = nemo_asr.models.ASRModel.from_pretrained("stt_en_conformer_ctc_large") - logger.info("[TTS Eval] NeMo ASR model loaded successfully") - return _nemo_asr_model - except ImportError as e: - logger.warning( - f"[TTS Eval] NeMo import failed: {e} – " - "WER/CER hallucination metrics will be skipped. " - "To enable, run:\n" - " pip install 'nemo_toolkit[asr]'\n" - " python -c \"import nemo.collections.asr as nemo_asr; " - "nemo_asr.models.ASRModel.from_pretrained('stt_en_conformer_ctc_large')\"" - ) - except Exception as e: - logger.error( - f"[TTS Eval] NeMo ASR model failed to load: {e} – " - "The model may not be cached yet. To download it manually, run:\n" - " python -c \"import nemo.collections.asr as nemo_asr; " - "nemo_asr.models.ASRModel.from_pretrained('stt_en_conformer_ctc_large')\"", - exc_info=True, - ) - - return None - - -def _transcribe_audio_for_eval(audio_path: str) -> str | None: - """Transcribe an audio file using NVIDIA NeMo Conformer CTC. - - Runs entirely on the worker – no API key needed. - """ - model = _get_nemo_asr_model() - if model is None: - return None - - try: - transcriptions = model.transcribe([audio_path]) - if transcriptions and len(transcriptions) > 0: - text = transcriptions[0] - # NeMo may return Hypothesis objects in some versions - if hasattr(text, "text"): - text = text.text - return str(text).strip() or None - return None - except Exception as e: - logger.warning(f"[TTS Eval] ASR transcription failed: {e}") - return None - - -@celery_app.task(name="evaluate_tts_comparison", bind=True, max_retries=1) -def evaluate_tts_comparison_task(self, comparison_id: str): - """ - Download each completed sample from S3 and run qualitative voice - metrics (MOS, Valence, Arousal, Prosody) plus ASR-based WER/CER - for hallucination detection. - """ - import tempfile - import os - from app.models.database import ( - TTSComparison, TTSSample, TTSComparisonStatus, TTSSampleStatus, - ) - from app.services.s3_service import s3_service - - db = SessionLocal() - try: - comp = db.query(TTSComparison).filter(TTSComparison.id == UUID(comparison_id)).first() - if not comp: - return {"error": "not found"} - - samples = ( - db.query(TTSSample) - .filter( - TTSSample.comparison_id == comp.id, - TTSSample.status == TTSSampleStatus.COMPLETED.value, - TTSSample.audio_s3_key.isnot(None), - ) - .all() - ) - - if not samples: - comp.status = TTSComparisonStatus.COMPLETED.value - db.commit() - return {"evaluated": 0} - - # Lazy-load qualitative service inside the worker - from app.services.qualitative_voice_service import qualitative_voice_service - - # Pre-warm the NeMo ASR model once for the batch (lazy singleton) - nemo_model = _get_nemo_asr_model() - - evaluated = 0 - for sample in samples: - tmp_path = None - try: - # Download from S3 to temp file - audio_bytes = s3_service.download_file_by_key(sample.audio_s3_key) - if not audio_bytes: - continue - - ext = ".mp3" - if sample.audio_s3_key: - key_ext = os.path.splitext(sample.audio_s3_key)[1].lower() - if key_ext in {".wav", ".mp3", ".flac", ".ogg", ".m4a"}: - ext = key_ext - tmp_fd, tmp_path = tempfile.mkstemp(suffix=ext) - os.close(tmp_fd) - with open(tmp_path, "wb") as f: - f.write(audio_bytes) - - metrics = qualitative_voice_service.calculate_all_metrics(tmp_path) - - # ASR-based hallucination detection (WER / CER) - if nemo_model is not None and sample.text: - asr_transcript = _transcribe_audio_for_eval(tmp_path) - if asr_transcript: - score_bundle = _compute_wer_cer(sample.text, asr_transcript) - metrics["WER Raw"] = score_bundle.get("raw_wer") - metrics["CER Raw"] = score_bundle.get("raw_cer") - metrics["WER Normalized"] = score_bundle.get("normalized_wer") - metrics["CER Normalized"] = score_bundle.get("normalized_cer") - metrics["WER"] = ( - score_bundle.get("normalized_wer") - if score_bundle.get("normalized_wer") is not None - else score_bundle.get("raw_wer") - ) - metrics["CER"] = ( - score_bundle.get("normalized_cer") - if score_bundle.get("normalized_cer") is not None - else score_bundle.get("raw_cer") - ) - metrics["ASR Transcript"] = asr_transcript - else: - metrics["WER"] = None - metrics["CER"] = None - metrics["WER Raw"] = None - metrics["CER Raw"] = None - metrics["WER Normalized"] = None - metrics["CER Normalized"] = None - metrics["ASR Transcript"] = None - - sample.evaluation_metrics = metrics - db.commit() - evaluated += 1 - - logger.info(f"[TTS Eval] Sample {sample.id} metrics: {metrics}") - - except Exception as e: - logger.warning(f"[TTS Eval] Sample {sample.id} eval failed: {e}") - finally: - if tmp_path and os.path.exists(tmp_path): - try: - os.unlink(tmp_path) - except Exception: - pass - - # Build summary - from app.api.v1.routes.voice_playground import _recompute_summary - _recompute_summary(comp, db) - - comp.status = TTSComparisonStatus.COMPLETED.value - db.commit() - - logger.info(f"[TTS Eval] Comparison {comparison_id} complete – {evaluated}/{len(samples)} evaluated") - return {"evaluated": evaluated} - - except Exception as exc: - logger.error(f"[TTS Eval] Task failed: {exc}", exc_info=True) - try: - comp = db.query(TTSComparison).filter(TTSComparison.id == UUID(comparison_id)).first() - if comp: - comp.status = TTSComparisonStatus.FAILED.value - comp.error_message = f"Evaluation failed: {str(exc)[:400]}" - db.commit() - except Exception: - pass - raise self.retry(exc=exc, countdown=30) - finally: - db.close() - - -@celery_app.task(name="generate_tts_report_pdf", bind=True, max_retries=1) -def generate_tts_report_pdf_task(self, report_job_id: str): - """Generate a Voice Playground PDF report and store it in S3.""" - from app.models.database import ( - TTSComparison, - TTSSample, - TTSReportJob, - TTSReportJobStatus, - ) - from app.services.s3_service import s3_service - from app.services.voice_playground_report_service import voice_playground_report_service - - db = SessionLocal() - try: - report_job = db.query(TTSReportJob).filter(TTSReportJob.id == UUID(report_job_id)).first() - if not report_job: - logger.error(f"[TTS Report] Job {report_job_id} not found") - return {"error": "report_job_not_found"} - - comparison = ( - db.query(TTSComparison) - .filter( - TTSComparison.id == report_job.comparison_id, - TTSComparison.organization_id == report_job.organization_id, - ) - .first() - ) - if not comparison: - report_job.status = TTSReportJobStatus.FAILED.value - report_job.error_message = "Comparison not found" - db.commit() - return {"error": "comparison_not_found"} - - report_job.status = TTSReportJobStatus.PROCESSING.value - report_job.celery_task_id = self.request.id - db.commit() - - samples = ( - db.query(TTSSample) - .filter(TTSSample.comparison_id == comparison.id) - .order_by(TTSSample.run_index, TTSSample.sample_index) - .all() - ) - - payload = voice_playground_report_service.build_payload(comparison, samples) - pdf_bytes = voice_playground_report_service.render_pdf(payload) - - report_filename = ( - f"voice-playground-report-{comparison.simulation_id or str(comparison.id)[:8]}.pdf" - ) - s3_key = ( - f"{s3_service.prefix}organizations/{report_job.organization_id}/voicePlayground/" - f"{comparison.id}/reports/{report_job.id}.pdf" - ) - s3_service.upload_file_by_key( - file_content=pdf_bytes, - key=s3_key, - content_type="application/pdf", - ) - - report_job.status = TTSReportJobStatus.COMPLETED.value - report_job.filename = report_filename - report_job.s3_key = s3_key - report_job.error_message = None - db.commit() +from app.workers.config import celery_app +from app.workers.tasks import ( + process_evaluation_task, + process_evaluator_result_task, + run_evaluator_task, + generate_tts_comparison_task, + evaluate_tts_comparison_task, + generate_tts_report_pdf_task, +) - return {"status": "completed", "s3_key": s3_key} - except Exception as exc: - logger.error(f"[TTS Report] Task failed: {exc}", exc_info=True) - try: - report_job = db.query(TTSReportJob).filter(TTSReportJob.id == UUID(report_job_id)).first() - if report_job: - report_job.status = TTSReportJobStatus.FAILED.value - report_job.error_message = str(exc)[:500] - db.commit() - except Exception: - pass - raise self.retry(exc=exc, countdown=30) - finally: - db.close() +__all__ = [ + "celery_app", + "process_evaluation_task", + "process_evaluator_result_task", + "run_evaluator_task", + "generate_tts_comparison_task", + "evaluate_tts_comparison_task", + "generate_tts_report_pdf_task", +] diff --git a/app/workers/config.py b/app/workers/config.py new file mode 100644 index 00000000..c7a0ae7c --- /dev/null +++ b/app/workers/config.py @@ -0,0 +1,37 @@ +"""Celery application configuration and creation.""" + +from pathlib import Path + +from celery import Celery +from loguru import logger + +from app.config import settings, load_config_from_file + +# Load config.yml if it exists (before using settings) +# This ensures the Celery worker has the same configuration as the main app +_config_path = Path("config.yml") +if _config_path.exists(): + try: + load_config_from_file(str(_config_path)) + logger.info(f"✅ Celery worker loaded configuration from {_config_path}") + except Exception as e: + logger.warning(f"⚠️ Celery worker: Could not load config.yml: {e}") + +# Create Celery app +celery_app = Celery( + "efficientai", + broker=settings.CELERY_BROKER_URL, + backend=settings.CELERY_RESULT_BACKEND, +) + +# Celery configuration +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_track_started=True, + task_time_limit=30 * 60, # 30 minutes + task_soft_time_limit=25 * 60, # 25 minutes +) diff --git a/app/workers/tasks/__init__.py b/app/workers/tasks/__init__.py new file mode 100644 index 00000000..3bcf3fb2 --- /dev/null +++ b/app/workers/tasks/__init__.py @@ -0,0 +1,27 @@ +"""Celery task modules - import to register tasks with the app.""" + +from app.workers.config import celery_app + +# Import task modules to register tasks with Celery +from . import process_evaluation +from . import process_evaluator_result +from . import run_evaluator +from . import tts_comparison +from . import tts_report + +__all__ = [ + "celery_app", + "process_evaluation_task", + "process_evaluator_result_task", + "run_evaluator_task", + "generate_tts_comparison_task", + "evaluate_tts_comparison_task", + "generate_tts_report_pdf_task", +] + +process_evaluation_task = process_evaluation.process_evaluation_task +process_evaluator_result_task = process_evaluator_result.process_evaluator_result_task +run_evaluator_task = run_evaluator.run_evaluator_task +generate_tts_comparison_task = tts_comparison.generate_tts_comparison_task +evaluate_tts_comparison_task = tts_comparison.evaluate_tts_comparison_task +generate_tts_report_pdf_task = tts_report.generate_tts_report_pdf_task diff --git a/app/workers/tasks/helpers/__init__.py b/app/workers/tasks/helpers/__init__.py new file mode 100644 index 00000000..6dd13ae9 --- /dev/null +++ b/app/workers/tasks/helpers/__init__.py @@ -0,0 +1,17 @@ +"""Helper modules for evaluator result processing.""" + +from .constants import REMOVED_EVALUATION_METRIC_NAMES, AUDIO_ONLY_METRIC_NAMES +from .score_utils import provider_matches, extract_score, find_matching_key +from .audio_evaluation import evaluate_audio_metrics +from .llm_evaluation import build_evaluation_prompt, evaluate_with_llm + +__all__ = [ + "REMOVED_EVALUATION_METRIC_NAMES", + "AUDIO_ONLY_METRIC_NAMES", + "provider_matches", + "extract_score", + "find_matching_key", + "evaluate_audio_metrics", + "build_evaluation_prompt", + "evaluate_with_llm", +] diff --git a/app/workers/tasks/helpers/audio_evaluation.py b/app/workers/tasks/helpers/audio_evaluation.py new file mode 100644 index 00000000..71a167ba --- /dev/null +++ b/app/workers/tasks/helpers/audio_evaluation.py @@ -0,0 +1,101 @@ +"""Audio metrics evaluation using Parselmouth and qualitative voice services.""" + +import os +import tempfile +from typing import Any + +from loguru import logger + +from .score_utils import get_metric_type_value + + +PARSELMOUTH_METRIC_NAMES = {"pitch variance", "jitter", "shimmer", "hnr"} + + +def evaluate_audio_metrics( + audio_s3_key: str, + audio_metrics: list, + result_id: str, +) -> dict[str, dict[str, Any]]: + """ + Evaluate audio-dependent metrics by downloading audio and running analysis. + + Args: + audio_s3_key: S3 key for the audio file + audio_metrics: List of Metric objects to evaluate + result_id: Result ID for logging + + Returns: + Dictionary mapping metric ID to score info + """ + from app.services.s3_service import s3_service + from app.services.voice_quality_service import calculate_audio_metrics + from app.services.qualitative_voice_service import qualitative_voice_service + + metric_scores: dict[str, dict[str, Any]] = {} + + logger.info(f"[EvaluatorResult {result_id}] Running audio analysis on {len(audio_metrics)} metrics") + + audio_bytes = s3_service.download_file_by_key(audio_s3_key) + if not audio_bytes: + logger.warning(f"[EvaluatorResult {result_id}] Could not download audio from S3") + for m in audio_metrics: + metric_scores[str(m.id)] = { + "value": None, + "type": get_metric_type_value(m), + "metric_name": m.name, + "error": "audio_download_failed", + } + return metric_scores + + tmp_fd, tmp_path = tempfile.mkstemp(suffix=".mp3") + os.close(tmp_fd) + + try: + with open(tmp_path, "wb") as f: + f.write(audio_bytes) + + audio_metric_names = [m.name for m in audio_metrics] + parselmouth_names = [n for n in audio_metric_names if n.lower() in PARSELMOUTH_METRIC_NAMES] + qualitative_names = [n for n in audio_metric_names if n not in parselmouth_names] + + raw_results: dict = {} + + if parselmouth_names: + raw_results.update(calculate_audio_metrics(tmp_path, parselmouth_names, is_url=False)) + + if qualitative_names: + raw_results.update( + qualitative_voice_service.calculate_metrics(tmp_path, qualitative_names, is_url=False) + ) + + for m in audio_metrics: + metric_scores[str(m.id)] = { + "value": raw_results.get(m.name), + "type": get_metric_type_value(m), + "metric_name": m.name, + } + + logger.info(f"[EvaluatorResult {result_id}] Audio analysis complete: {list(raw_results.keys())}") + + finally: + if os.path.exists(tmp_path): + os.unlink(tmp_path) + + return metric_scores + + +def handle_audio_evaluation_error( + audio_metrics: list, + error: Exception, +) -> dict[str, dict[str, Any]]: + """Build error response for all audio metrics when evaluation fails.""" + metric_scores: dict[str, dict[str, Any]] = {} + for m in audio_metrics: + metric_scores[str(m.id)] = { + "value": None, + "type": get_metric_type_value(m), + "metric_name": m.name, + "error": str(error), + } + return metric_scores diff --git a/app/workers/tasks/helpers/constants.py b/app/workers/tasks/helpers/constants.py new file mode 100644 index 00000000..56de8f4a --- /dev/null +++ b/app/workers/tasks/helpers/constants.py @@ -0,0 +1,17 @@ +"""Constants for evaluator result processing.""" + +REMOVED_EVALUATION_METRIC_NAMES = {"clarity and empathy"} + +AUDIO_ONLY_METRIC_NAMES = { + "pitch variance", + "jitter", + "shimmer", + "hnr", + "mos score", + "emotion category", + "emotion confidence", + "valence", + "arousal", + "speaker consistency", + "prosody score", +} diff --git a/app/workers/tasks/helpers/llm_evaluation.py b/app/workers/tasks/helpers/llm_evaluation.py new file mode 100644 index 00000000..8b1959e3 --- /dev/null +++ b/app/workers/tasks/helpers/llm_evaluation.py @@ -0,0 +1,318 @@ +"""LLM-based evaluation: prompt building and response parsing.""" + +import json +import re +import time +from typing import Any +from uuid import UUID + +from loguru import logger + +from app.models.database import ModelProvider + +from .score_utils import ( + provider_matches, + extract_score, + find_matching_key, + get_metric_type_value, + normalize_score, +) + + +def build_evaluation_prompt( + transcription: str, + llm_metrics: list, + evaluator=None, + agent=None, + persona=None, + scenario=None, +) -> str: + """ + Build the evaluation prompt for LLM-based metric evaluation. + + Args: + transcription: The conversation transcript + llm_metrics: List of Metric objects to evaluate + evaluator: Optional Evaluator with custom_prompt + agent: Optional Agent for context + persona: Optional Persona for language info + scenario: Optional Scenario for context + + Returns: + Complete evaluation prompt string + """ + is_custom_evaluator = evaluator and bool(evaluator.custom_prompt) + + if is_custom_evaluator: + prompt = f"""You are evaluating a conversation transcript against the agent's system prompt. You MUST evaluate ONLY the specific metrics listed below and use the EXACT metric keys provided. + +## Agent System Prompt +The following is the system prompt / instructions that the agent was configured with. Use this to understand the agent's goals, rules, and expected behavior when evaluating the conversation. + +{evaluator.custom_prompt} + +## Conversation Transcript +{transcription} + +## Metrics to Evaluate (use EXACT keys below) +""" + else: + call_type_val = ( + (agent.call_type.value if hasattr(agent.call_type, "value") else agent.call_type) + if agent and agent.call_type + else "conversations" + ) + language_val = ( + (persona.language.value if hasattr(persona.language, "value") else persona.language) + if persona and persona.language + else "N/A" + ) + agent_objective = ( + agent.description + if agent and agent.description + else f"The agent's objective is to handle {call_type_val}." + ) + scenario_context = scenario.description if scenario and scenario.description else "" + scenario_goals = scenario.required_info if scenario and scenario.required_info else {} + + prompt = f"""You are evaluating a conversation transcript. You MUST evaluate ONLY the specific metrics listed below and use the EXACT metric keys provided. + +## Agent Information +- Name: {agent.name if agent else 'Unknown'} +- Objective/Purpose: {agent_objective} +- Call Type: {call_type_val if agent and agent.call_type else 'N/A'} +- Language: {language_val} + +## Scenario Information +- Name: {scenario.name if scenario else 'Unknown'} +- Description: {scenario_context} +- Required Information: {json.dumps(scenario_goals) if scenario_goals else 'N/A'} + +## Conversation Transcript +{transcription} + +## Metrics to Evaluate (use EXACT keys below) +""" + + for metric in llm_metrics: + metric_key = metric.name.lower().replace(" ", "_") + metric_desc = metric.description or f"Evaluate {metric.name}" + m_type = get_metric_type_value(metric) + + if m_type == "rating": + prompt += f'\n- "{metric_key}" (rating 0.0-1.0): {metric_desc}' + elif m_type == "boolean": + prompt += f'\n- "{metric_key}" (true/false): {metric_desc}' + elif m_type == "number": + prompt += f'\n- "{metric_key}" (numeric value): {metric_desc}' + + prompt += _build_response_format_instructions(llm_metrics) + return prompt + + +def _build_response_format_instructions(llm_metrics: list) -> str: + """Build the response format section of the prompt.""" + instructions = """ + +## REQUIRED Response Format +You MUST respond with ONLY a JSON object using the EXACT metric keys listed above. No other keys allowed. + +Example format: +{ +""" + for metric in llm_metrics: + metric_key = metric.name.lower().replace(" ", "_") + m_type = get_metric_type_value(metric) + + if m_type == "rating": + instructions += f' "{metric_key}": 0.75,\n' + elif m_type == "boolean": + instructions += f' "{metric_key}": true,\n' + elif m_type == "number": + instructions += f' "{metric_key}": 5,\n' + + instructions += """} + +CRITICAL RULES: +1. Use the EXACT metric keys shown above - copy them character-for-character +2. Each value must be a SINGLE NUMBER (not an object with score/comments) +3. Do NOT wrap in "metrics" or any other object +4. Do NOT add comments or explanations +5. Return ONLY the JSON object, nothing else""" + + return instructions + + +def _build_system_message(llm_metrics: list) -> str: + """Build the system message for LLM evaluation.""" + exact_keys = [metric.name.lower().replace(" ", "_") for metric in llm_metrics] + + return f"""You are an expert conversation evaluator. You MUST follow these rules STRICTLY: + +1. Return ONLY valid JSON - no markdown, no explanations, no comments +2. Use ONLY these exact metric keys (copy-paste them exactly): {json.dumps(exact_keys)} +3. Each value must be a single number (0.0-1.0 for ratings, 0 or 1 for boolean) - NO nested objects, NO comments +4. Do NOT rename, abbreviate, or modify the metric keys in any way + +Example of CORRECT format: +{{"follow_instructions": 0.8, "clarity_and_empathy": 0.7}} + +Example of WRONG format (DO NOT do this): +{{"metrics": {{"Clarity": {{"score": 7}}}}}}""" + + +def _parse_llm_response(response_text: str, result_id: str) -> dict: + """Parse LLM response text to extract evaluation data.""" + text = response_text.strip() + + if text.startswith("```json"): + text = text.replace("```json", "").replace("```", "").strip() + elif text.startswith("```"): + text = text.replace("```", "").strip() + + try: + return json.loads(text) + except json.JSONDecodeError: + logger.warning(f"[EvaluatorResult {result_id}] JSON parsing failed, attempting regex extraction") + json_match = re.search(r"\{[\s\S]*\}", text) + if json_match: + try: + return json.loads(json_match.group()) + except json.JSONDecodeError: + raise ValueError("Could not parse extracted JSON") + raise ValueError("Could not parse LLM response as JSON") + + +def evaluate_with_llm( + transcription: str, + llm_metrics: list, + ai_providers: list, + organization_id: UUID, + result_id: str, + db, + evaluator=None, + agent=None, + persona=None, + scenario=None, +) -> tuple[dict[str, dict[str, Any]], float | None]: + """ + Evaluate metrics using LLM. + + Args: + transcription: The conversation transcript + llm_metrics: List of Metric objects to evaluate + ai_providers: List of configured AI providers + organization_id: Organization UUID + result_id: Result ID for logging + db: Database session + evaluator: Optional Evaluator with custom_prompt and LLM config + agent: Optional Agent for context + persona: Optional Persona for language info + scenario: Optional Scenario for context + + Returns: + Tuple of (metric_scores dict, evaluation_time in seconds) + """ + from app.services.llm_service import llm_service + + evaluation_prompt = build_evaluation_prompt( + transcription=transcription, + llm_metrics=llm_metrics, + evaluator=evaluator, + agent=agent, + persona=persona, + scenario=scenario, + ) + + evaluator_llm_provider = getattr(evaluator, "llm_provider", None) if evaluator else None + evaluator_llm_model = getattr(evaluator, "llm_model", None) if evaluator else None + + if evaluator_llm_provider and evaluator_llm_model: + if isinstance(evaluator_llm_provider, str): + llm_provider = ModelProvider(evaluator_llm_provider.lower()) + else: + llm_provider = evaluator_llm_provider + llm_model = evaluator_llm_model + else: + llm_provider = ModelProvider.OPENAI + llm_model = "gpt-4o" + + chosen_provider = next( + (p for p in ai_providers if provider_matches(p.provider, llm_provider)), + None, + ) + if not chosen_provider: + logger.warning( + f"[EvaluatorResult {result_id}] Provider {llm_provider.value} not configured, evaluation may fail" + ) + + messages = [ + {"role": "system", "content": _build_system_message(llm_metrics)}, + {"role": "user", "content": evaluation_prompt}, + ] + + evaluation_start_time = time.time() + llm_result = llm_service.generate_response( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + temperature=0.3, + max_tokens=2000, + ) + evaluation_time = time.time() - evaluation_start_time + + evaluation_data = _parse_llm_response(llm_result["text"], result_id) + + if "metrics" in evaluation_data and isinstance(evaluation_data["metrics"], dict): + evaluation_data = evaluation_data["metrics"] + + metric_scores = _map_evaluation_to_metrics(evaluation_data, llm_metrics) + return metric_scores, evaluation_time + + +def _map_evaluation_to_metrics( + evaluation_data: dict, + llm_metrics: list, +) -> dict[str, dict[str, Any]]: + """Map LLM evaluation response to metric scores.""" + metric_scores: dict[str, dict[str, Any]] = {} + response_keys = list(evaluation_data.keys()) + + for metric in llm_metrics: + metric_key = metric.name.lower().replace(" ", "_") + m_type = get_metric_type_value(metric) + + raw_score = evaluation_data.get(metric_key) + if raw_score is None: + matched_key = find_matching_key(metric.name, response_keys) + if matched_key: + raw_score = evaluation_data.get(matched_key) + + score = extract_score(raw_score) + score = normalize_score(score, m_type) + + metric_scores[str(metric.id)] = { + "value": score, + "type": m_type, + "metric_name": metric.name, + } + + return metric_scores + + +def handle_llm_evaluation_error( + llm_metrics: list, + error: Exception, +) -> dict[str, dict[str, Any]]: + """Build error response for all LLM metrics when evaluation fails.""" + metric_scores: dict[str, dict[str, Any]] = {} + for metric in llm_metrics: + metric_scores[str(metric.id)] = { + "value": None, + "type": get_metric_type_value(metric), + "metric_name": metric.name, + "error": str(error), + } + return metric_scores diff --git a/app/workers/tasks/helpers/score_utils.py b/app/workers/tasks/helpers/score_utils.py new file mode 100644 index 00000000..05afdc5a --- /dev/null +++ b/app/workers/tasks/helpers/score_utils.py @@ -0,0 +1,100 @@ +"""Utility functions for score extraction and key matching.""" + + +def provider_matches(db_provider, target_enum) -> bool: + """Compare provider field (could be string or enum) with target enum.""" + if db_provider is None: + return False + if isinstance(db_provider, str): + return db_provider.lower() == target_enum.value.lower() + if hasattr(db_provider, "value"): + return db_provider.value.lower() == target_enum.value.lower() + return db_provider == target_enum + + +def extract_score(value): + """Extract numeric/boolean score from various response formats.""" + if value is None: + return None + if isinstance(value, (int, float, bool)): + return value + if isinstance(value, dict): + if "score" in value: + return value["score"] + if "value" in value: + return value["value"] + if "rating" in value: + return value["rating"] + if isinstance(value, str): + try: + return float(value) + except ValueError: + if value.lower() in ("true", "yes"): + return True + if value.lower() in ("false", "no"): + return False + return None + + +def find_matching_key(target_key: str, response_keys: list[str]) -> str | None: + """Find a matching key in the response, with fuzzy matching.""" + target_lower = target_key.lower().replace(" ", "_").replace("-", "_") + target_words = set(target_lower.replace("_", " ").split()) + + for key in response_keys: + if key.lower().replace(" ", "_").replace("-", "_") == target_lower: + return key + + for key in response_keys: + key_lower = key.lower().replace(" ", "_").replace("-", "_") + if target_lower in key_lower or key_lower in target_lower: + return key + + best_match = None + best_overlap = 0 + for key in response_keys: + key_words = set(key.lower().replace("_", " ").replace("-", " ").split()) + overlap = len(target_words & key_words) + if overlap > best_overlap: + best_overlap = overlap + best_match = key + + if best_overlap >= 1: + return best_match + return None + + +def get_metric_type_value(metric) -> str: + """Extract metric type as lowercase string.""" + m_type = metric.metric_type.value if hasattr(metric.metric_type, "value") else metric.metric_type + return m_type.lower() if isinstance(m_type, str) else m_type + + +def normalize_score(score, metric_type: str): + """Normalize and validate score based on metric type.""" + if score is None: + return None + + if metric_type == "rating": + try: + score = float(score) + if score > 1.0: + score = score / 10.0 + return max(0.0, min(1.0, score)) + except (ValueError, TypeError): + return None + + if metric_type == "boolean": + if isinstance(score, bool): + return score + if isinstance(score, (int, float)): + return score > 0.5 if score <= 1 else score > 5 + return bool(score) + + if metric_type == "number": + try: + return float(score) + except (ValueError, TypeError): + return None + + return score diff --git a/app/workers/tasks/process_evaluation.py b/app/workers/tasks/process_evaluation.py new file mode 100644 index 00000000..99451ed7 --- /dev/null +++ b/app/workers/tasks/process_evaluation.py @@ -0,0 +1,31 @@ +"""Celery task: process evaluation.""" + +from uuid import UUID + +from app.database import SessionLocal +from app.services.evaluation_service import evaluation_service + +from app.workers.config import celery_app + + +@celery_app.task(name="process_evaluation", bind=True, max_retries=3) +def process_evaluation_task(self, evaluation_id: str): + """ + Celery task to process an evaluation. + + Args: + self: Task instance + evaluation_id: Evaluation ID as string + + Returns: + Dictionary with evaluation results + """ + db = SessionLocal() + try: + eval_id = UUID(evaluation_id) + result = evaluation_service.process_evaluation(eval_id, db) + return result + except Exception as exc: + raise self.retry(exc=exc, countdown=60) + finally: + db.close() diff --git a/app/workers/tasks/process_evaluator_result.py b/app/workers/tasks/process_evaluator_result.py new file mode 100644 index 00000000..1b0c0498 --- /dev/null +++ b/app/workers/tasks/process_evaluator_result.py @@ -0,0 +1,281 @@ +"""Celery task: process evaluator result (transcribe and evaluate metrics).""" + +import time +from uuid import UUID + +from loguru import logger + +from app.database import SessionLocal +from app.models.database import ModelProvider + +from app.workers.config import celery_app +from app.workers.tasks.helpers.constants import ( + REMOVED_EVALUATION_METRIC_NAMES, + AUDIO_ONLY_METRIC_NAMES, +) +from app.workers.tasks.helpers.score_utils import provider_matches, get_metric_type_value +from app.workers.tasks.helpers.audio_evaluation import ( + evaluate_audio_metrics, + handle_audio_evaluation_error, +) +from app.workers.tasks.helpers.llm_evaluation import ( + evaluate_with_llm, + handle_llm_evaluation_error, +) + + +def _load_related_entities(db, result): + """Load evaluator, agent, persona, scenario from database.""" + from app.models.database import Evaluator, Agent, Persona, Scenario + + evaluator = None + if result.evaluator_id: + evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() + if not evaluator: + logger.warning( + f"[EvaluatorResult {result.result_id}] Evaluator {result.evaluator_id} not found, " + "continuing without evaluator" + ) + + agent = None + if result.agent_id: + agent = db.query(Agent).filter(Agent.id == result.agent_id).first() + + persona = None + if result.persona_id: + persona = db.query(Persona).filter(Persona.id == result.persona_id).first() + + scenario = None + if result.scenario_id: + scenario = db.query(Scenario).filter(Scenario.id == result.scenario_id).first() + + return evaluator, agent, persona, scenario + + +def _transcribe_audio(result, ai_providers, db): + """Transcribe audio file and return transcript with timing info.""" + from app.services.transcription_service import transcription_service + + stt_provider = ModelProvider.OPENAI + stt_model = "whisper-1" + + openai_provider = next( + (p for p in ai_providers if provider_matches(p.provider, ModelProvider.OPENAI)), + None, + ) + if not openai_provider: + logger.warning( + f"[EvaluatorResult {result.result_id}] No OpenAI provider found, using default whisper-1" + ) + + transcription_start_time = time.time() + transcription_result = transcription_service.transcribe( + audio_file_key=result.audio_s3_key, + stt_provider=stt_provider, + stt_model=stt_model, + organization_id=result.organization_id, + db=db, + language=None, + enable_speaker_diarization=True, + ) + transcription_time = time.time() - transcription_start_time + + return ( + transcription_result.get("transcript", ""), + transcription_result.get("speaker_segments", []), + transcription_time, + ) + + +def _categorize_metrics(enabled_metrics, has_audio): + """Split metrics into LLM-evaluable and audio-only categories.""" + llm_metrics = [] + audio_metrics = [] + skipped_scores = {} + + for m in enabled_metrics: + if m.name.lower() in AUDIO_ONLY_METRIC_NAMES: + if has_audio: + audio_metrics.append(m) + else: + skipped_scores[str(m.id)] = { + "value": None, + "type": get_metric_type_value(m), + "metric_name": m.name, + "skipped": "audio_required", + } + else: + llm_metrics.append(m) + + return llm_metrics, audio_metrics, skipped_scores + + +@celery_app.task(name="process_evaluator_result", bind=True, max_retries=3) +def process_evaluator_result_task(self, result_id: str): + """ + Celery task to process an evaluator result: transcribe audio and evaluate metrics. + + Workflow: + 1. QUEUED -> Job is created and queued + 2. TRANSCRIBING -> Audio is being transcribed + 3. EVALUATING -> Transcription is being evaluated against metrics + 4. COMPLETED -> All processing is complete + 5. FAILED -> An error occurred + """ + db = SessionLocal() + task_start_time = time.time() + + try: + from app.models.database import ( + EvaluatorResult, + EvaluatorResultStatus, + Metric, + AIProvider, + ) + + result_uuid = UUID(result_id) + result = db.query(EvaluatorResult).filter(EvaluatorResult.id == result_uuid).first() + + if not result: + logger.error(f"[EvaluatorResult {result_id}] Job not found in database") + return {"error": "Evaluator result not found"} + + logger.info(f"[EvaluatorResult {result.result_id}] Starting processing task") + + result.celery_task_id = self.request.id + db.commit() + + has_existing_transcript = bool(result.transcription) + + try: + if not result.audio_s3_key and not has_existing_transcript: + raise ValueError("No audio S3 key or existing transcript found") + + evaluator, agent, persona, scenario = _load_related_entities(db, result) + is_custom_evaluator = evaluator and bool(evaluator.custom_prompt) + + if not is_custom_evaluator and not agent: + raise ValueError("Agent not found and no custom prompt available") + + ai_providers = db.query(AIProvider).filter( + AIProvider.organization_id == result.organization_id, + AIProvider.is_active == True, + ).all() + + # Step 1: Transcription + if has_existing_transcript: + transcription = result.transcription + speaker_segments = result.speaker_segments or [] + transcription_time = 0.0 + else: + result.status = EvaluatorResultStatus.TRANSCRIBING.value + db.commit() + + transcription, speaker_segments, transcription_time = _transcribe_audio( + result, ai_providers, db + ) + result.transcription = transcription + result.speaker_segments = speaker_segments if speaker_segments else None + db.commit() + + # Step 2: Load and categorize metrics + enabled_metrics = db.query(Metric).filter( + Metric.organization_id == result.organization_id, + Metric.enabled == True, + ).all() + enabled_metrics = [ + m for m in enabled_metrics + if (m.name or "").strip().lower() not in REMOVED_EVALUATION_METRIC_NAMES + ] + + has_audio = bool(result.audio_s3_key) + llm_metrics, audio_metrics, metric_scores = _categorize_metrics(enabled_metrics, has_audio) + + evaluation_time = None + + # Step 3: Audio metrics evaluation + if audio_metrics and has_audio: + try: + audio_scores = evaluate_audio_metrics( + audio_s3_key=result.audio_s3_key, + audio_metrics=audio_metrics, + result_id=result.result_id, + ) + metric_scores.update(audio_scores) + except Exception as audio_err: + logger.error( + f"[EvaluatorResult {result.result_id}] Audio analysis failed: {audio_err}", + exc_info=True, + ) + metric_scores.update(handle_audio_evaluation_error(audio_metrics, audio_err)) + + # Step 4: LLM metrics evaluation + if llm_metrics and transcription: + result.status = EvaluatorResultStatus.EVALUATING.value + db.commit() + + try: + llm_scores, evaluation_time = evaluate_with_llm( + transcription=transcription, + llm_metrics=llm_metrics, + ai_providers=ai_providers, + organization_id=result.organization_id, + result_id=result.result_id, + db=db, + evaluator=evaluator, + agent=agent, + persona=persona, + scenario=scenario, + ) + metric_scores.update(llm_scores) + except Exception as llm_err: + error_msg = str(llm_err).replace("{", "{{").replace("}", "}}") + logger.error( + f"[EvaluatorResult {result.result_id}] ✗ LLM evaluation failed: {error_msg}", + exc_info=True, + ) + metric_scores.update(handle_llm_evaluation_error(llm_metrics, llm_err)) + else: + if not llm_metrics: + logger.warning( + f"[EvaluatorResult {result.result_id}] No LLM-evaluable metrics found " + "(audio-only metrics were skipped), skipping evaluation" + ) + if not transcription: + logger.warning( + f"[EvaluatorResult {result.result_id}] No transcription available, " + "skipping evaluation" + ) + + # Step 5: Complete + result.metric_scores = metric_scores + result.status = EvaluatorResultStatus.COMPLETED.value + db.commit() + + total_time = time.time() - task_start_time + logger.info( + f"[EvaluatorResult {result.result_id}] Completed in {total_time:.2f}s, " + f"{len(metric_scores)} metrics evaluated" + ) + + return { + "result_id": result_id, + "status": "completed", + "transcription": transcription, + "metrics_evaluated": len(metric_scores), + "processing_time": total_time, + "transcription_time": transcription_time, + "evaluation_time": evaluation_time, + } + + except Exception as e: + logger.error(f"[EvaluatorResult {result.result_id}] Processing failed: {e}", exc_info=True) + result.status = EvaluatorResultStatus.FAILED.value + result.error_message = str(e) + db.commit() + raise + + except Exception as exc: + raise self.retry(exc=exc, countdown=60) + finally: + db.close() diff --git a/app/workers/tasks/run_evaluator.py b/app/workers/tasks/run_evaluator.py new file mode 100644 index 00000000..076fef8f --- /dev/null +++ b/app/workers/tasks/run_evaluator.py @@ -0,0 +1,143 @@ +"""Celery task: run evaluator (bridge test agent to voice AI and record conversation).""" + +import asyncio +import time +from uuid import UUID + +from loguru import logger + +from app.database import SessionLocal +from app.models.database import EvaluatorResult, EvaluatorResultStatus + +from app.workers.config import celery_app + + +@celery_app.task(name="run_evaluator", bind=True, max_retries=3) +def run_evaluator_task(self, evaluator_id: str, evaluator_result_id: str): + """ + Celery task to run an evaluator: bridge test agent to Voice AI agent and record conversation. + + Args: + self: Task instance + evaluator_id: Evaluator ID as string + evaluator_result_id: Pre-created EvaluatorResult ID as string + + Returns: + Dictionary with execution results + """ + db = SessionLocal() + task_start_time = time.time() + + try: + from app.models.database import Evaluator, Agent + from app.services.test_agent_bridge_service import test_agent_bridge_service + + evaluator_uuid = UUID(evaluator_id) + result_uuid = UUID(evaluator_result_id) + + evaluator = db.query(Evaluator).filter(Evaluator.id == evaluator_uuid).first() + if not evaluator: + logger.error(f"[RunEvaluator {evaluator_id}] Evaluator not found") + return {"error": "Evaluator not found"} + + result = db.query(EvaluatorResult).filter(EvaluatorResult.id == result_uuid).first() + if not result: + logger.error(f"[RunEvaluator {evaluator_id}] EvaluatorResult not found") + return {"error": "EvaluatorResult not found"} + + agent = db.query(Agent).filter(Agent.id == evaluator.agent_id).first() + if not agent: + logger.error(f"[RunEvaluator {evaluator_id}] Agent not found") + result.status = EvaluatorResultStatus.FAILED.value + result.error_message = "Agent not found" + db.commit() + return {"error": "Agent not found"} + + logger.info(f"[RunEvaluator {evaluator.evaluator_id}] Starting task (Result: {result.result_id})") + + result.celery_task_id = self.request.id + if result.status != EvaluatorResultStatus.QUEUED.value: + logger.warning(f"[RunEvaluator {evaluator.evaluator_id}] Status was {result.status}, expected QUEUED") + db.commit() + + has_voice_bundle = agent.voice_bundle_id is not None + has_voice_ai_integration = ( + agent.voice_ai_integration_id is not None and agent.voice_ai_agent_id is not None + ) + + if has_voice_bundle and has_voice_ai_integration: + try: + result.status = EvaluatorResultStatus.CALL_INITIATING.value + result.call_event = "task_started" + db.commit() + + loop = asyncio.get_event_loop() + if loop.is_closed(): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + bridge_result = loop.run_until_complete( + test_agent_bridge_service.bridge_test_agent_to_voice_agent( + evaluator_id=evaluator_uuid, + evaluator_result_id=result_uuid, + organization_id=evaluator.organization_id, + db=db, + ) + ) + + db.refresh(result) + + if result.error_message and result.error_message.startswith("call_id:"): + pass + + db.commit() + + return { + "evaluator_id": evaluator_id, + "result_id": evaluator_result_id, + "status": "initiated", + "bridge_result": bridge_result, + } + + except Exception as bridge_error: + logger.error( + f"[RunEvaluator {evaluator.evaluator_id}] Bridge service error: {bridge_error}", + exc_info=True, + ) + result.status = EvaluatorResultStatus.FAILED.value + result.error_message = str(bridge_error) + result.call_event = "bridge_error" + db.commit() + raise + + elif has_voice_bundle: + result.status = EvaluatorResultStatus.FAILED.value + result.error_message = "Standard voice agent flow not yet implemented for evaluator runs" + db.commit() + return {"error": "Standard flow not implemented"} + + else: + logger.error(f"[RunEvaluator {evaluator.evaluator_id}] Agent missing required configuration") + result.status = EvaluatorResultStatus.FAILED.value + result.error_message = ( + f"Agent missing required configuration: voice_bundle={has_voice_bundle}, " + f"voice_ai_integration={has_voice_ai_integration}" + ) + result.call_event = "configuration_error" + db.commit() + return {"error": "Agent does not have required configuration for bridging"} + + except Exception as exc: + logger.error(f"[RunEvaluator {evaluator_id}] Task failed: {exc}", exc_info=True) + try: + result = db.query(EvaluatorResult).filter( + EvaluatorResult.id == UUID(evaluator_result_id) + ).first() + if result: + result.status = EvaluatorResultStatus.FAILED.value + result.error_message = str(exc) + db.commit() + except Exception: + pass + raise self.retry(exc=exc, countdown=60) + finally: + db.close() diff --git a/app/workers/tasks/tts_comparison.py b/app/workers/tasks/tts_comparison.py new file mode 100644 index 00000000..4a8b6535 --- /dev/null +++ b/app/workers/tasks/tts_comparison.py @@ -0,0 +1,458 @@ +"""Celery tasks: TTS comparison generation and evaluation.""" + +import os +import re +import string +import tempfile +from uuid import UUID + +from loguru import logger + +from app.database import SessionLocal + +from app.workers.config import celery_app + +# Singleton for the NeMo ASR model (loaded once per worker process) +_nemo_asr_model = None + + +def _compute_wer_cer(ground_truth: str, predicted: str): + """Compute raw and normalized WER/CER between reference and ASR text. + + Normalized scores reduce false penalties on numeric/currency phrasing + differences (for example "$1,234.56" vs "one thousand two hundred..."). + """ + try: + from jiwer import cer, wer + except ImportError: + logger.warning("[TTS Eval] jiwer not installed – skipping WER/CER") + return { + "raw_wer": None, + "raw_cer": None, + "normalized_wer": None, + "normalized_cer": None, + } + + number_words = { + "zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", + "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", + "seventeen", "eighteen", "nineteen", "twenty", "thirty", "forty", "fifty", + "sixty", "seventy", "eighty", "ninety", "hundred", "thousand", "million", + "billion", "trillion", "point", "and", + } + currency_words = { + "dollar", "dollars", "usd", "cent", "cents", "rupee", "rupees", "inr", + "euro", "euros", "eur", "pound", "pounds", "gbp", + } + + def _is_numeric_token(token: str) -> bool: + return bool(re.fullmatch(r"\d+(?:\.\d+)?", token)) + + def _normalize_base(text: str) -> str: + punct_table = str.maketrans("", "", string.punctuation) + return text.lower().translate(punct_table).strip() + + def _normalize_entities(text: str) -> str: + tokens = _normalize_base(text).split() + normalized_tokens = [] + i = 0 + while i < len(tokens): + token = tokens[i] + is_entity_token = ( + _is_numeric_token(token) + or token in number_words + or token in currency_words + ) + if not is_entity_token: + normalized_tokens.append(token) + i += 1 + continue + + j = i + has_currency = token in currency_words + while j < len(tokens): + t = tokens[j] + if _is_numeric_token(t) or t in number_words or t in currency_words: + if t in currency_words: + has_currency = True + j += 1 + continue + break + + normalized_tokens.append("" if has_currency else "") + i = j + + return " ".join(normalized_tokens) + + ref = _normalize_base(ground_truth) + hyp = _normalize_base(predicted) + + if not ref: + return { + "raw_wer": None, + "raw_cer": None, + "normalized_wer": None, + "normalized_cer": None, + } + + try: + raw_wer = round(wer(ref, hyp), 4) + raw_cer = round(cer(ref, hyp), 4) + + norm_ref = _normalize_entities(ground_truth) + norm_hyp = _normalize_entities(predicted) + normalized_wer = round(wer(norm_ref, norm_hyp), 4) if norm_ref else None + normalized_cer = round(cer(norm_ref, norm_hyp), 4) if norm_ref else None + + return { + "raw_wer": raw_wer, + "raw_cer": raw_cer, + "normalized_wer": normalized_wer, + "normalized_cer": normalized_cer, + } + except Exception as e: + logger.warning(f"[TTS Eval] WER/CER calculation error: {e}") + return { + "raw_wer": None, + "raw_cer": None, + "normalized_wer": None, + "normalized_cer": None, + } + + +def _get_nemo_asr_model(): + """Lazy-load NVIDIA NeMo Conformer CTC model for hallucination detection. + + Requires: pip install efficientai[nemo-asr] + Returns the model instance, or None if NeMo is not installed. + """ + global _nemo_asr_model + + if _nemo_asr_model is not None: + return _nemo_asr_model + + try: + import nemo.collections.asr as nemo_asr + + logger.info("[TTS Eval] Loading NeMo ASR model (stt_en_conformer_ctc_large)...") + _nemo_asr_model = nemo_asr.models.ASRModel.from_pretrained("stt_en_conformer_ctc_large") + logger.info("[TTS Eval] NeMo ASR model loaded successfully") + return _nemo_asr_model + except ImportError as e: + logger.warning( + f"[TTS Eval] NeMo import failed: {e} – " + "WER/CER hallucination metrics will be skipped. " + "To enable, run:\n" + " pip install 'nemo_toolkit[asr]'\n" + " python -c \"import nemo.collections.asr as nemo_asr; " + "nemo_asr.models.ASRModel.from_pretrained('stt_en_conformer_ctc_large')\"" + ) + except Exception as e: + logger.error( + f"[TTS Eval] NeMo ASR model failed to load: {e} – " + "The model may not be cached yet. To download it manually, run:\n" + " python -c \"import nemo.collections.asr as nemo_asr; " + "nemo_asr.models.ASRModel.from_pretrained('stt_en_conformer_ctc_large')\"", + exc_info=True, + ) + + return None + + +def _transcribe_audio_for_eval(audio_path: str) -> str | None: + """Transcribe an audio file using NVIDIA NeMo Conformer CTC. + + Runs entirely on the worker – no API key needed. + """ + model = _get_nemo_asr_model() + if model is None: + return None + + try: + transcriptions = model.transcribe([audio_path]) + if transcriptions and len(transcriptions) > 0: + text = transcriptions[0] + if hasattr(text, "text"): + text = text.text + return str(text).strip() or None + return None + except Exception as e: + logger.warning(f"[TTS Eval] ASR transcription failed: {e}") + return None + + +@celery_app.task(name="generate_tts_comparison", bind=True, max_retries=1) +def generate_tts_comparison_task(self, comparison_id: str): + """ + Generate TTS audio for every sample in a comparison, upload to S3, + then dispatch evaluation. + """ + from app.models.database import ( + TTSComparison, + TTSSample, + TTSComparisonStatus, + TTSSampleStatus, + ModelProvider, + ) + from app.services.tts_service import tts_service, get_audio_file_extension + from app.services.s3_service import s3_service + + db = SessionLocal() + try: + comp = db.query(TTSComparison).filter(TTSComparison.id == UUID(comparison_id)).first() + if not comp: + logger.error(f"[TTS Generate] Comparison {comparison_id} not found") + return {"error": "not found"} + + comp.status = TTSComparisonStatus.GENERATING.value + db.commit() + + samples = ( + db.query(TTSSample) + .filter(TTSSample.comparison_id == comp.id) + .order_by(TTSSample.sample_index) + .all() + ) + + voice_configs_a = {v["id"]: v for v in (comp.voices_a or []) if isinstance(v, dict)} + voice_configs_b = {v["id"]: v for v in (comp.voices_b or []) if isinstance(v, dict)} + + def _resolve_voice_meta(sample_obj): + """Match sample to correct side's voice config (A or B).""" + if sample_obj.side == "A": + return voice_configs_a.get(sample_obj.voice_id) or {} + if sample_obj.side == "B": + return voice_configs_b.get(sample_obj.voice_id) or {} + is_side_a = ( + sample_obj.provider == comp.provider_a and sample_obj.model == comp.model_a + ) + is_side_b = ( + sample_obj.provider == comp.provider_b and sample_obj.model == comp.model_b + ) + if is_side_a and not is_side_b: + return voice_configs_a.get(sample_obj.voice_id) or {} + if is_side_b and not is_side_a: + return voice_configs_b.get(sample_obj.voice_id) or {} + return voice_configs_a.get(sample_obj.voice_id) or voice_configs_b.get(sample_obj.voice_id) or {} + + failed_count = 0 + for sample in samples: + try: + sample.status = TTSSampleStatus.GENERATING.value + db.commit() + + voice_meta = _resolve_voice_meta(sample) + tts_config = {} + sample_rate_hz = voice_meta.get("sample_rate_hz") + if sample_rate_hz: + tts_config["sample_rate_hz"] = int(sample_rate_hz) + language_code = voice_meta.get("language_code") + if language_code: + tts_config["language_code"] = language_code + + provider_enum = ModelProvider(sample.provider) + logger.info( + f"[TTS Generate] Sample {sample.id} – " + f"provider={sample.provider} voice={sample.voice_id} " + f"sample_rate_hz={sample_rate_hz} config={tts_config}" + ) + audio_bytes, latency_ms, ttfb_ms = tts_service.synthesize_timed( + text=sample.text, + tts_provider=provider_enum, + tts_model=sample.model, + organization_id=comp.organization_id, + db=db, + voice=sample.voice_id, + config=tts_config or None, + ) + + audio_ext = get_audio_file_extension( + sample.provider, int(sample_rate_hz) if sample_rate_hz else None + ) + s3_key = s3_service.upload_file_by_key( + file_content=audio_bytes, + key=f"{s3_service.prefix}organizations/{comp.organization_id}/voicePlayground/{comp.id}/{sample.id}.{audio_ext}", + ) + + if audio_ext == "wav" and len(audio_bytes) > 44: + import struct as _struct + + sr = _struct.unpack_from(" 0 else None + else: + duration_est = len(audio_bytes) / (128000 / 8) if audio_bytes else None + + sample.audio_s3_key = s3_key + sample.latency_ms = round(latency_ms, 1) + sample.ttfb_ms = round(ttfb_ms, 1) + sample.duration_seconds = round(duration_est, 2) if duration_est else None + sample.status = TTSSampleStatus.COMPLETED.value + db.commit() + + logger.info( + f"[TTS Generate] Sample {sample.id} done – " + f"{sample.provider}/{sample.voice_name} ttfb={ttfb_ms:.0f}ms total={latency_ms:.0f}ms" + ) + + except Exception as e: + logger.error(f"[TTS Generate] Sample {sample.id} failed: {e}") + sample.status = TTSSampleStatus.FAILED.value + sample.error_message = str(e)[:500] + db.commit() + failed_count += 1 + + total = len(samples) + if failed_count == total: + comp.status = TTSComparisonStatus.FAILED.value + comp.error_message = "All samples failed to generate" + db.commit() + return {"error": "all failed"} + + comp.status = TTSComparisonStatus.EVALUATING.value + db.commit() + evaluate_tts_comparison_task.delay(comparison_id) + + return {"generated": total - failed_count, "failed": failed_count} + + except Exception as exc: + logger.error(f"[TTS Generate] Task failed: {exc}", exc_info=True) + try: + comp = db.query(TTSComparison).filter(TTSComparison.id == UUID(comparison_id)).first() + if comp: + comp.status = TTSComparisonStatus.FAILED.value + comp.error_message = str(exc)[:500] + db.commit() + except Exception: + pass + raise self.retry(exc=exc, countdown=30) + finally: + db.close() + + +@celery_app.task(name="evaluate_tts_comparison", bind=True, max_retries=1) +def evaluate_tts_comparison_task(self, comparison_id: str): + """ + Download each completed sample from S3 and run qualitative voice + metrics (MOS, Valence, Arousal, Prosody) plus ASR-based WER/CER + for hallucination detection. + """ + from app.models.database import ( + TTSComparison, + TTSSample, + TTSComparisonStatus, + TTSSampleStatus, + ) + from app.services.s3_service import s3_service + from app.services.qualitative_voice_service import qualitative_voice_service + + db = SessionLocal() + try: + comp = db.query(TTSComparison).filter(TTSComparison.id == UUID(comparison_id)).first() + if not comp: + return {"error": "not found"} + + samples = ( + db.query(TTSSample) + .filter( + TTSSample.comparison_id == comp.id, + TTSSample.status == TTSSampleStatus.COMPLETED.value, + TTSSample.audio_s3_key.isnot(None), + ) + .all() + ) + + if not samples: + comp.status = TTSComparisonStatus.COMPLETED.value + db.commit() + return {"evaluated": 0} + + nemo_model = _get_nemo_asr_model() + + evaluated = 0 + for sample in samples: + tmp_path = None + try: + audio_bytes = s3_service.download_file_by_key(sample.audio_s3_key) + if not audio_bytes: + continue + + ext = ".mp3" + if sample.audio_s3_key: + key_ext = os.path.splitext(sample.audio_s3_key)[1].lower() + if key_ext in {".wav", ".mp3", ".flac", ".ogg", ".m4a"}: + ext = key_ext + tmp_fd, tmp_path = tempfile.mkstemp(suffix=ext) + os.close(tmp_fd) + with open(tmp_path, "wb") as f: + f.write(audio_bytes) + + metrics = qualitative_voice_service.calculate_all_metrics(tmp_path) + + if nemo_model is not None and sample.text: + asr_transcript = _transcribe_audio_for_eval(tmp_path) + if asr_transcript: + score_bundle = _compute_wer_cer(sample.text, asr_transcript) + metrics["WER Raw"] = score_bundle.get("raw_wer") + metrics["CER Raw"] = score_bundle.get("raw_cer") + metrics["WER Normalized"] = score_bundle.get("normalized_wer") + metrics["CER Normalized"] = score_bundle.get("normalized_cer") + metrics["WER"] = ( + score_bundle.get("normalized_wer") + if score_bundle.get("normalized_wer") is not None + else score_bundle.get("raw_wer") + ) + metrics["CER"] = ( + score_bundle.get("normalized_cer") + if score_bundle.get("normalized_cer") is not None + else score_bundle.get("raw_cer") + ) + metrics["ASR Transcript"] = asr_transcript + else: + metrics["WER"] = None + metrics["CER"] = None + metrics["WER Raw"] = None + metrics["CER Raw"] = None + metrics["WER Normalized"] = None + metrics["CER Normalized"] = None + metrics["ASR Transcript"] = None + + sample.evaluation_metrics = metrics + db.commit() + evaluated += 1 + + logger.info(f"[TTS Eval] Sample {sample.id} metrics: {metrics}") + + except Exception as e: + logger.warning(f"[TTS Eval] Sample {sample.id} eval failed: {e}") + finally: + if tmp_path and os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except Exception: + pass + + from app.api.v1.routes.voice_playground import _recompute_summary + + _recompute_summary(comp, db) + + comp.status = TTSComparisonStatus.COMPLETED.value + db.commit() + + logger.info( + f"[TTS Eval] Comparison {comparison_id} complete – {evaluated}/{len(samples)} evaluated" + ) + return {"evaluated": evaluated} + + except Exception as exc: + logger.error(f"[TTS Eval] Task failed: {exc}", exc_info=True) + try: + comp = db.query(TTSComparison).filter(TTSComparison.id == UUID(comparison_id)).first() + if comp: + comp.status = TTSComparisonStatus.FAILED.value + comp.error_message = f"Evaluation failed: {str(exc)[:400]}" + db.commit() + except Exception: + pass + raise self.retry(exc=exc, countdown=30) + finally: + db.close() diff --git a/app/workers/tasks/tts_report.py b/app/workers/tasks/tts_report.py new file mode 100644 index 00000000..233f30b0 --- /dev/null +++ b/app/workers/tasks/tts_report.py @@ -0,0 +1,91 @@ +"""Celery task: generate TTS report PDF.""" + +from uuid import UUID + +from loguru import logger + +from app.database import SessionLocal + +from app.workers.config import celery_app + + +@celery_app.task(name="generate_tts_report_pdf", bind=True, max_retries=1) +def generate_tts_report_pdf_task(self, report_job_id: str): + """Generate a Voice Playground PDF report and store it in S3.""" + from app.models.database import ( + TTSComparison, + TTSSample, + TTSReportJob, + TTSReportJobStatus, + ) + from app.services.s3_service import s3_service + from app.services.voice_playground_report_service import voice_playground_report_service + + db = SessionLocal() + try: + report_job = db.query(TTSReportJob).filter(TTSReportJob.id == UUID(report_job_id)).first() + if not report_job: + logger.error(f"[TTS Report] Job {report_job_id} not found") + return {"error": "report_job_not_found"} + + comparison = ( + db.query(TTSComparison) + .filter( + TTSComparison.id == report_job.comparison_id, + TTSComparison.organization_id == report_job.organization_id, + ) + .first() + ) + if not comparison: + report_job.status = TTSReportJobStatus.FAILED.value + report_job.error_message = "Comparison not found" + db.commit() + return {"error": "comparison_not_found"} + + report_job.status = TTSReportJobStatus.PROCESSING.value + report_job.celery_task_id = self.request.id + db.commit() + + samples = ( + db.query(TTSSample) + .filter(TTSSample.comparison_id == comparison.id) + .order_by(TTSSample.run_index, TTSSample.sample_index) + .all() + ) + + payload = voice_playground_report_service.build_payload(comparison, samples) + pdf_bytes = voice_playground_report_service.render_pdf(payload) + + report_filename = ( + f"voice-playground-report-{comparison.simulation_id or str(comparison.id)[:8]}.pdf" + ) + s3_key = ( + f"{s3_service.prefix}organizations/{report_job.organization_id}/voicePlayground/" + f"{comparison.id}/reports/{report_job.id}.pdf" + ) + s3_service.upload_file_by_key( + file_content=pdf_bytes, + key=s3_key, + content_type="application/pdf", + ) + + report_job.status = TTSReportJobStatus.COMPLETED.value + report_job.filename = report_filename + report_job.s3_key = s3_key + report_job.error_message = None + db.commit() + + return {"status": "completed", "s3_key": s3_key} + except Exception as exc: + logger.error(f"[TTS Report] Task failed: {exc}", exc_info=True) + try: + report_job = db.query(TTSReportJob).filter(TTSReportJob.id == UUID(report_job_id)).first() + if report_job: + report_job.status = TTSReportJobStatus.FAILED.value + report_job.error_message = str(exc)[:500] + db.commit() + except Exception: + pass + raise self.retry(exc=exc, countdown=30) + finally: + db.close() diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 024075db..20a7b652 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -23,6 +23,22 @@ import type { S3Status, } from '../types/api' +export interface EnterpriseFeatureMeta { + title: string + description?: string + category?: string +} + +export type EnterpriseFeatureCatalog = Record + +export interface LicenseInfoResponse { + is_enterprise: boolean + enabled_features: string[] + all_enterprise_features: string[] + feature_catalog?: EnterpriseFeatureCatalog + organization?: string +} + // When running in production (served from same origin), use relative path // Otherwise use environment variable or default const API_BASE_URL = import.meta.env.VITE_API_URL || @@ -1427,12 +1443,7 @@ class ApiClient { } // License / Enterprise - async getLicenseInfo(): Promise<{ - is_enterprise: boolean - enabled_features: string[] - all_enterprise_features: string[] - organization?: string - }> { + async getLicenseInfo(): Promise { const response = await this.client.get('/api/v1/settings/license-info') return response.data } diff --git a/frontend/src/pages/enterprise/EnterpriseUpgrade.tsx b/frontend/src/pages/enterprise/EnterpriseUpgrade.tsx index 8237692a..8c990df7 100644 --- a/frontend/src/pages/enterprise/EnterpriseUpgrade.tsx +++ b/frontend/src/pages/enterprise/EnterpriseUpgrade.tsx @@ -1,72 +1,70 @@ -import { Lock, Mail, ExternalLink } from 'lucide-react' - -const FEATURE_LABELS: Record = { - voice_playground: { title: 'Voice Playground' }, -} - -const FALLBACK_TITLE = 'Enterprise Feature' - -export default function EnterpriseUpgrade({ feature }: { feature: string }) { - const title = FEATURE_LABELS[feature]?.title ?? FALLBACK_TITLE - - return ( -
-
-
- -
- -

{title}

- - - Enterprise Feature - - -

- This feature is available with an EfficientAI Enterprise license. -

- -
-

- To unlock this feature: -

-
    -
  1. - 1. - Contact the EfficientAI team for an enterprise license key -
    2. -
  2. - 2. - Set the EFFICIENTAI_LICENSE environment variable on your server -
    3. -
  3. - 3. - Restart the EfficientAI backend to activate -
    4. -
-
- - -
-
- ) -} +import { Lock, Mail, ExternalLink } from 'lucide-react' +import { useLicenseStore } from '../../store/licenseStore' + +const FALLBACK_TITLE = 'Enterprise Feature' + +export default function EnterpriseUpgrade({ feature }: { feature: string }) { + const getFeatureMeta = useLicenseStore((state) => state.getFeatureMeta) + const title = getFeatureMeta(feature)?.title ?? FALLBACK_TITLE + + return ( +
+
+
+ +
+ +

{title}

+ + + Enterprise Feature + + +

+ This feature is available with an EfficientAI Enterprise license. +

+ +
+

+ To unlock this feature: +

+
    +
  1. + 1. + Contact the EfficientAI team for an enterprise license key +
    2. +
  2. + 2. + Set the EFFICIENTAI_LICENSE environment variable on your server +
    3. +
  3. + 3. + Restart the EfficientAI backend to activate +
    4. +
+
+ + +
+
+ ) +} diff --git a/frontend/src/store/licenseStore.ts b/frontend/src/store/licenseStore.ts index 4b707701..4081589b 100644 --- a/frontend/src/store/licenseStore.ts +++ b/frontend/src/store/licenseStore.ts @@ -1,41 +1,51 @@ -import { create } from 'zustand' -import { apiClient } from '../lib/api' - -interface LicenseState { - isEnterprise: boolean - enabledFeatures: string[] - allEnterpriseFeatures: string[] - isLoaded: boolean - fetchLicense: () => Promise - isFeatureEnabled: (feature: string) => boolean -} - -export const useLicenseStore = create((set, get) => ({ - isEnterprise: false, - enabledFeatures: [], - allEnterpriseFeatures: [], - isLoaded: false, - - fetchLicense: async () => { - try { - const info = await apiClient.getLicenseInfo() - set({ - isEnterprise: info.is_enterprise, - enabledFeatures: info.enabled_features, - allEnterpriseFeatures: info.all_enterprise_features, - isLoaded: true, - }) - } catch { - set({ - isEnterprise: false, - enabledFeatures: [], - allEnterpriseFeatures: [], - isLoaded: true, - }) - } - }, - - isFeatureEnabled: (feature: string) => { - return get().enabledFeatures.includes(feature) - }, -})) +import { create } from 'zustand' +import { apiClient } from '../lib/api' +import type { EnterpriseFeatureCatalog, EnterpriseFeatureMeta } from '../lib/api' + +interface LicenseState { + isEnterprise: boolean + enabledFeatures: string[] + allEnterpriseFeatures: string[] + featureCatalog: EnterpriseFeatureCatalog + isLoaded: boolean + fetchLicense: () => Promise + isFeatureEnabled: (feature: string) => boolean + getFeatureMeta: (feature: string) => EnterpriseFeatureMeta | undefined +} + +export const useLicenseStore = create((set, get) => ({ + isEnterprise: false, + enabledFeatures: [], + allEnterpriseFeatures: [], + featureCatalog: {}, + isLoaded: false, + + fetchLicense: async () => { + try { + const info = await apiClient.getLicenseInfo() + set({ + isEnterprise: info.is_enterprise, + enabledFeatures: info.enabled_features, + allEnterpriseFeatures: info.all_enterprise_features, + featureCatalog: info.feature_catalog ?? {}, + isLoaded: true, + }) + } catch { + set({ + isEnterprise: false, + enabledFeatures: [], + allEnterpriseFeatures: [], + featureCatalog: {}, + isLoaded: true, + }) + } + }, + + isFeatureEnabled: (feature: string) => { + return get().enabledFeatures.includes(feature) + }, + + getFeatureMeta: (feature: string) => { + return get().featureCatalog[feature] + }, +})) From 56b978041157edbd3f1e225ffce237e5e2225847 Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Sat, 14 Mar 2026 08:10:18 +0000 Subject: [PATCH 2/4] feat: updating backend services --- app/api/v1/routes/agents.py | 2 +- app/api/v1/routes/alerts.py | 4 +- app/api/v1/routes/audio.py | 6 +- app/api/v1/routes/chat.py | 2 +- app/api/v1/routes/conversation_evaluations.py | 6 +- app/api/v1/routes/data_sources.py | 2 +- app/api/v1/routes/evaluator_results.py | 2 +- app/api/v1/routes/evaluators.py | 2 +- app/api/v1/routes/manual_evaluations.py | 4 +- app/api/v1/routes/model_config.py | 2 +- app/api/v1/routes/playground.py | 4 +- app/api/v1/routes/prompt_partials.py | 4 +- app/api/v1/routes/test_agents.py | 4 +- app/api/v1/routes/voice_agent.py | 2 +- app/api/v1/routes/voice_playground.py | 8 +- app/services/ai/__init__.py | 18 + app/services/{ => ai}/llm_service.py | 4 +- app/services/{ => ai}/model_config_service.py | 49 +- .../{ => ai}/transcription_service.py | 1419 ++++++++--------- app/services/{ => ai}/tts_service.py | 21 +- app/services/alerts/__init__.py | 11 + .../{ => alerts}/alert_evaluation_service.py | 48 +- .../alert_notification_service.py | 60 +- app/services/audio/__init__.py | 33 + app/services/{ => audio}/audio_service.py | 2 - .../{ => audio}/qualitative_voice_service.py | 347 ++-- .../{ => audio}/voice_quality_service.py | 242 +-- app/services/evaluation/__init__.py | 11 + .../{ => evaluation}/evaluation_service.py | 8 +- .../{ => evaluation}/metrics_service.py | 1 - app/services/reporting/__init__.py | 8 + .../voice_playground_report_service.py | 4 +- app/services/storage/__init__.py | 11 + app/services/{ => storage}/s3_service.py | 199 +-- app/services/{ => storage}/storage_service.py | 3 - app/services/testing/__init__.py | 11 + .../test_agent_bridge_service.py | 827 ++++------ .../{ => testing}/test_agent_service.py | 324 ++-- app/services/voice_agent/utils/audio_merge.py | 2 +- app/services/voice_agent/voice_bundle.py | 2 +- app/workers/tasks/helpers/audio_evaluation.py | 6 +- app/workers/tasks/helpers/llm_evaluation.py | 2 +- app/workers/tasks/process_evaluation.py | 2 +- app/workers/tasks/process_evaluator_result.py | 2 +- app/workers/tasks/run_evaluator.py | 2 +- app/workers/tasks/tts_comparison.py | 8 +- app/workers/tasks/tts_report.py | 4 +- 47 files changed, 1548 insertions(+), 2197 deletions(-) create mode 100644 app/services/ai/__init__.py rename app/services/{ => ai}/llm_service.py (94%) rename app/services/{ => ai}/model_config_service.py (89%) rename app/services/{ => ai}/transcription_service.py (77%) rename app/services/{ => ai}/tts_service.py (93%) create mode 100644 app/services/alerts/__init__.py rename app/services/{ => alerts}/alert_evaluation_service.py (91%) rename app/services/{ => alerts}/alert_notification_service.py (87%) create mode 100644 app/services/audio/__init__.py rename app/services/{ => audio}/audio_service.py (95%) rename app/services/{ => audio}/qualitative_voice_service.py (76%) rename app/services/{ => audio}/voice_quality_service.py (66%) create mode 100644 app/services/evaluation/__init__.py rename app/services/{ => evaluation}/evaluation_service.py (94%) rename app/services/{ => evaluation}/metrics_service.py (96%) create mode 100644 app/services/reporting/__init__.py rename app/services/{ => reporting}/voice_playground_report_service.py (97%) create mode 100644 app/services/storage/__init__.py rename app/services/{ => storage}/s3_service.py (75%) rename app/services/{ => storage}/storage_service.py (95%) create mode 100644 app/services/testing/__init__.py rename app/services/{ => testing}/test_agent_bridge_service.py (68%) rename app/services/{ => testing}/test_agent_service.py (65%) diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index b5859252..8d481c95 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -95,7 +95,7 @@ async def generate_agent_description( db: Session = Depends(get_db), ): """Generate an agent description using AI from a brief description.""" - from app.services.llm_service import llm_service + from app.services.ai.llm_service import llm_service if not data.description.strip(): raise HTTPException(400, "Description is required") diff --git a/app/api/v1/routes/alerts.py b/app/api/v1/routes/alerts.py index 7ba30813..0a52e384 100644 --- a/app/api/v1/routes/alerts.py +++ b/app/api/v1/routes/alerts.py @@ -19,8 +19,8 @@ AlertHistoryResponse, AlertHistoryUpdate, ) -from app.services.alert_evaluation_service import alert_evaluation_service -from app.services.alert_notification_service import alert_notification_service +from app.services.alerts.alert_evaluation_service import alert_evaluation_service +from app.services.alerts.alert_notification_service import alert_notification_service router = APIRouter(prefix="/alerts", tags=["alerts"]) diff --git a/app/api/v1/routes/audio.py b/app/api/v1/routes/audio.py index 5e52c152..c0bc866b 100644 --- a/app/api/v1/routes/audio.py +++ b/app/api/v1/routes/audio.py @@ -8,9 +8,9 @@ from app.dependencies import get_api_key, get_organization_id from app.models.database import AudioFile from app.models.schemas import AudioFileResponse, MessageResponse -from app.services.storage_service import storage_service -from app.services.audio_service import AudioService -from app.services.s3_service import s3_service +from app.services.storage.storage_service import storage_service +from app.services.audio.audio_service import AudioService +from app.services.storage.s3_service import s3_service from app.core.exceptions import AudioFileNotFoundError, StorageError from uuid import UUID from loguru import logger diff --git a/app/api/v1/routes/chat.py b/app/api/v1/routes/chat.py index e32b2160..31b6184a 100644 --- a/app/api/v1/routes/chat.py +++ b/app/api/v1/routes/chat.py @@ -9,7 +9,7 @@ from pydantic import BaseModel from app.dependencies import get_db, get_organization_id -from app.services.llm_service import llm_service +from app.services.ai.llm_service import llm_service from app.models.schemas import ModelProvider router = APIRouter(prefix="/chat", tags=["chat"]) diff --git a/app/api/v1/routes/conversation_evaluations.py b/app/api/v1/routes/conversation_evaluations.py index 8414cfc0..ff0e4c12 100644 --- a/app/api/v1/routes/conversation_evaluations.py +++ b/app/api/v1/routes/conversation_evaluations.py @@ -11,9 +11,9 @@ from app.dependencies import get_api_key, get_organization_id from app.models.database import ConversationEvaluation, ManualTranscription, Agent, ModelProvider, Metric from app.models.schemas import ConversationEvaluationCreate, ConversationEvaluationResponse, MessageResponse -from app.services.llm_service import llm_service -from app.services.voice_quality_service import is_audio_metric, calculate_audio_metrics, AUDIO_METRICS -from app.services.s3_service import s3_service +from app.services.ai.llm_service import llm_service +from app.services.audio.voice_quality_service import is_audio_metric, calculate_audio_metrics, AUDIO_METRICS +from app.services.storage.s3_service import s3_service router = APIRouter(prefix="/conversation-evaluations", tags=["Conversation Evaluations"]) diff --git a/app/api/v1/routes/data_sources.py b/app/api/v1/routes/data_sources.py index 69a06f54..1c5ea8d7 100644 --- a/app/api/v1/routes/data_sources.py +++ b/app/api/v1/routes/data_sources.py @@ -8,7 +8,7 @@ from app.dependencies import get_api_key, get_organization_id from app.models.schemas import MessageResponse, S3ListFilesResponse, S3FileInfo, S3BrowseResponse, S3FolderInfo -from app.services.s3_service import s3_service +from app.services.storage.s3_service import s3_service from app.core.exceptions import StorageError from uuid import UUID diff --git a/app/api/v1/routes/evaluator_results.py b/app/api/v1/routes/evaluator_results.py index d94c743c..3d8f5e55 100644 --- a/app/api/v1/routes/evaluator_results.py +++ b/app/api/v1/routes/evaluator_results.py @@ -479,7 +479,7 @@ def re_evaluate_result( import uuid as _uuid from app.models.database import Agent, Integration from app.core.encryption import decrypt_api_key - from app.services.s3_service import s3_service + from app.services.storage.s3_service import s3_service call_data = result.call_data or {} platform = (result.provider_platform or "").lower() diff --git a/app/api/v1/routes/evaluators.py b/app/api/v1/routes/evaluators.py index dbcd60a7..b009eb11 100644 --- a/app/api/v1/routes/evaluators.py +++ b/app/api/v1/routes/evaluators.py @@ -55,7 +55,7 @@ def format_custom_prompt( db: Session = Depends(get_db), ): """Reformat a raw custom prompt into well-structured markdown using the org's LLM.""" - from app.services.llm_service import llm_service + from app.services.ai.llm_service import llm_service from app.models.database import AIProvider from app.models.enums import ModelProvider diff --git a/app/api/v1/routes/manual_evaluations.py b/app/api/v1/routes/manual_evaluations.py index 74e0e42f..8d2ccdef 100644 --- a/app/api/v1/routes/manual_evaluations.py +++ b/app/api/v1/routes/manual_evaluations.py @@ -10,8 +10,8 @@ from app.dependencies import get_api_key, get_organization_id from app.models.database import ManualTranscription, ModelProvider, AudioFile from app.models.schemas import MessageResponse, S3ListFilesResponse, S3FileInfo -from app.services.transcription_service import transcription_service -from app.services.s3_service import s3_service +from app.services.ai.transcription_service import transcription_service +from app.services.storage.s3_service import s3_service router = APIRouter(prefix="/manual-evaluations", tags=["Manual Evaluations"]) diff --git a/app/api/v1/routes/model_config.py b/app/api/v1/routes/model_config.py index 6a42697d..87f5e6c6 100644 --- a/app/api/v1/routes/model_config.py +++ b/app/api/v1/routes/model_config.py @@ -4,7 +4,7 @@ from typing import Dict, List, Any from app.dependencies import get_api_key from app.models.database import ModelProvider -from app.services.model_config_service import model_config_service +from app.services.ai.model_config_service import model_config_service router = APIRouter(prefix="/model-config", tags=["Model Config"]) diff --git a/app/api/v1/routes/playground.py b/app/api/v1/routes/playground.py index 5af58bb0..f5604262 100644 --- a/app/api/v1/routes/playground.py +++ b/app/api/v1/routes/playground.py @@ -314,7 +314,7 @@ def poll_call_metrics( try: import requests as _http import uuid as _uuid - from app.services.s3_service import s3_service + from app.services.storage.s3_service import s3_service recording_urls = call_metrics.get("recording_urls", {}) audio_bytes = None @@ -865,7 +865,7 @@ async def re_evaluate_call_recording( """ import requests as http_requests import uuid as _uuid - from app.services.s3_service import s3_service + from app.services.storage.s3_service import s3_service call_recording = db.query(CallRecording).filter( CallRecording.call_short_id == call_short_id, diff --git a/app/api/v1/routes/prompt_partials.py b/app/api/v1/routes/prompt_partials.py index 8d68d95f..90a6850b 100644 --- a/app/api/v1/routes/prompt_partials.py +++ b/app/api/v1/routes/prompt_partials.py @@ -112,7 +112,7 @@ async def generate_prompt_with_ai( db: Session = Depends(get_db), ): """Generate a new prompt using AI from a description.""" - from app.services.llm_service import llm_service + from app.services.ai.llm_service import llm_service if not data.description.strip(): raise HTTPException(400, "Description is required") @@ -158,7 +158,7 @@ async def improve_prompt_with_ai( db: Session = Depends(get_db), ): """Improve/reformat existing prompt content using AI.""" - from app.services.llm_service import llm_service + from app.services.ai.llm_service import llm_service if not data.content.strip(): raise HTTPException(400, "Content is required") diff --git a/app/api/v1/routes/test_agents.py b/app/api/v1/routes/test_agents.py index 968af347..5c750623 100644 --- a/app/api/v1/routes/test_agents.py +++ b/app/api/v1/routes/test_agents.py @@ -16,7 +16,7 @@ TestAgentConversationUpdate, TestAgentConversationResponse ) -from app.services.test_agent_service import test_agent_service +from app.services.testing.test_agent_service import test_agent_service router = APIRouter(prefix="/test-agents", tags=["test-agents"]) @@ -188,7 +188,7 @@ async def get_response_audio( raise HTTPException(status_code=404, detail="Audio segment key not found") # Download from S3 - from app.services.s3_service import s3_service + from app.services.storage.s3_service import s3_service try: audio_bytes = s3_service.download_file_by_key(audio_key) from fastapi.responses import Response diff --git a/app/api/v1/routes/voice_agent.py b/app/api/v1/routes/voice_agent.py index 1cdaf199..da8758b8 100644 --- a/app/api/v1/routes/voice_agent.py +++ b/app/api/v1/routes/voice_agent.py @@ -15,7 +15,7 @@ from app.core.encryption import decrypt_api_key from app.services.voice_agent.bot_fast_api import run_bot from app.services.voice_agent.voice_bundle import run_voice_bundle_fastapi -from app.services.s3_service import s3_service +from app.services.storage.s3_service import s3_service router = APIRouter(prefix="/voice-agent", tags=["voice-agent"]) diff --git a/app/api/v1/routes/voice_playground.py b/app/api/v1/routes/voice_playground.py index e222563f..1ded0ea8 100644 --- a/app/api/v1/routes/voice_playground.py +++ b/app/api/v1/routes/voice_playground.py @@ -27,10 +27,10 @@ ModelProvider, VoiceBundle, ) -from app.services.model_config_service import model_config_service -from app.services.s3_service import s3_service -from app.services.llm_service import llm_service -from app.services.voice_playground_report_service import voice_playground_report_service +from app.services.ai.model_config_service import model_config_service +from app.services.storage.s3_service import s3_service +from app.services.ai.llm_service import llm_service +from app.services.reporting.voice_playground_report_service import voice_playground_report_service router = APIRouter( prefix="/voice-playground", diff --git a/app/services/ai/__init__.py b/app/services/ai/__init__.py new file mode 100644 index 00000000..96ae5155 --- /dev/null +++ b/app/services/ai/__init__.py @@ -0,0 +1,18 @@ +"""AI service package exports.""" + +from app.services.ai.llm_service import LLMService, llm_service +from app.services.ai.model_config_service import ModelConfigService, model_config_service +from app.services.ai.transcription_service import TranscriptionService, transcription_service +from app.services.ai.tts_service import TTSService, get_audio_file_extension, tts_service + +__all__ = [ + "LLMService", + "llm_service", + "ModelConfigService", + "model_config_service", + "TranscriptionService", + "transcription_service", + "TTSService", + "get_audio_file_extension", + "tts_service", +] diff --git a/app/services/llm_service.py b/app/services/ai/llm_service.py similarity index 94% rename from app/services/llm_service.py rename to app/services/ai/llm_service.py index 657df351..1b1089fa 100644 --- a/app/services/llm_service.py +++ b/app/services/ai/llm_service.py @@ -2,8 +2,8 @@ LLM service for generating text responses using various LLM providers. Uses LiteLLM as a unified gateway so every provider (OpenAI, Anthropic, -Google, DeepSeek, Groq, Azure, AWS Bedrock, …) is accessed through a -single interface. LiteLLM handles message-format translation, parameter +Google, DeepSeek, Groq, Azure, AWS Bedrock, ...) is accessed through a +single interface. LiteLLM handles message-format translation, parameter mapping, and endpoint selection (e.g. OpenAI Responses API vs Chat Completions) automatically. """ diff --git a/app/services/model_config_service.py b/app/services/ai/model_config_service.py similarity index 89% rename from app/services/model_config_service.py rename to app/services/ai/model_config_service.py index d3086d09..424c2661 100644 --- a/app/services/model_config_service.py +++ b/app/services/ai/model_config_service.py @@ -9,43 +9,43 @@ class ModelConfigService: """Service to load and manage model configurations from JSON file.""" - + def __init__(self, config_path: Optional[Path] = None): """Initialize the service with config file path.""" if config_path is None: # Default to app/config/models.json - config_path = Path(__file__).parent.parent / "config" / "models.json" + config_path = Path(__file__).parent.parent.parent / "config" / "models.json" self.config_path = config_path self._config: Optional[Dict[str, Any]] = None self._load_config() - + def _load_config(self) -> None: """Load configuration from JSON file.""" try: - with open(self.config_path, 'r', encoding='utf-8') as f: + with open(self.config_path, "r", encoding="utf-8") as f: self._config = json.load(f) except FileNotFoundError: raise FileNotFoundError(f"Model config file not found: {self.config_path}") except json.JSONDecodeError as e: raise ValueError(f"Invalid JSON in config file: {e}") - + def reload_config(self) -> None: """Reload configuration from file.""" self._load_config() - + def get_all_models(self) -> Dict[str, Any]: """Get all model configurations.""" if self._config is None: self._load_config() # Exclude sample_spec return {k: v for k, v in self._config.items() if k != "sample_spec"} - + def get_model_config(self, model_name: str) -> Optional[Dict[str, Any]]: """Get configuration for a specific model.""" if self._config is None: self._load_config() return self._config.get(model_name) - + def get_models_by_provider(self, provider: ModelProvider) -> List[str]: """Get all model names for a specific provider.""" if self._config is None: @@ -58,11 +58,11 @@ def get_models_by_provider(self, provider: ModelProvider) -> List[str]: if config.get("provider") == provider_str: models.append(model_name) return models - + def get_models_by_type(self, provider: ModelProvider, model_type: str) -> List[str]: """ Get models by provider and type (stt, llm, tts, s2s). - + Args: provider: The model provider model_type: One of 'stt', 'llm', 'tts', 's2s' @@ -77,11 +77,11 @@ def get_models_by_type(self, provider: ModelProvider, model_type: str) -> List[s if config.get("provider") == provider_str and config.get("model_type") == model_type: models.append(model_name) return models - + def get_model_options_by_provider(self, provider: ModelProvider) -> Dict[str, List[str]]: """ Get model options organized by type for a provider. - + Returns: Dict with keys 'stt', 'llm', 'tts', 's2s' and values as lists of model names """ @@ -91,24 +91,24 @@ def get_model_options_by_provider(self, provider: ModelProvider) -> Dict[str, Li "tts": self.get_models_by_type(provider, "tts"), "s2s": self.get_models_by_type(provider, "s2s"), } - + def get_model_info(self, model_name: str) -> Optional[Dict[str, Any]]: """ Get detailed information about a model. - + Returns: Dict with model configuration (provider and model_type) """ return self.get_model_config(model_name) - + def validate_model(self, provider: ModelProvider, model_name: str, model_type: str) -> bool: """Validate that a model exists and matches the provider and type.""" config = self.get_model_config(model_name) if config is None: return False return ( - config.get("provider") == provider.value and - config.get("model_type") == model_type + config.get("provider") == provider.value + and config.get("model_type") == model_type ) def get_voices_for_model(self, model_name: str) -> List[Dict[str, Any]]: @@ -164,12 +164,14 @@ def get_voices_for_model(self, model_name: str) -> List[Dict[str, Any]]: if not voice_id: continue display_name = item.get("name") or item.get("displayName") or voice_id - normalized.append({ - "id": str(voice_id), - "name": str(display_name), - "gender": str(item.get("gender") or "Unknown"), - "accent": str(item.get("accent") or "Unknown"), - }) + normalized.append( + { + "id": str(voice_id), + "name": str(display_name), + "gender": str(item.get("gender") or "Unknown"), + "accent": str(item.get("accent") or "Unknown"), + } + ) return normalized def get_tts_voices_by_provider(self, provider: ModelProvider) -> Dict[str, List[Dict[str, Any]]]: @@ -194,4 +196,3 @@ def get_tts_voices_by_provider(self, provider: ModelProvider) -> Dict[str, List[ # Singleton instance model_config_service = ModelConfigService() - diff --git a/app/services/transcription_service.py b/app/services/ai/transcription_service.py similarity index 77% rename from app/services/transcription_service.py rename to app/services/ai/transcription_service.py index cdae1314..3b962038 100644 --- a/app/services/transcription_service.py +++ b/app/services/ai/transcription_service.py @@ -1,720 +1,699 @@ -""" -Transcription service for converting audio to text using various STT providers. - -Response Format: -All providers return a standardized format: -{ - "text": str, # Full transcript text - "language": str, # Language code (e.g., "en", "es") - "segments": [ # List of segments with timestamps - { - "start": float, # Start time in seconds - "end": float, # End time in seconds - "text": str # Text for this segment - } - ], - "speaker_segments": [ # Segments with speaker labels (if diarization enabled) - { - "speaker": str, # Speaker label (e.g., "Speaker 1") - "text": str, - "start": float, - "end": float - } - ], - "processing_time": float, - "raw_output": dict # Original provider response -} - -Provider-Specific Formats: -- OpenAI Whisper API: Uses verbose_json format which includes segments with timestamps -- Local Whisper: Returns segments by default with word-level timestamps available -- Google/Azure/AWS: Formats documented but not yet implemented - -Speaker Diarization: -- Whisper does NOT provide speaker diarization natively (only transcription) -- We use pyannote.audio (speaker-diarization-3.1) for accurate ML-based diarization -- Whisper provides word-level timestamps, pyannote identifies speakers, then we align -- Requires: pyannote.audio installed + diarization.huggingface_token in config.yml -- Falls back to unreliable gap-based heuristics if pyannote is unavailable -""" - -import time -import tempfile -import os -import logging -from typing import Optional, Dict, Any, List -from uuid import UUID -from pathlib import Path - -from app.models.database import ModelProvider, AIProvider -from app.services.s3_service import s3_service -from app.core.exceptions import StorageError -from sqlalchemy.orm import Session - -logger = logging.getLogger(__name__) - - -class TranscriptionService: - """Service for transcribing audio files using various STT providers.""" - - def __init__(self): - """Initialize transcription service.""" - self._pyannote_pipeline = None - - def _get_ai_provider(self, provider: ModelProvider, db: Session, organization_id: UUID) -> Optional[AIProvider]: - """Get AI provider configuration from database.""" - from sqlalchemy import func - - # Handle both string and enum comparisons (database might have uppercase or lowercase) - provider_value = provider.value if hasattr(provider, 'value') else provider - - # Try exact match first - ai_provider = db.query(AIProvider).filter( - AIProvider.provider == provider_value, - AIProvider.organization_id == organization_id, - AIProvider.is_active == True - ).first() - - # If not found, try case-insensitive match - if not ai_provider: - ai_provider = db.query(AIProvider).filter( - func.lower(AIProvider.provider) == provider_value.lower(), - AIProvider.organization_id == organization_id, - AIProvider.is_active == True - ).first() - - return ai_provider - - def _download_audio_to_temp(self, audio_file_key: str, db: Optional[Session] = None) -> str: - """ - Download audio from S3 to temporary file, or use local file if S3 is not available. - - Args: - audio_file_key: S3 key or local file path - db: Optional database session to look up local file paths - - Returns: - Path to temporary file (or original file if local) - """ - import os - - # First, try S3 if enabled - if s3_service.is_enabled(): - try: - # Download from S3 - audio_bytes = s3_service.download_file_by_key(audio_file_key) - - # Determine file extension from key - file_ext = Path(audio_file_key).suffix.lstrip('.') or 'wav' - - # Create temporary file - with tempfile.NamedTemporaryFile(delete=False, suffix=f'.{file_ext}') as temp_file: - temp_file.write(audio_bytes) - return temp_file.name - except Exception as e: - # If S3 download fails, fall through to local file check - logger.warning(f"S3 download failed for {audio_file_key}: {e}, trying local file...") - - # Fallback: Try to find local file - # Check if it's already a local file path - if os.path.exists(audio_file_key): - # It's a local file path, return it directly - return audio_file_key - - # Try to look up in database if db session is provided - if db: - try: - from app.models.database import AudioFile - # Try to find by S3 key or file path - audio_file = db.query(AudioFile).filter( - (AudioFile.file_path == audio_file_key) | - (AudioFile.file_path.like(f"%{audio_file_key}%")) - ).first() - - if audio_file and os.path.exists(audio_file.file_path): - return audio_file.file_path - except Exception as e: - logger.warning(f"Database lookup failed for {audio_file_key}: {e}") - - # If all else fails, raise error - raise StorageError(f"Failed to download audio file: S3 is not enabled and local file not found for key: {audio_file_key}") - - def _transcribe_with_openai(self, audio_file_path: str, model: str, api_key: str, language: Optional[str] = None) -> Dict[str, Any]: - """Transcribe audio using OpenAI Whisper API with word-level timestamps.""" - try: - from openai import OpenAI - - client = OpenAI(api_key=api_key) - - with open(audio_file_path, 'rb') as audio_file: - transcript = client.audio.transcriptions.create( - model=model, - file=audio_file, - language=language, - response_format="verbose_json", - timestamp_granularities=["word", "segment"] - ) - - result = { - "text": transcript.text if hasattr(transcript, 'text') else str(transcript), - "language": getattr(transcript, 'language', language) if language else getattr(transcript, 'language', 'en'), - "segments": [], - "words": [] - } - - if hasattr(transcript, 'segments') and transcript.segments: - for seg in transcript.segments: - result["segments"].append({ - "start": getattr(seg, 'start', 0), - "end": getattr(seg, 'end', 0), - "text": getattr(seg, 'text', '') - }) - elif isinstance(transcript, dict) and 'segments' in transcript: - for seg in transcript['segments']: - result["segments"].append({ - "start": seg.get('start', 0), - "end": seg.get('end', 0), - "text": seg.get('text', '') - }) - - if hasattr(transcript, 'words') and transcript.words: - for w in transcript.words: - # Handle both object attributes and dict-like access - if isinstance(w, dict): - word_text = w.get('word', '') - word_start = w.get('start', 0) or 0 - word_end = w.get('end', 0) or 0 - else: - word_text = getattr(w, 'word', '') or '' - word_start = getattr(w, 'start', None) - word_end = getattr(w, 'end', None) - # Some SDK versions may use 'start_time'/'end_time' - if word_start is None: - word_start = getattr(w, 'start_time', 0) or 0 - if word_end is None: - word_end = getattr(w, 'end_time', 0) or 0 - word_start = float(word_start) if word_start else 0.0 - word_end = float(word_end) if word_end else 0.0 - result["words"].append({ - "word": word_text, - "start": word_start, - "end": word_end - }) - elif isinstance(transcript, dict) and 'words' in transcript: - for w in transcript['words']: - result["words"].append({ - "word": w.get('word', ''), - "start": w.get('start', 0) or 0, - "end": w.get('end', 0) or 0 - }) - - if not result["segments"] and result["text"]: - import re - sentences = re.split(r'[.!?]+\s+', result["text"].strip()) - sentences = [s.strip() for s in sentences if s.strip()] - - if sentences: - total_words = len(result["text"].split()) - estimated_duration = max(1.0, (total_words / 150.0) * 60.0) - - current_time = 0.0 - for sentence in sentences: - sentence_words = len(sentence.split()) - sentence_duration = max(0.5, (sentence_words / 150.0) * 60.0) - result["segments"].append({ - "start": current_time, - "end": current_time + sentence_duration, - "text": sentence - }) - current_time += sentence_duration - else: - word_count = len(result["text"].split()) - estimated_duration = max(1.0, (word_count / 150.0) * 60.0) - result["segments"] = [{ - "start": 0.0, - "end": estimated_duration, - "text": result["text"] - }] - - return result - except ImportError: - raise RuntimeError("OpenAI library not installed. Install with: pip install openai") - except Exception as e: - import traceback - error_details = traceback.format_exc() - raise RuntimeError(f"OpenAI transcription failed: {str(e)}\nDetails: {error_details}") - - def _transcribe_with_whisper_local(self, audio_file_path: str, model_name: str = "base") -> Dict[str, Any]: - """Transcribe audio using local Whisper model.""" - try: - import whisper - - model = whisper.load_model(model_name) - result = model.transcribe(audio_file_path) - - return { - "text": result.get("text", ""), - "language": result.get("language", "en"), - "segments": [ - { - "start": seg.get("start", 0), - "end": seg.get("end", 0), - "text": seg.get("text", "") - } - for seg in result.get("segments", []) - ] - } - except ImportError: - raise RuntimeError("Whisper library not installed. Install with: pip install openai-whisper") - except Exception as e: - raise RuntimeError(f"Whisper transcription failed: {str(e)}") - - def _get_pyannote_pipeline(self): - """Load and cache the pyannote diarization pipeline.""" - if self._pyannote_pipeline is not None: - return self._pyannote_pipeline - - # Compatibility shim: list_audio_backends was removed in torchaudio 2.4+ - import torchaudio - if not hasattr(torchaudio, 'list_audio_backends'): - torchaudio.list_audio_backends = lambda: ["ffmpeg"] - - from pyannote.audio import Pipeline - from app.config import settings - - hf_token = settings.HUGGINGFACE_TOKEN - if not hf_token: - raise RuntimeError( - "HUGGINGFACE_TOKEN not configured. Set it under 'diarization.huggingface_token' " - "in config.yml. Required for pyannote speaker diarization." - ) - - logger.info("Loading pyannote speaker-diarization-3.1 pipeline (first call, will be cached)...") - self._pyannote_pipeline = Pipeline.from_pretrained( - "pyannote/speaker-diarization-3.1", - token=hf_token - ) - logger.info("Pyannote pipeline loaded successfully") - return self._pyannote_pipeline - - def _detect_speakers_with_pyannote( - self, audio_file_path: str, segments: List[Dict[str, Any]], - words: Optional[List[Dict[str, Any]]] = None, - num_speakers: Optional[int] = 2, - min_speakers: Optional[int] = None, - max_speakers: Optional[int] = None - ) -> List[Dict[str, Any]]: - """ - Use pyannote.audio for ML-based speaker diarization, aligned with - Whisper word timestamps for accurate speaker-text mapping. - - Args: - num_speakers: If set, forces pyannote to produce exactly this many - speaker clusters. Defaults to 2 (agent + customer), which greatly - improves accuracy on mono phone recordings. - min_speakers: Optional lower bound on speaker count. - max_speakers: Optional upper bound on speaker count. - - Raises exceptions on failure so the caller can handle fallback and logging. - """ - pipeline = self._get_pyannote_pipeline() - - # Build pipeline kwargs for speaker count hints - pipeline_kwargs = {} - if num_speakers is not None: - pipeline_kwargs["num_speakers"] = num_speakers - if min_speakers is not None: - pipeline_kwargs["min_speakers"] = min_speakers - if max_speakers is not None: - pipeline_kwargs["max_speakers"] = max_speakers - - logger.info( - f"Running pyannote diarization on {audio_file_path} " - f"(speaker hints: {pipeline_kwargs or 'auto'})" - ) - raw_output = pipeline(audio_file_path, **pipeline_kwargs) - - # Handle different return types across pyannote versions: - # - Older versions: Annotation directly (has itertracks) - # - Newer versions: DiarizeOutput with .speaker_diarization attribute - if hasattr(raw_output, 'itertracks'): - annotation = raw_output - elif hasattr(raw_output, 'speaker_diarization'): - annotation = raw_output.speaker_diarization - elif hasattr(raw_output, 'annotation'): - annotation = raw_output.annotation - elif isinstance(raw_output, tuple): - annotation = raw_output[0] - else: - attrs = [a for a in dir(raw_output) if not a.startswith('_')] - raise TypeError( - f"Unexpected pyannote output type: {type(raw_output).__name__}. " - f"Available attributes: {attrs}" - ) - - # Collect diarization turns as a sorted list for fast lookup - diar_turns = [] - raw_labels = set() - for turn, _, speaker_label in annotation.itertracks(yield_label=True): - diar_turns.append((turn.start, turn.end, speaker_label)) - raw_labels.add(speaker_label) - - if not diar_turns: - logger.warning("Pyannote returned no speaker turns") - return [] - - sorted_labels = sorted(raw_labels) - label_map = {lbl: f"Speaker {i + 1}" for i, lbl in enumerate(sorted_labels)} - logger.info(f"Pyannote detected {len(sorted_labels)} speakers, {len(diar_turns)} turns") - - def find_speaker(midpoint: float) -> str: - """Find which speaker is active at a given timestamp.""" - for t_start, t_end, lbl in diar_turns: - if t_start <= midpoint <= t_end: - return label_map[lbl] - # No exact match -- find the closest turn - min_dist = float('inf') - closest_label = label_map[diar_turns[0][2]] - for t_start, t_end, lbl in diar_turns: - dist = min(abs(midpoint - t_start), abs(midpoint - t_end)) - if dist < min_dist: - min_dist = dist - closest_label = label_map[lbl] - return closest_label - - if words and len(words) > 0: - speaker_segments = [] - current_speaker = None - current_words: List[str] = [] - current_start = 0.0 - current_end = 0.0 - - for w in words: - word_text = w.get("word", "").strip() - w_start = w.get("start", 0) - w_end = w.get("end", 0) - if not word_text: - continue - - midpoint = (w_start + w_end) / 2.0 - speaker = find_speaker(midpoint) - - if speaker != current_speaker: - if current_words and current_speaker: - speaker_segments.append({ - "speaker": current_speaker, - "text": " ".join(current_words).strip(), - "start": round(current_start, 3), - "end": round(current_end, 3) - }) - current_speaker = speaker - current_words = [word_text] - current_start = w_start - current_end = w_end - else: - current_words.append(word_text) - current_end = w_end - - if current_words and current_speaker: - speaker_segments.append({ - "speaker": current_speaker, - "text": " ".join(current_words).strip(), - "start": round(current_start, 3), - "end": round(current_end, 3) - }) - - return speaker_segments - - # Fallback: align at segment level when word timestamps are not available - speaker_segments = [] - for seg in segments: - seg_start = seg.get("start", 0) - seg_end = seg.get("end", 0) - seg_text = seg.get("text", "").strip() - if not seg_text: - continue - - midpoint = (seg_start + seg_end) / 2.0 - speaker = find_speaker(midpoint) - - speaker_segments.append({ - "speaker": speaker, - "text": seg_text, - "start": round(seg_start, 3), - "end": round(seg_end, 3) - }) - - return speaker_segments - - def _detect_speakers_heuristic(self, segments: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Heuristic-based speaker diarization fallback. Uses gap-based detection - which is unreliable -- pyannote.audio should be used for accurate results. - """ - logger.warning( - "Using heuristic speaker diarization (unreliable). For accurate results, " - "install pyannote.audio and configure diarization.huggingface_token in config.yml" - ) - if not segments: - return [] - - speaker_segments = [] - current_speaker = "Speaker 1" - - # Thresholds for speaker change detection - GAP_THRESHOLD_LARGE = 0.8 # seconds - large gap definitely suggests change - GAP_THRESHOLD_MEDIUM = 0.4 # seconds - medium gap suggests change - GAP_THRESHOLD_SMALL = 0.2 # seconds - small gap, but combined with other factors - MIN_SEGMENT_DURATION = 0.2 # seconds - minimum segment to consider - - # Calculate average gap size to adapt thresholds - gaps = [] - for i in range(1, len(segments)): - gap = segments[i].get("start", 0) - segments[i-1].get("end", 0) - if gap > 0: - gaps.append(gap) - - avg_gap = sum(gaps) / len(gaps) if gaps else 0.5 - # Adaptive threshold based on conversation pace - adaptive_threshold = min(max(avg_gap * 1.5, 0.3), 1.0) - - # Track recent speaker assignments for pattern detection - recent_assignments = [] - assignment_window = 5 # Look at last N segments for patterns - - for i, seg in enumerate(segments): - seg_start = seg.get("start", 0) - seg_end = seg.get("end", 0) - seg_duration = seg_end - seg_start - seg_text = seg.get("text", "").strip() - - # Skip very short segments (likely noise or artifacts) - if seg_duration < MIN_SEGMENT_DURATION or not seg_text: - continue - - # Check for speaker change indicators - should_switch = False - gap = 0 - - if i > 0: - prev_seg = segments[i-1] - gap = seg_start - prev_seg.get("end", 0) - - # Large gap definitely suggests speaker change - if gap > GAP_THRESHOLD_LARGE: - should_switch = True - # Medium gap suggests change - elif gap > GAP_THRESHOLD_MEDIUM: - should_switch = True - # Small gap but check for alternating pattern - elif gap > GAP_THRESHOLD_SMALL: - # If we've been alternating, continue the pattern - if len(recent_assignments) >= 2: - # Check if last two were the same speaker (suggests we should switch) - if recent_assignments[-1] == recent_assignments[-2] == current_speaker: - should_switch = True - # Or if we see a pattern of quick back-and-forth - elif len(recent_assignments) >= 3: - # If pattern is A-A-A, switch to B - if all(a == current_speaker for a in recent_assignments[-3:]): - should_switch = True - # Very small or no gap - use alternating pattern if established - elif gap >= 0: - # If we have an established alternating pattern, continue it - if len(recent_assignments) >= 2: - # Check if we should alternate based on recent pattern - if recent_assignments[-1] == current_speaker: - # If last segment was same speaker, consider switching - # But only if we have a pattern suggesting alternation - if len(recent_assignments) >= 4: - # Check for A-B-A-B pattern - pattern = recent_assignments[-4:] - if pattern[0] != pattern[1] and pattern[1] != pattern[2] and pattern[2] != pattern[3]: - # We have alternating pattern, continue it - should_switch = True - - # Additional heuristics for first few segments - if i < 3 and i > 0: - # Early in conversation, be more aggressive about switching - if gap > 0.1: # Any noticeable gap - should_switch = True - - # If we have multiple consecutive segments from same speaker, force alternation - if len(recent_assignments) >= 2: - # If last 2 segments were both the same speaker (and it's the current speaker), switch - # This prevents one speaker from getting too many consecutive segments - if recent_assignments[-1] == current_speaker and recent_assignments[-2] == current_speaker: - should_switch = True - - # Switch speaker if needed - if should_switch: - current_speaker = "Speaker 2" if current_speaker == "Speaker 1" else "Speaker 1" - - speaker_segments.append({ - "speaker": current_speaker, - "text": seg_text, - "start": seg_start, - "end": seg_end - }) - - # Track recent assignments for pattern detection - recent_assignments.append(current_speaker) - if len(recent_assignments) > assignment_window: - recent_assignments.pop(0) - - # Post-process: Balance speakers if one dominates too much - speaker_1_count = sum(1 for s in speaker_segments if s["speaker"] == "Speaker 1") - speaker_2_count = sum(1 for s in speaker_segments if s["speaker"] == "Speaker 2") - total_segments = len(speaker_segments) - - # If one speaker has more than 70% of segments, redistribute by alternating - if total_segments > 0: - speaker_1_ratio = speaker_1_count / total_segments - if speaker_1_ratio > 0.7: - # Redistribute: alternate segments starting from index 1 - # This assumes the first speaker is correct, then alternates - for i in range(1, len(speaker_segments)): - expected_speaker = "Speaker 2" if i % 2 == 1 else "Speaker 1" - if speaker_segments[i]["speaker"] != expected_speaker: - speaker_segments[i]["speaker"] = expected_speaker - elif speaker_1_ratio < 0.3: - # Speaker 2 dominates, redistribute - for i in range(1, len(speaker_segments)): - expected_speaker = "Speaker 1" if i % 2 == 1 else "Speaker 2" - if speaker_segments[i]["speaker"] != expected_speaker: - speaker_segments[i]["speaker"] = expected_speaker - - return speaker_segments - - def transcribe( - self, - audio_file_key: str, - stt_provider: ModelProvider, - stt_model: str, - organization_id: UUID, - db: Session, - language: Optional[str] = None, - enable_speaker_diarization: bool = True - ) -> Dict[str, Any]: - """ - Transcribe audio file from S3. - - Args: - audio_file_key: S3 key of the audio file - stt_provider: STT provider to use - stt_model: STT model name - organization_id: Organization ID - db: Database session - language: Optional language code (e.g., 'en', 'es') - enable_speaker_diarization: Whether to detect multiple speakers - - Returns: - Dictionary with transcript and metadata - """ - start_time = time.time() - temp_file_path = None - - try: - # Download audio to temporary file - temp_file_path = self._download_audio_to_temp(audio_file_key, db=db) - - # Get provider API key - ai_provider = self._get_ai_provider(stt_provider, db, organization_id) - if not ai_provider: - raise RuntimeError(f"AI provider {stt_provider} not configured for this organization. Please configure an AI provider in the settings.") - - # Decrypt API key - from app.core.encryption import decrypt_api_key - try: - api_key = decrypt_api_key(ai_provider.api_key) - except Exception as e: - raise RuntimeError(f"Failed to decrypt API key for provider {stt_provider}: {str(e)}") - - # Transcribe based on provider - if stt_provider == ModelProvider.OPENAI: - if stt_model.startswith("whisper-"): - # Use OpenAI API - result = self._transcribe_with_openai(temp_file_path, stt_model, api_key, language) - else: - # Fallback to local Whisper - model_name = stt_model.replace("whisper-", "") if stt_model.startswith("whisper-") else "base" - result = self._transcribe_with_whisper_local(temp_file_path, model_name) - elif stt_provider == ModelProvider.GOOGLE: - # TODO: Implement Google Speech-to-Text - raise NotImplementedError("Google Speech-to-Text not yet implemented") - elif stt_provider == ModelProvider.AZURE: - # TODO: Implement Azure Speech Services - raise NotImplementedError("Azure Speech Services not yet implemented") - elif stt_provider == ModelProvider.AWS: - # TODO: Implement AWS Transcribe - raise NotImplementedError("AWS Transcribe not yet implemented") - else: - # Default to local Whisper - result = self._transcribe_with_whisper_local(temp_file_path, "base") - - # Apply speaker diarization if enabled - speaker_segments = None - if enable_speaker_diarization: - segments = result.get("segments", []) - - # If no segments from transcription, create a single segment from the full text - if not segments and result.get("text"): - estimated_duration = 0.0 - if hasattr(result, 'duration') and result.get('duration'): - estimated_duration = result.get('duration') - elif audio_file_path and os.path.exists(audio_file_path): - try: - import librosa - duration = librosa.get_duration(path=audio_file_path) - estimated_duration = duration - except: - pass - - segments = [{ - "start": 0.0, - "end": estimated_duration if estimated_duration > 0 else 10.0, # Default to 10s if unknown - "text": result.get("text", "") - }] - - if segments: - words = result.get("words", []) - # Check if words actually have valid timestamps - valid_word_count = sum(1 for w in words if w.get("start", 0) > 0 or w.get("end", 0) > 0) - if words and valid_word_count == 0: - words = [] - - try: - from app.config import settings as app_settings - num_spk = getattr(app_settings, 'DIARIZATION_NUM_SPEAKERS', 2) - speaker_segments = self._detect_speakers_with_pyannote( - temp_file_path, segments, words, num_speakers=num_spk - ) - if not speaker_segments: - speaker_segments = self._detect_speakers_heuristic(segments) - except Exception as e: - logger.warning(f"Pyannote diarization failed: {e}, falling back to heuristic") - speaker_segments = self._detect_speakers_heuristic(segments) - - processing_time = time.time() - start_time - - return { - "transcript": result["text"], - "language": result.get("language", language), - "speaker_segments": speaker_segments, - "segments": result.get("segments", []), - "processing_time": processing_time, - "raw_output": result - } - - finally: - # Clean up temporary file - if temp_file_path and os.path.exists(temp_file_path): - try: - os.unlink(temp_file_path) - except Exception: - pass - - -# Singleton instance -transcription_service = TranscriptionService() - +""" +Transcription service for converting audio to text using various STT providers. + +Response Format: +All providers return a standardized format: +{ + "text": str, # Full transcript text + "language": str, # Language code (e.g., "en", "es") + "segments": [ # List of segments with timestamps + { + "start": float, # Start time in seconds + "end": float, # End time in seconds + "text": str # Text for this segment + } + ], + "speaker_segments": [ # Segments with speaker labels (if diarization enabled) + { + "speaker": str, # Speaker label (e.g., "Speaker 1") + "text": str, + "start": float, + "end": float + } + ], + "processing_time": float, + "raw_output": dict # Original provider response +} + +Provider-Specific Formats: +- OpenAI Whisper API: Uses verbose_json format which includes segments with timestamps +- Local Whisper: Returns segments by default with word-level timestamps available +- Google/Azure/AWS: Formats documented but not yet implemented + +Speaker Diarization: +- Whisper does NOT provide speaker diarization natively (only transcription) +- We use pyannote.audio (speaker-diarization-3.1) for accurate ML-based diarization +- Whisper provides word-level timestamps, pyannote identifies speakers, then we align +- Requires: pyannote.audio installed + diarization.huggingface_token in config.yml +- Falls back to unreliable gap-based heuristics if pyannote is unavailable +""" + +import time +import tempfile +import os +import logging +from typing import Optional, Dict, Any, List +from uuid import UUID +from pathlib import Path + +from app.models.database import ModelProvider, AIProvider +from app.services.storage.s3_service import s3_service +from app.core.exceptions import StorageError +from sqlalchemy.orm import Session + +logger = logging.getLogger(__name__) + + +class TranscriptionService: + """Service for transcribing audio files using various STT providers.""" + + def __init__(self): + """Initialize transcription service.""" + self._pyannote_pipeline = None + + def _get_ai_provider(self, provider: ModelProvider, db: Session, organization_id: UUID) -> Optional[AIProvider]: + """Get AI provider configuration from database.""" + from sqlalchemy import func + + # Handle both string and enum comparisons (database might have uppercase or lowercase) + provider_value = provider.value if hasattr(provider, "value") else provider + + # Try exact match first + ai_provider = db.query(AIProvider).filter( + AIProvider.provider == provider_value, + AIProvider.organization_id == organization_id, + AIProvider.is_active == True, + ).first() + + # If not found, try case-insensitive match + if not ai_provider: + ai_provider = db.query(AIProvider).filter( + func.lower(AIProvider.provider) == provider_value.lower(), + AIProvider.organization_id == organization_id, + AIProvider.is_active == True, + ).first() + + return ai_provider + + def _download_audio_to_temp(self, audio_file_key: str, db: Optional[Session] = None) -> str: + """ + Download audio from S3 to temporary file, or use local file if S3 is not available. + """ + import os + + # First, try S3 if enabled + if s3_service.is_enabled(): + try: + # Download from S3 + audio_bytes = s3_service.download_file_by_key(audio_file_key) + + # Determine file extension from key + file_ext = Path(audio_file_key).suffix.lstrip(".") or "wav" + + # Create temporary file + with tempfile.NamedTemporaryFile(delete=False, suffix=f".{file_ext}") as temp_file: + temp_file.write(audio_bytes) + return temp_file.name + except Exception as e: + # If S3 download fails, fall through to local file check + logger.warning(f"S3 download failed for {audio_file_key}: {e}, trying local file...") + + # Fallback: Try to find local file + # Check if it's already a local file path + if os.path.exists(audio_file_key): + # It's a local file path, return it directly + return audio_file_key + + # Try to look up in database if db session is provided + if db: + try: + from app.models.database import AudioFile + # Try to find by S3 key or file path + audio_file = db.query(AudioFile).filter( + (AudioFile.file_path == audio_file_key) | (AudioFile.file_path.like(f"%{audio_file_key}%")) + ).first() + + if audio_file and os.path.exists(audio_file.file_path): + return audio_file.file_path + except Exception as e: + logger.warning(f"Database lookup failed for {audio_file_key}: {e}") + + # If all else fails, raise error + raise StorageError(f"Failed to download audio file: S3 is not enabled and local file not found for key: {audio_file_key}") + + def _transcribe_with_openai(self, audio_file_path: str, model: str, api_key: str, language: Optional[str] = None) -> Dict[str, Any]: + """Transcribe audio using OpenAI Whisper API with word-level timestamps.""" + try: + from openai import OpenAI + + client = OpenAI(api_key=api_key) + + with open(audio_file_path, "rb") as audio_file: + transcript = client.audio.transcriptions.create( + model=model, + file=audio_file, + language=language, + response_format="verbose_json", + timestamp_granularities=["word", "segment"], + ) + + result = { + "text": transcript.text if hasattr(transcript, "text") else str(transcript), + "language": getattr(transcript, "language", language) if language else getattr(transcript, "language", "en"), + "segments": [], + "words": [], + } + + if hasattr(transcript, "segments") and transcript.segments: + for seg in transcript.segments: + result["segments"].append( + { + "start": getattr(seg, "start", 0), + "end": getattr(seg, "end", 0), + "text": getattr(seg, "text", ""), + } + ) + elif isinstance(transcript, dict) and "segments" in transcript: + for seg in transcript["segments"]: + result["segments"].append( + { + "start": seg.get("start", 0), + "end": seg.get("end", 0), + "text": seg.get("text", ""), + } + ) + + if hasattr(transcript, "words") and transcript.words: + for w in transcript.words: + # Handle both object attributes and dict-like access + if isinstance(w, dict): + word_text = w.get("word", "") + word_start = w.get("start", 0) or 0 + word_end = w.get("end", 0) or 0 + else: + word_text = getattr(w, "word", "") or "" + word_start = getattr(w, "start", None) + word_end = getattr(w, "end", None) + # Some SDK versions may use 'start_time'/'end_time' + if word_start is None: + word_start = getattr(w, "start_time", 0) or 0 + if word_end is None: + word_end = getattr(w, "end_time", 0) or 0 + word_start = float(word_start) if word_start else 0.0 + word_end = float(word_end) if word_end else 0.0 + result["words"].append( + { + "word": word_text, + "start": word_start, + "end": word_end, + } + ) + elif isinstance(transcript, dict) and "words" in transcript: + for w in transcript["words"]: + result["words"].append( + { + "word": w.get("word", ""), + "start": w.get("start", 0) or 0, + "end": w.get("end", 0) or 0, + } + ) + + if not result["segments"] and result["text"]: + import re + sentences = re.split(r"[.!?]+\s+", result["text"].strip()) + sentences = [s.strip() for s in sentences if s.strip()] + + if sentences: + total_words = len(result["text"].split()) + estimated_duration = max(1.0, (total_words / 150.0) * 60.0) + + current_time = 0.0 + for sentence in sentences: + sentence_words = len(sentence.split()) + sentence_duration = max(0.5, (sentence_words / 150.0) * 60.0) + result["segments"].append( + { + "start": current_time, + "end": current_time + sentence_duration, + "text": sentence, + } + ) + current_time += sentence_duration + else: + word_count = len(result["text"].split()) + estimated_duration = max(1.0, (word_count / 150.0) * 60.0) + result["segments"] = [{"start": 0.0, "end": estimated_duration, "text": result["text"]}] + + return result + except ImportError: + raise RuntimeError("OpenAI library not installed. Install with: pip install openai") + except Exception as e: + import traceback + error_details = traceback.format_exc() + raise RuntimeError(f"OpenAI transcription failed: {str(e)}\nDetails: {error_details}") + + def _transcribe_with_whisper_local(self, audio_file_path: str, model_name: str = "base") -> Dict[str, Any]: + """Transcribe audio using local Whisper model.""" + try: + import whisper + + model = whisper.load_model(model_name) + result = model.transcribe(audio_file_path) + + return { + "text": result.get("text", ""), + "language": result.get("language", "en"), + "segments": [ + {"start": seg.get("start", 0), "end": seg.get("end", 0), "text": seg.get("text", "")} + for seg in result.get("segments", []) + ], + } + except ImportError: + raise RuntimeError("Whisper library not installed. Install with: pip install openai-whisper") + except Exception as e: + raise RuntimeError(f"Whisper transcription failed: {str(e)}") + + def _get_pyannote_pipeline(self): + """Load and cache the pyannote diarization pipeline.""" + if self._pyannote_pipeline is not None: + return self._pyannote_pipeline + + # Compatibility shim: list_audio_backends was removed in torchaudio 2.4+ + import torchaudio + if not hasattr(torchaudio, "list_audio_backends"): + torchaudio.list_audio_backends = lambda: ["ffmpeg"] + + from pyannote.audio import Pipeline + from app.config import settings + + hf_token = settings.HUGGINGFACE_TOKEN + if not hf_token: + raise RuntimeError( + "HUGGINGFACE_TOKEN not configured. Set it under 'diarization.huggingface_token' " + "in config.yml. Required for pyannote speaker diarization." + ) + + logger.info("Loading pyannote speaker-diarization-3.1 pipeline (first call, will be cached)...") + self._pyannote_pipeline = Pipeline.from_pretrained("pyannote/speaker-diarization-3.1", token=hf_token) + logger.info("Pyannote pipeline loaded successfully") + return self._pyannote_pipeline + + def _detect_speakers_with_pyannote( + self, + audio_file_path: str, + segments: List[Dict[str, Any]], + words: Optional[List[Dict[str, Any]]] = None, + num_speakers: Optional[int] = 2, + min_speakers: Optional[int] = None, + max_speakers: Optional[int] = None, + ) -> List[Dict[str, Any]]: + """ + Use pyannote.audio for ML-based speaker diarization, aligned with + Whisper word timestamps for accurate speaker-text mapping. + """ + pipeline = self._get_pyannote_pipeline() + + # Build pipeline kwargs for speaker count hints + pipeline_kwargs = {} + if num_speakers is not None: + pipeline_kwargs["num_speakers"] = num_speakers + if min_speakers is not None: + pipeline_kwargs["min_speakers"] = min_speakers + if max_speakers is not None: + pipeline_kwargs["max_speakers"] = max_speakers + + logger.info( + f"Running pyannote diarization on {audio_file_path} " f"(speaker hints: {pipeline_kwargs or 'auto'})" + ) + raw_output = pipeline(audio_file_path, **pipeline_kwargs) + + # Handle different return types across pyannote versions: + # - Older versions: Annotation directly (has itertracks) + # - Newer versions: DiarizeOutput with .speaker_diarization attribute + if hasattr(raw_output, "itertracks"): + annotation = raw_output + elif hasattr(raw_output, "speaker_diarization"): + annotation = raw_output.speaker_diarization + elif hasattr(raw_output, "annotation"): + annotation = raw_output.annotation + elif isinstance(raw_output, tuple): + annotation = raw_output[0] + else: + attrs = [a for a in dir(raw_output) if not a.startswith("_")] + raise TypeError( + f"Unexpected pyannote output type: {type(raw_output).__name__}. " f"Available attributes: {attrs}" + ) + + # Collect diarization turns as a sorted list for fast lookup + diar_turns = [] + raw_labels = set() + for turn, _, speaker_label in annotation.itertracks(yield_label=True): + diar_turns.append((turn.start, turn.end, speaker_label)) + raw_labels.add(speaker_label) + + if not diar_turns: + logger.warning("Pyannote returned no speaker turns") + return [] + + sorted_labels = sorted(raw_labels) + label_map = {lbl: f"Speaker {i + 1}" for i, lbl in enumerate(sorted_labels)} + logger.info(f"Pyannote detected {len(sorted_labels)} speakers, {len(diar_turns)} turns") + + def find_speaker(midpoint: float) -> str: + """Find which speaker is active at a given timestamp.""" + for t_start, t_end, lbl in diar_turns: + if t_start <= midpoint <= t_end: + return label_map[lbl] + # No exact match -- find the closest turn + min_dist = float("inf") + closest_label = label_map[diar_turns[0][2]] + for t_start, t_end, lbl in diar_turns: + dist = min(abs(midpoint - t_start), abs(midpoint - t_end)) + if dist < min_dist: + min_dist = dist + closest_label = label_map[lbl] + return closest_label + + if words and len(words) > 0: + speaker_segments = [] + current_speaker = None + current_words: List[str] = [] + current_start = 0.0 + current_end = 0.0 + + for w in words: + word_text = w.get("word", "").strip() + w_start = w.get("start", 0) + w_end = w.get("end", 0) + if not word_text: + continue + + midpoint = (w_start + w_end) / 2.0 + speaker = find_speaker(midpoint) + + if speaker != current_speaker: + if current_words and current_speaker: + speaker_segments.append( + { + "speaker": current_speaker, + "text": " ".join(current_words).strip(), + "start": round(current_start, 3), + "end": round(current_end, 3), + } + ) + current_speaker = speaker + current_words = [word_text] + current_start = w_start + current_end = w_end + else: + current_words.append(word_text) + current_end = w_end + + if current_words and current_speaker: + speaker_segments.append( + { + "speaker": current_speaker, + "text": " ".join(current_words).strip(), + "start": round(current_start, 3), + "end": round(current_end, 3), + } + ) + + return speaker_segments + + # Fallback: align at segment level when word timestamps are not available + speaker_segments = [] + for seg in segments: + seg_start = seg.get("start", 0) + seg_end = seg.get("end", 0) + seg_text = seg.get("text", "").strip() + if not seg_text: + continue + + midpoint = (seg_start + seg_end) / 2.0 + speaker = find_speaker(midpoint) + + speaker_segments.append( + { + "speaker": speaker, + "text": seg_text, + "start": round(seg_start, 3), + "end": round(seg_end, 3), + } + ) + + return speaker_segments + + def _detect_speakers_heuristic(self, segments: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Heuristic-based speaker diarization fallback. Uses gap-based detection + which is unreliable -- pyannote.audio should be used for accurate results. + """ + logger.warning( + "Using heuristic speaker diarization (unreliable). For accurate results, " + "install pyannote.audio and configure diarization.huggingface_token in config.yml" + ) + if not segments: + return [] + + speaker_segments = [] + current_speaker = "Speaker 1" + + # Thresholds for speaker change detection + GAP_THRESHOLD_LARGE = 0.8 # seconds - large gap definitely suggests change + GAP_THRESHOLD_MEDIUM = 0.4 # seconds - medium gap suggests change + GAP_THRESHOLD_SMALL = 0.2 # seconds - small gap, but combined with other factors + MIN_SEGMENT_DURATION = 0.2 # seconds - minimum segment to consider + + # Calculate average gap size to adapt thresholds + gaps = [] + for i in range(1, len(segments)): + gap = segments[i].get("start", 0) - segments[i - 1].get("end", 0) + if gap > 0: + gaps.append(gap) + + avg_gap = sum(gaps) / len(gaps) if gaps else 0.5 + # Adaptive threshold based on conversation pace + adaptive_threshold = min(max(avg_gap * 1.5, 0.3), 1.0) + + # Track recent speaker assignments for pattern detection + recent_assignments = [] + assignment_window = 5 # Look at last N segments for patterns + + for i, seg in enumerate(segments): + seg_start = seg.get("start", 0) + seg_end = seg.get("end", 0) + seg_duration = seg_end - seg_start + seg_text = seg.get("text", "").strip() + + # Skip very short segments (likely noise or artifacts) + if seg_duration < MIN_SEGMENT_DURATION or not seg_text: + continue + + # Check for speaker change indicators + should_switch = False + gap = 0 + + if i > 0: + prev_seg = segments[i - 1] + gap = seg_start - prev_seg.get("end", 0) + + # Large gap definitely suggests speaker change + if gap > GAP_THRESHOLD_LARGE: + should_switch = True + # Medium gap suggests change + elif gap > GAP_THRESHOLD_MEDIUM: + should_switch = True + # Small gap but check for alternating pattern + elif gap > GAP_THRESHOLD_SMALL: + # If we've been alternating, continue the pattern + if len(recent_assignments) >= 2: + # Check if last two were the same speaker (suggests we should switch) + if recent_assignments[-1] == recent_assignments[-2] == current_speaker: + should_switch = True + # Or if we see a pattern of quick back-and-forth + elif len(recent_assignments) >= 3: + # If pattern is A-A-A, switch to B + if all(a == current_speaker for a in recent_assignments[-3:]): + should_switch = True + # Very small or no gap - use alternating pattern if established + elif gap >= 0: + # If we have an established alternating pattern, continue it + if len(recent_assignments) >= 2: + # Check if we should alternate based on recent pattern + if recent_assignments[-1] == current_speaker: + # If last segment was same speaker, consider switching + # But only if we have a pattern suggesting alternation + if len(recent_assignments) >= 4: + # Check for A-B-A-B pattern + pattern = recent_assignments[-4:] + if pattern[0] != pattern[1] and pattern[1] != pattern[2] and pattern[2] != pattern[3]: + # We have alternating pattern, continue it + should_switch = True + + # Additional heuristics for first few segments + if i < 3 and i > 0: + # Early in conversation, be more aggressive about switching + if gap > 0.1: # Any noticeable gap + should_switch = True + + # If we have multiple consecutive segments from same speaker, force alternation + if len(recent_assignments) >= 2: + # If last 2 segments were both the same speaker (and it's the current speaker), switch + # This prevents one speaker from getting too many consecutive segments + if recent_assignments[-1] == current_speaker and recent_assignments[-2] == current_speaker: + should_switch = True + + # Switch speaker if needed + if should_switch: + current_speaker = "Speaker 2" if current_speaker == "Speaker 1" else "Speaker 1" + + speaker_segments.append( + { + "speaker": current_speaker, + "text": seg_text, + "start": seg_start, + "end": seg_end, + } + ) + + # Track recent assignments for pattern detection + recent_assignments.append(current_speaker) + if len(recent_assignments) > assignment_window: + recent_assignments.pop(0) + + # Post-process: Balance speakers if one dominates too much + speaker_1_count = sum(1 for s in speaker_segments if s["speaker"] == "Speaker 1") + speaker_2_count = sum(1 for s in speaker_segments if s["speaker"] == "Speaker 2") + total_segments = len(speaker_segments) + + # If one speaker has more than 70% of segments, redistribute by alternating + if total_segments > 0: + speaker_1_ratio = speaker_1_count / total_segments + if speaker_1_ratio > 0.7: + # Redistribute: alternate segments starting from index 1 + # This assumes the first speaker is correct, then alternates + for i in range(1, len(speaker_segments)): + expected_speaker = "Speaker 2" if i % 2 == 1 else "Speaker 1" + if speaker_segments[i]["speaker"] != expected_speaker: + speaker_segments[i]["speaker"] = expected_speaker + elif speaker_1_ratio < 0.3: + # Speaker 2 dominates, redistribute + for i in range(1, len(speaker_segments)): + expected_speaker = "Speaker 1" if i % 2 == 1 else "Speaker 2" + if speaker_segments[i]["speaker"] != expected_speaker: + speaker_segments[i]["speaker"] = expected_speaker + + return speaker_segments + + def transcribe( + self, + audio_file_key: str, + stt_provider: ModelProvider, + stt_model: str, + organization_id: UUID, + db: Session, + language: Optional[str] = None, + enable_speaker_diarization: bool = True, + ) -> Dict[str, Any]: + """ + Transcribe audio file from S3. + """ + start_time = time.time() + temp_file_path = None + + try: + # Download audio to temporary file + temp_file_path = self._download_audio_to_temp(audio_file_key, db=db) + + # Get provider API key + ai_provider = self._get_ai_provider(stt_provider, db, organization_id) + if not ai_provider: + raise RuntimeError(f"AI provider {stt_provider} not configured for this organization. Please configure an AI provider in the settings.") + + # Decrypt API key + from app.core.encryption import decrypt_api_key + try: + api_key = decrypt_api_key(ai_provider.api_key) + except Exception as e: + raise RuntimeError(f"Failed to decrypt API key for provider {stt_provider}: {str(e)}") + + # Transcribe based on provider + if stt_provider == ModelProvider.OPENAI: + if stt_model.startswith("whisper-"): + # Use OpenAI API + result = self._transcribe_with_openai(temp_file_path, stt_model, api_key, language) + else: + # Fallback to local Whisper + model_name = stt_model.replace("whisper-", "") if stt_model.startswith("whisper-") else "base" + result = self._transcribe_with_whisper_local(temp_file_path, model_name) + elif stt_provider == ModelProvider.GOOGLE: + # TODO: Implement Google Speech-to-Text + raise NotImplementedError("Google Speech-to-Text not yet implemented") + elif stt_provider == ModelProvider.AZURE: + # TODO: Implement Azure Speech Services + raise NotImplementedError("Azure Speech Services not yet implemented") + elif stt_provider == ModelProvider.AWS: + # TODO: Implement AWS Transcribe + raise NotImplementedError("AWS Transcribe not yet implemented") + else: + # Default to local Whisper + result = self._transcribe_with_whisper_local(temp_file_path, "base") + + # Apply speaker diarization if enabled + speaker_segments = None + if enable_speaker_diarization: + segments = result.get("segments", []) + + # If no segments from transcription, create a single segment from the full text + if not segments and result.get("text"): + estimated_duration = 0.0 + if hasattr(result, "duration") and result.get("duration"): + estimated_duration = result.get("duration") + elif audio_file_path and os.path.exists(audio_file_path): + try: + import librosa + duration = librosa.get_duration(path=audio_file_path) + estimated_duration = duration + except Exception: + pass + + segments = [ + { + "start": 0.0, + "end": estimated_duration if estimated_duration > 0 else 10.0, # Default to 10s if unknown + "text": result.get("text", ""), + } + ] + + if segments: + words = result.get("words", []) + # Check if words actually have valid timestamps + valid_word_count = sum(1 for w in words if w.get("start", 0) > 0 or w.get("end", 0) > 0) + if words and valid_word_count == 0: + words = [] + + try: + from app.config import settings as app_settings + num_spk = getattr(app_settings, "DIARIZATION_NUM_SPEAKERS", 2) + speaker_segments = self._detect_speakers_with_pyannote( + temp_file_path, segments, words, num_speakers=num_spk + ) + if not speaker_segments: + speaker_segments = self._detect_speakers_heuristic(segments) + except Exception as e: + logger.warning(f"Pyannote diarization failed: {e}, falling back to heuristic") + speaker_segments = self._detect_speakers_heuristic(segments) + + processing_time = time.time() - start_time + + return { + "transcript": result["text"], + "language": result.get("language", language), + "speaker_segments": speaker_segments, + "segments": result.get("segments", []), + "processing_time": processing_time, + "raw_output": result, + } + + finally: + # Clean up temporary file + if temp_file_path and os.path.exists(temp_file_path): + try: + os.unlink(temp_file_path) + except Exception: + pass + + +# Singleton instance +transcription_service = TranscriptionService() diff --git a/app/services/tts_service.py b/app/services/ai/tts_service.py similarity index 93% rename from app/services/tts_service.py rename to app/services/ai/tts_service.py index 3b81deb7..f865dfee 100644 --- a/app/services/tts_service.py +++ b/app/services/ai/tts_service.py @@ -7,7 +7,7 @@ from uuid import UUID from app.models.database import ModelProvider, AIProvider, Integration -from app.services.s3_service import s3_service +from app.services.storage.s3_service import s3_service from efficientai.services.cartesia.http_tts import synthesize_cartesia_bytes from efficientai.services.deepgram.http_tts import synthesize_deepgram_bytes from efficientai.services.elevenlabs.http_tts import synthesize_elevenlabs_bytes @@ -34,6 +34,7 @@ "voicemaker": [8000, 16000, 22050, 24000, 44100, 48000], } + def get_audio_file_extension(provider: str, sample_rate_hz: Optional[int] = None) -> str: """Determine audio file extension based on provider and requested sample rate.""" if provider == "sarvam": @@ -56,19 +57,19 @@ def _get_ai_provider(self, provider: ModelProvider, db: Session, organization_id """Get AI provider configuration from database.""" from sqlalchemy import func - provider_value = provider.value if hasattr(provider, 'value') else provider + provider_value = provider.value if hasattr(provider, "value") else provider ai_provider = db.query(AIProvider).filter( AIProvider.provider == provider_value, AIProvider.organization_id == organization_id, - AIProvider.is_active == True + AIProvider.is_active == True, ).first() if not ai_provider: ai_provider = db.query(AIProvider).filter( func.lower(AIProvider.provider) == provider_value.lower(), AIProvider.organization_id == organization_id, - AIProvider.is_active == True + AIProvider.is_active == True, ).first() return ai_provider @@ -85,11 +86,11 @@ def _get_api_key_for_provider( return decrypt_api_key(ai_provider.api_key) # Fallback: check Integration table for cartesia/elevenlabs/deepgram - provider_value = provider.value if hasattr(provider, 'value') else provider + provider_value = provider.value if hasattr(provider, "value") else provider integration = db.query(Integration).filter( func.lower(Integration.platform) == provider_value.lower(), Integration.organization_id == organization_id, - Integration.is_active == True + Integration.is_active == True, ).first() if integration: return decrypt_api_key(integration.api_key) @@ -107,7 +108,7 @@ def _synthesize_with_openai( return synthesize_openai_bytes(text=text, model=model, api_key=api_key, voice=voice, config=config) # ------------------------------------------------------------------ - # Google (unary RPC – no streaming; TTFB ≈ total API time) + # Google (unary RPC - no streaming; TTFB ~= total API time) # ------------------------------------------------------------------ def _synthesize_with_google( @@ -229,7 +230,7 @@ def synthesize( ) -> bytes: """ Synthesize speech from text. - + Args: text: Text to convert to speech tts_provider: TTS provider to use @@ -238,7 +239,7 @@ def synthesize( db: Database session voice: Voice selection (if applicable) config: Additional provider-specific configuration - + Returns: Audio bytes (MP3 format) """ @@ -280,6 +281,7 @@ def synthesize_and_upload( audio_bytes = self.synthesize(text, tts_provider, tts_model, organization_id, db, voice, config) import uuid as _uuid + file_id = _uuid.uuid4() s3_key = s3_service.upload_file( file_id=file_id, @@ -292,4 +294,3 @@ def synthesize_and_upload( # Singleton instance tts_service = TTSService() - diff --git a/app/services/alerts/__init__.py b/app/services/alerts/__init__.py new file mode 100644 index 00000000..930c3d5b --- /dev/null +++ b/app/services/alerts/__init__.py @@ -0,0 +1,11 @@ +"""Alerts service package exports.""" + +from app.services.alerts.alert_evaluation_service import AlertEvaluationService, alert_evaluation_service +from app.services.alerts.alert_notification_service import AlertNotificationService, alert_notification_service + +__all__ = [ + "AlertEvaluationService", + "alert_evaluation_service", + "AlertNotificationService", + "alert_notification_service", +] diff --git a/app/services/alert_evaluation_service.py b/app/services/alerts/alert_evaluation_service.py similarity index 91% rename from app/services/alert_evaluation_service.py rename to app/services/alerts/alert_evaluation_service.py index 8d8261c0..f142f64d 100644 --- a/app/services/alert_evaluation_service.py +++ b/app/services/alerts/alert_evaluation_service.py @@ -2,7 +2,7 @@ import operator as op_module from datetime import datetime, timedelta -from typing import Dict, Any, List, Optional +from typing import Dict, Any, Optional from uuid import UUID from loguru import logger @@ -24,7 +24,7 @@ AlertNotifyFrequency, EvaluatorResultStatus, ) -from app.services.alert_notification_service import alert_notification_service +from app.services.alerts.alert_notification_service import alert_notification_service # Operator mapping @@ -63,12 +63,6 @@ class AlertEvaluationService: def evaluate_all_alerts(self, db: Session) -> Dict[str, Any]: """ Evaluate all active alerts across all organizations. - - Args: - db: Database session - - Returns: - Summary of evaluation results """ logger.info("[AlertEvaluation] Starting evaluation of all active alerts") @@ -132,13 +126,6 @@ def evaluate_single_alert( ) -> Dict[str, Any]: """ Evaluate a single alert's condition. - - Args: - alert: Alert ORM object - db: Database session - - Returns: - Evaluation result dict """ alert_name = alert.name logger.debug(f"[AlertEvaluation] Evaluating alert '{alert_name}'") @@ -217,14 +204,6 @@ def evaluate_alert_by_id( ) -> Dict[str, Any]: """ Evaluate a specific alert by ID (manual trigger). - - Args: - alert_id: Alert UUID - organization_id: Organization UUID - db: Database session - - Returns: - Evaluation result dict """ alert = ( db.query(Alert) @@ -251,13 +230,6 @@ def _compute_metric( ) -> Optional[float]: """ Compute the aggregated metric value for an alert. - - Args: - alert: Alert ORM object - db: Database session - - Returns: - Computed metric value, or None if no data """ metric_type = alert.metric_type aggregation = alert.aggregation @@ -450,7 +422,6 @@ def _compute_custom_metric( ) -> Optional[float]: """ Compute custom metric - counts all evaluator results. - Custom metrics can be extended based on specific requirements. """ query = self._build_evaluator_result_query( db, organization_id, agent_ids, window_start @@ -499,13 +470,6 @@ def _apply_aggregation( def _should_notify(self, alert: Alert, db: Session) -> bool: """ Check if the alert should send a notification based on frequency cooldown. - - Args: - alert: Alert ORM object - db: Database session - - Returns: - True if notification should be sent """ frequency = alert.notify_frequency cooldown_seconds = FREQUENCY_COOLDOWN.get(frequency, 0) @@ -549,14 +513,6 @@ def _trigger_alert( ) -> Dict[str, Any]: """ Trigger an alert: create history record and send notifications. - - Args: - alert: Alert ORM object - triggered_value: The value that triggered the alert - db: Database session - - Returns: - Trigger result dict """ triggered_at = datetime.utcnow() diff --git a/app/services/alert_notification_service.py b/app/services/alerts/alert_notification_service.py similarity index 87% rename from app/services/alert_notification_service.py rename to app/services/alerts/alert_notification_service.py index b41ea1a9..55a52f66 100644 --- a/app/services/alert_notification_service.py +++ b/app/services/alerts/alert_notification_service.py @@ -35,24 +35,6 @@ def send_slack_notification( ) -> Dict[str, Any]: """ Send a Slack notification via incoming webhook. - - Args: - webhook_url: Slack incoming webhook URL - alert_name: Name of the alert - alert_description: Description of the alert - metric_type: Type of metric (e.g., "number_of_calls") - aggregation: Aggregation type (e.g., "sum") - operator: Comparison operator (e.g., ">") - threshold_value: The configured threshold - triggered_value: The actual value that triggered the alert - time_window_minutes: Time window for the metric - triggered_at: When the alert was triggered - agent_names: Optional list of agent names in scope - alert_id: Optional alert ID for reference - history_id: Optional alert history ID for reference - - Returns: - Dict with success status and details """ try: # Build Slack Block Kit message @@ -198,24 +180,6 @@ def send_email_notification( ) -> Dict[str, Any]: """ Send an email notification for a triggered alert. - - Args: - to_email: Recipient email address - alert_name: Name of the alert - alert_description: Description of the alert - metric_type: Type of metric - aggregation: Aggregation type - operator: Comparison operator - threshold_value: The configured threshold - triggered_value: The actual value that triggered the alert - time_window_minutes: Time window for the metric - triggered_at: When the alert was triggered - agent_names: Optional list of agent names in scope - alert_id: Optional alert ID for reference - history_id: Optional alert history ID for reference - - Returns: - Dict with success status and details """ from app.config import settings @@ -343,16 +307,6 @@ def send_all_notifications( ) -> List[Dict[str, Any]]: """ Send notifications to all configured channels for an alert. - - Args: - alert: Alert ORM object with notification configuration - triggered_value: The actual value that triggered the alert - triggered_at: When the alert was triggered - agent_names: Optional list of agent names in scope - history_id: Optional alert history ID for reference - - Returns: - List of notification results """ results = [] @@ -410,20 +364,20 @@ def _get_severity_emoji( if operator in (">", ">="): ratio = actual / threshold if threshold > 0 else 2.0 if ratio >= 2.0: - return "\U0001f6a8" # rotating light + return "🚨" # rotating light elif ratio >= 1.5: - return "\u26a0\ufe0f" # warning + return "⚠️" # warning else: - return "\U0001f514" # bell + return "🔔" # bell elif operator in ("<", "<="): ratio = threshold / actual if actual > 0 else 2.0 if ratio >= 2.0: - return "\U0001f6a8" + return "🚨" elif ratio >= 1.5: - return "\u26a0\ufe0f" + return "⚠️" else: - return "\U0001f514" - return "\U0001f514" + return "🔔" + return "🔔" def _get_severity_label( self, operator: str, threshold: float, actual: float diff --git a/app/services/audio/__init__.py b/app/services/audio/__init__.py new file mode 100644 index 00000000..2894552b --- /dev/null +++ b/app/services/audio/__init__.py @@ -0,0 +1,33 @@ +"""Audio service package exports.""" + +from app.services.audio.audio_service import AudioService +from app.services.audio.qualitative_voice_service import ( + QualitativeVoiceMetricsService, + calculate_qualitative_metrics, + calculate_qualitative_metrics_from_call_data, + is_qualitative_audio_metric, + qualitative_voice_service, +) +from app.services.audio.voice_quality_service import ( + AUDIO_METRICS, + calculate_audio_metrics, + calculate_audio_metrics_from_call_data, + download_audio, + get_recording_url, + is_audio_metric, +) + +__all__ = [ + "AudioService", + "QualitativeVoiceMetricsService", + "qualitative_voice_service", + "is_qualitative_audio_metric", + "calculate_qualitative_metrics", + "calculate_qualitative_metrics_from_call_data", + "AUDIO_METRICS", + "is_audio_metric", + "get_recording_url", + "download_audio", + "calculate_audio_metrics", + "calculate_audio_metrics_from_call_data", +] diff --git a/app/services/audio_service.py b/app/services/audio/audio_service.py similarity index 95% rename from app/services/audio_service.py rename to app/services/audio/audio_service.py index 08549a7e..53adfdc6 100644 --- a/app/services/audio_service.py +++ b/app/services/audio/audio_service.py @@ -3,7 +3,6 @@ import librosa import soundfile as sf from pathlib import Path -from typing import Optional, Tuple from app.core.exceptions import AudioFileNotFoundError @@ -80,4 +79,3 @@ def is_valid_audio_file(self, file_path: str) -> bool: return True except Exception: return False - diff --git a/app/services/qualitative_voice_service.py b/app/services/audio/qualitative_voice_service.py similarity index 76% rename from app/services/qualitative_voice_service.py rename to app/services/audio/qualitative_voice_service.py index 61400926..397de142 100644 --- a/app/services/qualitative_voice_service.py +++ b/app/services/audio/qualitative_voice_service.py @@ -11,9 +11,7 @@ """ import os -import tempfile from typing import Dict, Any, Optional, List, Set, Tuple -from pathlib import Path import numpy as np from loguru import logger @@ -28,12 +26,14 @@ # Try importing required libraries try: import torch + TORCH_AVAILABLE = True except ImportError: logger.warning("torch not installed. Qualitative voice metrics will not be available.") try: import librosa + LIBROSA_AVAILABLE = True except ImportError: logger.warning("librosa not installed. Audio loading may be limited.") @@ -41,12 +41,14 @@ try: import parselmouth from parselmouth.praat import call + PARSELMOUTH_AVAILABLE = True except ImportError: logger.warning("praat-parselmouth not installed. Prosody metrics will not be available.") try: from transformers import pipeline, AutoProcessor, AutoModelForAudioClassification + TRANSFORMERS_AVAILABLE = True except ImportError: logger.warning("transformers not installed. Emotion metrics will not be available.") @@ -56,6 +58,7 @@ SPEECHBRAIN_AVAILABLE = False try: from speechbrain.inference.speaker import EncoderClassifier + SPEECHBRAIN_AVAILABLE = True logger.info("speechbrain loaded successfully. Speaker consistency metric available.") except ImportError: @@ -74,13 +77,13 @@ # Qualitative metrics names QUALITATIVE_AUDIO_METRICS: Set[str] = { - "MOS Score", # Mean Opinion Score (1.0-5.0) - "Emotion Category", # Categorical emotion (angry, happy, sad, neutral, etc.) - "Emotion Confidence", # Confidence of emotion prediction - "Valence", # Emotional positivity (-1.0 to 1.0) - "Arousal", # Emotional intensity (0.0 to 1.0) + "MOS Score", # Mean Opinion Score (1.0-5.0) + "Emotion Category", # Categorical emotion (angry, happy, sad, neutral, etc.) + "Emotion Confidence", # Confidence of emotion prediction + "Valence", # Emotional positivity (-1.0 to 1.0) + "Arousal", # Emotional intensity (0.0 to 1.0) "Speaker Consistency", # Same voice throughout (0.0-1.0) - "Prosody Score", # Expressiveness (0.0-1.0) + "Prosody Score", # Expressiveness (0.0-1.0) } @@ -91,7 +94,7 @@ def is_qualitative_audio_metric(metric_name: str) -> bool: class QualitativeVoiceMetricsService: """Service for calculating qualitative voice metrics.""" - + def __init__(self): """Initialize the service with lazy-loaded models.""" self._emotion_classifier = None @@ -101,17 +104,10 @@ def __init__(self): self._mos_predictor = None self._device = "cuda" if TORCH_AVAILABLE and torch.cuda.is_available() else "cpu" logger.info(f"[QualitativeVoice] Initialized with device: {self._device}") - + def _load_audio(self, audio_path: str, target_sr: int = 16000) -> Optional[Tuple[np.ndarray, int]]: """ Load audio file and resample to target sample rate. - - Args: - audio_path: Path to audio file - target_sr: Target sample rate (default 16kHz for speech models) - - Returns: - Tuple of (audio_array, sample_rate) or None if loading failed """ try: if LIBROSA_AVAILABLE: @@ -120,34 +116,31 @@ def _load_audio(self, audio_path: str, target_sr: int = 16000) -> Optional[Tuple else: # Fallback using soundfile if librosa not available import soundfile as sf + audio, sr = sf.read(audio_path) if len(audio.shape) > 1: audio = audio.mean(axis=1) # Convert to mono if sr != target_sr: # Simple resampling (not ideal but works) import scipy.signal + audio = scipy.signal.resample(audio, int(len(audio) * target_sr / sr)) return audio, target_sr except Exception as e: logger.error(f"[QualitativeVoice] Failed to load audio: {e}") return None - + # ========================================================================= # MOS (Mean Opinion Score) - Human-Likeness & Audio Fidelity # ========================================================================= - + def _get_mos_predictor(self): """Lazy load MOS predictor model (UTMOS-based).""" if self._mos_predictor is None and TORCH_AVAILABLE: try: # Try to use UTMOS model from torch hub - # UTMOS: UTokyo-SaruLab MOS predictor logger.info("[QualitativeVoice] Loading MOS predictor (UTMOS)...") - self._mos_predictor = torch.hub.load( - "tarepan/SpeechMOS:v1.2.0", - "utmos22_strong", - trust_repo=True - ) + self._mos_predictor = torch.hub.load("tarepan/SpeechMOS:v1.2.0", "utmos22_strong", trust_repo=True) self._mos_predictor.eval() if self._device == "cuda": self._mos_predictor = self._mos_predictor.cuda() @@ -157,95 +150,81 @@ def _get_mos_predictor(self): # Fallback: We'll estimate MOS from SNR and other acoustic features self._mos_predictor = "fallback" return self._mos_predictor - + def calculate_mos(self, audio_path: str) -> Optional[float]: """ Calculate Mean Opinion Score (1.0-5.0). - - MOS predicts human perception of audio quality: - - 1.0-2.0: Poor quality (robotic, tin can, bad reception) - - 3.0: Standard telephone quality - - 4.0-5.0: Studio/high fidelity quality - - Args: - audio_path: Path to audio file - - Returns: - MOS score (1.0-5.0) or None if calculation failed """ try: predictor = self._get_mos_predictor() - + if predictor == "fallback": # Fallback: Estimate MOS from acoustic features return self._estimate_mos_from_acoustics(audio_path) - + if predictor is None: logger.warning("[QualitativeVoice] MOS predictor not available") return None - + # Load audio at 16kHz (model requirement) audio_data = self._load_audio(audio_path, target_sr=16000) if audio_data is None: return None - + audio, sr = audio_data - + # Convert to torch tensor audio_tensor = torch.from_numpy(audio).float().unsqueeze(0) if self._device == "cuda": audio_tensor = audio_tensor.cuda() - + # Predict MOS with torch.no_grad(): mos = predictor(audio_tensor, sr) mos_value = float(mos.item()) - + # Clamp to valid range mos_value = max(1.0, min(5.0, mos_value)) logger.info(f"[QualitativeVoice] MOS Score: {mos_value:.2f}") return round(mos_value, 2) - + except Exception as e: logger.error(f"[QualitativeVoice] MOS calculation failed: {e}") return self._estimate_mos_from_acoustics(audio_path) - + def _estimate_mos_from_acoustics(self, audio_path: str) -> Optional[float]: """Fallback MOS estimation using acoustic features.""" try: if not PARSELMOUTH_AVAILABLE: return None - + sound = parselmouth.Sound(audio_path) - + # Get HNR (Harmonics-to-Noise Ratio) - higher = cleaner voice harmonicity = sound.to_harmonicity() hnr_values = harmonicity.values[harmonicity.values != -200] mean_hnr = np.mean(hnr_values) if len(hnr_values) > 0 else 10 - + # Estimate MOS based on HNR - # HNR < 10 dB: Poor (MOS ~2) - # HNR 10-20 dB: Medium (MOS ~3) - # HNR > 20 dB: Good (MOS ~4-5) if mean_hnr < 10: mos = 1.5 + (mean_hnr / 10) * 1.0 elif mean_hnr < 20: mos = 2.5 + ((mean_hnr - 10) / 10) * 1.5 else: mos = 4.0 + min(1.0, (mean_hnr - 20) / 20) - + mos = max(1.0, min(5.0, mos)) logger.info(f"[QualitativeVoice] Estimated MOS (from HNR): {mos:.2f}") return round(mos, 2) - + except Exception as e: logger.error(f"[QualitativeVoice] Fallback MOS estimation failed: {e}") return None - + # ========================================================================= # Emotional Match Accuracy - Categorical + Valence/Arousal # ========================================================================= - + def _get_emotion_classifier(self): """Lazy load emotion classification model.""" if self._emotion_classifier is None and TRANSFORMERS_AVAILABLE: @@ -254,13 +233,13 @@ def _get_emotion_classifier(self): self._emotion_classifier = pipeline( "audio-classification", model="ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition", - device=0 if self._device == "cuda" else -1 + device=0 if self._device == "cuda" else -1, ) logger.info("[QualitativeVoice] Emotion classifier loaded successfully") except Exception as e: logger.error(f"[QualitativeVoice] Failed to load emotion classifier: {e}") return self._emotion_classifier - + def _get_valence_arousal_model(self): """Lazy load valence/arousal model.""" if self._valence_arousal_model is None and TRANSFORMERS_AVAILABLE: @@ -276,106 +255,79 @@ def _get_valence_arousal_model(self): except Exception as e: logger.error(f"[QualitativeVoice] Failed to load valence/arousal model: {e}") return self._valence_arousal_model - + def calculate_emotion_category(self, audio_path: str) -> Tuple[Optional[str], Optional[float]]: """ Classify the dominant emotion in the audio. - - Like the "Sorting Hat" - listens and declares one emotion: - angry, happy, sad, neutral, fearful, disgusted, surprised - - Args: - audio_path: Path to audio file - - Returns: - Tuple of (emotion_label, confidence) or (None, None) if failed """ try: classifier = self._get_emotion_classifier() if classifier is None: return None, None - + # Run classification results = classifier(audio_path) - + if results and len(results) > 0: top_result = results[0] - emotion = top_result['label'] - confidence = top_result['score'] + emotion = top_result["label"] + confidence = top_result["score"] logger.info(f"[QualitativeVoice] Emotion: {emotion} (confidence: {confidence:.2f})") return emotion, round(confidence, 3) - + return None, None - + except Exception as e: logger.error(f"[QualitativeVoice] Emotion classification failed: {e}") return None, None - + def calculate_valence_arousal(self, audio_path: str) -> Tuple[Optional[float], Optional[float]]: """ Calculate Valence and Arousal scores. - - Valence (Left/Right): How positive/negative the emotion is (-1.0 to 1.0) - -1.0 = Very negative (sad, angry) - +1.0 = Very positive (happy, excited) - - Arousal (Up/Down): How intense/activated the emotion is (0.0 to 1.0) - 0.0 = Low energy (sleepy, calm) - 1.0 = High energy (excited, angry) - - Args: - audio_path: Path to audio file - - Returns: - Tuple of (valence, arousal) or (None, None) if failed """ try: model = self._get_valence_arousal_model() if model is None or self._valence_arousal_processor is None: return None, None - + # Load audio audio_data = self._load_audio(audio_path, target_sr=16000) if audio_data is None: return None, None - + audio, sr = audio_data - + # Process audio - inputs = self._valence_arousal_processor( - audio, - sampling_rate=sr, - return_tensors="pt" - ) + inputs = self._valence_arousal_processor(audio, sampling_rate=sr, return_tensors="pt") if self._device == "cuda": inputs = {k: v.cuda() for k, v in inputs.items()} - + # Get predictions with torch.no_grad(): outputs = model(**inputs) # Model outputs: arousal, dominance, valence predictions = outputs.logits.cpu().numpy()[0] - + # audeering model outputs: [arousal, dominance, valence] arousal = float(predictions[0]) valence = float(predictions[2]) - + # Normalize to expected ranges # Model outputs are typically in range [0, 1], convert valence to [-1, 1] valence = (valence - 0.5) * 2 # Convert from [0,1] to [-1,1] arousal = max(0.0, min(1.0, arousal)) # Keep in [0,1] - + logger.info(f"[QualitativeVoice] Valence: {valence:.2f}, Arousal: {arousal:.2f}") return round(valence, 3), round(arousal, 3) - + except Exception as e: logger.error(f"[QualitativeVoice] Valence/Arousal calculation failed: {e}") return None, None - + # ========================================================================= # Speaker Consistency - Voice Identity Stability # ========================================================================= - + def _get_speaker_encoder(self): """Lazy load speaker encoder (ECAPA-TDNN).""" if self._speaker_encoder is None and SPEECHBRAIN_AVAILABLE: @@ -384,220 +336,169 @@ def _get_speaker_encoder(self): self._speaker_encoder = EncoderClassifier.from_hparams( source="speechbrain/spkrec-ecapa-voxceleb", savedir="pretrained_models/spkrec-ecapa-voxceleb", - run_opts={"device": self._device} + run_opts={"device": self._device}, ) logger.info("[QualitativeVoice] Speaker encoder loaded successfully") except Exception as e: logger.error(f"[QualitativeVoice] Failed to load speaker encoder: {e}") return self._speaker_encoder - + def calculate_speaker_consistency(self, audio_path: str, segment_duration: float = 5.0) -> Optional[float]: """ Calculate speaker consistency score. - - Compares voice "fingerprints" from the start and end of the audio - to detect if the voice changed mid-call (glitch/hallucination). - - Score interpretation: - - > 0.8: Same person throughout (PASS) - - 0.5-0.8: Possible variation - - < 0.5: Different person detected (FAIL - possible voice glitch) - - Args: - audio_path: Path to audio file - segment_duration: Duration of segments to compare (default 5s) - - Returns: - Cosine similarity score (0.0-1.0) or None if failed """ try: encoder = self._get_speaker_encoder() if encoder is None: return None - + # Load full audio audio_data = self._load_audio(audio_path, target_sr=16000) if audio_data is None: return None - + audio, sr = audio_data total_duration = len(audio) / sr - + # Need at least 2x segment_duration for comparison if total_duration < segment_duration * 2: - logger.warning(f"[QualitativeVoice] Audio too short for speaker consistency check") + logger.warning("[QualitativeVoice] Audio too short for speaker consistency check") # If audio is short, assume it's consistent return 1.0 - + # Extract start and end segments segment_samples = int(segment_duration * sr) start_segment = audio[:segment_samples] end_segment = audio[-segment_samples:] - + # Convert to torch tensors start_tensor = torch.from_numpy(start_segment).float().unsqueeze(0) end_tensor = torch.from_numpy(end_segment).float().unsqueeze(0) - + # Get embeddings with torch.no_grad(): start_embedding = encoder.encode_batch(start_tensor) end_embedding = encoder.encode_batch(end_tensor) - + # Calculate cosine similarity start_emb = start_embedding.squeeze().cpu().numpy() end_emb = end_embedding.squeeze().cpu().numpy() - - similarity = np.dot(start_emb, end_emb) / ( - np.linalg.norm(start_emb) * np.linalg.norm(end_emb) - ) - + + similarity = np.dot(start_emb, end_emb) / (np.linalg.norm(start_emb) * np.linalg.norm(end_emb)) + # Convert to 0-1 range (cosine similarity can be negative) similarity = (similarity + 1) / 2 similarity = max(0.0, min(1.0, similarity)) - + logger.info(f"[QualitativeVoice] Speaker Consistency: {similarity:.3f}") return round(similarity, 3) - + except Exception as e: logger.error(f"[QualitativeVoice] Speaker consistency calculation failed: {e}") return None - + # ========================================================================= # Prosody Score - Expressiveness/Drama # ========================================================================= - + def calculate_prosody_score(self, audio_path: str, arousal: Optional[float] = None) -> Optional[float]: """ Calculate prosody/expressiveness score. - - Measures "Boring (Monotone)" vs "Dramatic (Storyteller)" by combining: - - Pitch Variance: Standard deviation of F0 (fundamental frequency) - - Arousal: Energy/intensity from emotion model - - Formula: Expressiveness = (PitchVariance_norm × 0.5) + (Arousal × 0.5) - - Args: - audio_path: Path to audio file - arousal: Pre-calculated arousal score (if available) - - Returns: - Prosody score (0.0-1.0) or None if failed """ try: if not PARSELMOUTH_AVAILABLE: logger.warning("[QualitativeVoice] Parselmouth not available for prosody calculation") return None - + # Load sound sound = parselmouth.Sound(audio_path) - + # Extract pitch pitch = sound.to_pitch() pitch_values = pitch.selected_array["frequency"] voiced_values = pitch_values[pitch_values > 0] - + if len(voiced_values) < 2: logger.warning("[QualitativeVoice] Not enough voiced frames for prosody") return None - + # Calculate pitch variance pitch_std = np.std(voiced_values) pitch_mean = np.mean(voiced_values) - + # Normalize pitch variance (coefficient of variation) # Typical CV for speech is 0.1-0.3, higher = more expressive cv = pitch_std / pitch_mean if pitch_mean > 0 else 0 - + # Normalize to 0-1 range # CV < 0.1 = monotone, CV > 0.3 = very expressive pitch_norm = min(1.0, cv / 0.3) - + # Get arousal if not provided if arousal is None: _, arousal = self.calculate_valence_arousal(audio_path) - + if arousal is None: # Use only pitch variance if arousal unavailable prosody = pitch_norm else: # Combine pitch variance and arousal prosody = (pitch_norm * 0.5) + (arousal * 0.5) - + prosody = max(0.0, min(1.0, prosody)) logger.info(f"[QualitativeVoice] Prosody Score: {prosody:.3f} (pitch_norm={pitch_norm:.3f}, arousal={arousal})") return round(prosody, 3) - + except Exception as e: logger.error(f"[QualitativeVoice] Prosody calculation failed: {e}") return None - + # ========================================================================= # Main Entry Point # ========================================================================= - + def calculate_all_metrics(self, audio_path: str) -> Dict[str, Any]: """ Calculate all qualitative voice metrics. - - This is the main entry point that runs all metrics and returns - a comprehensive JSON object. - - Args: - audio_path: Path to audio file - - Returns: - Dictionary with all metric values """ results: Dict[str, Any] = {} - + logger.info(f"[QualitativeVoice] Analyzing audio: {audio_path}") - + # MOS Score results["MOS Score"] = self.calculate_mos(audio_path) - + # Emotion Category emotion, confidence = self.calculate_emotion_category(audio_path) results["Emotion Category"] = emotion results["Emotion Confidence"] = confidence - + # Valence & Arousal valence, arousal = self.calculate_valence_arousal(audio_path) results["Valence"] = valence results["Arousal"] = arousal - + # Speaker Consistency results["Speaker Consistency"] = self.calculate_speaker_consistency(audio_path) - + # Prosody Score (uses arousal if available) results["Prosody Score"] = self.calculate_prosody_score(audio_path, arousal=arousal) - + logger.info(f"[QualitativeVoice] All metrics calculated: {results}") return results - - def calculate_metrics( - self, - audio_source: str, - metric_names: List[str], - is_url: bool = True - ) -> Dict[str, Any]: + + def calculate_metrics(self, audio_source: str, metric_names: List[str], is_url: bool = True) -> Dict[str, Any]: """ Calculate specific qualitative metrics from audio. - - Args: - audio_source: URL or file path to audio - metric_names: List of metric names to calculate - is_url: If True, download from URL first - - Returns: - Dictionary mapping metric names to values """ results: Dict[str, Any] = {} temp_file = None - + try: # Get audio file path if is_url: - from app.services.voice_quality_service import download_audio + from app.services.audio.voice_quality_service import download_audio + temp_file = download_audio(audio_source) if not temp_file: logger.error("[QualitativeVoice] Failed to download audio") @@ -608,50 +509,50 @@ def calculate_metrics( if not os.path.exists(audio_path): logger.error(f"[QualitativeVoice] Audio file not found: {audio_path}") return {name: None for name in metric_names} - + # Calculate requested metrics valence, arousal = None, None - + for metric_name in metric_names: if metric_name not in QUALITATIVE_AUDIO_METRICS: logger.warning(f"[QualitativeVoice] Unknown metric: {metric_name}") results[metric_name] = None continue - + if metric_name == "MOS Score": results[metric_name] = self.calculate_mos(audio_path) - + elif metric_name == "Emotion Category": emotion, confidence = self.calculate_emotion_category(audio_path) results["Emotion Category"] = emotion results["Emotion Confidence"] = confidence - + elif metric_name == "Emotion Confidence": if "Emotion Category" not in results: emotion, confidence = self.calculate_emotion_category(audio_path) results["Emotion Category"] = emotion results["Emotion Confidence"] = confidence - + elif metric_name in ("Valence", "Arousal"): if valence is None and arousal is None: valence, arousal = self.calculate_valence_arousal(audio_path) results["Valence"] = valence results["Arousal"] = arousal - + elif metric_name == "Speaker Consistency": results[metric_name] = self.calculate_speaker_consistency(audio_path) - + elif metric_name == "Prosody Score": if arousal is None: _, arousal = self.calculate_valence_arousal(audio_path) results[metric_name] = self.calculate_prosody_score(audio_path, arousal=arousal) - + return results - + except Exception as e: logger.error(f"[QualitativeVoice] Error calculating metrics: {e}") return {name: None for name in metric_names} - + finally: # Clean up temp file if temp_file and os.path.exists(temp_file): @@ -665,21 +566,9 @@ def calculate_metrics( qualitative_voice_service = QualitativeVoiceMetricsService() -def calculate_qualitative_metrics( - audio_source: str, - metric_names: List[str], - is_url: bool = True -) -> Dict[str, Any]: +def calculate_qualitative_metrics(audio_source: str, metric_names: List[str], is_url: bool = True) -> Dict[str, Any]: """ Convenience function to calculate qualitative voice metrics. - - Args: - audio_source: URL or file path to audio - metric_names: List of metric names to calculate - is_url: If True, download from URL first - - Returns: - Dictionary mapping metric names to values """ return qualitative_voice_service.calculate_metrics(audio_source, metric_names, is_url) @@ -687,26 +576,18 @@ def calculate_qualitative_metrics( def calculate_qualitative_metrics_from_call_data( call_data: Optional[Dict[str, Any]], provider_platform: Optional[str], - metric_names: List[str] + metric_names: List[str], ) -> Dict[str, Any]: """ Calculate qualitative metrics from provider call data. - - Args: - call_data: Call data from voice provider - provider_platform: Provider name ('retell', 'vapi') - metric_names: List of metric names to calculate - - Returns: - Dictionary mapping metric names to values """ - from app.services.voice_quality_service import get_recording_url - + from app.services.audio.voice_quality_service import get_recording_url + recording_url = get_recording_url(call_data, provider_platform) - + if not recording_url: logger.warning(f"[QualitativeVoice] No recording URL found for {provider_platform}") return {name: None for name in metric_names} - + logger.info(f"[QualitativeVoice] Found recording URL: {recording_url[:80]}...") return calculate_qualitative_metrics(recording_url, metric_names, is_url=True) diff --git a/app/services/voice_quality_service.py b/app/services/audio/voice_quality_service.py similarity index 66% rename from app/services/voice_quality_service.py rename to app/services/audio/voice_quality_service.py index 5edf4aa1..48ffe3e2 100644 --- a/app/services/voice_quality_service.py +++ b/app/services/audio/voice_quality_service.py @@ -3,7 +3,7 @@ This service calculates acoustic/voice quality metrics from audio files: - Pitch Variance: F0 (fundamental frequency) variation - Jitter: Cycle-to-cycle pitch period variation -- Shimmer: Cycle-to-cycle amplitude variation +- Shimmer: Cycle-to-cycle amplitude variation - HNR: Harmonics-to-Noise Ratio These metrics are industry-standard measures used in voice quality assessment. @@ -11,7 +11,6 @@ import tempfile import os -from pathlib import Path from typing import Dict, Any, Optional, Set, List import numpy as np from loguru import logger @@ -19,6 +18,7 @@ try: import parselmouth from parselmouth.praat import call + PARSELMOUTH_AVAILABLE = True except ImportError: PARSELMOUTH_AVAILABLE = False @@ -34,25 +34,19 @@ "Shimmer", "HNR", # Qualitative Voice AI metrics (new) - "MOS Score", # Mean Opinion Score (1.0-5.0) - Human-likeness - "Emotion Category", # Categorical emotion (angry, happy, etc.) - "Emotion Confidence", # Confidence of emotion prediction - "Valence", # Emotional positivity (-1.0 to 1.0) - "Arousal", # Emotional intensity (0.0 to 1.0) + "MOS Score", # Mean Opinion Score (1.0-5.0) - Human-likeness + "Emotion Category", # Categorical emotion (angry, happy, etc.) + "Emotion Confidence", # Confidence of emotion prediction + "Valence", # Emotional positivity (-1.0 to 1.0) + "Arousal", # Emotional intensity (0.0 to 1.0) "Speaker Consistency", # Same voice throughout (0.0-1.0) - "Prosody Score", # Expressiveness (0.0-1.0) + "Prosody Score", # Expressiveness (0.0-1.0) } def is_audio_metric(metric_name: str) -> bool: """ Check if a metric should be evaluated from audio (not LLM). - - Args: - metric_name: Name of the metric - - Returns: - True if this metric requires audio analysis """ return metric_name in AUDIO_METRICS @@ -60,57 +54,35 @@ def is_audio_metric(metric_name: str) -> bool: def get_recording_url(call_data: Optional[Dict[str, Any]], provider_platform: Optional[str]) -> Optional[str]: """ Extract recording URL from provider call_data. - - Args: - call_data: Call data from voice provider (Retell/Vapi) - provider_platform: Provider name ('retell', 'vapi') - - Returns: - Recording URL or None if not available """ if not call_data: return None - + if provider_platform == "vapi": # Vapi stores recording URLs in recording_urls object recording_urls = call_data.get("recording_urls", {}) - return ( - recording_urls.get("combined_url") or - recording_urls.get("stereo_url") or - call_data.get("recordingUrl") - ) + return recording_urls.get("combined_url") or recording_urls.get("stereo_url") or call_data.get("recordingUrl") elif provider_platform == "retell": # Retell stores recording URL directly return call_data.get("recording_url") else: # Try common patterns for unknown providers - return ( - call_data.get("recording_url") or - call_data.get("recordingUrl") or - call_data.get("recording_urls", {}).get("combined_url") - ) + return call_data.get("recording_url") or call_data.get("recordingUrl") or call_data.get("recording_urls", {}).get("combined_url") def download_audio(url: str, timeout: float = 60.0) -> Optional[str]: """ Download audio from URL to a temporary file. - - Args: - url: URL of the audio file - timeout: Download timeout in seconds - - Returns: - Path to temporary file, or None if download failed """ import httpx - + try: logger.info(f"[VoiceQuality] Downloading audio from URL: {url[:100]}...") - + with httpx.Client(timeout=timeout, follow_redirects=True) as client: response = client.get(url) response.raise_for_status() - + # Determine file extension from content-type or URL content_type = response.headers.get("content-type", "") if "wav" in content_type or url.endswith(".wav"): @@ -121,17 +93,17 @@ def download_audio(url: str, timeout: float = 60.0) -> Optional[str]: suffix = ".ogg" else: suffix = ".wav" # Default to wav - + # Create temporary file fd, temp_path = tempfile.mkstemp(suffix=suffix) try: os.write(fd, response.content) finally: os.close(fd) - + logger.info(f"[VoiceQuality] Downloaded audio to: {temp_path} ({len(response.content)} bytes)") return temp_path - + except Exception as e: logger.error(f"[VoiceQuality] Failed to download audio: {e}") return None @@ -140,17 +112,11 @@ def download_audio(url: str, timeout: float = 60.0) -> Optional[str]: def _load_sound(audio_path: str) -> Optional["parselmouth.Sound"]: """ Load audio file as Parselmouth Sound object. - - Args: - audio_path: Path to audio file - - Returns: - Parselmouth Sound object or None if loading failed """ if not PARSELMOUTH_AVAILABLE: logger.error("[VoiceQuality] Parselmouth not available") return None - + try: sound = parselmouth.Sound(audio_path) logger.info(f"[VoiceQuality] Loaded audio: duration={sound.duration:.2f}s, sample_rate={sound.sampling_frequency}") @@ -161,148 +127,90 @@ def _load_sound(audio_path: str) -> Optional["parselmouth.Sound"]: def calculate_pitch_variance(sound: "parselmouth.Sound") -> Optional[float]: - """ - Calculate pitch (F0) variance. - - Pitch variance measures the variation in fundamental frequency, - indicating prosodic expressiveness. Higher values suggest more - expressive speech, while very low values may indicate monotone speech. - - Args: - sound: Parselmouth Sound object - - Returns: - Pitch variance in Hz, or None if calculation failed - """ + """Calculate pitch (F0) variance.""" try: # Extract pitch using default settings (75-600 Hz range, suitable for speech) pitch = sound.to_pitch() - + # Get pitch values, filtering out unvoiced frames (which have value 0) pitch_values = pitch.selected_array["frequency"] voiced_values = pitch_values[pitch_values > 0] - + if len(voiced_values) < 2: logger.warning("[VoiceQuality] Not enough voiced frames for pitch variance") return None - + variance = float(np.std(voiced_values)) logger.debug(f"[VoiceQuality] Pitch variance: {variance:.2f} Hz") return round(variance, 2) - + except Exception as e: logger.error(f"[VoiceQuality] Pitch variance calculation failed: {e}") return None def calculate_jitter(sound: "parselmouth.Sound") -> Optional[float]: - """ - Calculate local jitter (pitch period perturbation). - - Jitter measures cycle-to-cycle variation in pitch period, - indicating vocal stability. Lower values (< 1%) indicate - stable voice, while higher values may indicate voice disorders. - - Args: - sound: Parselmouth Sound object - - Returns: - Jitter as percentage (0-100), or None if calculation failed - """ + """Calculate local jitter (pitch period perturbation).""" try: # Extract pitch and create point process - pitch = sound.to_pitch() point_process = call(sound, "To PointProcess (periodic, cc)", 75, 600) - + # Calculate local jitter jitter = call(point_process, "Get jitter (local)", 0, 0, 0.0001, 0.02, 1.3) - + # Convert to percentage jitter_percent = jitter * 100 logger.debug(f"[VoiceQuality] Jitter: {jitter_percent:.4f}%") return round(jitter_percent, 4) - + except Exception as e: logger.error(f"[VoiceQuality] Jitter calculation failed: {e}") return None def calculate_shimmer(sound: "parselmouth.Sound") -> Optional[float]: - """ - Calculate local shimmer (amplitude perturbation). - - Shimmer measures cycle-to-cycle variation in amplitude, - indicating voice quality. Lower values (< 3%) indicate - consistent voice, while higher values may indicate breathiness. - - Args: - sound: Parselmouth Sound object - - Returns: - Shimmer as percentage (0-100), or None if calculation failed - """ + """Calculate local shimmer (amplitude perturbation).""" try: # Create point process for shimmer calculation point_process = call(sound, "To PointProcess (periodic, cc)", 75, 600) - + # Calculate local shimmer - shimmer = call( - [sound, point_process], - "Get shimmer (local)", - 0, 0, 0.0001, 0.02, 1.3, 1.6 - ) - + shimmer = call([sound, point_process], "Get shimmer (local)", 0, 0, 0.0001, 0.02, 1.3, 1.6) + # Convert to percentage shimmer_percent = shimmer * 100 logger.debug(f"[VoiceQuality] Shimmer: {shimmer_percent:.4f}%") return round(shimmer_percent, 4) - + except Exception as e: logger.error(f"[VoiceQuality] Shimmer calculation failed: {e}") return None def calculate_hnr(sound: "parselmouth.Sound") -> Optional[float]: - """ - Calculate Harmonics-to-Noise Ratio (HNR). - - HNR measures the ratio of periodic (harmonic) to aperiodic (noise) - components in the voice signal. Higher values (> 20 dB) indicate - cleaner voice with less breathiness or hoarseness. - - Args: - sound: Parselmouth Sound object - - Returns: - HNR in dB, or None if calculation failed - """ + """Calculate Harmonics-to-Noise Ratio (HNR).""" try: # Calculate harmonicity harmonicity = sound.to_harmonicity() - + # Get HNR values, excluding undefined values (-200 dB) hnr_values = harmonicity.values[harmonicity.values != -200] - + if len(hnr_values) == 0: logger.warning("[VoiceQuality] No valid HNR values found") return None - + mean_hnr = float(np.mean(hnr_values)) logger.debug(f"[VoiceQuality] HNR: {mean_hnr:.2f} dB") return round(mean_hnr, 2) - + except Exception as e: logger.error(f"[VoiceQuality] HNR calculation failed: {e}") return None # Traditional Parselmouth metrics -PARSELMOUTH_METRICS: Set[str] = { - "Pitch Variance", - "Jitter", - "Shimmer", - "HNR", -} +PARSELMOUTH_METRICS: Set[str] = {"Pitch Variance", "Jitter", "Shimmer", "HNR"} # Qualitative AI metrics QUALITATIVE_METRICS: Set[str] = { @@ -316,32 +224,16 @@ def calculate_hnr(sound: "parselmouth.Sound") -> Optional[float]: } -def calculate_audio_metrics( - audio_source: str, - metric_names: List[str], - is_url: bool = True -) -> Dict[str, Any]: +def calculate_audio_metrics(audio_source: str, metric_names: List[str], is_url: bool = True) -> Dict[str, Any]: """ Calculate voice quality metrics from audio. - - This is the main entry point for voice quality analysis. - Handles both traditional Parselmouth metrics and new qualitative AI metrics. - - Args: - audio_source: URL or file path to audio - metric_names: List of metric names to calculate (from AUDIO_METRICS) - is_url: If True, audio_source is a URL to download; if False, it's a file path - - Returns: - Dictionary mapping metric names to their values. - Values are None if calculation failed. """ results: Dict[str, Any] = {} - + # Separate metrics into Parselmouth and Qualitative parselmouth_metrics = [m for m in metric_names if m in PARSELMOUTH_METRICS] qualitative_metrics = [m for m in metric_names if m in QUALITATIVE_METRICS] - + # Calculate Parselmouth metrics if parselmouth_metrics: if not PARSELMOUTH_AVAILABLE: @@ -350,11 +242,12 @@ def calculate_audio_metrics( else: parselmouth_results = _calculate_parselmouth_metrics(audio_source, parselmouth_metrics, is_url) results.update(parselmouth_results) - + # Calculate Qualitative metrics if qualitative_metrics: try: - from app.services.qualitative_voice_service import calculate_qualitative_metrics + from app.services.audio.qualitative_voice_service import calculate_qualitative_metrics + qualitative_results = calculate_qualitative_metrics(audio_source, qualitative_metrics, is_url) results.update(qualitative_results) except ImportError as e: @@ -363,36 +256,24 @@ def calculate_audio_metrics( except Exception as e: logger.error(f"[VoiceQuality] Error calculating qualitative metrics: {e}") results.update({name: None for name in qualitative_metrics}) - + # Handle any unknown metrics unknown_metrics = [m for m in metric_names if m not in AUDIO_METRICS] for metric_name in unknown_metrics: logger.warning(f"[VoiceQuality] Unknown audio metric: {metric_name}") results[metric_name] = None - + logger.info(f"[VoiceQuality] Calculated metrics: {results}") return results -def _calculate_parselmouth_metrics( - audio_source: str, - metric_names: List[str], - is_url: bool = True -) -> Dict[str, Any]: +def _calculate_parselmouth_metrics(audio_source: str, metric_names: List[str], is_url: bool = True) -> Dict[str, Any]: """ Calculate traditional Parselmouth-based metrics. - - Args: - audio_source: URL or file path to audio - metric_names: List of Parselmouth metric names to calculate - is_url: If True, download from URL first - - Returns: - Dictionary mapping metric names to values """ results: Dict[str, Any] = {} temp_file = None - + try: # Get audio file path if is_url: @@ -406,12 +287,12 @@ def _calculate_parselmouth_metrics( if not os.path.exists(audio_path): logger.error(f"[VoiceQuality] Audio file not found: {audio_path}") return {name: None for name in metric_names} - + # Load sound sound = _load_sound(audio_path) if sound is None: return {name: None for name in metric_names} - + # Calculate requested metrics for metric_name in metric_names: if metric_name == "Pitch Variance": @@ -422,13 +303,13 @@ def _calculate_parselmouth_metrics( results[metric_name] = calculate_shimmer(sound) elif metric_name == "HNR": results[metric_name] = calculate_hnr(sound) - + return results - + except Exception as e: logger.error(f"[VoiceQuality] Error calculating Parselmouth metrics: {e}") return {name: None for name in metric_names} - + finally: # Clean up temporary file if temp_file and os.path.exists(temp_file): @@ -442,27 +323,16 @@ def _calculate_parselmouth_metrics( def calculate_audio_metrics_from_call_data( call_data: Optional[Dict[str, Any]], provider_platform: Optional[str], - metric_names: List[str] + metric_names: List[str], ) -> Dict[str, Any]: """ Calculate voice quality metrics from provider call data. - - Convenience function that extracts the recording URL from call_data - and calculates the requested metrics. - - Args: - call_data: Call data from voice provider (Retell/Vapi) - provider_platform: Provider name ('retell', 'vapi') - metric_names: List of metric names to calculate - - Returns: - Dictionary mapping metric names to their values """ recording_url = get_recording_url(call_data, provider_platform) - + if not recording_url: logger.warning(f"[VoiceQuality] No recording URL found in call_data for {provider_platform}") return {name: None for name in metric_names} - + logger.info(f"[VoiceQuality] Found recording URL for {provider_platform}: {recording_url[:80]}...") return calculate_audio_metrics(recording_url, metric_names, is_url=True) diff --git a/app/services/evaluation/__init__.py b/app/services/evaluation/__init__.py new file mode 100644 index 00000000..1d7fd248 --- /dev/null +++ b/app/services/evaluation/__init__.py @@ -0,0 +1,11 @@ +"""Evaluation service package exports.""" + +from app.services.evaluation.evaluation_service import EvaluationService, evaluation_service +from app.services.evaluation.metrics_service import MetricsService, metrics_service + +__all__ = [ + "EvaluationService", + "evaluation_service", + "MetricsService", + "metrics_service", +] diff --git a/app/services/evaluation_service.py b/app/services/evaluation/evaluation_service.py similarity index 94% rename from app/services/evaluation_service.py rename to app/services/evaluation/evaluation_service.py index 97b1ae0c..8c5fe5eb 100644 --- a/app/services/evaluation_service.py +++ b/app/services/evaluation/evaluation_service.py @@ -7,19 +7,21 @@ from sqlalchemy.orm import Session from app.models.database import Evaluation, EvaluationResult, EvaluationStatus, AudioFile -from app.services.metrics_service import metrics_service -from app.services.audio_service import AudioService +from app.services.evaluation.metrics_service import metrics_service +from app.services.audio.audio_service import AudioService from app.core.exceptions import EvaluationNotFoundError, AudioFileNotFoundError # Lazy import for whisper (optional dependency) whisper = None + def _get_whisper(): """Lazy load whisper module.""" global whisper if whisper is None: try: import whisper as _whisper + whisper = _whisper except ImportError: raise ImportError( @@ -28,6 +30,7 @@ def _get_whisper(): ) return whisper + audio_service = AudioService() @@ -188,4 +191,3 @@ def cancel_evaluation(self, evaluation_id: UUID, db: Session) -> bool: # Singleton instance evaluation_service = EvaluationService() - diff --git a/app/services/metrics_service.py b/app/services/evaluation/metrics_service.py similarity index 96% rename from app/services/metrics_service.py rename to app/services/evaluation/metrics_service.py index b489be6b..b003843d 100644 --- a/app/services/metrics_service.py +++ b/app/services/evaluation/metrics_service.py @@ -197,4 +197,3 @@ def calculate_metrics( # Singleton instance metrics_service = MetricsService() - diff --git a/app/services/reporting/__init__.py b/app/services/reporting/__init__.py new file mode 100644 index 00000000..f6789b9f --- /dev/null +++ b/app/services/reporting/__init__.py @@ -0,0 +1,8 @@ +"""Reporting service package exports.""" + +from app.services.reporting.voice_playground_report_service import ( + VoicePlaygroundReportService, + voice_playground_report_service, +) + +__all__ = ["VoicePlaygroundReportService", "voice_playground_report_service"] diff --git a/app/services/voice_playground_report_service.py b/app/services/reporting/voice_playground_report_service.py similarity index 97% rename from app/services/voice_playground_report_service.py rename to app/services/reporting/voice_playground_report_service.py index 6e277d03..a1d4f368 100644 --- a/app/services/voice_playground_report_service.py +++ b/app/services/reporting/voice_playground_report_service.py @@ -17,7 +17,7 @@ class VoicePlaygroundReportService: """Build and render comprehensive Voice Playground benchmark reports.""" def __init__(self) -> None: - templates_dir = Path(__file__).parent.parent / "templates" + templates_dir = Path(__file__).parent.parent.parent / "templates" self._jinja_env = Environment( loader=FileSystemLoader(str(templates_dir)), autoescape=select_autoescape(["html", "xml"]), @@ -29,7 +29,7 @@ def __init__(self) -> None: @staticmethod def _build_logo_data_uri() -> str | None: """Load frontend favicon and convert it to an embeddable data URI.""" - project_root = Path(__file__).parent.parent.parent + project_root = Path(__file__).parent.parent.parent.parent candidate_paths = [ project_root / "frontend" / "public" / "favicon_dark.png", project_root / "frontend" / "public" / "favicon_light.png", diff --git a/app/services/storage/__init__.py b/app/services/storage/__init__.py new file mode 100644 index 00000000..c9aec558 --- /dev/null +++ b/app/services/storage/__init__.py @@ -0,0 +1,11 @@ +"""Storage service package exports.""" + +from app.services.storage.s3_service import S3Service, s3_service +from app.services.storage.storage_service import StorageService, storage_service + +__all__ = [ + "S3Service", + "s3_service", + "StorageService", + "storage_service", +] diff --git a/app/services/s3_service.py b/app/services/storage/s3_service.py similarity index 75% rename from app/services/s3_service.py rename to app/services/storage/s3_service.py index b8a343e5..368e047a 100644 --- a/app/services/s3_service.py +++ b/app/services/storage/s3_service.py @@ -2,7 +2,7 @@ import boto3 from botocore.exceptions import ClientError, NoCredentialsError -from typing import Optional, List, BinaryIO +from typing import Optional, List from pathlib import Path import uuid from app.config import settings @@ -16,22 +16,22 @@ def __init__(self): """Initialize S3 service with configuration.""" self.s3_client = None self._initialization_error = None - + @property def enabled(self) -> bool: """Get S3 enabled status from settings.""" return settings.S3_ENABLED - + @property def bucket_name(self) -> Optional[str]: """Get S3 bucket name from settings.""" return settings.S3_BUCKET_NAME - + @property def region(self) -> str: """Get S3 region from settings.""" return settings.S3_REGION - + @property def prefix(self) -> str: """Get S3 prefix from settings.""" @@ -41,20 +41,20 @@ def _ensure_initialized(self): """Lazily initialize S3 client if not already initialized.""" if self.s3_client is not None: return - + if not self.enabled: return - + if not self.bucket_name: self._initialization_error = "S3 is enabled but bucket_name is not configured" return - + try: # Initialize S3 client s3_kwargs = { "region_name": self.region, } - + # Add credentials if provided if settings.S3_ACCESS_KEY_ID and settings.S3_SECRET_ACCESS_KEY: s3_kwargs["aws_access_key_id"] = settings.S3_ACCESS_KEY_ID @@ -62,13 +62,13 @@ def _ensure_initialized(self): else: self._initialization_error = "S3 credentials not configured" return - + # Add endpoint URL for S3-compatible services (e.g., MinIO, DigitalOcean Spaces) if settings.S3_ENDPOINT_URL: s3_kwargs["endpoint_url"] = settings.S3_ENDPOINT_URL - + self.s3_client = boto3.client("s3", **s3_kwargs) - + # Test connection by checking if bucket exists (non-blocking) try: self.s3_client.head_bucket(Bucket=self.bucket_name) @@ -86,7 +86,7 @@ def _ensure_initialized(self): except NoCredentialsError: self._initialization_error = "S3 credentials not found. Check your configuration." self.s3_client = None - + except Exception as e: self._initialization_error = f"Failed to initialize S3 service: {str(e)}" self.s3_client = None @@ -97,7 +97,7 @@ def is_enabled(self) -> bool: return False self._ensure_initialized() return self.s3_client is not None - + def get_status_message(self) -> Optional[str]: """Get status message if there's an initialization error.""" if not self.enabled: @@ -106,65 +106,36 @@ def get_status_message(self) -> Optional[str]: return self._initialization_error def _get_key( - self, - file_id: uuid.UUID, - file_format: str, + self, + file_id: uuid.UUID, + file_format: str, organization_id: Optional[str] = None, evaluator_id: Optional[str] = None, meaningful_id: Optional[str] = None ) -> str: """ Generate S3 key for a file. - - Args: - file_id: Unique file identifier (UUID) - file_format: File format extension - organization_id: Optional organization ID - evaluator_id: Optional evaluator ID for organizing by evaluator - meaningful_id: Optional meaningful identifier (e.g., result_id, timestamp-based ID) - - Returns: - S3 key path """ # Use meaningful_id if provided, otherwise use file_id file_identifier = meaningful_id if meaningful_id else str(file_id) base_key = f"{file_identifier}.{file_format}" - + if organization_id: if evaluator_id: - # Organize by evaluator: prefix/organizations/{org_id}/evaluators/{evaluator_id}/audio/{meaningful_id}.{format} return f"{self.prefix}organizations/{organization_id}/evaluators/{evaluator_id}/audio/{base_key}" - else: - # Organize files by organization: prefix/organizations/{org_id}/audio/{file_id}.{format} - return f"{self.prefix}organizations/{organization_id}/audio/{base_key}" + return f"{self.prefix}organizations/{organization_id}/audio/{base_key}" return f"{self.prefix}{base_key}" def upload_file( - self, - file_content: bytes, - file_id: uuid.UUID, - file_format: str, + self, + file_content: bytes, + file_id: uuid.UUID, + file_format: str, organization_id: Optional[str] = None, evaluator_id: Optional[str] = None, meaningful_id: Optional[str] = None ) -> str: - """ - Upload file to S3. - - Args: - file_content: File content as bytes - file_id: Unique identifier for the file - file_format: File format extension - organization_id: Optional organization ID to organize files in folders - evaluator_id: Optional evaluator ID to organize files by evaluator - meaningful_id: Optional meaningful identifier (e.g., result_id, timestamp-based ID) - - Returns: - S3 key (path) of the uploaded file - - Raises: - StorageError: If upload fails - """ + """Upload file to S3.""" self._ensure_initialized() if not self.is_enabled(): error_msg = self._initialization_error or "S3 is not enabled or not configured" @@ -172,7 +143,7 @@ def upload_file( try: key = self._get_key(file_id, file_format, organization_id, evaluator_id, meaningful_id) - + # Determine content type based on file format content_type_map = { "wav": "audio/wav", @@ -188,7 +159,6 @@ def upload_file( Body=file_content, ContentType=content_type, ) - return key except ClientError as e: raise StorageError(f"Failed to upload file to S3: {str(e)}") @@ -216,19 +186,7 @@ def upload_file_by_key(self, file_content: bytes, key: str, content_type: str = raise StorageError(f"Unexpected error uploading file to S3: {str(e)}") def download_file(self, file_id: uuid.UUID, file_format: str) -> bytes: - """ - Download file from S3. - - Args: - file_id: File identifier - file_format: File format extension - - Returns: - File content as bytes - - Raises: - StorageError: If download fails or file not found - """ + """Download file from S3.""" self._ensure_initialized() if not self.is_enabled(): error_msg = self._initialization_error or "S3 is not enabled or not configured" @@ -247,19 +205,7 @@ def download_file(self, file_id: uuid.UUID, file_format: str) -> bytes: raise StorageError(f"Unexpected error downloading file from S3: {str(e)}") def delete_file(self, file_id: uuid.UUID, file_format: str) -> bool: - """ - Delete file from S3. - - Args: - file_id: File identifier - file_format: File format extension - - Returns: - True if file was deleted, False if it didn't exist - - Raises: - StorageError: If delete fails - """ + """Delete file from S3.""" self._ensure_initialized() if not self.is_enabled(): error_msg = self._initialization_error or "S3 is not enabled or not configured" @@ -278,18 +224,7 @@ def delete_file(self, file_id: uuid.UUID, file_format: str) -> bool: raise StorageError(f"Unexpected error deleting file from S3: {str(e)}") def delete_file_by_key(self, key: str) -> bool: - """ - Delete file from S3 by key. - - Args: - key: S3 key (path) of the file - - Returns: - True if file was deleted, False if it didn't exist - - Raises: - StorageError: If delete fails - """ + """Delete file from S3 by key.""" self._ensure_initialized() if not self.is_enabled(): error_msg = self._initialization_error or "S3 is not enabled or not configured" @@ -307,16 +242,7 @@ def delete_file_by_key(self, key: str) -> bool: raise StorageError(f"Unexpected error deleting file from S3: {str(e)}") def file_exists(self, file_id: uuid.UUID, file_format: str) -> bool: - """ - Check if file exists in S3. - - Args: - file_id: File identifier - file_format: File format extension - - Returns: - True if file exists, False otherwise - """ + """Check if file exists in S3.""" self._ensure_initialized() if not self.is_enabled(): return False @@ -329,26 +255,12 @@ def file_exists(self, file_id: uuid.UUID, file_format: str) -> bool: error_code = e.response.get("Error", {}).get("Code", "") if error_code == "404": return False - # For other errors, assume file doesn't exist return False except Exception: return False def list_audio_files(self, prefix: Optional[str] = None, max_keys: int = 1000, organization_id: Optional[str] = None) -> List[dict]: - """ - List audio files in S3 bucket. - - Args: - prefix: Optional prefix to filter files (defaults to configured prefix) - max_keys: Maximum number of keys to return - organization_id: Optional organization ID to filter files for a specific organization - - Returns: - List of file metadata dictionaries with keys: key, size, last_modified - - Raises: - StorageError: If listing fails - """ + """List audio files in S3 bucket.""" self._ensure_initialized() if not self.is_enabled(): error_msg = self._initialization_error or "S3 is not enabled or not configured" @@ -396,14 +308,6 @@ def browse_folder( """ Browse a folder within an organization's S3 namespace. Uses S3 delimiter to return folders and files at the current level only. - - Args: - organization_id: Organization ID to scope the browsing - path: Relative path within the org folder (e.g. "audio/" or "evaluators/abc/") - max_keys: Maximum number of keys to return - - Returns: - Dict with 'folders' (list of folder names) and 'files' (list of file dicts) """ self._ensure_initialized() if not self.is_enabled(): @@ -459,18 +363,7 @@ def browse_folder( raise StorageError(f"Unexpected error browsing S3 folder: {str(e)}") def download_file_by_key(self, key: str) -> bytes: - """ - Download file from S3 by key. - - Args: - key: S3 key (path) of the file - - Returns: - File content as bytes - - Raises: - StorageError: If download fails or file not found - """ + """Download file from S3 by key.""" self._ensure_initialized() if not self.is_enabled(): error_msg = self._initialization_error or "S3 is not enabled or not configured" @@ -488,20 +381,7 @@ def download_file_by_key(self, key: str) -> bytes: raise StorageError(f"Unexpected error downloading file from S3: {str(e)}") def generate_presigned_url(self, file_id: uuid.UUID, file_format: str, expiration: int = 3600) -> str: - """ - Generate a presigned URL for temporary file access. - - Args: - file_id: File identifier - file_format: File format extension - expiration: URL expiration time in seconds (default: 1 hour) - - Returns: - Presigned URL string - - Raises: - StorageError: If URL generation fails - """ + """Generate a presigned URL for temporary file access.""" self._ensure_initialized() if not self.is_enabled(): error_msg = self._initialization_error or "S3 is not enabled or not configured" @@ -521,19 +401,7 @@ def generate_presigned_url(self, file_id: uuid.UUID, file_format: str, expiratio raise StorageError(f"Unexpected error generating presigned URL: {str(e)}") def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str: - """ - Generate a presigned URL for temporary file access by key. - - Args: - key: S3 key (path) of the file - expiration: URL expiration time in seconds (default: 1 hour) - - Returns: - Presigned URL string - - Raises: - StorageError: If URL generation fails - """ + """Generate a presigned URL for temporary file access by key.""" self._ensure_initialized() if not self.is_enabled(): error_msg = self._initialization_error or "S3 is not enabled or not configured" @@ -554,4 +422,3 @@ def generate_presigned_url_by_key(self, key: str, expiration: int = 3600) -> str # Singleton instance s3_service = S3Service() - diff --git a/app/services/storage_service.py b/app/services/storage/storage_service.py similarity index 95% rename from app/services/storage_service.py rename to app/services/storage/storage_service.py index 17672224..7dd268a1 100644 --- a/app/services/storage_service.py +++ b/app/services/storage/storage_service.py @@ -1,9 +1,7 @@ """File storage service for handling audio file uploads and storage.""" -import os import uuid from pathlib import Path -from typing import BinaryIO from fastapi import UploadFile from app.config import settings from app.core.exceptions import StorageError, InvalidAudioFormatError @@ -141,4 +139,3 @@ def delete_file(self, file_id: uuid.UUID, file_format: str) -> bool: # Singleton instance storage_service = StorageService() - diff --git a/app/services/testing/__init__.py b/app/services/testing/__init__.py new file mode 100644 index 00000000..7d7597dc --- /dev/null +++ b/app/services/testing/__init__.py @@ -0,0 +1,11 @@ +"""Testing service package exports.""" + +from app.services.testing.test_agent_bridge_service import TestAgentBridgeService, test_agent_bridge_service +from app.services.testing.test_agent_service import TestAgentService, test_agent_service + +__all__ = [ + "TestAgentBridgeService", + "test_agent_bridge_service", + "TestAgentService", + "test_agent_service", +] diff --git a/app/services/test_agent_bridge_service.py b/app/services/testing/test_agent_bridge_service.py similarity index 68% rename from app/services/test_agent_bridge_service.py rename to app/services/testing/test_agent_bridge_service.py index 3adb1559..9610b50b 100644 --- a/app/services/test_agent_bridge_service.py +++ b/app/services/testing/test_agent_bridge_service.py @@ -5,9 +5,7 @@ """ import asyncio -import tempfile import os -import time from typing import Dict, Any, Optional from uuid import UUID from loguru import logger @@ -15,17 +13,17 @@ from app.models.database import Agent, Integration, VoiceBundle, EvaluatorResult, EvaluatorResultStatus from app.core.encryption import decrypt_api_key from app.services.voice_providers import get_voice_provider -from app.services.s3_service import s3_service +from app.services.storage.s3_service import s3_service from app.workers.celery_app import process_evaluator_result_task class TestAgentBridgeService: """Service to bridge test voice AI agent with Voice AI agent.""" - + def __init__(self): """Initialize the bridge service.""" pass - + async def bridge_test_agent_to_voice_agent( self, evaluator_id: UUID, @@ -35,75 +33,71 @@ async def bridge_test_agent_to_voice_agent( ) -> Dict[str, Any]: """ Main bridging logic to connect test agent to Voice AI agent. - - Args: - evaluator_id: Evaluator ID - evaluator_result_id: EvaluatorResult ID (pre-created) - organization_id: Organization ID - db: Database session - - Returns: - Dictionary with call metadata including s3_key, duration, etc. """ logger.info( f"[Bridge] Starting bridge_test_agent_to_voice_agent: " f"evaluator_id={evaluator_id}, result_id={evaluator_result_id}" ) - + from app.models.database import Evaluator, Persona, Scenario - + # Load evaluator and related entities evaluator = db.query(Evaluator).filter(Evaluator.id == evaluator_id).first() if not evaluator: raise ValueError(f"Evaluator {evaluator_id} not found") - + agent = db.query(Agent).filter(Agent.id == evaluator.agent_id).first() if not agent: raise ValueError(f"Agent {evaluator.agent_id} not found") - + persona = db.query(Persona).filter(Persona.id == evaluator.persona_id).first() if not persona: raise ValueError(f"Persona {evaluator.persona_id} not found") - + scenario = db.query(Scenario).filter(Scenario.id == evaluator.scenario_id).first() if not scenario: raise ValueError(f"Scenario {evaluator.scenario_id} not found") - + # Verify agent has both voice_bundle_id and voice_ai_integration_id if not agent.voice_bundle_id: raise ValueError(f"Agent {agent.id} does not have voice_bundle_id configured") - + if not agent.voice_ai_integration_id or not agent.voice_ai_agent_id: raise ValueError( f"Agent {agent.id} does not have voice_ai_integration_id and voice_ai_agent_id configured" ) - + # Get voice bundle - voice_bundle = db.query(VoiceBundle).filter( - VoiceBundle.id == agent.voice_bundle_id, - VoiceBundle.organization_id == organization_id - ).first() + voice_bundle = ( + db.query(VoiceBundle) + .filter(VoiceBundle.id == agent.voice_bundle_id, VoiceBundle.organization_id == organization_id) + .first() + ) if not voice_bundle: raise ValueError(f"VoiceBundle {agent.voice_bundle_id} not found") - + # Get integration - integration = db.query(Integration).filter( - Integration.id == agent.voice_ai_integration_id, - Integration.organization_id == organization_id, - Integration.is_active == True - ).first() + integration = ( + db.query(Integration) + .filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ) + .first() + ) if not integration: raise ValueError(f"Integration {agent.voice_ai_integration_id} not found or inactive") - + # Decrypt API key try: api_key = decrypt_api_key(integration.api_key) except Exception as e: raise ValueError(f"Failed to decrypt integration API key: {e}") - + # Get voice provider # Handle platform being either enum or string - platform_value = integration.platform.value if hasattr(integration.platform, 'value') else integration.platform + platform_value = integration.platform.value if hasattr(integration.platform, "value") else integration.platform try: provider_class = get_voice_provider(platform_value) # For Vapi, pass the public_key as well (needed for web call creation) @@ -112,18 +106,18 @@ async def bridge_test_agent_to_voice_agent( provider = provider_class(api_key=api_key, public_key=integration.public_key) else: provider = provider_class(api_key=api_key) - except ValueError as e: + except ValueError: raise ValueError(f"Unsupported voice provider platform: {platform_value}") - + logger.info( f"[Bridge] Starting bridge for evaluator {evaluator.evaluator_id}, " f"agent {agent.name}, integration {platform_value}" ) - + # Step 1: Create web call to Voice AI agent (Retell/Vapi/ElevenLabs) logger.info(f"[Bridge] Step 1: Creating web call to {platform_value} agent {agent.voice_ai_agent_id}") try: - logger.info(f"[Bridge] Calling provider.create_web_call()...") + logger.info("[Bridge] Calling provider.create_web_call()...") web_call_response = provider.create_web_call( agent_id=agent.voice_ai_agent_id, metadata={ @@ -133,10 +127,10 @@ async def bridge_test_agent_to_voice_agent( "test_agent_mode": True, "persona_id": str(persona.id), "scenario_id": str(scenario.id), - } + }, ) logger.info(f"[Bridge] provider.create_web_call() returned: {list(web_call_response.keys()) if web_call_response else 'None'}") - + call_id = web_call_response.get("call_id") # For Retell: access_token (LiveKit token) # For Vapi: web_call_url (Daily.co URL) - we pass it in the access_token field @@ -147,31 +141,31 @@ async def bridge_test_agent_to_voice_agent( or web_call_response.get("signed_url") ) sample_rate = web_call_response.get("sample_rate", 24000) - + # ElevenLabs uses 16kHz if platform_value.lower() == "elevenlabs": sample_rate = 16000 - + logger.info(f"[Bridge] Extracted: call_id={call_id}, access_token={'set' if access_token else 'NOT SET'}, sample_rate={sample_rate}") - + # ElevenLabs returns call_id=None; the conversation_id is obtained # after WebSocket connection. Skip the call_id check for ElevenLabs. if not call_id and platform_value.lower() != "elevenlabs": logger.error(f"[Bridge] ❌ No call_id in response. Full response: {web_call_response}") raise ValueError("No call_id received from web call creation") - + if not access_token: logger.error(f"[Bridge] ❌ No access_token/web_call_url/signed_url in response. Full response: {web_call_response}") raise ValueError("No access_token/web_call_url/signed_url received from web call creation") - + logger.info(f"[Bridge] ✅ Created web call: call_id={call_id}, token/url={'***' if access_token else 'None'}") - + # Step 2: Store call info in evaluator result and update status - logger.info(f"[Bridge] Step 2: Updating evaluator result status to CALL_INITIATING") + logger.info("[Bridge] Step 2: Updating evaluator result status to CALL_INITIATING") result = db.query(EvaluatorResult).filter(EvaluatorResult.id == evaluator_result_id).first() if not result: raise ValueError(f"EvaluatorResult {evaluator_result_id} not found") - + result.status = EvaluatorResultStatus.CALL_INITIATING.value result.call_event = "call_initiating" if call_id: @@ -179,17 +173,11 @@ async def bridge_test_agent_to_voice_agent( result.error_message = None db.commit() logger.info(f"[Bridge] ✅ Status updated to CALL_INITIATING for result {result.result_id}: call_id={call_id}") - + # Step 3: Connect to Retell via WebRTC and run the bridge # This runs synchronously and waits for the call to complete - logger.info(f"[Bridge] Initiating WebRTC bridge connection") - - # Run the WebRTC bridge - this will wait for the call to complete - # The bridge handles: - # 1. Connecting to Retell via LiveKit - # 2. Bridging audio to/from test agent - # 3. Recording the conversation - # 4. Detecting call end + logger.info("[Bridge] Initiating WebRTC bridge connection") + bridge_task = asyncio.create_task( self._connect_and_bridge_with_webrtc( evaluator_id=evaluator.id, @@ -203,10 +191,10 @@ async def bridge_test_agent_to_voice_agent( provider_platform=platform_value, evaluator_result_id=evaluator_result_id, voice_bundle_id=agent.voice_bundle_id, - db=db + db=db, ) ) - + # Step 4: Start polling for call results (runs concurrently with bridge) poll_task = asyncio.create_task( self._poll_call_results( @@ -215,26 +203,19 @@ async def bridge_test_agent_to_voice_agent( provider_platform=platform_value, evaluator_result_id=evaluator_result_id, organization_id=organization_id, - db=db + db=db, ) ) - + logger.info( f"[Bridge] Call initiated: call_id={call_id}. " f"Waiting for bridge connection and call to complete..." ) - + # Wait for both tasks to complete - # The bridge task will complete when the call ends (WebRTC disconnect) - # The poll task will complete when it gets results from voice provider and triggers evaluation - # IMPORTANT: We must wait for BOTH tasks because: - # - bridge_task detects call end via WebRTC (fast) - # - poll_task fetches results from voice provider API and triggers evaluation (needs time) try: - # Wait for bridge task first (it detects call end) - # Give poll task additional time to fetch results after bridge completes - logger.info(f"[Bridge] Waiting for bridge task to complete...") - + logger.info("[Bridge] Waiting for bridge task to complete...") + try: await asyncio.wait_for(bridge_task, timeout=600) # 10 minute max call duration logger.info(f"[Bridge] Bridge task completed for call {call_id}") @@ -243,11 +224,9 @@ async def bridge_test_agent_to_voice_agent( bridge_task.cancel() except Exception as bridge_error: logger.error(f"[Bridge] Bridge task error: {bridge_error}") - - # Now wait for poll task to finish fetching results and triggering evaluation - # Give it extra time after bridge completes (Retell API may need time) - logger.info(f"[Bridge] Waiting for poll task to fetch results and trigger evaluation...") - + + logger.info("[Bridge] Waiting for poll task to fetch results and trigger evaluation...") + try: # Give poll task up to 2 minutes to fetch results after call ends await asyncio.wait_for(poll_task, timeout=120) @@ -256,49 +235,41 @@ async def bridge_test_agent_to_voice_agent( logger.warning(f"[Bridge] Poll task timed out for call {call_id}") poll_task.cancel() except asyncio.CancelledError: - logger.info(f"[Bridge] Poll task was cancelled") + logger.info("[Bridge] Poll task was cancelled") except Exception as poll_error: logger.error(f"[Bridge] Poll task error: {poll_error}") - + logger.info(f"[Bridge] All tasks completed for call {call_id}") - + except asyncio.TimeoutError: logger.warning(f"[Bridge] Call {call_id} timed out after 10 minutes") bridge_task.cancel() poll_task.cancel() - + return { "call_id": call_id, "access_token": access_token, "sample_rate": sample_rate, "web_call_response": web_call_response, "status": "completed", - "message": "Call completed. Results should be available." + "message": "Call completed. Results should be available.", } - + except Exception as e: logger.error(f"[Bridge] Error creating web call: {e}", exc_info=True) raise - + def initiate_voice_agent_call( self, provider, agent_id: str, - metadata: Optional[Dict[str, Any]] = None + metadata: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Initiate a call to Voice AI agent via integration API. - - Args: - provider: Voice provider instance - agent_id: Voice AI agent ID - metadata: Optional metadata - - Returns: - Call response with connection details """ return provider.create_web_call(agent_id=agent_id, metadata=metadata or {}) - + async def _connect_and_bridge_with_webrtc( self, evaluator_id: UUID, @@ -312,50 +283,26 @@ async def _connect_and_bridge_with_webrtc( provider_platform: str, evaluator_result_id: UUID, voice_bundle_id: UUID, - db + db, ): """ Connect test agent and bridge to Retell/Vapi (WebRTC) or ElevenLabs (WebSocket). - - This implementation: - 1. Creates a connection to the voice provider (LiveKit/Daily/WebSocket) - 2. Initializes the in-process test agent (LLM + TTS) - 3. Bridges audio streams bidirectionally - 4. Records the conversation - - Args: - evaluator_id: Evaluator ID - agent_id: Agent ID - persona_id: Persona ID - scenario_id: Scenario ID - organization_id: Organization ID - call_id: Retell/Vapi call ID - access_token: Retell/Vapi access token or ElevenLabs signed URL - sample_rate: Audio sample rate - provider_platform: Platform name (retell, vapi, elevenlabs) - evaluator_result_id: EvaluatorResult ID - voice_bundle_id: Voice bundle ID for test agent - db: Database session """ from app.services.webrtc_bridge.retell_webrtc_bridge import RetellWebRTCBridge from app.services.webrtc_bridge.vapi_webrtc_bridge import VapiWebRTCBridge from app.services.webrtc_bridge.elevenlabs_ws_bridge import ElevenLabsWSBridge - from app.config import settings - import websockets - import json - + webrtc_bridge = None test_agent = None - + # Helper function to update status async def update_status(new_status: str, event: str = None, error: str = None): """Update evaluator result status.""" from app.database import SessionLocal + status_db = SessionLocal() try: - result = status_db.query(EvaluatorResult).filter( - EvaluatorResult.id == evaluator_result_id - ).first() + result = status_db.query(EvaluatorResult).filter(EvaluatorResult.id == evaluator_result_id).first() if result: result.status = new_status if event: @@ -368,70 +315,48 @@ async def update_status(new_status: str, event: str = None, error: str = None): logger.error(f"[Bridge WebRTC] Error updating status: {e}", exc_info=True) finally: status_db.close() - + try: logger.info( f"[Bridge WebRTC] Starting WebRTC bridge for evaluator {evaluator_id}, " f"bridging to {provider_platform} call {call_id}" ) - + # Update status to connecting - await update_status( - EvaluatorResultStatus.CALL_CONNECTING.value, - "call_connecting" - ) - + await update_status(EvaluatorResultStatus.CALL_CONNECTING.value, "call_connecting") + # Step 1: Create WebRTC bridge to Retell/Vapi # Set up call ended callback (shared between providers) async def on_call_ended(): logger.info("[Bridge WebRTC] Call ended, cleaning up") - await update_status( - EvaluatorResultStatus.CALL_ENDED.value, - "call_ended" - ) + await update_status(EvaluatorResultStatus.CALL_ENDED.value, "call_ended") # Recording and result processing will be handled by polling - + if provider_platform == "retell": - webrtc_bridge = RetellWebRTCBridge( - call_id=call_id, - access_token=access_token, - sample_rate=sample_rate - ) + webrtc_bridge = RetellWebRTCBridge(call_id=call_id, access_token=access_token, sample_rate=sample_rate) webrtc_bridge.on_call_ended = on_call_ended - + # Connect to Retell connected = await webrtc_bridge.connect_to_retell() if not connected: - await update_status( - EvaluatorResultStatus.FAILED.value, - "call_connection_failed", - "Failed to connect to Retell WebRTC call" - ) + await update_status(EvaluatorResultStatus.FAILED.value, "call_connection_failed", "Failed to connect to Retell WebRTC call") raise Exception("Failed to connect to Retell WebRTC call") - + logger.info("[Bridge WebRTC] ✅ Connected to Retell WebRTC call") - + elif provider_platform == "vapi": # For Vapi, access_token is actually the web_call_url (Daily.co URL) web_call_url = access_token # Passed as access_token from bridge_test_agent_to_voice_agent - - webrtc_bridge = VapiWebRTCBridge( - call_id=call_id, - web_call_url=web_call_url, - sample_rate=sample_rate - ) + + webrtc_bridge = VapiWebRTCBridge(call_id=call_id, web_call_url=web_call_url, sample_rate=sample_rate) webrtc_bridge.on_call_ended = on_call_ended - + # Connect to Vapi via Daily.co connected = await webrtc_bridge.connect_to_vapi() if not connected: - await update_status( - EvaluatorResultStatus.FAILED.value, - "call_connection_failed", - "Failed to connect to Vapi WebRTC call" - ) + await update_status(EvaluatorResultStatus.FAILED.value, "call_connection_failed", "Failed to connect to Vapi WebRTC call") raise Exception("Failed to connect to Vapi WebRTC call") - + logger.info("[Bridge WebRTC] ✅ Connected to Vapi WebRTC call via Daily.co") elif provider_platform == "elevenlabs": @@ -446,11 +371,7 @@ async def on_call_ended(): connected = await webrtc_bridge.connect_to_elevenlabs() if not connected: - await update_status( - EvaluatorResultStatus.FAILED.value, - "call_connection_failed", - "Failed to connect to ElevenLabs WebSocket" - ) + await update_status(EvaluatorResultStatus.FAILED.value, "call_connection_failed", "Failed to connect to ElevenLabs WebSocket") raise Exception("Failed to connect to ElevenLabs WebSocket") logger.info("[Bridge WebRTC] ✅ Connected to ElevenLabs WebSocket") @@ -459,11 +380,10 @@ async def on_call_ended(): # Update the evaluator result so that _poll_call_results can use it. if webrtc_bridge.conversation_id: from app.database import SessionLocal + id_db = SessionLocal() try: - r = id_db.query(EvaluatorResult).filter( - EvaluatorResult.id == evaluator_result_id - ).first() + r = id_db.query(EvaluatorResult).filter(EvaluatorResult.id == evaluator_result_id).first() if r: r.provider_call_id = webrtc_bridge.conversation_id id_db.commit() @@ -476,62 +396,65 @@ async def on_call_ended(): else: raise ValueError(f"WebRTC bridging not yet implemented for platform: {provider_platform}") - + # Update status to in progress - await update_status( - EvaluatorResultStatus.CALL_IN_PROGRESS.value, - "call_started" - ) - + await update_status(EvaluatorResultStatus.CALL_IN_PROGRESS.value, "call_started") + # Step 2: Initialize in-process test agent (LLM + TTS) - # This runs the test agent directly without WebSocket from app.services.webrtc_bridge.test_agent_processor import TestAgentProcessor, TestAgentConfig from app.models.database import Persona, Scenario, AIProvider, ModelProvider, Integration, IntegrationPlatform, Agent - from app.core.encryption import decrypt_api_key - + # Load agent, persona and scenario for the test agent agent = db.query(Agent).filter(Agent.id == agent_id).first() persona = db.query(Persona).filter(Persona.id == persona_id).first() scenario = db.query(Scenario).filter(Scenario.id == scenario_id).first() - + if not agent: raise ValueError(f"Agent not found: {agent_id}") if not persona or not scenario: - raise ValueError(f"Persona or scenario not found") - + raise ValueError("Persona or scenario not found") + # Helper function to resolve API key (same logic as voice_agent.py + env fallback) def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: """Resolve API key from AIProvider (preferred), Integration, or environment.""" import os - - # 1) Check AIProvider table first (handle both string and enum comparisons) from sqlalchemy import func - provider_value = provider.value if hasattr(provider, 'value') else provider - - ai_provider_rec = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - AIProvider.provider == provider_value, - AIProvider.is_active == True, - ).first() - - # If not found, try case-insensitive match - if not ai_provider_rec: - ai_provider_rec = db.query(AIProvider).filter( + + # 1) Check AIProvider table first + provider_value = provider.value if hasattr(provider, "value") else provider + + ai_provider_rec = ( + db.query(AIProvider) + .filter( AIProvider.organization_id == organization_id, - func.lower(AIProvider.provider) == provider_value.lower(), + AIProvider.provider == provider_value, AIProvider.is_active == True, - ).first() + ) + .first() + ) + + # If not found, try case-insensitive match + if not ai_provider_rec: + ai_provider_rec = ( + db.query(AIProvider) + .filter( + AIProvider.organization_id == organization_id, + func.lower(AIProvider.provider) == provider_value.lower(), + AIProvider.is_active == True, + ) + .first() + ) if ai_provider_rec: try: key = decrypt_api_key(ai_provider_rec.api_key) if key: - provider_val = provider.value if hasattr(provider, 'value') else provider + provider_val = provider.value if hasattr(provider, "value") else provider logger.debug(f"[Bridge WebRTC] Found API key for {provider_val} in AIProvider table") return key except Exception as e: logger.error(f"[Bridge WebRTC] Failed to decrypt AIProvider key for {provider}: {e}") - - # 2) Check Integration table (for platforms that exist in IntegrationPlatform) + + # 2) Check Integration table platform_map = { ModelProvider.DEEPGRAM: IntegrationPlatform.DEEPGRAM, ModelProvider.CARTESIA: IntegrationPlatform.CARTESIA, @@ -539,32 +462,39 @@ def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: } plat = platform_map.get(provider) if plat: - # Handle both string and enum comparisons for platform - plat_value = plat.value if hasattr(plat, 'value') else plat - integ = db.query(Integration).filter( - Integration.organization_id == organization_id, - Integration.platform == plat_value, - Integration.is_active == True, - ).first() - - # If not found, try case-insensitive match - if not integ: - integ = db.query(Integration).filter( + plat_value = plat.value if hasattr(plat, "value") else plat + integ = ( + db.query(Integration) + .filter( Integration.organization_id == organization_id, - func.lower(Integration.platform) == plat_value.lower(), + Integration.platform == plat_value, Integration.is_active == True, - ).first() + ) + .first() + ) + + # If not found, try case-insensitive match + if not integ: + integ = ( + db.query(Integration) + .filter( + Integration.organization_id == organization_id, + func.lower(Integration.platform) == plat_value.lower(), + Integration.is_active == True, + ) + .first() + ) if integ: try: key = decrypt_api_key(integ.api_key) if key: - provider_val = provider.value if hasattr(provider, 'value') else provider + provider_val = provider.value if hasattr(provider, "value") else provider logger.debug(f"[Bridge WebRTC] Found API key for {provider_val} in Integration table") return key except Exception as e: logger.error(f"[Bridge WebRTC] Failed to decrypt Integration key for {provider}: {e}") - - # 3) Fallback to environment variables (same as voice_bundle.py) + + # 3) Fallback to environment variables env_map = { ModelProvider.OPENAI: "OPENAI_API_KEY", ModelProvider.CARTESIA: "CARTESIA_API_KEY", @@ -576,18 +506,19 @@ def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: if env_var: env_key = os.getenv(env_var) if env_key: - provider_val = provider.value if hasattr(provider, 'value') else provider + provider_val = provider.value if hasattr(provider, "value") else provider logger.debug(f"[Bridge WebRTC] Found API key for {provider_val} in environment variable {env_var}") return env_key - + return None - + # Resolve the TTS provider from the voice bundle - voice_bundle = db.query(VoiceBundle).filter( - VoiceBundle.id == voice_bundle_id, - VoiceBundle.organization_id == organization_id, - ).first() - + voice_bundle = ( + db.query(VoiceBundle) + .filter(VoiceBundle.id == voice_bundle_id, VoiceBundle.organization_id == organization_id) + .first() + ) + tts_voice_id = None tts_model = None tts_provider_str = None @@ -603,7 +534,7 @@ def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: f"Voice bundle {voice_bundle_id} is missing tts_provider. " f"Please configure the TTS provider in the voice bundle settings." ) - + tts_provider_enum_map = { "cartesia": ModelProvider.CARTESIA, "elevenlabs": ModelProvider.ELEVENLABS, @@ -619,9 +550,9 @@ def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: logger.info(f"[Bridge WebRTC] Resolving API keys for test agent (org: {organization_id}, tts_provider={tts_provider_str})") llm_api_key = resolve_api_key_for_provider(ModelProvider.OPENAI) tts_api_key = resolve_api_key_for_provider(tts_model_provider) - + logger.info(f"[Bridge WebRTC] API keys found: OpenAI={'yes' if llm_api_key else 'no'}, {tts_provider_str}={'yes' if tts_api_key else 'no'}") - + missing_keys = [] if not llm_api_key: missing_keys.append("OpenAI (LLM) - check AIProvider table or OPENAI_API_KEY env var") @@ -629,43 +560,39 @@ def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: env_hints = {"cartesia": "CARTESIA_API_KEY", "elevenlabs": "ELEVENLABS_API_KEY", "openai": "OPENAI_API_KEY"} env_hint = env_hints.get(tts_provider_str, f"{tts_provider_str.upper()}_API_KEY") missing_keys.append(f"{tts_provider_str} (TTS) - check AIProvider/Integration table or {env_hint} env var") - + if missing_keys: logger.warning( f"[Bridge WebRTC] Missing API keys for test agent: {', '.join(missing_keys)}. " - f"Running without test agent." + "Running without test agent." ) test_agent = None else: # Build test agent config from persona/scenario - # Extract goal from required_info or scenario description scenario_goal = "Complete the test call successfully" first_message = f"Hello, this is {persona.name} calling." - + if scenario.required_info: - # Check if required_info contains goal or first_message if isinstance(scenario.required_info, dict): scenario_goal = scenario.required_info.get("goal", scenario_goal) first_message = scenario.required_info.get("first_message", first_message) - + # Build persona description from available fields - # Persona model has: name, language, accent, gender, background_noise - # Handle enum values being either enum or string persona_traits = [] - if hasattr(persona, 'gender') and persona.gender: - gender_val = persona.gender.value if hasattr(persona.gender, 'value') else persona.gender + if hasattr(persona, "gender") and persona.gender: + gender_val = persona.gender.value if hasattr(persona.gender, "value") else persona.gender persona_traits.append(f"{gender_val} caller") - if hasattr(persona, 'accent') and persona.accent: - accent_val = persona.accent.value if hasattr(persona.accent, 'value') else persona.accent + if hasattr(persona, "accent") and persona.accent: + accent_val = persona.accent.value if hasattr(persona.accent, "value") else persona.accent persona_traits.append(f"with {accent_val} accent") - if hasattr(persona, 'language') and persona.language: - language_val = persona.language.value if hasattr(persona.language, 'value') else persona.language + if hasattr(persona, "language") and persona.language: + language_val = persona.language.value if hasattr(persona.language, "value") else persona.language persona_traits.append(f"speaking {language_val}") - + persona_description = f"A caller named {persona.name}" if persona_traits: persona_description += " (" + ", ".join(persona_traits) + ")" - + test_agent_config = TestAgentConfig( # Who we are calling (the voice AI agent) agent_name=agent.name or "Voice AI Agent", @@ -674,7 +601,7 @@ def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: persona_name=persona.name, persona_description=persona_description, # The test scenario - scenario_description=getattr(scenario, 'description', None) or scenario.name or "Test call scenario", + scenario_description=getattr(scenario, "description", None) or scenario.name or "Test call scenario", scenario_goal=scenario_goal, first_message=first_message, llm_api_key=llm_api_key, @@ -683,50 +610,42 @@ def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: tts_voice_id=tts_voice_id, tts_model=tts_model, sample_rate=sample_rate, - max_turns=20 + max_turns=20, ) - + test_agent = TestAgentProcessor(test_agent_config) await test_agent.initialize() - + logger.info(f"[Bridge WebRTC] ✅ Test agent initialized: {persona.name}") - + # Step 3: Start recording and connect test agent to Retell webrtc_bridge.is_bridging = True - + # Start recording await webrtc_bridge.start_recording() - + logger.info("[Bridge WebRTC] ✅ Recording started") - + if test_agent: # Set up callbacks to connect test agent with voice provider - # Vapi uses 40ms chunks (640 samples at 16kHz), Retell uses 20ms chunks - # ElevenLabs uses 250ms recommended but 20ms works fine for streaming chunk_ms = 40 if provider_platform == "vapi" else 20 - + async def send_audio_chunks(audio: bytes): """Stream audio to voice provider in real-time chunks.""" - await test_agent.stream_audio_chunks( - audio, - webrtc_bridge.receive_audio_from_test_agent, - chunk_duration_ms=chunk_ms - ) - # ElevenLabs needs trailing silence so its VAD detects - # end-of-speech (a real browser mic always streams audio). + await test_agent.stream_audio_chunks(audio, webrtc_bridge.receive_audio_from_test_agent, chunk_duration_ms=chunk_ms) + # ElevenLabs needs trailing silence so its VAD detects end-of-speech if provider_platform == "elevenlabs" and hasattr(webrtc_bridge, "send_silence"): await webrtc_bridge.send_silence(duration_ms=600) - + async def on_transcript_received(transcript: str): """When voice agent finishes speaking, process with test agent.""" logger.info(f"[Bridge WebRTC] Received transcript from {provider_platform}: {transcript[:50]}...") audio = await test_agent.process_agent_transcript(transcript) if audio: - # Stream test agent's response audio to voice agent in chunks logger.info(f"[Bridge WebRTC] Streaming {len(audio)} bytes of audio to {provider_platform}...") await send_audio_chunks(audio) logger.info("[Bridge WebRTC] Audio streaming complete") - + async def on_agent_start_talking(): """Voice AI agent started speaking -- test agent should wait.""" logger.info(f"[Bridge WebRTC] {provider_platform} agent started speaking") @@ -736,7 +655,7 @@ async def on_agent_stop_talking(): """Voice AI agent stopped speaking -- test agent can respond.""" logger.info(f"[Bridge WebRTC] {provider_platform} agent stopped speaking") test_agent.agent_is_talking = False - + # Process any transcript that was queued while agent was talking pending = test_agent._pending_transcript if pending: @@ -752,22 +671,15 @@ async def on_call_should_end(): """Test agent decided to end the call.""" logger.info("[Bridge WebRTC] Test agent requested call end") webrtc_bridge.is_bridging = False - + webrtc_bridge.on_transcript_received = on_transcript_received webrtc_bridge.on_agent_start_talking = on_agent_start_talking webrtc_bridge.on_agent_stop_talking = on_agent_stop_talking test_agent.on_call_should_end = on_call_should_end - - # ElevenLabs agents have a built-in greeting — they start talking - # as soon as the WebSocket connects. Sending a first_message would - # collide with the greeting, cause an interruption, and confuse the - # agent. So for ElevenLabs we skip the first message and let the - # on_transcript_received callback handle the response to the greeting. + + # ElevenLabs agents have a built-in greeting if provider_platform == "elevenlabs": - logger.info( - "[Bridge WebRTC] ElevenLabs: skipping first message — " - "waiting for agent greeting" - ) + logger.info("[Bridge WebRTC] ElevenLabs: skipping first message — waiting for agent greeting") else: # Retell / Vapi: test agent initiates the conversation logger.info("[Bridge WebRTC] Sending test agent's first message...") @@ -776,31 +688,31 @@ async def on_call_should_end(): logger.info(f"[Bridge WebRTC] Streaming first message ({len(first_audio)} bytes)...") await send_audio_chunks(first_audio) logger.info(f"[Bridge WebRTC] ✅ First message sent to {provider_platform}") - + logger.info("[Bridge WebRTC] ✅ Test agent connected, conversation starting...") else: - logger.info("[Bridge WebRTC] Running without test agent - Retell will handle the call") - + logger.info("[Bridge WebRTC] Running without test agent - provider will handle the call") + # Wait for the call to end call_timeout = 300 # 5 minutes max call duration start_time = asyncio.get_event_loop().time() - + while webrtc_bridge.is_connected and webrtc_bridge.is_bridging: await asyncio.sleep(1) - + # Check for timeout elapsed = asyncio.get_event_loop().time() - start_time if elapsed > call_timeout: logger.warning(f"[Bridge WebRTC] Call timeout after {call_timeout} seconds") break - + logger.info("[Bridge WebRTC] Call ended") - + # Get conversation transcript from test agent (for debugging) if test_agent: conversation_log = test_agent.get_conversation_transcript() logger.info(f"[Bridge WebRTC] Test agent conversation log:\n{conversation_log}") - + # Stop local recording (for backup/debugging only) recording_path = await webrtc_bridge.stop_recording() if recording_path: @@ -812,80 +724,44 @@ async def on_call_should_end(): logger.info(f"[Bridge WebRTC] Local recording cleaned up (using {provider_platform}'s recording)") except Exception as e: logger.warning(f"[Bridge WebRTC] Could not clean up local recording: {e}") - + logger.info(f"[Bridge WebRTC] Bridge completed. Waiting for _poll_call_results to fetch call data from {provider_platform}...") - - # NOTE: We do NOT upload to S3 or trigger transcription here. - # The _poll_call_results task handles everything: - # - Fetches call_data from voice provider (includes transcript, recording_url, cost, latency) - # - Stores call_data in database - # - Triggers evaluation task - # This approach avoids: - # - Redundant S3 storage costs - # - Re-transcription (Retell already transcribed) - + except Exception as e: logger.error(f"[Bridge WebRTC] Error in WebRTC bridge: {e}", exc_info=True) - + # Update result status - await update_status( - EvaluatorResultStatus.FAILED.value, - "call_error", - str(e) - ) + await update_status(EvaluatorResultStatus.FAILED.value, "call_error", str(e)) finally: # Cleanup if webrtc_bridge: await webrtc_bridge.disconnect() if test_agent: await test_agent.cleanup() - + logger.info("[Bridge WebRTC] Bridge cleanup completed") - + async def connect_test_agent( self, websocket_url: str, evaluator_id: UUID, persona_id: UUID, scenario_id: UUID, - agent_id: UUID + agent_id: UUID, ) -> Any: """ Connect test agent via WebSocket. - - Args: - websocket_url: WebSocket URL for test agent - evaluator_id: Evaluator ID - persona_id: Persona ID - scenario_id: Scenario ID - agent_id: Agent ID - - Returns: - WebSocket connection """ - # This would create a WebSocket connection - # Implementation depends on the WebSocket client library used - # For now, this is a placeholder logger.info(f"[Bridge] Connecting test agent via WebSocket: {websocket_url}") return None - - async def bridge_audio_streams( - self, - test_agent_ws: Any, - voice_agent_call: Any - ) -> None: + + async def bridge_audio_streams(self, test_agent_ws: Any, voice_agent_call: Any) -> None: """ Bridge audio streams between test agent and voice agent. - - Args: - test_agent_ws: Test agent WebSocket connection - voice_agent_call: Voice AI agent call connection """ - # This would handle real-time audio bridging - # Implementation would use audio streaming libraries logger.info("[Bridge] Bridging audio streams") pass - + async def _poll_call_results( self, call_id: str, @@ -895,53 +771,32 @@ async def _poll_call_results( organization_id: UUID, db, max_attempts: int = 120, # Poll for up to 10 minutes (5 second intervals) - poll_interval: int = 5 + poll_interval: int = 5, ): """ Poll for call results from the provider after call ends. - - Status Flow: - - CALL_IN_PROGRESS (set by bridge) → CALL_ENDED → FETCHING_DETAILS → EVALUATING → COMPLETED - - This method uses the transcript directly from the provider (Retell/Vapi) - instead of downloading audio to S3 and re-transcribing. This approach: - - Avoids redundant S3 storage costs - - Uses provider's high-quality transcription (with speaker diarization) - - Enables faster evaluation since no transcription step is needed - - Args: - call_id: Provider call ID - provider: Voice provider instance - provider_platform: Platform name (retell, vapi, etc.) - evaluator_result_id: EvaluatorResult ID - organization_id: Organization ID - db: Database session - max_attempts: Maximum polling attempts - poll_interval: Seconds between polls """ from app.database import SessionLocal - + # Use a new database session for polling (important for long-running task) poll_db = SessionLocal() try: - result = poll_db.query(EvaluatorResult).filter( - EvaluatorResult.id == evaluator_result_id - ).first() - + result = poll_db.query(EvaluatorResult).filter(EvaluatorResult.id == evaluator_result_id).first() + if not result: logger.error(f"[Bridge Poll] EvaluatorResult {evaluator_result_id} not found") return - + # Store provider info immediately if call_id: result.provider_call_id = call_id result.provider_platform = provider_platform poll_db.commit() logger.info(f"[Bridge Poll] Starting to poll {provider_platform} call {call_id} for results") - + # Wait a bit before starting to poll (call might be starting) await asyncio.sleep(10) - + # For ElevenLabs the call_id (conversation_id) is set by the bridge # after the WebSocket connects. Wait for it to appear in the DB. if not call_id and provider_platform == "elevenlabs": @@ -952,67 +807,66 @@ async def _poll_call_results( logger.info(f"[Bridge Poll] Got ElevenLabs conversation_id from DB: {call_id}") break await asyncio.sleep(1) - + if not call_id: logger.error("[Bridge Poll] ElevenLabs conversation_id never appeared") result.status = EvaluatorResultStatus.FAILED.value result.error_message = "ElevenLabs conversation_id was never set" poll_db.commit() return - + call_completed = False call_metrics = None - + for attempt in range(max_attempts): try: # Wait before polling (except first attempt) if attempt > 0: await asyncio.sleep(poll_interval) - + # Retrieve call metrics from provider if provider_platform in ["retell", "vapi", "elevenlabs"] and hasattr(provider, "retrieve_call_metrics"): call_metrics = provider.retrieve_call_metrics(call_id) else: logger.warning(f"[Bridge Poll] Platform {provider_platform} polling not yet implemented") continue - + # Check call status call_status = call_metrics.get("call_status", "") end_timestamp = call_metrics.get("end_timestamp") transcript = call_metrics.get("transcript", "") - + logger.info( f"[Bridge Poll] Attempt {attempt + 1}: " f"status={call_status}, has_end={bool(end_timestamp)}, " f"transcript_len={len(transcript) if transcript else 0}" ) - + # ElevenLabs may stay "processing" briefly before "ended" if call_status == "processing" and provider_platform == "elevenlabs": - logger.info(f"[Bridge Poll] ElevenLabs call still processing, waiting...") + logger.info("[Bridge Poll] ElevenLabs call still processing, waiting...") continue # If call is complete, process results if end_timestamp or call_status in ["ended", "completed", "failed", "done"]: call_completed = True logger.info(f"[Bridge Poll] ✅ Call completed: status={call_status}") - + # === Status: CALL_ENDED === result.status = EvaluatorResultStatus.CALL_ENDED.value result.call_event = "call_ended" poll_db.commit() - logger.info(f"[Bridge Poll] Status: CALL_ENDED") - + logger.info("[Bridge Poll] Status: CALL_ENDED") + # === Status: FETCHING_DETAILS === result.status = EvaluatorResultStatus.FETCHING_DETAILS.value poll_db.commit() - logger.info(f"[Bridge Poll] Status: FETCHING_DETAILS") - + logger.info("[Bridge Poll] Status: FETCHING_DETAILS") + # Store FULL call_data from provider - # This contains: transcript, transcript_object, recording_url, latency, cost, etc. result.call_data = call_metrics logger.info(f"[Bridge Poll] ✅ Stored call_data with {len(call_metrics)} keys: {list(call_metrics.keys())}") - + # Extract duration (Retell uses duration_ms, Vapi uses duration_seconds) duration_ms = call_metrics.get("duration_ms") duration_seconds = call_metrics.get("duration_seconds") @@ -1022,24 +876,21 @@ async def _poll_call_results( result.duration_seconds = duration_seconds if result.duration_seconds: logger.info(f"[Bridge Poll] Duration: {result.duration_seconds:.1f}s") - + # Extract transcript and speaker segments - transcript_text, speaker_segments = self._extract_transcript_from_call_data( - call_metrics, provider_platform - ) - + transcript_text, speaker_segments = self._extract_transcript_from_call_data(call_metrics, provider_platform) + if transcript_text: result.transcription = transcript_text logger.info(f"[Bridge Poll] ✅ Extracted transcript: {len(transcript_text)} characters") else: - logger.warning(f"[Bridge Poll] ⚠️ No transcript extracted from call_data") - + logger.warning("[Bridge Poll] ⚠️ No transcript extracted from call_data") + if speaker_segments: result.speaker_segments = speaker_segments logger.info(f"[Bridge Poll] ✅ Extracted {len(speaker_segments)} speaker segments") - + # Download call audio from provider and upload to S3 - # (same logic as agent playground – needed for qualitative audio metrics) audio_s3_key = None try: import requests as _http @@ -1052,11 +903,7 @@ async def _poll_call_results( if plat == "elevenlabs": audio_url = recording_urls.get("conversation_audio") if audio_url: - resp = _http.get( - audio_url, - headers={"xi-api-key": provider.api_key}, - timeout=120, - ) + resp = _http.get(audio_url, headers={"xi-api-key": provider.api_key}, timeout=120) if resp.status_code == 200: audio_bytes = resp.content elif plat == "retell": @@ -1077,45 +924,30 @@ async def _poll_call_results( audio_bytes = resp.content if audio_bytes: - content_type = getattr(resp, "headers", {}).get( - "content-type", "audio/mpeg" - ) + content_type = getattr(resp, "headers", {}).get("content-type", "audio/mpeg") ext = "wav" if "wav" in content_type else "mp3" org_id = str(organization_id) - audio_s3_key = ( - f"audio/organizations/{org_id}/evaluations/" - f"{result.provider_call_id}/{_uuid.uuid4()}.{ext}" - ) - s3_service.upload_file_by_key( - audio_bytes, audio_s3_key, content_type=content_type - ) + audio_s3_key = f"audio/organizations/{org_id}/evaluations/{result.provider_call_id}/{_uuid.uuid4()}.{ext}" + s3_service.upload_file_by_key(audio_bytes, audio_s3_key, content_type=content_type) result.audio_s3_key = audio_s3_key - logger.info( - f"[Bridge Poll] ✅ Uploaded call audio to S3: " - f"{audio_s3_key} ({len(audio_bytes)} bytes)" - ) + logger.info(f"[Bridge Poll] ✅ Uploaded call audio to S3: {audio_s3_key} ({len(audio_bytes)} bytes)") else: - logger.warning( - f"[Bridge Poll] Could not download audio for call " - f"{result.provider_call_id}" - ) + logger.warning(f"[Bridge Poll] Could not download audio for call {result.provider_call_id}") except Exception as audio_err: - logger.warning( - f"[Bridge Poll] Audio download/upload failed: {audio_err}" - ) - + logger.warning(f"[Bridge Poll] Audio download/upload failed: {audio_err}") + # Commit all the data poll_db.commit() - logger.info(f"[Bridge Poll] ✅ All call data committed to database") - + logger.info("[Bridge Poll] ✅ All call data committed to database") + # === Status: EVALUATING === if transcript_text: result.status = EvaluatorResultStatus.EVALUATING.value result.error_message = None poll_db.commit() - logger.info(f"[Bridge Poll] Status: EVALUATING") - - # Trigger evaluation task (will skip transcription step since we have transcript) + logger.info("[Bridge Poll] Status: EVALUATING") + + # Trigger evaluation task try: process_evaluator_result_task.delay(str(result.id)) logger.info(f"[Bridge Poll] ✅ Triggered evaluation task for result {result.id}") @@ -1129,114 +961,88 @@ async def _poll_call_results( result.status = EvaluatorResultStatus.FAILED.value result.error_message = "Call completed but no transcript available from provider" poll_db.commit() - logger.warning(f"[Bridge Poll] ❌ No transcript in call_data") - + logger.warning("[Bridge Poll] ❌ No transcript in call_data") + break - + except Exception as e: logger.warning(f"[Bridge Poll] Error on attempt {attempt + 1}: {e}") continue - + if not call_completed: logger.warning(f"[Bridge Poll] ❌ Call {call_id} did not complete within polling window") result.status = EvaluatorResultStatus.FAILED.value result.error_message = "Call did not complete within expected time (10 min timeout)" poll_db.commit() - + except Exception as e: logger.error(f"[Bridge Poll] ❌ Fatal error in polling: {e}", exc_info=True) try: - result = poll_db.query(EvaluatorResult).filter( - EvaluatorResult.id == evaluator_result_id - ).first() + result = poll_db.query(EvaluatorResult).filter(EvaluatorResult.id == evaluator_result_id).first() if result: result.status = EvaluatorResultStatus.FAILED.value result.error_message = f"Polling error: {str(e)}" poll_db.commit() - except: + except Exception: pass finally: poll_db.close() - - def _extract_transcript_from_call_data( - self, - call_data: dict, - provider_platform: str - ) -> tuple[str, list[dict]]: + + def _extract_transcript_from_call_data(self, call_data: dict, provider_platform: str) -> tuple[str, list[dict]]: """ Extract transcript text and speaker segments from provider call data. - - Retell format: - - transcript: Plain text transcript - - transcript_object: List of {role, content, words: [{word, start, end}]} - - Args: - call_data: Full call data from provider - provider_platform: Provider name (retell, vapi, etc.) - - Returns: - Tuple of (transcript_text, speaker_segments) """ transcript_text = "" speaker_segments = [] - + if provider_platform == "retell": # Get plain text transcript transcript_text = call_data.get("transcript", "") - + # Get structured transcript with speaker info transcript_object = call_data.get("transcript_object", []) - + if transcript_object: for msg in transcript_object: role = msg.get("role", "unknown") content = msg.get("content", "") words = msg.get("words", []) - + # Map Retell roles to speaker labels speaker = "Speaker 1" if role == "user" else "Speaker 2" - + # Calculate start/end times from words if available start_time = 0.0 end_time = 0.0 if words: start_time = words[0].get("start", 0.0) end_time = words[-1].get("end", 0.0) - + if content.strip(): - speaker_segments.append({ - "speaker": speaker, - "text": content.strip(), - "start": start_time, - "end": end_time - }) - + speaker_segments.append( + {"speaker": speaker, "text": content.strip(), "start": start_time, "end": end_time} + ) + # If we have transcript_object but no plain transcript, build it if not transcript_text and speaker_segments: - transcript_text = "\n".join( - f"{seg['speaker']}: {seg['text']}" - for seg in speaker_segments - ) - + transcript_text = "\n".join(f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments) + elif provider_platform == "vapi": - # Vapi format: - # - transcript: Plain text transcript (AI: ... \nUser: ...) - # - transcript_object: List of {role, content, seconds_from_start, duration_ms, words} - # - messages: Raw messages from Vapi + # Vapi format transcript_text = call_data.get("transcript", "") - + # Get structured transcript object (preferred) or fall back to messages transcript_obj = call_data.get("transcript_object", []) messages = call_data.get("messages", []) - - # Use transcript_object if available (it's already cleaned up) + + # Use transcript_object if available if transcript_obj: for entry in transcript_obj: role = entry.get("role", "unknown") content = entry.get("content", "") seconds_from_start = entry.get("seconds_from_start", 0) duration_ms = entry.get("duration_ms", 0) - + # Map Vapi roles to speaker labels if role == "user": speaker = "Speaker 1" # Test agent / caller @@ -1244,16 +1050,18 @@ def _extract_transcript_from_call_data( speaker = "Speaker 2" # Vapi agent else: continue - + if content and content.strip(): - speaker_segments.append({ - "speaker": speaker, - "text": content.strip(), - "start": seconds_from_start, - "end": seconds_from_start + (duration_ms / 1000) if duration_ms else seconds_from_start, - "words": entry.get("words") # Word-level timing if available - }) - + speaker_segments.append( + { + "speaker": speaker, + "text": content.strip(), + "start": seconds_from_start, + "end": seconds_from_start + (duration_ms / 1000) if duration_ms else seconds_from_start, + "words": entry.get("words"), # Word-level timing if available + } + ) + # Fall back to messages if no transcript_object elif messages: for msg in messages: @@ -1261,11 +1069,11 @@ def _extract_transcript_from_call_data( content = msg.get("message", "") or msg.get("content", "") seconds_from_start = msg.get("secondsFromStart", 0) duration_ms = msg.get("duration", 0) - + # Skip system messages if role == "system": continue - + # Map Vapi roles to speaker labels if role == "user": speaker = "Speaker 1" # Test agent / caller @@ -1273,26 +1081,23 @@ def _extract_transcript_from_call_data( speaker = "Speaker 2" # Vapi agent else: continue - + if content and content.strip(): - speaker_segments.append({ - "speaker": speaker, - "text": content.strip(), - "start": seconds_from_start, - "end": seconds_from_start + (duration_ms / 1000) if duration_ms else seconds_from_start - }) - + speaker_segments.append( + { + "speaker": speaker, + "text": content.strip(), + "start": seconds_from_start, + "end": seconds_from_start + (duration_ms / 1000) if duration_ms else seconds_from_start, + } + ) + # If we have segments but no plain transcript, build it if not transcript_text and speaker_segments: - transcript_text = "\n".join( - f"{seg['speaker']}: {seg['text']}" - for seg in speaker_segments - ) + transcript_text = "\n".join(f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments) elif provider_platform == "elevenlabs": - # ElevenLabs format (from retrieve_call_metrics): - # - transcript: "Agent: ...\nUser: ..." plain text - # - transcript_object: [{speaker, text, start, end}, ...] + # ElevenLabs format transcript_text = call_data.get("transcript", "") transcript_obj = call_data.get("transcript_object", []) @@ -1307,51 +1112,37 @@ def _extract_transcript_from_call_data( speaker = "Speaker 2" # ElevenLabs agent else: speaker = "Speaker 1" # Test agent / caller - speaker_segments.append({ - "speaker": speaker, - "text": text.strip(), - "start": entry.get("start", 0), - "end": entry.get("end", 0), - }) + speaker_segments.append( + { + "speaker": speaker, + "text": text.strip(), + "start": entry.get("start", 0), + "end": entry.get("end", 0), + } + ) if not transcript_text and speaker_segments: - transcript_text = "\n".join( - f"{seg['speaker']}: {seg['text']}" - for seg in speaker_segments - ) + transcript_text = "\n".join(f"{seg['speaker']}: {seg['text']}" for seg in speaker_segments) return transcript_text, speaker_segments - + def record_conversation( self, audio_data: bytes, organization_id: UUID, evaluator_id: UUID, - result_id: str + result_id: str, ) -> str: """ Record the bridged conversation and upload to S3. - - Args: - audio_data: Merged audio data - organization_id: Organization ID - evaluator_id: Evaluator ID - result_id: Result ID - - Returns: - S3 key of uploaded audio """ import uuid + file_id = uuid.uuid4() - s3_key = s3_service.upload_file( - file_content=audio_data, - file_id=file_id, - file_format="wav" - ) + s3_key = s3_service.upload_file(file_content=audio_data, file_id=file_id, file_format="wav") logger.info(f"[Bridge] Recorded conversation uploaded to S3: {s3_key}") return s3_key # Singleton instance test_agent_bridge_service = TestAgentBridgeService() - diff --git a/app/services/test_agent_service.py b/app/services/testing/test_agent_service.py similarity index 65% rename from app/services/test_agent_service.py rename to app/services/testing/test_agent_service.py index 9e8eec40..ae3a6a4b 100644 --- a/app/services/test_agent_service.py +++ b/app/services/testing/test_agent_service.py @@ -2,26 +2,29 @@ Test Agent Service - Orchestrates conversations between test AI agent and voice AI agent. """ -import time import tempfile import os import uuid import warnings import subprocess -from typing import Optional, Dict, Any, List +from typing import Optional, Dict, Any from uuid import UUID from datetime import datetime, timezone import librosa import soundfile as sf from app.models.database import ( - TestAgentConversation, TestAgentConversationStatus, - Agent, Persona, Scenario, VoiceBundle + TestAgentConversation, + TestAgentConversationStatus, + Agent, + Persona, + Scenario, + VoiceBundle, ) -from app.services.transcription_service import transcription_service -from app.services.llm_service import llm_service -from app.services.tts_service import tts_service -from app.services.s3_service import s3_service +from app.services.ai.transcription_service import transcription_service +from app.services.ai.llm_service import llm_service +from app.services.ai.tts_service import tts_service +from app.services.storage.s3_service import s3_service from sqlalchemy.orm import Session @@ -37,18 +40,18 @@ def _build_system_prompt( agent: Agent, persona: Persona, scenario: Scenario, - db: Session + db: Session, ) -> str: """Build system prompt from agent, persona, and scenario.""" - prompt_parts = [] - + prompt_parts = [] + # Agent information prompt_parts.append(f"You are a test agent interacting with: {agent.name}") if agent.description: prompt_parts.append(f"Agent description: {agent.description}") prompt_parts.append(f"Agent phone number: {agent.phone_number}") prompt_parts.append(f"Agent language: {agent.language.value}") - + # Persona information prompt_parts.append(f"\nYou are role-playing as: {persona.name}") prompt_parts.append(f"Persona language: {persona.language.value}") @@ -56,21 +59,21 @@ def _build_system_prompt( prompt_parts.append(f"Persona gender: {persona.gender.value}") if persona.background_noise: prompt_parts.append(f"Background noise: {persona.background_noise.value}") - + # Scenario information prompt_parts.append(f"\nScenario: {scenario.name}") if scenario.description: prompt_parts.append(f"Scenario description: {scenario.description}") if scenario.required_info: prompt_parts.append(f"Required information to collect: {scenario.required_info}") - + # Instructions prompt_parts.append("\nInstructions:") prompt_parts.append("- Respond naturally and in character as the persona") prompt_parts.append("- Follow the scenario objectives") prompt_parts.append("- Keep responses concise and conversational") prompt_parts.append("- Do not break character") - + return "\n".join(prompt_parts) def _convert_webm_to_wav(self, webm_bytes: bytes) -> bytes: @@ -79,38 +82,38 @@ def _convert_webm_to_wav(self, webm_bytes: bytes) -> bytes: wav_path = None try: # Save WebM to temporary file - with tempfile.NamedTemporaryFile(delete=False, suffix='.webm') as webm_file: + with tempfile.NamedTemporaryFile(delete=False, suffix=".webm") as webm_file: webm_file.write(webm_bytes) webm_path = webm_file.name - + # Create WAV output path - with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as wav_file: + with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as wav_file: wav_path = wav_file.name - + # Try ffmpeg directly first (most reliable) try: result = subprocess.run( - ['ffmpeg', '-i', webm_path, '-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1', '-y', wav_path], + ["ffmpeg", "-i", webm_path, "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", "-y", wav_path], capture_output=True, text=True, - timeout=30 + timeout=30, ) if result.returncode == 0 and os.path.exists(wav_path): - with open(wav_path, 'rb') as f: + with open(wav_path, "rb") as f: wav_bytes = f.read() return wav_bytes except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError): # ffmpeg not available or failed, try pydub pass - + # Try using pydub (also requires ffmpeg but handles it better) try: from pydub import AudioSegment audio = AudioSegment.from_file(webm_path, format="webm") audio.export(wav_path, format="wav", parameters=["-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1"]) - + if os.path.exists(wav_path): - with open(wav_path, 'rb') as f: + with open(wav_path, "rb") as f: wav_bytes = f.read() return wav_bytes except ImportError: @@ -119,30 +122,30 @@ def _convert_webm_to_wav(self, webm_bytes: bytes) -> bytes: except Exception: # pydub failed pass - + # Fallback to librosa (requires ffmpeg via audioread) with warnings.catch_warnings(): - warnings.filterwarnings('ignore', category=FutureWarning) - warnings.filterwarnings('ignore', category=UserWarning, message='PySoundFile failed') - + warnings.filterwarnings("ignore", category=FutureWarning) + warnings.filterwarnings("ignore", category=UserWarning, message="PySoundFile failed") + try: y, sr = librosa.load(webm_path, sr=16000, mono=True) except Exception as e: raise RuntimeError( - f"Could not convert WebM to WAV. Please install ffmpeg:\n" - f" Ubuntu/Debian: sudo apt-get install ffmpeg\n" - f" macOS: brew install ffmpeg\n" - f" Windows: Download from https://ffmpeg.org/download.html\n" + "Could not convert WebM to WAV. Please install ffmpeg:\n" + " Ubuntu/Debian: sudo apt-get install ffmpeg\n" + " macOS: brew install ffmpeg\n" + " Windows: Download from https://ffmpeg.org/download.html\n" f"Error: {str(e)}" ) - + # Save as WAV using soundfile - sf.write(wav_path, y, sr, format='WAV', subtype='PCM_16') - + sf.write(wav_path, y, sr, format="WAV", subtype="PCM_16") + # Read WAV bytes - with open(wav_path, 'rb') as f: + with open(wav_path, "rb") as f: wav_bytes = f.read() - + return wav_bytes except Exception as e: raise RuntimeError(f"Failed to convert WebM to WAV: {str(e)}") @@ -163,39 +166,34 @@ def create_conversation( voice_bundle_id: UUID, organization_id: UUID, db: Session, - conversation_metadata: Optional[Dict[str, Any]] = None + conversation_metadata: Optional[Dict[str, Any]] = None, ) -> TestAgentConversation: """Create a new test agent conversation.""" # Verify all entities exist - agent = db.query(Agent).filter( - Agent.id == agent_id, - Agent.organization_id == organization_id - ).first() + agent = db.query(Agent).filter(Agent.id == agent_id, Agent.organization_id == organization_id).first() if not agent: raise ValueError(f"Agent {agent_id} not found") - - persona = db.query(Persona).filter( - Persona.id == persona_id, - Persona.organization_id == organization_id - ).first() + + persona = db.query(Persona).filter(Persona.id == persona_id, Persona.organization_id == organization_id).first() if not persona: raise ValueError(f"Persona {persona_id} not found") - - scenario = db.query(Scenario).filter( - Scenario.id == scenario_id, - Scenario.organization_id == organization_id - ).first() + + scenario = db.query(Scenario).filter(Scenario.id == scenario_id, Scenario.organization_id == organization_id).first() if not scenario: raise ValueError(f"Scenario {scenario_id} not found") - - voice_bundle = db.query(VoiceBundle).filter( - VoiceBundle.id == voice_bundle_id, - VoiceBundle.organization_id == organization_id, - VoiceBundle.is_active == True - ).first() + + voice_bundle = ( + db.query(VoiceBundle) + .filter( + VoiceBundle.id == voice_bundle_id, + VoiceBundle.organization_id == organization_id, + VoiceBundle.is_active == True, + ) + .first() + ) if not voice_bundle: raise ValueError(f"VoiceBundle {voice_bundle_id} not found or inactive") - + # Create conversation conversation = TestAgentConversation( organization_id=organization_id, @@ -205,13 +203,13 @@ def create_conversation( voice_bundle_id=voice_bundle_id, status=TestAgentConversationStatus.INITIALIZING, live_transcription=[], - conversation_metadata=conversation_metadata or {} + conversation_metadata=conversation_metadata or {}, ) - + db.add(conversation) db.commit() db.refresh(conversation) - + return conversation def process_audio_chunk( @@ -220,54 +218,38 @@ def process_audio_chunk( audio_chunk: bytes, organization_id: UUID, db: Session, - chunk_timestamp: Optional[float] = None + chunk_timestamp: Optional[float] = None, ) -> Dict[str, Any]: """ Process an audio chunk from the voice AI agent. - - This method: - 1. Saves audio chunk temporarily - 2. Transcribes using STT from voice bundle - 3. Generates response using LLM with system prompt - 4. Converts response to speech using TTS - 5. Updates conversation with new turn - 6. Returns response audio and transcription - - Args: - conversation_id: Conversation ID - audio_chunk: Audio bytes from voice AI agent - organization_id: Organization ID - db: Database session - chunk_timestamp: Timestamp of this chunk (seconds from start) - - Returns: - Dictionary with response audio bytes, transcription, and metadata """ # Get conversation - conversation = db.query(TestAgentConversation).filter( - TestAgentConversation.id == conversation_id, - TestAgentConversation.organization_id == organization_id - ).first() + conversation = ( + db.query(TestAgentConversation) + .filter( + TestAgentConversation.id == conversation_id, + TestAgentConversation.organization_id == organization_id, + ) + .first() + ) if not conversation: raise ValueError(f"Conversation {conversation_id} not found") - + if conversation.status != TestAgentConversationStatus.ACTIVE: raise ValueError(f"Conversation is not active (status: {conversation.status})") - + # Get voice bundle and related entities - voice_bundle = db.query(VoiceBundle).filter( - VoiceBundle.id == conversation.voice_bundle_id - ).first() + voice_bundle = db.query(VoiceBundle).filter(VoiceBundle.id == conversation.voice_bundle_id).first() if not voice_bundle: raise ValueError("Voice bundle not found") - + agent = db.query(Agent).filter(Agent.id == conversation.agent_id).first() persona = db.query(Persona).filter(Persona.id == conversation.persona_id).first() scenario = db.query(Scenario).filter(Scenario.id == conversation.scenario_id).first() - + if not all([agent, persona, scenario]): raise ValueError("Missing agent, persona, or scenario") - + # Calculate timestamp if chunk_timestamp is None: if conversation.started_at: @@ -280,7 +262,7 @@ def process_audio_chunk( chunk_timestamp = (now - started_at).total_seconds() else: chunk_timestamp = 0.0 - + # Convert WebM audio to WAV (OpenAI requires WAV/MP3) try: wav_audio_bytes = self._convert_webm_to_wav(audio_chunk) @@ -288,25 +270,21 @@ def process_audio_chunk( return { "response_audio": None, "transcription": None, - "error": f"Failed to convert audio format: {str(e)}" + "error": f"Failed to convert audio format: {str(e)}", } - + # Save audio chunk temporarily for transcription temp_file_path = None try: # Save converted WAV to temp file for transcription - with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as temp_file: + with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_file: temp_file.write(wav_audio_bytes) temp_file_path = temp_file.name - + # Upload to S3 temporarily for transcription service (it needs S3 key) chunk_file_id = uuid.uuid4() - chunk_s3_key = s3_service.upload_file( - file_content=wav_audio_bytes, - file_id=chunk_file_id, - file_format="wav" - ) - + chunk_s3_key = s3_service.upload_file(file_content=wav_audio_bytes, file_id=chunk_file_id, file_format="wav") + # Transcribe using STT transcription_result = transcription_service.transcribe( audio_file_key=chunk_s3_key, @@ -315,45 +293,35 @@ def process_audio_chunk( organization_id=organization_id, db=db, language=agent.language.value if agent.language else None, - enable_speaker_diarization=False + enable_speaker_diarization=False, ) - + voice_agent_text = transcription_result.get("transcript", "").strip() - + if not voice_agent_text: return { "response_audio": None, "transcription": None, - "error": "No speech detected in audio chunk" + "error": "No speech detected in audio chunk", } - + # Add voice agent turn to conversation conversation_turns = conversation.live_transcription or [] - conversation_turns.append({ - "speaker": "voice_agent", - "text": voice_agent_text, - "timestamp": chunk_timestamp - }) - + conversation_turns.append({"speaker": "voice_agent", "text": voice_agent_text, "timestamp": chunk_timestamp}) + # Build conversation history for LLM messages = [] - + # System prompt system_prompt = self._build_system_prompt(agent, persona, scenario, db) - messages.append({ - "role": "system", - "content": system_prompt - }) - + messages.append({"role": "system", "content": system_prompt}) + # Add conversation history (last 10 turns for context) recent_turns = conversation_turns[-10:] for turn in recent_turns: role = "user" if turn["speaker"] == "voice_agent" else "assistant" - messages.append({ - "role": role, - "content": turn["text"] - }) - + messages.append({"role": role, "content": turn["text"]}) + # Generate response using LLM llm_result = llm_service.generate_response( messages=messages, @@ -363,18 +331,18 @@ def process_audio_chunk( db=db, temperature=voice_bundle.llm_temperature or 0.7, max_tokens=voice_bundle.llm_max_tokens, - config=voice_bundle.llm_config + config=voice_bundle.llm_config, ) - + test_agent_text = llm_result.get("text", "").strip() - + if not test_agent_text: return { "response_audio": None, "transcription": None, - "error": "LLM did not generate a response" + "error": "LLM did not generate a response", } - + # Convert response to speech using TTS response_audio_bytes = tts_service.synthesize( text=test_agent_text, @@ -383,45 +351,35 @@ def process_audio_chunk( organization_id=organization_id, db=db, voice=voice_bundle.tts_voice, - config=voice_bundle.tts_config + config=voice_bundle.tts_config, ) - + # Upload response audio to S3 (temporarily, for reference) response_file_id = uuid.uuid4() - response_s3_key = s3_service.upload_file( - file_content=response_audio_bytes, - file_id=response_file_id, - file_format="mp3" - ) - + response_s3_key = s3_service.upload_file(file_content=response_audio_bytes, file_id=response_file_id, file_format="mp3") + # Add test agent turn to conversation - conversation_turns.append({ - "speaker": "test_agent", - "text": test_agent_text, - "timestamp": chunk_timestamp + transcription_result.get("processing_time", 0) + llm_result.get("processing_time", 0) - }) - + conversation_turns.append( + { + "speaker": "test_agent", + "text": test_agent_text, + "timestamp": chunk_timestamp + transcription_result.get("processing_time", 0) + llm_result.get("processing_time", 0), + } + ) + # Update conversation conversation.live_transcription = conversation_turns - conversation.full_transcript = "\n".join([ - f"{turn['speaker']}: {turn['text']}" for turn in conversation_turns - ]) + conversation.full_transcript = "\n".join([f"{turn['speaker']}: {turn['text']}" for turn in conversation_turns]) db.commit() - + return { "response_audio": response_audio_bytes, - "transcription": { - "voice_agent": voice_agent_text, - "test_agent": test_agent_text - }, + "transcription": {"voice_agent": voice_agent_text, "test_agent": test_agent_text}, "metadata": { - "processing_times": { - "stt": transcription_result.get("processing_time", 0), - "llm": llm_result.get("processing_time", 0) - } - } + "processing_times": {"stt": transcription_result.get("processing_time", 0), "llm": llm_result.get("processing_time", 0)} + }, } - + finally: # Clean up temp file if temp_file_path and os.path.exists(temp_file_path): @@ -430,25 +388,21 @@ def process_audio_chunk( except Exception: pass - def start_conversation( - self, - conversation_id: UUID, - organization_id: UUID, - db: Session - ) -> TestAgentConversation: + def start_conversation(self, conversation_id: UUID, organization_id: UUID, db: Session) -> TestAgentConversation: """Start a conversation (change status to ACTIVE).""" - conversation = db.query(TestAgentConversation).filter( - TestAgentConversation.id == conversation_id, - TestAgentConversation.organization_id == organization_id - ).first() + conversation = ( + db.query(TestAgentConversation) + .filter(TestAgentConversation.id == conversation_id, TestAgentConversation.organization_id == organization_id) + .first() + ) if not conversation: raise ValueError(f"Conversation {conversation_id} not found") - + conversation.status = TestAgentConversationStatus.ACTIVE conversation.started_at = datetime.now(timezone.utc) db.commit() db.refresh(conversation) - + return conversation def end_conversation( @@ -456,19 +410,20 @@ def end_conversation( conversation_id: UUID, organization_id: UUID, db: Session, - final_audio_key: Optional[str] = None + final_audio_key: Optional[str] = None, ) -> TestAgentConversation: """End a conversation and save final audio.""" - conversation = db.query(TestAgentConversation).filter( - TestAgentConversation.id == conversation_id, - TestAgentConversation.organization_id == organization_id - ).first() + conversation = ( + db.query(TestAgentConversation) + .filter(TestAgentConversation.id == conversation_id, TestAgentConversation.organization_id == organization_id) + .first() + ) if not conversation: raise ValueError(f"Conversation {conversation_id} not found") - + conversation.status = TestAgentConversationStatus.COMPLETED conversation.ended_at = datetime.now(timezone.utc) - + if conversation.started_at: # Ensure both datetimes are timezone-aware ended_at = conversation.ended_at @@ -476,19 +431,16 @@ def end_conversation( # If started_at is timezone-naive, assume UTC if started_at.tzinfo is None: started_at = started_at.replace(tzinfo=timezone.utc) - conversation.duration_seconds = ( - ended_at - started_at - ).total_seconds() - + conversation.duration_seconds = (ended_at - started_at).total_seconds() + if final_audio_key: conversation.conversation_audio_key = final_audio_key - + db.commit() db.refresh(conversation) - + return conversation # Singleton instance test_agent_service = TestAgentService() - diff --git a/app/services/voice_agent/utils/audio_merge.py b/app/services/voice_agent/utils/audio_merge.py index 5a6569d8..153346b6 100644 --- a/app/services/voice_agent/utils/audio_merge.py +++ b/app/services/voice_agent/utils/audio_merge.py @@ -4,7 +4,7 @@ import time import uuid from loguru import logger -from app.services.s3_service import s3_service +from app.services.storage.s3_service import s3_service def merge_and_upload_audio( diff --git a/app/services/voice_agent/voice_bundle.py b/app/services/voice_agent/voice_bundle.py index a5290812..c7931d8e 100644 --- a/app/services/voice_agent/voice_bundle.py +++ b/app/services/voice_agent/voice_bundle.py @@ -49,7 +49,7 @@ from efficientai.transports.base_transport import BaseTransport, TransportParams from efficientai.turns.bot.turn_analyzer_bot_turn_start_strategy import TurnAnalyzerBotTurnStartStrategy from efficientai.turns.turn_start_strategies import TurnStartStrategies -from app.services.s3_service import s3_service +from app.services.storage.s3_service import s3_service load_dotenv(override=True) diff --git a/app/workers/tasks/helpers/audio_evaluation.py b/app/workers/tasks/helpers/audio_evaluation.py index 71a167ba..0b7777f3 100644 --- a/app/workers/tasks/helpers/audio_evaluation.py +++ b/app/workers/tasks/helpers/audio_evaluation.py @@ -28,9 +28,9 @@ def evaluate_audio_metrics( Returns: Dictionary mapping metric ID to score info """ - from app.services.s3_service import s3_service - from app.services.voice_quality_service import calculate_audio_metrics - from app.services.qualitative_voice_service import qualitative_voice_service + from app.services.storage.s3_service import s3_service + from app.services.audio.voice_quality_service import calculate_audio_metrics + from app.services.audio.qualitative_voice_service import qualitative_voice_service metric_scores: dict[str, dict[str, Any]] = {} diff --git a/app/workers/tasks/helpers/llm_evaluation.py b/app/workers/tasks/helpers/llm_evaluation.py index 8b1959e3..5ac2c603 100644 --- a/app/workers/tasks/helpers/llm_evaluation.py +++ b/app/workers/tasks/helpers/llm_evaluation.py @@ -213,7 +213,7 @@ def evaluate_with_llm( Returns: Tuple of (metric_scores dict, evaluation_time in seconds) """ - from app.services.llm_service import llm_service + from app.services.ai.llm_service import llm_service evaluation_prompt = build_evaluation_prompt( transcription=transcription, diff --git a/app/workers/tasks/process_evaluation.py b/app/workers/tasks/process_evaluation.py index 99451ed7..f0350994 100644 --- a/app/workers/tasks/process_evaluation.py +++ b/app/workers/tasks/process_evaluation.py @@ -3,7 +3,7 @@ from uuid import UUID from app.database import SessionLocal -from app.services.evaluation_service import evaluation_service +from app.services.evaluation.evaluation_service import evaluation_service from app.workers.config import celery_app diff --git a/app/workers/tasks/process_evaluator_result.py b/app/workers/tasks/process_evaluator_result.py index 1b0c0498..5d3a2bda 100644 --- a/app/workers/tasks/process_evaluator_result.py +++ b/app/workers/tasks/process_evaluator_result.py @@ -54,7 +54,7 @@ def _load_related_entities(db, result): def _transcribe_audio(result, ai_providers, db): """Transcribe audio file and return transcript with timing info.""" - from app.services.transcription_service import transcription_service + from app.services.ai.transcription_service import transcription_service stt_provider = ModelProvider.OPENAI stt_model = "whisper-1" diff --git a/app/workers/tasks/run_evaluator.py b/app/workers/tasks/run_evaluator.py index 076fef8f..536e8c53 100644 --- a/app/workers/tasks/run_evaluator.py +++ b/app/workers/tasks/run_evaluator.py @@ -30,7 +30,7 @@ def run_evaluator_task(self, evaluator_id: str, evaluator_result_id: str): try: from app.models.database import Evaluator, Agent - from app.services.test_agent_bridge_service import test_agent_bridge_service + from app.services.testing.test_agent_bridge_service import test_agent_bridge_service evaluator_uuid = UUID(evaluator_id) result_uuid = UUID(evaluator_result_id) diff --git a/app/workers/tasks/tts_comparison.py b/app/workers/tasks/tts_comparison.py index 4a8b6535..b028db26 100644 --- a/app/workers/tasks/tts_comparison.py +++ b/app/workers/tasks/tts_comparison.py @@ -194,8 +194,8 @@ def generate_tts_comparison_task(self, comparison_id: str): TTSSampleStatus, ModelProvider, ) - from app.services.tts_service import tts_service, get_audio_file_extension - from app.services.s3_service import s3_service + from app.services.ai.tts_service import tts_service, get_audio_file_extension + from app.services.storage.s3_service import s3_service db = SessionLocal() try: @@ -342,8 +342,8 @@ def evaluate_tts_comparison_task(self, comparison_id: str): TTSComparisonStatus, TTSSampleStatus, ) - from app.services.s3_service import s3_service - from app.services.qualitative_voice_service import qualitative_voice_service + from app.services.storage.s3_service import s3_service + from app.services.audio.qualitative_voice_service import qualitative_voice_service db = SessionLocal() try: diff --git a/app/workers/tasks/tts_report.py b/app/workers/tasks/tts_report.py index 233f30b0..a71ea47c 100644 --- a/app/workers/tasks/tts_report.py +++ b/app/workers/tasks/tts_report.py @@ -18,8 +18,8 @@ def generate_tts_report_pdf_task(self, report_job_id: str): TTSReportJob, TTSReportJobStatus, ) - from app.services.s3_service import s3_service - from app.services.voice_playground_report_service import voice_playground_report_service + from app.services.storage.s3_service import s3_service + from app.services.reporting.voice_playground_report_service import voice_playground_report_service db = SessionLocal() try: From 4f50d92794f18c000aa50a12d8fbd580a8d12c77 Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Sun, 15 Mar 2026 08:18:47 +0000 Subject: [PATCH 3/4] feat: updating customisations to reports --- app/api/v1/routes/voice_playground.py | 177 ++++++- ...nd_threshold_overrides_to_organizations.py | 44 ++ app/models/database.py | 1 + .../voice_playground_report_service.py | 349 ++++++++++++- .../reports/voice_playground_report.html | 327 ++++++------ app/workers/tasks/tts_report.py | 8 +- frontend/src/lib/api.ts | 65 ++- .../components/ComparisonResultsView.tsx | 38 +- .../voice/components/MetricCard.tsx | 38 +- .../voice/components/PlaygroundTab.tsx | 4 +- .../voice/components/ReportConfigModal.tsx | 475 ++++++++++++++++++ .../voice/components/SimulationsTab.tsx | 4 +- .../voice/context/VoicePlaygroundContext.tsx | 21 +- frontend/src/pages/playground/voice/types.ts | 69 +++ 14 files changed, 1428 insertions(+), 192 deletions(-) create mode 100644 app/migrations/013_add_voice_playground_threshold_overrides_to_organizations.py create mode 100644 frontend/src/pages/playground/voice/components/ReportConfigModal.tsx diff --git a/app/api/v1/routes/voice_playground.py b/app/api/v1/routes/voice_playground.py index 1ded0ea8..2a2ed876 100644 --- a/app/api/v1/routes/voice_playground.py +++ b/app/api/v1/routes/voice_playground.py @@ -3,6 +3,7 @@ TTS A/B comparison: generate audio, blind test, and quality evaluation. """ +import json import random from fastapi import APIRouter, Depends, HTTPException, Response, status @@ -18,6 +19,7 @@ AIProvider, CustomTTSVoice, Integration, + Organization, TTSComparison, TTSSample, TTSComparisonStatus, @@ -207,6 +209,47 @@ class CustomVoiceUpdate(BaseModel): description: Optional[str] = None +class TTSReportOptions(BaseModel): + show_runs: bool = True + min_runs_to_show: int = 100 + include_latency: bool = True + include_ttfb: bool = True + include_endpoint: bool = True + include_naturalness: bool = True + include_hallucination: bool = True + include_prosody: bool = True + include_arousal: bool = True + include_valence: bool = True + include_cer: bool = True + include_wer: bool = True + include_hallucination_examples: bool = True + hallucination_examples_limit: int = 5 + include_disclaimer_sections: bool = True + include_methodology_sections: bool = False + zone_threshold_overrides: Optional[Dict[str, Dict[str, float]]] = None + + +class TTSReportJobCreate(BaseModel): + report_options: Optional[TTSReportOptions] = None + + +class VoicePlaygroundThresholdDefaultsUpdate(BaseModel): + zone_threshold_overrides: Optional[Dict[str, Dict[str, float]]] = None + reset_to_system_defaults: bool = False + + +DEFAULT_ZONE_THRESHOLD_OVERRIDES: Dict[str, Dict[str, float]] = { + "avg_mos": {"neutral_min": 3.0, "good_min": 4.0}, + "avg_prosody": {"neutral_min": 0.4, "good_min": 0.7}, + "avg_valence": {"neutral_min": -0.2, "good_min": 0.3}, + "avg_arousal": {"neutral_min": 0.4, "good_min": 0.7}, + "avg_wer": {"good_max": 0.1, "neutral_max": 0.25}, + "avg_cer": {"good_max": 0.08, "neutral_max": 0.2}, + "avg_ttfb_ms": {"good_max": 350.0, "neutral_max": 800.0}, + "avg_latency_ms": {"good_max": 1500.0, "neutral_max": 3000.0}, +} + + SAMPLE_GENERATION_SYSTEM_PROMPT = """You are an expert at creating realistic text-to-speech sample scripts. \ Generate natural-sounding text that would be spoken aloud by a voice AI agent, \ varied in tone and content, and suitable for evaluating TTS voice quality. \ @@ -871,10 +914,58 @@ async def get_tts_analytics( return result +@router.get("/report-threshold-defaults", operation_id="getVoicePlaygroundReportThresholdDefaults") +async def get_voice_playground_report_threshold_defaults( + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + org = db.query(Organization).filter(Organization.id == organization_id).first() + if not org: + raise HTTPException(404, "Organization not found") + + stored = _sanitize_zone_threshold_overrides(org.voice_playground_threshold_overrides) + is_custom = bool(stored) + return { + "zone_threshold_overrides": stored if is_custom else DEFAULT_ZONE_THRESHOLD_OVERRIDES, + "is_custom": is_custom, + } + + +@router.put("/report-threshold-defaults", operation_id="updateVoicePlaygroundReportThresholdDefaults") +async def update_voice_playground_report_threshold_defaults( + data: VoicePlaygroundThresholdDefaultsUpdate, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + org = db.query(Organization).filter(Organization.id == organization_id).first() + if not org: + raise HTTPException(404, "Organization not found") + + if data.reset_to_system_defaults: + org.voice_playground_threshold_overrides = None + else: + sanitized = _sanitize_zone_threshold_overrides(data.zone_threshold_overrides or {}) + org.voice_playground_threshold_overrides = sanitized + + db.commit() + db.refresh(org) + + stored = _sanitize_zone_threshold_overrides(org.voice_playground_threshold_overrides) + is_custom = bool(stored) + return { + "zone_threshold_overrides": stored if is_custom else DEFAULT_ZONE_THRESHOLD_OVERRIDES, + "is_custom": is_custom, + "message": "Voice Playground threshold defaults updated", + } + + @router.get("/comparisons/{comparison_id}/report.pdf", operation_id="downloadTTSComparisonReport") async def download_tts_comparison_report( comparison_id: UUID, include_unfinished_samples: bool = False, + report_options: Optional[str] = None, organization_id: UUID = Depends(get_organization_id), api_key: str = Depends(get_api_key), db: Session = Depends(get_db), @@ -895,8 +986,16 @@ async def download_tts_comparison_report( raise HTTPException(400, "No samples found to generate report") try: - payload = voice_playground_report_service.build_payload(comparison, samples) + options_dict = _parse_report_options(report_options) + options_dict = _merge_org_threshold_defaults(db, organization_id, options_dict) + payload = voice_playground_report_service.build_payload( + comparison, + samples, + report_options=options_dict, + ) pdf_bytes = voice_playground_report_service.render_pdf(payload) + except HTTPException: + raise except Exception as e: raise HTTPException(500, f"Failed to generate PDF report: {str(e)}") @@ -911,6 +1010,7 @@ async def download_tts_comparison_report( @router.post("/comparisons/{comparison_id}/reports", operation_id="createTTSComparisonReportJob") async def create_tts_comparison_report_job( comparison_id: UUID, + data: Optional[TTSReportJobCreate] = None, organization_id: UUID = Depends(get_organization_id), api_key: str = Depends(get_api_key), db: Session = Depends(get_db), @@ -932,7 +1032,13 @@ async def create_tts_comparison_report_job( try: from app.workers.celery_app import generate_tts_report_pdf_task - task = generate_tts_report_pdf_task.delay(str(report_job.id)) + options_dict = ( + data.report_options.model_dump(exclude_none=True) + if (data and data.report_options is not None) + else {} + ) + options_dict = _merge_org_threshold_defaults(db, organization_id, options_dict) + task = generate_tts_report_pdf_task.delay(str(report_job.id), options_dict) report_job.celery_task_id = task.id db.commit() except Exception as e: @@ -947,6 +1053,7 @@ async def create_tts_comparison_report_job( "status": report_job.status, "format": report_job.format, "task_id": report_job.celery_task_id, + "report_options": options_dict, "created_at": report_job.created_at.isoformat() if report_job.created_at else None, } @@ -996,6 +1103,72 @@ async def get_tts_comparison_report_job( # ====================================================================== +def _sanitize_zone_threshold_overrides(raw: Any) -> Dict[str, Dict[str, float]]: + if not isinstance(raw, dict): + return {} + + allowed_metric_keys = set(DEFAULT_ZONE_THRESHOLD_OVERRIDES.keys()) + allowed_threshold_keys = {"good_min", "neutral_min", "good_max", "neutral_max"} + sanitized: Dict[str, Dict[str, float]] = {} + + for metric_key, metric_values in raw.items(): + if metric_key not in allowed_metric_keys or not isinstance(metric_values, dict): + continue + bucket: Dict[str, float] = {} + for threshold_key, raw_val in metric_values.items(): + if threshold_key not in allowed_threshold_keys: + continue + try: + bucket[threshold_key] = float(raw_val) + except (TypeError, ValueError): + continue + if bucket: + sanitized[metric_key] = bucket + return sanitized + + +def _merge_org_threshold_defaults(db: Session, organization_id: UUID, report_options: Dict[str, Any]) -> Dict[str, Any]: + merged = dict(report_options or {}) + + org = db.query(Organization).filter(Organization.id == organization_id).first() + org_defaults = _sanitize_zone_threshold_overrides( + org.voice_playground_threshold_overrides if org else None + ) + base_thresholds = org_defaults or DEFAULT_ZONE_THRESHOLD_OVERRIDES + + incoming_overrides = _sanitize_zone_threshold_overrides(merged.get("zone_threshold_overrides")) + merged_thresholds: Dict[str, Dict[str, float]] = {} + for metric_key, default_values in base_thresholds.items(): + merged_thresholds[metric_key] = dict(default_values) + if metric_key in incoming_overrides: + merged_thresholds[metric_key].update(incoming_overrides[metric_key]) + for metric_key, override_values in incoming_overrides.items(): + if metric_key not in merged_thresholds: + merged_thresholds[metric_key] = dict(override_values) + + merged["zone_threshold_overrides"] = merged_thresholds + return merged + + +def _parse_report_options(report_options_raw: Optional[str]) -> Dict[str, Any]: + if not report_options_raw: + return {} + try: + parsed = json.loads(report_options_raw) + except json.JSONDecodeError: + raise HTTPException(400, "Invalid report_options JSON") + + if not isinstance(parsed, dict): + raise HTTPException(400, "report_options must be a JSON object") + + try: + validated = TTSReportOptions(**parsed) + except Exception as exc: + raise HTTPException(400, f"Invalid report options: {str(exc)}") + + return validated.model_dump(exclude_none=True) + + def _get_comparison_or_404(comparison_id: UUID, organization_id: UUID, db: Session) -> TTSComparison: c = db.query(TTSComparison).filter( TTSComparison.id == comparison_id, diff --git a/app/migrations/013_add_voice_playground_threshold_overrides_to_organizations.py b/app/migrations/013_add_voice_playground_threshold_overrides_to_organizations.py new file mode 100644 index 00000000..6f02bcd5 --- /dev/null +++ b/app/migrations/013_add_voice_playground_threshold_overrides_to_organizations.py @@ -0,0 +1,44 @@ +""" +Migration: Add organization-level Voice Playground threshold overrides. + +Stores per-organization defaults for metric zone thresholds used in report legends/bar colors. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add voice_playground_threshold_overrides JSON column to organizations" + + +def upgrade(db: Session): + result = db.execute( + text( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_name = 'organizations' + AND column_name = 'voice_playground_threshold_overrides' + """ + ) + ) + + if result.fetchone() is not None: + print("Column voice_playground_threshold_overrides already exists on organizations, skipping...") + return + + db.execute( + text( + """ + ALTER TABLE organizations + ADD COLUMN voice_playground_threshold_overrides JSON + """ + ) + ) + + db.commit() + print("Added voice_playground_threshold_overrides column to organizations") + + +def downgrade(db: Session): + db.execute(text("ALTER TABLE organizations DROP COLUMN IF EXISTS voice_playground_threshold_overrides")) + db.commit() diff --git a/app/models/database.py b/app/models/database.py index 6c2d6117..e03c2793 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -31,6 +31,7 @@ class Organization(Base): id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) name = Column(String(255), nullable=False) + voice_playground_threshold_overrides = Column(JSON, nullable=True) created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) diff --git a/app/services/reporting/voice_playground_report_service.py b/app/services/reporting/voice_playground_report_service.py index a1d4f368..cc4ac0ef 100644 --- a/app/services/reporting/voice_playground_report_service.py +++ b/app/services/reporting/voice_playground_report_service.py @@ -15,6 +15,25 @@ class VoicePlaygroundReportService: """Build and render comprehensive Voice Playground benchmark reports.""" + DEFAULT_REPORT_OPTIONS: dict[str, Any] = { + "show_runs": True, + "min_runs_to_show": 100, + "include_latency": True, + "include_ttfb": True, + "include_endpoint": True, + "include_naturalness": True, + "include_hallucination": True, + "include_prosody": True, + "include_arousal": True, + "include_valence": True, + "include_cer": True, + "include_wer": True, + "include_hallucination_examples": True, + "hallucination_examples_limit": 5, + "include_disclaimer_sections": True, + "include_methodology_sections": False, + "zone_threshold_overrides": {}, + } def __init__(self) -> None: templates_dir = Path(__file__).parent.parent.parent / "templates" @@ -153,8 +172,96 @@ def _endpoint_type(avg_ttfb_ms: float | None, avg_latency_ms: float | None) -> s return "Streaming (inferred)" return "Non-streaming (inferred)" - def build_payload(self, comparison: Any, samples: list[Any]) -> dict[str, Any]: + @staticmethod + def _to_bool(value: Any, default: bool) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"1", "true", "yes", "on"}: + return True + if lowered in {"0", "false", "no", "off"}: + return False + if isinstance(value, (int, float)): + return bool(value) + return default + + def _normalize_report_options(self, report_options: dict[str, Any] | None) -> dict[str, Any]: + normalized = dict(self.DEFAULT_REPORT_OPTIONS) + if not isinstance(report_options, dict): + return normalized + + bool_keys = { + "show_runs", + "include_latency", + "include_ttfb", + "include_endpoint", + "include_naturalness", + "include_hallucination", + "include_prosody", + "include_arousal", + "include_valence", + "include_cer", + "include_wer", + "include_hallucination_examples", + "include_disclaimer_sections", + "include_methodology_sections", + } + for key in bool_keys: + if key in report_options: + normalized[key] = self._to_bool(report_options.get(key), bool(normalized[key])) + + try: + normalized["min_runs_to_show"] = max(0, int(report_options.get("min_runs_to_show", normalized["min_runs_to_show"]))) + except (TypeError, ValueError): + normalized["min_runs_to_show"] = int(self.DEFAULT_REPORT_OPTIONS["min_runs_to_show"]) + + try: + normalized["hallucination_examples_limit"] = max( + 0, min(50, int(report_options.get("hallucination_examples_limit", normalized["hallucination_examples_limit"]))) + ) + except (TypeError, ValueError): + normalized["hallucination_examples_limit"] = int(self.DEFAULT_REPORT_OPTIONS["hallucination_examples_limit"]) + + raw_threshold_overrides = report_options.get("zone_threshold_overrides") + clean_threshold_overrides: dict[str, dict[str, float]] = {} + if isinstance(raw_threshold_overrides, dict): + allowed_metric_keys = { + "avg_mos", + "avg_prosody", + "avg_valence", + "avg_arousal", + "avg_wer", + "avg_cer", + "avg_ttfb_ms", + "avg_latency_ms", + } + allowed_threshold_keys = {"good_min", "neutral_min", "good_max", "neutral_max"} + for metric_key, maybe_values in raw_threshold_overrides.items(): + if metric_key not in allowed_metric_keys or not isinstance(maybe_values, dict): + continue + cleaned_metric_values: dict[str, float] = {} + for threshold_key, raw_value in maybe_values.items(): + if threshold_key not in allowed_threshold_keys: + continue + try: + cleaned_metric_values[threshold_key] = float(raw_value) + except (TypeError, ValueError): + continue + if cleaned_metric_values: + clean_threshold_overrides[metric_key] = cleaned_metric_values + normalized["zone_threshold_overrides"] = clean_threshold_overrides + + return normalized + + def build_payload( + self, + comparison: Any, + samples: list[Any], + report_options: dict[str, Any] | None = None, + ) -> dict[str, Any]: """Build template context from comparison and sample rows.""" + normalized_options = self._normalize_report_options(report_options) grouped: dict[tuple[str, str, str, str], list[Any]] = defaultdict(list) for sample in samples: grouped[ @@ -262,7 +369,14 @@ def build_payload(self, comparison: Any, samples: list[Any]) -> dict[str, Any]: provider_rows.sort(key=lambda r: ((r["avg_mos"] is None), -(r["avg_mos"] or 0))) hallucination_examples.sort(key=lambda e: e["wer"], reverse=True) - hallucination_examples = hallucination_examples[:5] + if ( + normalized_options["include_hallucination"] + and normalized_options["include_hallucination_examples"] + and normalized_options["include_wer"] + ): + hallucination_examples = hallucination_examples[: normalized_options["hallucination_examples_limit"]] + else: + hallucination_examples = [] run_variability_rows: list[dict[str, Any]] = [] for (text, provider, model, voice_name), run_samples in run_groups.items(): @@ -436,10 +550,18 @@ def _winner(rows: list[dict[str, Any]], key: str, mode: str) -> dict[str, Any] | return max(valid, key=lambda r: float(r[key])) return min(valid, key=lambda r: float(r[key])) - top_naturalness = _winner(provider_rows, "avg_mos", "max") - lowest_latency = _winner(provider_rows, "avg_latency_ms", "min") - lowest_hallucination = _winner(provider_rows, "avg_wer", "min") - best_context = _winner(provider_rows, "avg_prosody", "max") + has_comparison_variants = len(provider_rows) > 1 + top_naturalness = _winner(provider_rows, "avg_mos", "max") if normalized_options["include_naturalness"] else None + lowest_latency = _winner(provider_rows, "avg_latency_ms", "min") if normalized_options["include_latency"] else None + lowest_hallucination = ( + _winner(provider_rows, "avg_wer", "min") + if ( + normalized_options["include_hallucination"] + and normalized_options["include_wer"] + ) + else None + ) + best_context = _winner(provider_rows, "avg_prosody", "max") if normalized_options["include_prosody"] else None recommendations = [] if top_naturalness: @@ -484,94 +606,142 @@ def _winner(rows: list[dict[str, Any]], key: str, mode: str) -> dict[str, Any] | "title": "MOS Score Ranking", "full_form": "Mean Opinion Score (MOS)", "subtitle": "Higher is better", + "definition": "Perceived speech naturalness and quality on a 1-5 scale.", "higher_is_better": True, "min_value": 1.0, "max_value": 5.0, "format_kind": "score", "range_label": "Measured range: 1.0 to 5.0", + "zone_thresholds": {"good_min": 4.0, "neutral_min": 3.0}, }, { "key": "avg_prosody", "title": "Prosody Score Ranking", "full_form": "Prosody Score", "subtitle": "Higher is better", + "definition": "Expressiveness proxy based on rhythm, stress, and intonation stability.", "higher_is_better": True, "min_value": 0.0, "max_value": 1.0, "format_kind": "score", "range_label": "Measured range: 0.0 to 1.0", + "zone_thresholds": {"good_min": 0.7, "neutral_min": 0.4}, }, { "key": "avg_valence", "title": "Valence Ranking", "full_form": "Valence", "subtitle": "Higher is better", + "definition": "Estimated emotional polarity from negative to positive.", "higher_is_better": True, "min_value": -1.0, "max_value": 1.0, "format_kind": "score", "range_label": "Measured range: -1.0 to 1.0", + "zone_thresholds": {"good_min": 0.3, "neutral_min": -0.2}, }, { "key": "avg_arousal", "title": "Arousal Ranking", "full_form": "Arousal", "subtitle": "Higher is better", + "definition": "Estimated emotional intensity from calm to energetic.", "higher_is_better": True, "min_value": 0.0, "max_value": 1.0, "format_kind": "score", "range_label": "Measured range: 0.0 to 1.0", + "zone_thresholds": {"good_min": 0.7, "neutral_min": 0.4}, }, { "key": "avg_wer", "title": "WER Ranking", "full_form": "Word Error Rate (WER)", "subtitle": "Lower is better", + "definition": "Word-level transcription mismatch between prompt and ASR output.", "higher_is_better": False, "min_value": 0.0, "max_value": 1.0, "format_kind": "pct", "range_label": "Measured range: 0.0 to 1.0 (0% to 100%)", + "zone_thresholds": {"good_max": 0.1, "neutral_max": 0.25}, }, { "key": "avg_cer", "title": "CER Ranking", "full_form": "Character Error Rate (CER)", "subtitle": "Lower is better", + "definition": "Character-level transcription mismatch between prompt and ASR output.", "higher_is_better": False, "min_value": 0.0, "max_value": 1.0, "format_kind": "pct", "range_label": "Measured range: 0.0 to 1.0 (0% to 100%)", + "zone_thresholds": {"good_max": 0.08, "neutral_max": 0.2}, }, { "key": "avg_ttfb_ms", "title": "TTFB Ranking", "full_form": "Time to First Byte (TTFB)", "subtitle": "Lower is better", + "definition": "Time from request start to first audio byte received.", "higher_is_better": False, "min_value": None, "max_value": None, "format_kind": "ms", "range_label": "Measured range: dataset-dependent (milliseconds)", + "zone_thresholds": {"good_max": 350.0, "neutral_max": 800.0}, }, { "key": "avg_latency_ms", "title": "Total Latency Ranking", "full_form": "Total Synthesis Latency", "subtitle": "Lower is better", + "definition": "End-to-end time from request start to complete audio payload.", "higher_is_better": False, "min_value": None, "max_value": None, "format_kind": "ms", "range_label": "Measured range: dataset-dependent (milliseconds)", + "zone_thresholds": {"good_max": 1500.0, "neutral_max": 3000.0}, }, ] + threshold_overrides = normalized_options.get("zone_threshold_overrides") or {} + if isinstance(threshold_overrides, dict): + for metric in metric_definitions: + metric_key = metric.get("key") + if not metric_key: + continue + override_values = threshold_overrides.get(metric_key) + if not isinstance(override_values, dict): + continue + merged_thresholds = dict(metric.get("zone_thresholds") or {}) + for threshold_key in ("good_min", "neutral_min", "good_max", "neutral_max"): + if threshold_key not in override_values: + continue + try: + merged_thresholds[threshold_key] = float(override_values[threshold_key]) + except (TypeError, ValueError): + continue + metric["zone_thresholds"] = merged_thresholds + metric_sections: list[dict[str, Any]] = [] for metric in metric_definitions: key = metric["key"] + metric_visibility_option = { + "avg_mos": "include_naturalness", + "avg_prosody": "include_prosody", + "avg_valence": "include_valence", + "avg_arousal": "include_arousal", + "avg_wer": "include_wer", + "avg_cer": "include_cer", + "avg_ttfb_ms": "include_ttfb", + "avg_latency_ms": "include_latency", + }.get(key) + if metric_visibility_option and not normalized_options.get(metric_visibility_option, True): + continue + valid_rows = [ { "provider": r["provider"], @@ -600,6 +770,7 @@ def _winner(rows: list[dict[str, Any]], key: str, mode: str) -> dict[str, Any] | spread = max_bound - min_bound section_rows = [] + thresholds = metric.get("zone_thresholds") or {} for row in valid_rows: value = float(row["value"]) if spread <= 0: @@ -612,6 +783,26 @@ def _winner(rows: list[dict[str, Any]], key: str, mode: str) -> dict[str, Any] | pct = normalized * 100.0 pct = max(8.0, min(100.0, pct)) + zone_class = "zone-neutral" + if metric["higher_is_better"]: + good_min = thresholds.get("good_min") + neutral_min = thresholds.get("neutral_min") + if good_min is not None and value >= float(good_min): + zone_class = "zone-good" + elif neutral_min is not None and value >= float(neutral_min): + zone_class = "zone-neutral" + else: + zone_class = "zone-bad" + else: + good_max = thresholds.get("good_max") + neutral_max = thresholds.get("neutral_max") + if good_max is not None and value <= float(good_max): + zone_class = "zone-good" + elif neutral_max is not None and value <= float(neutral_max): + zone_class = "zone-neutral" + else: + zone_class = "zone-bad" + section_rows.append( { "provider": row["provider"], @@ -621,17 +812,102 @@ def _winner(rows: list[dict[str, Any]], key: str, mode: str) -> dict[str, Any] | "value": value, "display_value": self._format_metric_value(value, metric["format_kind"]), "bar_pct": pct, + "zone_class": zone_class, } ) + zone_gradient = "linear-gradient(90deg, #ef4444 0%, #f59e0b 50%, #16a34a 100%)" + zone_labels = { + "start": "Low", + "middle": "Neutral", + "end": "High", + } + zone_ticks: list[dict[str, Any]] = [] + + def _threshold_to_pct(raw_threshold: Any) -> float | None: + if raw_threshold is None: + return None + try: + threshold = float(raw_threshold) + except (TypeError, ValueError): + return None + if spread <= 0: + return 50.0 + pct = ((threshold - min_bound) / spread) * 100.0 + return max(0.0, min(100.0, pct)) + + if metric["higher_is_better"]: + good_min = thresholds.get("good_min") + neutral_min = thresholds.get("neutral_min") + zone_labels = { + "start": ( + f"Red: < {self._format_metric_value(float(neutral_min), metric['format_kind'])}" + if neutral_min is not None + else "Red: low values" + ), + "middle": ( + "Neutral: " + f"{self._format_metric_value(float(neutral_min), metric['format_kind'])}" + f" - {self._format_metric_value(float(good_min), metric['format_kind'])}" + if (neutral_min is not None and good_min is not None) + else "Neutral: mid-range values" + ), + "end": ( + f"Green: >= {self._format_metric_value(float(good_min), metric['format_kind'])}" + if good_min is not None + else "Green: high values" + ), + } + neutral_tick = _threshold_to_pct(neutral_min) + good_tick = _threshold_to_pct(good_min) + if neutral_tick is not None: + zone_ticks.append({"position_pct": neutral_tick, "label": "Neutral threshold"}) + if good_tick is not None: + zone_ticks.append({"position_pct": good_tick, "label": "Good threshold"}) + else: + zone_gradient = "linear-gradient(90deg, #16a34a 0%, #f59e0b 50%, #ef4444 100%)" + good_max = thresholds.get("good_max") + neutral_max = thresholds.get("neutral_max") + zone_labels = { + "start": ( + f"Green: <= {self._format_metric_value(float(good_max), metric['format_kind'])}" + if good_max is not None + else "Green: low values" + ), + "middle": ( + "Neutral: " + f"{self._format_metric_value(float(good_max), metric['format_kind'])}" + f" - {self._format_metric_value(float(neutral_max), metric['format_kind'])}" + if (good_max is not None and neutral_max is not None) + else "Neutral: mid-range values" + ), + "end": ( + f"Red: > {self._format_metric_value(float(neutral_max), metric['format_kind'])}" + if neutral_max is not None + else "Red: high values" + ), + } + good_tick = _threshold_to_pct(good_max) + neutral_tick = _threshold_to_pct(neutral_max) + if good_tick is not None: + zone_ticks.append({"position_pct": good_tick, "label": "Good threshold"}) + if neutral_tick is not None: + zone_ticks.append({"position_pct": neutral_tick, "label": "Neutral threshold"}) + + zone_ticks.sort(key=lambda t: float(t["position_pct"])) + metric_sections.append( { "title": metric["title"], "full_form": metric["full_form"], "subtitle": metric["subtitle"], + "definition": metric.get("definition"), "range_label": ( f"{metric.get('range_label')} | Bar length represents raw metric magnitude" ), + "zone_gradient": zone_gradient, + "zone_labels": zone_labels, + "zone_ticks": zone_ticks, "rows": section_rows, } ) @@ -645,6 +921,43 @@ def _winner(rows: list[dict[str, Any]], key: str, mode: str) -> dict[str, Any] | endpoint_modes = sorted({r["endpoint_type"] for r in provider_rows if r.get("endpoint_type")}) + aggregate_metric_columns = [] + if normalized_options["include_naturalness"]: + aggregate_metric_columns.append({"key": "avg_mos", "label": "MOS", "format_kind": "score"}) + if normalized_options["include_valence"]: + aggregate_metric_columns.append({"key": "avg_valence", "label": "Valence", "format_kind": "score"}) + if normalized_options["include_arousal"]: + aggregate_metric_columns.append({"key": "avg_arousal", "label": "Arousal", "format_kind": "score"}) + if normalized_options["include_prosody"]: + aggregate_metric_columns.append({"key": "avg_prosody", "label": "Prosody", "format_kind": "score"}) + if normalized_options["include_wer"]: + aggregate_metric_columns.append({"key": "avg_wer", "label": "WER", "format_kind": "pct"}) + if normalized_options["include_cer"]: + aggregate_metric_columns.append({"key": "avg_cer", "label": "CER", "format_kind": "pct"}) + if normalized_options["include_latency"]: + aggregate_metric_columns.append({"key": "avg_latency_ms", "label": "Latency", "format_kind": "ms"}) + if normalized_options["include_ttfb"]: + aggregate_metric_columns.append({"key": "avg_ttfb_ms", "label": "TTFB", "format_kind": "ms"}) + + evaluation_dimensions: list[str] = [] + if normalized_options["include_naturalness"]: + evaluation_dimensions.append("Naturalness (MOS - Mean Opinion Score)") + if normalized_options["include_latency"] or normalized_options["include_ttfb"]: + evaluation_dimensions.append("Latency performance") + if normalized_options["include_wer"] or normalized_options["include_cer"]: + evaluation_dimensions.append("Speech accuracy (WER/CER)") + if normalized_options["include_valence"] or normalized_options["include_arousal"]: + evaluation_dimensions.append("Emotional quality (Valence/Arousal)") + if normalized_options["include_prosody"]: + evaluation_dimensions.append("Prosody / context stability proxy") + if normalized_options["include_endpoint"]: + evaluation_dimensions.append("Endpoint behavior (streaming vs non-streaming, inferred)") + + total_runs_observed = len({s.run_index for s in samples}) + show_run_summary = normalized_options["show_runs"] and ( + total_runs_observed >= normalized_options["min_runs_to_show"] + ) + metrics_glossary = [ { "major_metric": "Naturalness", @@ -729,13 +1042,6 @@ def _winner(rows: list[dict[str, Any]], key: str, mode: str) -> dict[str, Any] | "For repeated transcripts, this report shows a representative run plus best/worst outliers to highlight variability without duplicating every sample.", ], }, - { - "title": "Cost and Recommendation Disclaimer", - "points": [ - "Cost values are included only when available in current benchmark metadata and may not include provider minimums, surcharges, or burst pricing.", - "Recommendations are deterministic outputs from configured metric rules and should be validated against your product constraints, compliance needs, and customer profile.", - ], - }, ] methodology_sections = [ @@ -820,12 +1126,18 @@ def _winner(rows: list[dict[str, Any]], key: str, mode: str) -> dict[str, Any] | "tested_variants": sorted({r["variant_label"] for r in provider_rows}), "provider_overview": provider_overview, "num_runs": comparison.num_runs or 1, - "total_runs_observed": len({s.run_index for s in samples}), + "total_runs_observed": total_runs_observed, + "show_run_summary": show_run_summary, "sample_count": len(samples), "endpoint_modes": endpoint_modes, "summary": summary, + "has_comparison_variants": has_comparison_variants, "provider_rows": provider_rows, "provider_aggregate_rows": provider_aggregate_rows, + "aggregate_metric_columns": aggregate_metric_columns, + "evaluation_dimensions": evaluation_dimensions, + "show_endpoint_column": normalized_options["include_endpoint"], + "show_hallucination_section": normalized_options["include_hallucination"], "metric_sections": metric_sections, "hallucination_examples": hallucination_examples, "metrics_glossary": metrics_glossary, @@ -834,8 +1146,13 @@ def _winner(rows: list[dict[str, Any]], key: str, mode: str) -> dict[str, Any] | "run_variability_rows": run_variability_rows, "has_run_variability": len(run_variability_rows) > 0, "recommendations": recommendations, - "disclaimer_sections": disclaimer_sections, - "methodology_sections": methodology_sections, + "disclaimer_sections": ( + disclaimer_sections if normalized_options["include_disclaimer_sections"] else [] + ), + "methodology_sections": ( + methodology_sections if normalized_options["include_methodology_sections"] else [] + ), + "report_options": normalized_options, } def render_pdf(self, payload: dict[str, Any]) -> bytes: diff --git a/app/templates/reports/voice_playground_report.html b/app/templates/reports/voice_playground_report.html index e9c1c034..51f35b4a 100644 --- a/app/templates/reports/voice_playground_report.html +++ b/app/templates/reports/voice_playground_report.html @@ -178,17 +178,37 @@ color: #6b3c2a; } .report-footer { - margin-top: 22px; + margin-top: 24px; border-top: 1px solid #1f2937; - padding-top: 8px; + padding-top: 10px; display: flex; justify-content: space-between; - color: #666; + align-items: flex-start; + gap: 16px; + color: #4b5563; font-size: 10px; } + .footer-left { + display: flex; + flex-direction: column; + gap: 3px; + } + .footer-right { + text-align: right; + display: flex; + flex-direction: column; + gap: 3px; + } + .report-footer .brand-title { + color: #111827; + font-weight: 800; + font-size: 11px; + letter-spacing: 0.2px; + } .report-footer .brand-link { color: #d16532; font-weight: 700; + text-decoration: none; } .run-row { border: 1px solid #e2e2e2; @@ -232,6 +252,19 @@ font-size: 14px; font-weight: 700; } + .metric-title-row { + display: flex; + align-items: center; + gap: 8px; + } + .metric-color-chip { + width: 10px; + height: 10px; + border-radius: 999px; + border: 1px solid rgba(17, 24, 39, 0.15); + flex-shrink: 0; + background: linear-gradient(90deg, #ef4444 0%, #f59e0b 50%, #16a34a 100%); + } .metric-section .sub { margin: 2px 0 8px; color: #666; @@ -261,6 +294,9 @@ height: 100%; background: #f97316; } + .metric-fill.zone-good { background: #16a34a; } + .metric-fill.zone-neutral { background: #f59e0b; } + .metric-fill.zone-bad { background: #ef4444; } .metric-value { width: 60px; text-align: right; @@ -278,6 +314,39 @@ flex-direction: column; gap: 2px; } + .metric-zone-gradient { + margin-top: 6px; + } + .metric-zone-track-wrap { + position: relative; + } + .metric-zone-track { + height: 8px; + border-radius: 999px; + border: 1px solid #d1d5db; + } + .metric-zone-tick { + position: absolute; + top: -2px; + bottom: -2px; + width: 2px; + background: #111827; + opacity: 0.55; + transform: translateX(-1px); + border-radius: 999px; + } + .metric-zone-labels { + margin-top: 4px; + display: grid; + grid-template-columns: 1fr 1fr 1fr; + gap: 6px; + font-size: 9px; + color: #4b5563; + align-items: start; + } + .metric-zone-labels span:nth-child(1) { text-align: left; } + .metric-zone-labels span:nth-child(2) { text-align: center; } + .metric-zone-labels span:nth-child(3) { text-align: right; } .brand-header { display: flex; align-items: center; @@ -344,10 +413,20 @@

{{ title }}

Comparison ID: {{ simulation_id or comparison_id }}
Providers tested: {{ providers_tested | join(", ") }}
Voices tested: {{ voices_tested | join(", ") }}
+ {% if show_run_summary %}
Total runs / Samples: {{ total_runs_observed }} runs observed, {{ sample_count }} samples
+ {% else %} +
Total Samples: {{ sample_count }} samples
+ {% endif %} + {% if report_options.include_endpoint %}
Endpoint modes: {{ endpoint_modes | join(", ") if endpoint_modes else "Unknown" }}
+ {% endif %} + {% if report_options.include_latency or report_options.include_ttfb %}
Latency formula: Total Latency = TTFB + remaining audio delivery time
+ {% endif %} + {% if report_options.include_ttfb %}
TTFB: Time to First Byte (request start to first audio byte)
+ {% endif %}
Provider / Model / Voice tested:
@@ -372,19 +451,17 @@

{{ title }}

Key evaluation dimensions:

    -
  • Naturalness (MOS - Mean Opinion Score)
    • -
  • Latency
    • -
  • Speech accuracy (WER/CER)
    • -
  • Emotional quality (Valence/Arousal)
    • -
  • Prosody / context stability proxy
    • -
  • Endpoint behavior (streaming vs non-streaming, inferred)
    • + {% for dimension in evaluation_dimensions %} +
  • {{ dimension }}
    • + {% endfor %}

Summary Insights

+ {% if summary.top_naturalness %}
-
Top Naturalness
+
{{ "Top Naturalness" if has_comparison_variants else "Naturalness Score" }}
{{ service._to_score(summary.top_naturalness.avg_mos) if summary.top_naturalness else "N/A" }} MOS
{% if summary.top_naturalness %}
@@ -396,8 +473,10 @@

Summary Insights

N/A
{% endif %}
+ {% endif %} + {% if summary.lowest_latency %}
-
Lowest Latency
+
{{ "Lowest Latency" if has_comparison_variants else "Latency" }}
{{ service._to_ms(summary.lowest_latency.avg_latency_ms) if summary.lowest_latency else "N/A" }}
{% if summary.lowest_latency %}
@@ -408,10 +487,14 @@

Summary Insights

{% else %}
N/A
{% endif %} + {% if report_options.include_endpoint %}
{{ summary.lowest_latency.endpoint_type if summary.lowest_latency else "N/A" }}
+ {% endif %}
+ {% endif %} + {% if summary.lowest_hallucination %}
-
Lowest Hallucination
+
{{ "Lowest Hallucination" if has_comparison_variants else "Hallucination Rate" }}
{{ service._to_pct(summary.lowest_hallucination.avg_wer) if summary.lowest_hallucination else "N/A" }} WER
{% if summary.lowest_hallucination %}
@@ -423,8 +506,10 @@

Summary Insights

N/A
{% endif %}
+ {% endif %} + {% if summary.best_context %}
-
Best Prosody
+
{{ "Best Prosody" if has_comparison_variants else "Prosody Score" }}
{{ service._to_score(summary.best_context.avg_prosody, 3) if summary.best_context else "N/A" }}
{% if summary.best_context %}
@@ -436,6 +521,10 @@

Summary Insights

N/A
{% endif %}
+ {% endif %} + {% if not summary.top_naturalness and not summary.lowest_latency and not summary.lowest_hallucination and not summary.best_context %} +
No summary cards selected for this report configuration.
+ {% endif %}
@@ -448,15 +537,12 @@

1. Aggregate Benchmark Metrics

- - - - - - - - + {% for column in aggregate_metric_columns %} + + {% endfor %} + {% if show_endpoint_column %} + {% endif %} @@ -465,41 +551,78 @@

1. Aggregate Benchmark Metrics

- - - - - - - - + {% for column in aggregate_metric_columns %} + + {% endfor %} + {% if show_endpoint_column %} + {% endif %} {% endfor %}
Provider Model VoiceMOSValenceArousalProsodyWERCERLatencyTTFB{{ column.label }}Endpoint
{{ row.provider_display }} {{ row.model_display }} {{ row.voice_display }}{{ service._to_score(row.avg_mos) }}{{ service._to_score(row.avg_valence) }}{{ service._to_score(row.avg_arousal) }}{{ service._to_score(row.avg_prosody, 3) }}{{ service._to_pct(row.avg_wer) }}{{ service._to_pct(row.avg_cer) }}{{ service._to_ms(row.avg_latency_ms) }}{{ service._to_ms(row.avg_ttfb_ms) }} + {% if column.format_kind == 'ms' %} + {{ service._to_ms(row[column.key]) }} + {% elif column.format_kind == 'pct' %} + {{ service._to_pct(row[column.key]) }} + {% else %} + {{ service._to_score(row[column.key], 3 if column.key == 'avg_prosody' else 2) }} + {% endif %} + {{ row.endpoint_type }}

2. Metric Rankings by Category

+ {% if metric_sections %} {% for metric in metric_sections %}
-

{{ metric.title }}

+
+ +

{{ metric.title }}

+
{{ metric.full_form }}
{{ metric.subtitle }}
+ {% if metric.definition %} +
Definition: {{ metric.definition }}
+ {% endif %} {% for row in metric.rows %}
{{ row.variant_label }}
-
+
{{ row.display_value }}
{% endfor %}
Legend: {{ metric.range_label }}
+ {% if metric.zone_labels %} +
+
+
+ {% if metric.zone_ticks %} + {% for tick in metric.zone_ticks %} + + {% endfor %} + {% endif %} +
+
+ {{ metric.zone_labels.start }} + {{ metric.zone_labels.middle }} + {{ metric.zone_labels.end }} +
+
+ {% endif %}
{% endfor %} + {% else %} +
No ranking metrics selected for this report configuration.
+ {% endif %} + {% if show_hallucination_section %}

3. Speech Accuracy & Hallucination

Accuracy metrics are computed from existing Voice Playground evaluation outputs (WER/CER and ASR transcript where available). @@ -510,8 +633,12 @@

3. Speech Accuracy & Hallucination

Provider Model Voice + {% if report_options.include_wer %} WER + {% endif %} + {% if report_options.include_cer %} CER + {% endif %} @@ -520,15 +647,22 @@

3. Speech Accuracy & Hallucination

{{ row.provider_display }} {{ row.model_display }} {{ row.voice_display }} + {% if report_options.include_wer %} {{ service._to_pct(row.avg_wer) }} + {% endif %} + {% if report_options.include_cer %} {{ service._to_pct(row.avg_cer) }} + {% endif %} {% endfor %} + {% if report_options.include_hallucination_examples %}

Hallucination Examples

- {% if hallucination_examples %} + {% if not report_options.include_wer %} +
Enable WER to include hallucination example ranking in the report.
+ {% elif hallucination_examples %} {% for example in hallucination_examples %}
{{ example.provider_display }} • {{ example.model_display }} • {{ example.voice_display }} WER {{ service._to_pct(example.wer) }}
@@ -539,129 +673,36 @@

Hallucination Examples

{% else %}
No ASR transcript examples available in current comparison outputs.
{% endif %} - -

4. Metric Categories: Qualitative vs Quantitative

-

- Metrics are separated into qualitative perception metrics and quantitative measurement metrics. -

-
-
-

Qualitative Metrics

- {% for metric in qualitative_metrics %} -
-
{{ metric.major_metric }}
-
{{ metric.sub_metric }}
-
{{ metric.full_form }}
-
{{ metric.description }}
-
- {% endfor %} -
-
-

Quantitative Metrics

- {% for metric in quantitative_metrics %} -
-
{{ metric.major_metric }}
-
{{ metric.sub_metric }}
-
{{ metric.full_form }}
-
{{ metric.description }}
-
- {% endfor %} -
-
- -

5. Run-to-Run Variability (Representative + Outliers)

- {% if has_run_variability %} - {% for row in run_variability_rows %} -
-
Transcript: {{ row.text }}
-
- {{ row.provider_display }} • {{ row.model_display }} • {{ row.voice_display }} · {{ row.runs }} runs -
-
-
-
Representative (Run {{ row.representative.run_index }})
-
MOS: {{ service._to_score(row.representative.mos) }}
-
Valence: {{ service._to_score(row.representative.valence) }}
-
Arousal: {{ service._to_score(row.representative.arousal) }}
-
Prosody: {{ service._to_score(row.representative.prosody, 3) }}
-
WER/CER: {{ service._to_pct(row.representative.wer) }} / {{ service._to_pct(row.representative.cer) }}
-
Latency/TTFB: {{ service._to_ms(row.representative.latency_ms) }} / {{ service._to_ms(row.representative.ttfb_ms) }}
-
-
-
Best (Run {{ row.best.run_index }})
-
MOS: {{ service._to_score(row.best.mos) }}
-
Valence: {{ service._to_score(row.best.valence) }}
-
Arousal: {{ service._to_score(row.best.arousal) }}
-
Prosody: {{ service._to_score(row.best.prosody, 3) }}
-
WER/CER: {{ service._to_pct(row.best.wer) }} / {{ service._to_pct(row.best.cer) }}
-
Latency/TTFB: {{ service._to_ms(row.best.latency_ms) }} / {{ service._to_ms(row.best.ttfb_ms) }}
-
-
-
Worst (Run {{ row.worst.run_index }})
-
MOS: {{ service._to_score(row.worst.mos) }}
-
Valence: {{ service._to_score(row.worst.valence) }}
-
Arousal: {{ service._to_score(row.worst.arousal) }}
-
Prosody: {{ service._to_score(row.worst.prosody, 3) }}
-
WER/CER: {{ service._to_pct(row.worst.wer) }} / {{ service._to_pct(row.worst.cer) }}
-
Latency/TTFB: {{ service._to_ms(row.worst.latency_ms) }} / {{ service._to_ms(row.worst.ttfb_ms) }}
-
-
-
- {% endfor %} - {% else %} -
No repeated transcript runs were detected in this comparison, so outlier analysis is not shown.
+ {% endif %} {% endif %} -

6. Recommended Providers by Use Case

- - - - - - {% for rec in recommendations %} - - - - - - {% endfor %} - -
Use CaseRecommended ProviderReason
{{ rec.use_case }}{{ rec.provider }}{{ rec.reason }}
- -

7. Disclaimer & Interpretation Guidance

- {% for disclaimer in disclaimer_sections %} + {% if disclaimer_sections %} +

4. Disclaimer & Interpretation Guidance

+ {% for section in disclaimer_sections %}
-

{{ disclaimer.title }}

-
    - {% for point in disclaimer.points %} -
  • {{ point }}
    • - {% endfor %} -
-
- {% endfor %} - -

8. Benchmark Methodology

- {% for section in methodology_sections %} -

{{ section.title }}

-

{{ section.body }}

- {% if section.bullets %}
    - {% for bullet in section.bullets %} -
  • {{ bullet }}
    • + {% for point in section.points %} +
  • {{ point }}
    • {% endfor %}
- {% endif %}
{% endfor %} + {% endif %}
-
- Generated using EfficientAI
- Voice AI Evaluation Platform +
+
Powered by EfficientAI
+
Voice AI Evaluation Platform
+
+ Website: + https://efficientai.cloud +
-
- Run your own benchmark at EfficientAI +
+
Comparison ID: {{ comparison_id }}
+
Generated: {{ generated_at.strftime("%Y-%m-%d %H:%M UTC") }}
+
Benchmark reports by EfficientAI Cloud
diff --git a/app/workers/tasks/tts_report.py b/app/workers/tasks/tts_report.py index a71ea47c..c738244f 100644 --- a/app/workers/tasks/tts_report.py +++ b/app/workers/tasks/tts_report.py @@ -10,7 +10,7 @@ @celery_app.task(name="generate_tts_report_pdf", bind=True, max_retries=1) -def generate_tts_report_pdf_task(self, report_job_id: str): +def generate_tts_report_pdf_task(self, report_job_id: str, report_options: dict | None = None): """Generate a Voice Playground PDF report and store it in S3.""" from app.models.database import ( TTSComparison, @@ -53,7 +53,11 @@ def generate_tts_report_pdf_task(self, report_job_id: str): .all() ) - payload = voice_playground_report_service.build_payload(comparison, samples) + payload = voice_playground_report_service.build_payload( + comparison, + samples, + report_options=report_options or {}, + ) pdf_bytes = voice_playground_report_service.render_pdf(payload) report_filename = ( diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 20a7b652..6e989d9d 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -39,6 +39,31 @@ export interface LicenseInfoResponse { organization?: string } +type TTSReportOptionsPayload = { + show_runs?: boolean + min_runs_to_show?: number + include_latency?: boolean + include_ttfb?: boolean + include_endpoint?: boolean + include_naturalness?: boolean + include_hallucination?: boolean + include_prosody?: boolean + include_arousal?: boolean + include_valence?: boolean + include_cer?: boolean + include_wer?: boolean + include_hallucination_examples?: boolean + hallucination_examples_limit?: number + include_disclaimer_sections?: boolean + include_methodology_sections?: boolean + zone_threshold_overrides?: Record +} + // When running in production (served from same origin), use relative path // Otherwise use environment variable or default const API_BASE_URL = import.meta.env.VITE_API_URL || @@ -1323,27 +1348,58 @@ class ApiClient { async downloadTTSComparisonReport( comparisonId: string, - includeUnfinishedSamples = false + includeUnfinishedSamples = false, + reportOptions?: TTSReportOptionsPayload ): Promise { const response = await this.client.get( `/api/v1/voice-playground/comparisons/${comparisonId}/report.pdf`, { - params: { include_unfinished_samples: includeUnfinishedSamples }, + params: { + include_unfinished_samples: includeUnfinishedSamples, + ...(reportOptions ? { report_options: JSON.stringify(reportOptions) } : {}), + }, responseType: 'blob', } ) return response.data } - async createTTSComparisonReportJob(comparisonId: string): Promise<{ + async createTTSComparisonReportJob( + comparisonId: string, + reportOptions?: TTSReportOptionsPayload + ): Promise<{ id: string comparison_id: string status: string format: string task_id?: string + report_options?: TTSReportOptionsPayload created_at?: string | null }> { - const response = await this.client.post(`/api/v1/voice-playground/comparisons/${comparisonId}/reports`) + const response = await this.client.post( + `/api/v1/voice-playground/comparisons/${comparisonId}/reports`, + reportOptions ? { report_options: reportOptions } : {} + ) + return response.data + } + + async getVoicePlaygroundReportThresholdDefaults(): Promise<{ + zone_threshold_overrides: NonNullable + is_custom: boolean + }> { + const response = await this.client.get('/api/v1/voice-playground/report-threshold-defaults') + return response.data + } + + async updateVoicePlaygroundReportThresholdDefaults(data: { + zone_threshold_overrides?: NonNullable + reset_to_system_defaults?: boolean + }): Promise<{ + zone_threshold_overrides: NonNullable + is_custom: boolean + message: string + }> { + const response = await this.client.put('/api/v1/voice-playground/report-threshold-defaults', data) return response.data } @@ -1356,6 +1412,7 @@ class ApiClient { error_message?: string | null task_id?: string | null download_url?: string | null + report_options?: TTSReportOptionsPayload created_at?: string | null updated_at?: string | null }> { diff --git a/frontend/src/pages/playground/voice/components/ComparisonResultsView.tsx b/frontend/src/pages/playground/voice/components/ComparisonResultsView.tsx index 5c57b40d..7f865913 100644 --- a/frontend/src/pages/playground/voice/components/ComparisonResultsView.tsx +++ b/frontend/src/pages/playground/voice/components/ComparisonResultsView.tsx @@ -1,10 +1,12 @@ import { Trophy, Download, FileText, Loader2, Headphones } from 'lucide-react' +import { useState } from 'react' import Button from '../../../../components/Button' import ProviderLogo, { getProviderInfo } from '../../../../components/shared/ProviderLogo' -import { TTSComparison, TTSReportJob } from '../types' +import { DEFAULT_TTS_REPORT_OPTIONS, TTSComparison, TTSReportJob, TTSReportOptions } from '../types' import MetricCard from './MetricCard' import SampleGroup from './SampleGroup' import StatusBadge from './StatusBadge' +import ReportConfigModal from './ReportConfigModal' function hasSecondProvider(comp: { provider_b?: string | null @@ -35,8 +37,8 @@ interface ComparisonResultsViewProps { isDownloading: boolean isCreatingReport: boolean reportJob: TTSReportJob | null - onDownloadPdf: () => void - onGenerateAsync: () => void + onDownloadPdf: (options?: TTSReportOptions) => void + onGenerateAsync: (options?: TTSReportOptions) => void onOpenAsyncReport: () => void } @@ -52,6 +54,20 @@ export default function ComparisonResultsView({ onOpenAsyncReport, }: ComparisonResultsViewProps) { const hzMaps = buildVoiceHzMaps(comparison) + const [showReportConfig, setShowReportConfig] = useState(false) + const [lastOptions, setLastOptions] = useState(DEFAULT_TTS_REPORT_OPTIONS) + + const handleDownload = (options: TTSReportOptions) => { + setLastOptions(options) + onDownloadPdf(options) + setShowReportConfig(false) + } + + const handleGenerateAsync = (options: TTSReportOptions) => { + setLastOptions(options) + onGenerateAsync(options) + setShowReportConfig(false) + } return (
@@ -86,7 +102,7 @@ export default function ComparisonResultsView({ ) } - onClick={onDownloadPdf} + onClick={() => setShowReportConfig(true)} disabled={comparison.status !== 'completed' || isDownloading} > Download PDF @@ -101,7 +117,7 @@ export default function ComparisonResultsView({ ) } - onClick={onGenerateAsync} + onClick={() => setShowReportConfig(true)} disabled={isCreatingReport} > Generate Async @@ -161,7 +177,7 @@ export default function ComparisonResultsView({ const sumB = comparison.evaluation_summary.provider_b || {} const hasTwoProviders = hasSecondProvider(comparison) return ( -
+
+ + setShowReportConfig(false)} + onDownloadPdf={handleDownload} + onGenerateAsync={handleGenerateAsync} + />
) } diff --git a/frontend/src/pages/playground/voice/components/MetricCard.tsx b/frontend/src/pages/playground/voice/components/MetricCard.tsx index 6e3a33a5..a35f6b0f 100644 --- a/frontend/src/pages/playground/voice/components/MetricCard.tsx +++ b/frontend/src/pages/playground/voice/components/MetricCard.tsx @@ -19,18 +19,42 @@ export default function MetricCard({ const aWins = aVal !== null && bVal !== null && (higherIsBetter ? aVal > bVal : aVal < bVal) const bWins = aVal !== null && bVal !== null && (higherIsBetter ? bVal > aVal : bVal < aVal) + const formatValue = (value: number | null): string => { + if (value === null) return '—' + const abs = Math.abs(value) + const maximumFractionDigits = + unit === 'ms' + ? 1 + : abs >= 100 + ? 1 + : abs >= 10 + ? 2 + : 3 + const pretty = value.toLocaleString(undefined, { maximumFractionDigits }) + return `${pretty}${unit || ''}` + } + + const aDisplay = formatValue(aVal) + const bDisplay = formatValue(bVal) + return ( -
-

{label}

-
- - {aVal !== null ? `${aVal}${unit || ''}` : '—'} +
+

{label}

+
+ + {aDisplay} {!isSingleProvider && ( <> vs - - {bVal !== null ? `${bVal}${unit || ''}` : '—'} + + {bDisplay} )} diff --git a/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx b/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx index c4cfd894..2043de92 100644 --- a/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx +++ b/frontend/src/pages/playground/voice/components/PlaygroundTab.tsx @@ -387,8 +387,8 @@ export default function PlaygroundTab() { isDownloading={isDownloading} isCreatingReport={isCreatingReport} reportJob={activeReportJob || null} - onDownloadPdf={() => downloadReport(comparison.id)} - onGenerateAsync={() => createReportJob(comparison.id)} + onDownloadPdf={(options) => downloadReport(comparison.id, options)} + onGenerateAsync={(options) => createReportJob(comparison.id, options)} onOpenAsyncReport={() => openAsyncReport(activeReportJob)} /> diff --git a/frontend/src/pages/playground/voice/components/ReportConfigModal.tsx b/frontend/src/pages/playground/voice/components/ReportConfigModal.tsx new file mode 100644 index 00000000..a16124bb --- /dev/null +++ b/frontend/src/pages/playground/voice/components/ReportConfigModal.tsx @@ -0,0 +1,475 @@ +import { useEffect, useState, type ReactNode } from 'react' +import { Loader2, X } from 'lucide-react' +import { createPortal } from 'react-dom' +import Button from '../../../../components/Button' +import { apiClient } from '../../../../lib/api' +import { + DEFAULT_TTS_REPORT_OPTIONS, + TTSReportOptions, + TTSZoneThresholdOverrides, +} from '../types' + +interface ReportConfigModalProps { + isOpen: boolean + initialOptions?: TTSReportOptions + isDownloading: boolean + isCreatingReport: boolean + onClose: () => void + onDownloadPdf: (options: TTSReportOptions) => void + onGenerateAsync: (options: TTSReportOptions) => void +} + +type BooleanOptionKey = { + [K in keyof TTSReportOptions]: TTSReportOptions[K] extends boolean ? K : never +}[keyof TTSReportOptions] + +type ThresholdMetricKey = keyof TTSZoneThresholdOverrides +type ThresholdFieldKey = 'good_min' | 'neutral_min' | 'good_max' | 'neutral_max' + +const THRESHOLD_CONFIGS: Array<{ + key: ThresholdMetricKey + label: string + firstField: ThresholdFieldKey + firstLabel: string + secondField: ThresholdFieldKey + secondLabel: string + step?: string +}> = [ + { + key: 'avg_mos', + label: 'MOS', + firstField: 'neutral_min', + firstLabel: 'Neutral starts', + secondField: 'good_min', + secondLabel: 'Good starts', + step: '0.01', + }, + { + key: 'avg_prosody', + label: 'Prosody', + firstField: 'neutral_min', + firstLabel: 'Neutral starts', + secondField: 'good_min', + secondLabel: 'Good starts', + step: '0.01', + }, + { + key: 'avg_valence', + label: 'Valence', + firstField: 'neutral_min', + firstLabel: 'Neutral starts', + secondField: 'good_min', + secondLabel: 'Good starts', + step: '0.01', + }, + { + key: 'avg_arousal', + label: 'Arousal', + firstField: 'neutral_min', + firstLabel: 'Neutral starts', + secondField: 'good_min', + secondLabel: 'Good starts', + step: '0.01', + }, + { + key: 'avg_wer', + label: 'WER', + firstField: 'good_max', + firstLabel: 'Good max', + secondField: 'neutral_max', + secondLabel: 'Neutral max', + step: '0.001', + }, + { + key: 'avg_cer', + label: 'CER', + firstField: 'good_max', + firstLabel: 'Good max', + secondField: 'neutral_max', + secondLabel: 'Neutral max', + step: '0.001', + }, + { + key: 'avg_ttfb_ms', + label: 'TTFB (ms)', + firstField: 'good_max', + firstLabel: 'Good max', + secondField: 'neutral_max', + secondLabel: 'Neutral max', + step: '1', + }, + { + key: 'avg_latency_ms', + label: 'Latency (ms)', + firstField: 'good_max', + firstLabel: 'Good max', + secondField: 'neutral_max', + secondLabel: 'Neutral max', + step: '1', + }, +] + +export default function ReportConfigModal({ + isOpen, + initialOptions, + isDownloading, + isCreatingReport, + onClose, + onDownloadPdf, + onGenerateAsync, +}: ReportConfigModalProps) { + const [options, setOptions] = useState(initialOptions || DEFAULT_TTS_REPORT_OPTIONS) + const [showAdvancedThresholds, setShowAdvancedThresholds] = useState(false) + const [isLoadingOrgDefaults, setIsLoadingOrgDefaults] = useState(false) + const [isSavingOrgDefaults, setIsSavingOrgDefaults] = useState(false) + const [orgDefaultsMessage, setOrgDefaultsMessage] = useState(null) + + useEffect(() => { + if (!isOpen) return + setOptions(initialOptions || DEFAULT_TTS_REPORT_OPTIONS) + setShowAdvancedThresholds(false) + setOrgDefaultsMessage(null) + }, [isOpen, initialOptions]) + + useEffect(() => { + if (!isOpen) return + let cancelled = false + ;(async () => { + setIsLoadingOrgDefaults(true) + try { + const res = await apiClient.getVoicePlaygroundReportThresholdDefaults() + if (cancelled) return + setOptions((prev) => ({ + ...prev, + zone_threshold_overrides: (res.zone_threshold_overrides || + DEFAULT_TTS_REPORT_OPTIONS.zone_threshold_overrides) as TTSZoneThresholdOverrides, + })) + } catch (_err) { + if (!cancelled) { + setOrgDefaultsMessage('Could not load org threshold defaults. Using local defaults.') + } + } finally { + if (!cancelled) setIsLoadingOrgDefaults(false) + } + })() + return () => { + cancelled = true + } + }, [isOpen]) + + if (!isOpen) return null + + const renderModal = (content: ReactNode) => { + if (typeof document === 'undefined') return null + return createPortal(content, document.body) + } + + const toggle = (key: BooleanOptionKey) => { + setOptions((prev) => ({ ...prev, [key]: !prev[key] })) + } + + const updateThreshold = (metricKey: ThresholdMetricKey, fieldKey: ThresholdFieldKey, rawValue: string) => { + const cleanedValue = rawValue.trim() + const parsed = cleanedValue === '' ? undefined : Number(cleanedValue) + setOptions((prev) => ({ + ...prev, + zone_threshold_overrides: { + ...(prev.zone_threshold_overrides || {}), + [metricKey]: { + ...(prev.zone_threshold_overrides?.[metricKey] || {}), + [fieldKey]: Number.isFinite(parsed as number) ? (parsed as number) : undefined, + }, + }, + })) + } + + const resetThresholdsToDefaults = () => { + setOptions((prev) => ({ + ...prev, + zone_threshold_overrides: JSON.parse( + JSON.stringify(DEFAULT_TTS_REPORT_OPTIONS.zone_threshold_overrides) + ) as TTSZoneThresholdOverrides, + })) + setOrgDefaultsMessage('Thresholds reset to system defaults locally.') + } + + const saveThresholdsAsOrgDefaults = async () => { + setIsSavingOrgDefaults(true) + setOrgDefaultsMessage(null) + try { + await apiClient.updateVoicePlaygroundReportThresholdDefaults({ + zone_threshold_overrides: options.zone_threshold_overrides as Record, + }) + setOrgDefaultsMessage('Saved threshold defaults for this organization.') + } catch (_err) { + setOrgDefaultsMessage('Failed to save org threshold defaults.') + } finally { + setIsSavingOrgDefaults(false) + } + } + + return renderModal( +
+
e.stopPropagation()} + > +
+
+

Customize Report Generation

+

+ Configure which metrics and sections should be included in the PDF. +

+
+ +
+ +
+
+

Run Count Display

+ +
+ + + setOptions((prev) => ({ + ...prev, + min_runs_to_show: Math.max(0, Number(e.target.value || 0)), + })) + } + className="mt-1 w-40 rounded-md border border-gray-300 px-3 py-1.5 text-sm focus:border-primary-500 focus:outline-none focus:ring-1 focus:ring-primary-500 disabled:bg-gray-100 disabled:text-gray-400" + /> +
+
+ +
+

Performance Metrics

+
+ + + +
+
+ +
+

Quality Metrics (default ON)

+
+ + + + + + + +
+
+ +
+

Hallucination Examples

+ +
+ + + setOptions((prev) => ({ + ...prev, + hallucination_examples_limit: Math.max(0, Math.min(50, Number(e.target.value || 0))), + })) + } + className="mt-1 w-32 rounded-md border border-gray-300 px-3 py-1.5 text-sm focus:border-primary-500 focus:outline-none focus:ring-1 focus:ring-primary-500 disabled:bg-gray-100 disabled:text-gray-400" + /> +
+
+ +
+

Report Sections

+
+ +
+
+ +
+ +

+ Tune red / neutral / green cutoffs used by metric ranking bars and legends. +

+ {showAdvancedThresholds && ( +
+
+ + + {isLoadingOrgDefaults && ( + Loading org defaults... + )} +
+ {orgDefaultsMessage && ( +
{orgDefaultsMessage}
+ )} + {THRESHOLD_CONFIGS.map((cfg) => { + const values = options.zone_threshold_overrides?.[cfg.key] || {} + const firstValue = values[cfg.firstField] + const secondValue = values[cfg.secondField] + return ( +
+
{cfg.label}
+ + +
+ ) + })} +
+ )} +
+ +
+ Legacy explanatory sections remain excluded from PDF. Disclaimers are enabled by default and can be toggled. +
+
+ +
+ + + +
+
+
+ ) +} diff --git a/frontend/src/pages/playground/voice/components/SimulationsTab.tsx b/frontend/src/pages/playground/voice/components/SimulationsTab.tsx index f43db575..bf6a2afc 100644 --- a/frontend/src/pages/playground/voice/components/SimulationsTab.tsx +++ b/frontend/src/pages/playground/voice/components/SimulationsTab.tsx @@ -66,8 +66,8 @@ export default function SimulationsTab() { isDownloading={isDownloading} isCreatingReport={isCreatingReport} reportJob={viewedReportJob || null} - onDownloadPdf={() => downloadReport(viewedComparison.id)} - onGenerateAsync={() => createReportJob(viewedComparison.id)} + onDownloadPdf={(options) => downloadReport(viewedComparison.id, options)} + onGenerateAsync={(options) => createReportJob(viewedComparison.id, options)} onOpenAsyncReport={() => openAsyncReport(viewedReportJob)} /> diff --git a/frontend/src/pages/playground/voice/context/VoicePlaygroundContext.tsx b/frontend/src/pages/playground/voice/context/VoicePlaygroundContext.tsx index 8f3c9b98..9a78fb28 100644 --- a/frontend/src/pages/playground/voice/context/VoicePlaygroundContext.tsx +++ b/frontend/src/pages/playground/voice/context/VoicePlaygroundContext.tsx @@ -11,6 +11,7 @@ import { TTSAnalyticsRow, CustomTTSVoice, TTSReportJob, + TTSReportOptions, AnalyticsSortKey, BenchmarkMetricKey, DEFAULT_SAMPLE_TEXTS, @@ -131,9 +132,9 @@ interface VoicePlaygroundContextType { isCreating: boolean submitBlindTest: () => void isSubmittingBlindTest: boolean - downloadReport: (comparisonId: string) => void + downloadReport: (comparisonId: string, options?: TTSReportOptions) => void isDownloading: boolean - createReportJob: (comparisonId: string) => void + createReportJob: (comparisonId: string, options?: TTSReportOptions) => void isCreatingReport: boolean deleteComparison: (id: string) => void isDeleting: boolean @@ -472,9 +473,10 @@ export function VoicePlaygroundProvider({ children }: { children: ReactNode }) { }) const downloadReportMutation = useMutation({ - mutationFn: (comparisonId: string) => apiClient.downloadTTSComparisonReport(comparisonId), - onSuccess: (blob, comparisonId) => { - const filename = `voice-playground-report-${comparisonId.slice(0, 8)}.pdf` + mutationFn: ({ comparisonId, options }: { comparisonId: string; options?: TTSReportOptions }) => + apiClient.downloadTTSComparisonReport(comparisonId, false, options), + onSuccess: (blob, variables) => { + const filename = `voice-playground-report-${variables.comparisonId.slice(0, 8)}.pdf` const url = window.URL.createObjectURL(blob) const link = document.createElement('a') link.href = url @@ -487,7 +489,8 @@ export function VoicePlaygroundProvider({ children }: { children: ReactNode }) { }) const createReportJobMutation = useMutation({ - mutationFn: (comparisonId: string) => apiClient.createTTSComparisonReportJob(comparisonId), + mutationFn: ({ comparisonId, options }: { comparisonId: string; options?: TTSReportOptions }) => + apiClient.createTTSComparisonReportJob(comparisonId, options), onSuccess: (job) => { setReportJobByComparisonId(prev => ({ ...prev, [job.comparison_id]: job.id })) }, @@ -710,9 +713,11 @@ export function VoicePlaygroundProvider({ children }: { children: ReactNode }) { isCreating: createMutation.isPending, submitBlindTest: () => blindTestMutation.mutate(), isSubmittingBlindTest: blindTestMutation.isPending, - downloadReport: (id: string) => downloadReportMutation.mutate(id), + downloadReport: (id: string, options?: TTSReportOptions) => + downloadReportMutation.mutate({ comparisonId: id, options }), isDownloading: downloadReportMutation.isPending, - createReportJob: (id: string) => createReportJobMutation.mutate(id), + createReportJob: (id: string, options?: TTSReportOptions) => + createReportJobMutation.mutate({ comparisonId: id, options }), isCreatingReport: createReportJobMutation.isPending, deleteComparison: (id: string) => deleteMutation.mutate(id), isDeleting: deleteMutation.isPending, diff --git a/frontend/src/pages/playground/voice/types.ts b/frontend/src/pages/playground/voice/types.ts index cd0d6514..097d103c 100644 --- a/frontend/src/pages/playground/voice/types.ts +++ b/frontend/src/pages/playground/voice/types.ts @@ -109,10 +109,79 @@ export interface TTSReportJob { error_message?: string | null task_id?: string | null download_url?: string | null + report_options?: Partial created_at?: string | null updated_at?: string | null } +export type TTSMetricThresholdOverride = { + good_min?: number + neutral_min?: number + good_max?: number + neutral_max?: number +} + +export type TTSZoneThresholdOverrides = Partial> + +export interface TTSReportOptions { + show_runs: boolean + min_runs_to_show: number + include_latency: boolean + include_ttfb: boolean + include_endpoint: boolean + include_naturalness: boolean + include_hallucination: boolean + include_prosody: boolean + include_arousal: boolean + include_valence: boolean + include_cer: boolean + include_wer: boolean + include_hallucination_examples: boolean + hallucination_examples_limit: number + include_disclaimer_sections: boolean + include_methodology_sections: boolean + zone_threshold_overrides: TTSZoneThresholdOverrides +} + +export const DEFAULT_TTS_REPORT_OPTIONS: TTSReportOptions = { + show_runs: true, + min_runs_to_show: 100, + include_latency: true, + include_ttfb: true, + include_endpoint: true, + include_naturalness: true, + include_hallucination: true, + include_prosody: true, + include_arousal: true, + include_valence: true, + include_cer: true, + include_wer: true, + include_hallucination_examples: true, + hallucination_examples_limit: 5, + include_disclaimer_sections: true, + include_methodology_sections: false, + zone_threshold_overrides: { + avg_mos: { neutral_min: 3.0, good_min: 4.0 }, + avg_prosody: { neutral_min: 0.4, good_min: 0.7 }, + avg_valence: { neutral_min: -0.2, good_min: 0.3 }, + avg_arousal: { neutral_min: 0.4, good_min: 0.7 }, + avg_wer: { good_max: 0.1, neutral_max: 0.25 }, + avg_cer: { good_max: 0.08, neutral_max: 0.2 }, + avg_ttfb_ms: { good_max: 350, neutral_max: 800 }, + avg_latency_ms: { good_max: 1500, neutral_max: 3000 }, + }, +} + export type AnalyticsSortKey = | 'provider' | 'model' From 04020fd7d23b003f9a9af99bbcabc0ff384891b4 Mon Sep 17 00:00:00 2001 From: Tejas Narayan Date: Sun, 15 Mar 2026 08:48:38 +0000 Subject: [PATCH 4/4] fix: updating symantic versioning --- .github/workflows/pr-label-semver-release.yml | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 .github/workflows/pr-label-semver-release.yml diff --git a/.github/workflows/pr-label-semver-release.yml b/.github/workflows/pr-label-semver-release.yml new file mode 100644 index 00000000..9230135b --- /dev/null +++ b/.github/workflows/pr-label-semver-release.yml @@ -0,0 +1,109 @@ +name: PR Label Semantic Release + +on: + pull_request_target: + types: [closed] + branches: + - main + - master + +permissions: + contents: write + +jobs: + release: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + + steps: + - name: Determine bump type from PR labels + id: bump + uses: actions/github-script@v7 + with: + script: | + const labels = (context.payload.pull_request.labels || []).map((label) => + label.name.toLowerCase() + ); + core.info(`PR labels: ${labels.join(", ") || "none"}`); + + let bump = "patch"; + if (labels.includes("major")) { + bump = "major"; + } else if (labels.includes("minor")) { + bump = "minor"; + } else if (labels.includes("fix") || labels.includes("patch")) { + bump = "patch"; + } else { + core.info("No semver label found. Defaulting to patch bump."); + } + + core.setOutput("bump", bump); + + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Compute next semantic version + id: version + shell: bash + run: | + set -euo pipefail + + git fetch --tags --force + latest_tag="$(git tag -l 'v*' --sort=-v:refname | head -n 1)" + + if [[ -z "$latest_tag" ]]; then + latest_tag="v0.0.0" + fi + + current="${latest_tag#v}" + IFS='.' read -r major minor patch <<< "$current" + + if [[ -z "${major:-}" || -z "${minor:-}" || -z "${patch:-}" ]]; then + echo "Latest tag '$latest_tag' is not semver format (v..)." + exit 1 + fi + + case "${{ steps.bump.outputs.bump }}" in + major) + major=$((major + 1)) + minor=0 + patch=0 + ;; + minor) + minor=$((minor + 1)) + patch=0 + ;; + patch) + patch=$((patch + 1)) + ;; + *) + echo "Unknown bump type: ${{ steps.bump.outputs.bump }}" + exit 1 + ;; + esac + + next_tag="v${major}.${minor}.${patch}" + echo "latest_tag=$latest_tag" >> "$GITHUB_OUTPUT" + echo "next_tag=$next_tag" >> "$GITHUB_OUTPUT" + echo "Next tag: $next_tag" + + - name: Create GitHub release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NEXT_TAG: ${{ steps.version.outputs.next_tag }} + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + shell: bash + run: | + set -euo pipefail + + if gh release view "$NEXT_TAG" >/dev/null 2>&1; then + echo "Release $NEXT_TAG already exists. Skipping." + exit 0 + fi + + gh release create "$NEXT_TAG" \ + --target "$MERGE_SHA" \ + --title "$NEXT_TAG" \ + --generate-notes