diff --git a/.codebase-memory/.gitattributes b/.codebase-memory/.gitattributes new file mode 100644 index 00000000..c12d9ab6 --- /dev/null +++ b/.codebase-memory/.gitattributes @@ -0,0 +1,3 @@ +# Auto-generated by codebase-memory-mcp +# Prevent merge conflicts on compressed artifact +graph.db.zst binary merge=ours diff --git a/.codebase-memory/graph.db.zst b/.codebase-memory/graph.db.zst new file mode 100644 index 00000000..72eb823c Binary files /dev/null and b/.codebase-memory/graph.db.zst differ diff --git a/.github/workflows/backend-tests-postgres.yml b/.github/workflows/backend-tests-postgres.yml index 6fc18bc4..30be09f4 100644 --- a/.github/workflows/backend-tests-postgres.yml +++ b/.github/workflows/backend-tests-postgres.yml @@ -41,6 +41,8 @@ jobs: REDIS_URL: redis://localhost:6379/0 CELERY_BROKER_URL: redis://localhost:6379/0 CELERY_RESULT_BACKEND: redis://localhost:6379/0 + EFFICIENTAI_PYTEST: "1" + FLEXPRICE_ENABLED: "false" steps: - name: Checkout repository @@ -83,6 +85,8 @@ jobs: SHARD_DATABASE_URL_02: postgresql://postgres:postgres@localhost:5432/efficientai_data_02 SHARDING_INTEGRATION_TEST: "1" REDIS_URL: redis://localhost:6379/0 + EFFICIENTAI_PYTEST: "1" + FLEXPRICE_ENABLED: "false" steps: - name: Checkout repository diff --git a/app/api/v1/api.py b/app/api/v1/api.py index 055bc147..b5966554 100644 --- a/app/api/v1/api.py +++ b/app/api/v1/api.py @@ -1,103 +1,105 @@ -"""API v1 router aggregation.""" - -from fastapi import APIRouter -from app.api.v1.routes import ( - auth, - audio, - evaluations, - results, - agents, - personas, - scenarios, - iam, - profile, - integrations, - data_sources, - voicebundles, - aiproviders, - model_config, - manual_evaluations, - test_agents, - conversation_evaluations, - voice_agent, - evaluators, - evaluator_suites, - metrics, - evaluator_results, - chat, - playground, - settings, - observability, - alerts, - cron_jobs, - voice_playground, - public_blind_test, - prompt_partials, - prompt_optimization, - telephony, - vobiz_telephony, - call_imports, - call_import_schemas, - call_import_tags, - call_import_evaluations, - judge_alignment, - metric_studio, - workspaces, - workspace_iam, - dashboard, - llm_gateway, - platform_admin, - org_usage, - usage_pricing, -) - -api_router = APIRouter() - -# Include all route routers -api_router.include_router(auth.router) -api_router.include_router(audio.router) -api_router.include_router(evaluations.router) -api_router.include_router(results.router) -api_router.include_router(agents.router) -api_router.include_router(personas.router) -api_router.include_router(scenarios.router) -api_router.include_router(iam.router) -api_router.include_router(profile.router) -api_router.include_router(integrations.router) -api_router.include_router(data_sources.router) -api_router.include_router(voicebundles.router) -api_router.include_router(aiproviders.router) -api_router.include_router(model_config.router) -api_router.include_router(manual_evaluations.router) -api_router.include_router(test_agents.router) -api_router.include_router(conversation_evaluations.router) -api_router.include_router(voice_agent.router) -api_router.include_router(evaluators.router) -api_router.include_router(evaluator_suites.router) -api_router.include_router(metrics.router) -api_router.include_router(evaluator_results.router) -api_router.include_router(chat.router) -api_router.include_router(playground.router) -api_router.include_router(settings.router) -api_router.include_router(observability.router) -api_router.include_router(alerts.router) -api_router.include_router(cron_jobs.router) -api_router.include_router(voice_playground.router) -api_router.include_router(public_blind_test.router) -api_router.include_router(prompt_partials.router) -api_router.include_router(prompt_optimization.router) -api_router.include_router(telephony.router) -api_router.include_router(vobiz_telephony.router) -api_router.include_router(call_imports.router) -api_router.include_router(call_import_schemas.router) -api_router.include_router(call_import_tags.router) -api_router.include_router(call_import_evaluations.router) -api_router.include_router(judge_alignment.router) -api_router.include_router(metric_studio.router) -api_router.include_router(workspaces.router) -api_router.include_router(workspace_iam.router) -api_router.include_router(dashboard.router) -api_router.include_router(llm_gateway.router) -api_router.include_router(platform_admin.router) -api_router.include_router(org_usage.router) -api_router.include_router(usage_pricing.router) +"""API v1 router aggregation.""" + +from fastapi import APIRouter +from app.api.v1.routes import ( + auth, + audio, + evaluations, + results, + agents, + personas, + scenarios, + iam, + profile, + integrations, + data_sources, + voicebundles, + aiproviders, + model_config, + manual_evaluations, + test_agents, + conversation_evaluations, + voice_agent, + evaluators, + evaluator_suites, + metrics, + evaluator_results, + chat, + playground, + settings, + observability, + alerts, + cron_jobs, + voice_playground, + public_blind_test, + prompt_partials, + prompt_optimization, + telephony, + vobiz_telephony, + call_imports, + call_import_schemas, + call_import_tags, + call_import_evaluations, + judge_alignment, + metric_studio, + workspaces, + workspace_iam, + dashboard, + llm_gateway, + platform_admin, + org_usage, + usage_pricing, + synthetic_traces, +) + +api_router = APIRouter() + +# Include all route routers +api_router.include_router(auth.router) +api_router.include_router(audio.router) +api_router.include_router(evaluations.router) +api_router.include_router(results.router) +api_router.include_router(agents.router) +api_router.include_router(personas.router) +api_router.include_router(scenarios.router) +api_router.include_router(iam.router) +api_router.include_router(profile.router) +api_router.include_router(integrations.router) +api_router.include_router(data_sources.router) +api_router.include_router(voicebundles.router) +api_router.include_router(aiproviders.router) +api_router.include_router(model_config.router) +api_router.include_router(manual_evaluations.router) +api_router.include_router(test_agents.router) +api_router.include_router(conversation_evaluations.router) +api_router.include_router(voice_agent.router) +api_router.include_router(evaluators.router) +api_router.include_router(evaluator_suites.router) +api_router.include_router(metrics.router) +api_router.include_router(evaluator_results.router) +api_router.include_router(chat.router) +api_router.include_router(playground.router) +api_router.include_router(settings.router) +api_router.include_router(observability.router) +api_router.include_router(alerts.router) +api_router.include_router(cron_jobs.router) +api_router.include_router(voice_playground.router) +api_router.include_router(public_blind_test.router) +api_router.include_router(prompt_partials.router) +api_router.include_router(prompt_optimization.router) +api_router.include_router(telephony.router) +api_router.include_router(vobiz_telephony.router) +api_router.include_router(call_imports.router) +api_router.include_router(call_import_schemas.router) +api_router.include_router(call_import_tags.router) +api_router.include_router(call_import_evaluations.router) +api_router.include_router(judge_alignment.router) +api_router.include_router(metric_studio.router) +api_router.include_router(workspaces.router) +api_router.include_router(workspace_iam.router) +api_router.include_router(dashboard.router) +api_router.include_router(llm_gateway.router) +api_router.include_router(platform_admin.router) +api_router.include_router(org_usage.router) +api_router.include_router(usage_pricing.router) +api_router.include_router(synthetic_traces.router) diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index 17094877..f02f45a7 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -2,16 +2,19 @@ Agents API Routes Complete CRUD operations for test agents """ -from fastapi import APIRouter, Depends, HTTPException, status, Query +from uuid import uuid4 + +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status, Query from fastapi.responses import JSONResponse, Response from sqlalchemy.orm import Session from typing import List, Optional -from uuid import UUID +from uuid import UUID, uuid4 import random from pydantic import BaseModel from loguru import logger from app.dependencies import get_db, get_organization_id, get_workspace_id, get_api_key +from app.services.billing.flexprice_service import record_agent_test_setup_generated from app.models.database import ( Agent, ConversationEvaluation, TestAgentConversation, VoiceBundle, AIProvider, Integration, IntegrationPlatform, CallMediumEnum, @@ -254,6 +257,7 @@ def _scenario_draft_responses(scenarios) -> list[GeneratedScenarioDraftResponse] @router.post("/generate-test-prompt", response_model=GenerateTestPromptResponse) async def generate_test_prompt( data: GenerateTestPromptRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), @@ -297,6 +301,14 @@ async def generate_test_prompt( llm_config=data.llm_config, credential_id=data.credential_id, ) + background_tasks.add_task( + record_agent_test_setup_generated, + organization_id, + uuid4(), + workspace_id=workspace_id, + purpose="test_prompt", + model=result.model, + ) return GenerateTestPromptResponse( sections=_test_prompt_section_responses(result.sections), test_agent_prompt=result.test_agent_prompt, @@ -313,6 +325,7 @@ async def generate_test_prompt( @router.post("/generate-scenarios-from-prompt", response_model=GenerateScenariosFromPromptResponse) async def generate_scenarios_from_prompt( data: GenerateScenariosFromPromptRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), @@ -357,6 +370,15 @@ async def generate_scenarios_from_prompt( llm_config=data.llm_config, credential_id=data.credential_id, ) + background_tasks.add_task( + record_agent_test_setup_generated, + organization_id, + uuid4(), + workspace_id=workspace_id, + purpose="scenarios", + model=result.model, + scenario_count=len(result.scenarios), + ) return GenerateScenariosFromPromptResponse( scenarios=_scenario_draft_responses(result.scenarios), provider=result.provider, @@ -372,6 +394,7 @@ async def generate_scenarios_from_prompt( @router.post("/generate-test-setup", response_model=GenerateTestSetupResponse) async def generate_test_setup( data: GenerateTestSetupRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), @@ -430,6 +453,15 @@ async def generate_test_setup( llm_config=data.llm_config, credential_id=data.credential_id, ) + background_tasks.add_task( + record_agent_test_setup_generated, + organization_id, + uuid4(), + workspace_id=workspace_id, + purpose="full_setup", + model=scenario_result.model, + scenario_count=len(scenario_result.scenarios), + ) return GenerateTestSetupResponse( sections=_test_prompt_section_responses(prompt_result.sections), test_agent_prompt=prompt_result.test_agent_prompt, diff --git a/app/api/v1/routes/auth.py b/app/api/v1/routes/auth.py index 9077f120..11c1474d 100644 --- a/app/api/v1/routes/auth.py +++ b/app/api/v1/routes/auth.py @@ -287,6 +287,18 @@ def _extract_bearer(authorization: Optional[str]) -> Optional[str]: return token.strip() +def _revoke_local_password_access_token(bearer: str) -> None: + try: + claims = decode_access_token(bearer) + jti = claims.get("jti") + exp = claims.get("exp") + if jti and exp: + ttl = max(int(exp) - int(datetime.now(timezone.utc).timestamp()), 1) + revoke_access_jti(jti, ttl) + except JWTError: + pass + + def _issue_session_tokens( db: Session, *, @@ -611,15 +623,7 @@ def logout( """Revoke the current session's refresh token and blacklist the access token.""" bearer = _extract_bearer(authorization) if bearer and principal.auth_method == AuthMethod.LOCAL_PASSWORD: - try: - claims = decode_access_token(bearer) - jti = claims.get("jti") - exp = claims.get("exp") - if jti and exp: - ttl = max(int(exp) - int(datetime.now(timezone.utc).timestamp()), 1) - revoke_access_jti(jti, ttl) - except JWTError: - pass + _revoke_local_password_access_token(bearer) if payload and payload.refresh_token: revoke_refresh_token(db, payload.refresh_token) @@ -697,12 +701,14 @@ def refresh_session(payload: RefreshRequest, db: Session = Depends(get_db)) -> T class SwitchOrgRequest(BaseModel): organization_id: str + refresh_token: Optional[str] = None @router.post("/switch-org", response_model=TokenResponse) def switch_organization( payload: SwitchOrgRequest, principal: Principal = Depends(get_principal), + authorization: Optional[str] = Header(None, alias="Authorization"), db: Session = Depends(get_db), ) -> TokenResponse: """ @@ -766,6 +772,13 @@ def switch_organization( detail="User is no longer active.", ) + if principal.auth_method == AuthMethod.LOCAL_PASSWORD: + bearer = _extract_bearer(authorization) + if bearer: + _revoke_local_password_access_token(bearer) + if payload.refresh_token: + revoke_refresh_token(db, payload.refresh_token) + user.last_login_at = datetime.now(timezone.utc) db.commit() db.refresh(user) diff --git a/app/api/v1/routes/call_import_evaluations.py b/app/api/v1/routes/call_import_evaluations.py index 18749f6d..54188b08 100644 --- a/app/api/v1/routes/call_import_evaluations.py +++ b/app/api/v1/routes/call_import_evaluations.py @@ -1281,6 +1281,20 @@ def _name_for_source(source: str) -> Optional[str]: transcribe_overwrite=payload.transcribe_overwrite, ) + from app.services.billing.flexprice_service import ( + record_call_import_evaluation_started, + ) + + for evaluation in created_evaluations: + record_call_import_evaluation_started( + organization_id, + evaluation.id, + workspace_id=evaluation.workspace_id, + call_import_id=call_import.id, + total_rows=int(evaluation.total_rows or 0), + metric_count=len(leaf_metric_ids), + ) + for evaluation in created_evaluations: db.refresh(evaluation) @@ -3980,6 +3994,18 @@ async def generate_call_import_evaluation_pdf_report( detail="Failed to store PDF report due to a concurrent duplicate request.", ) from None db.refresh(pdf_report) + from app.services.billing.flexprice_service import ( + record_call_import_pdf_report_generated, + ) + + record_call_import_pdf_report_generated( + organization_id, + pdf_report.id, + workspace_id=pdf_report.workspace_id, + evaluation_id=evaluation.id, + call_import_id=call_import_id, + report_type=pdf_report.report_type, + ) return _pdf_report_response_from_row(pdf_report) @@ -6926,6 +6952,16 @@ def _resolve_alias(alias_map: Dict[str, str], key: str) -> str: # in ``app/workers/tasks/helpers/llm_evaluation.py`` — kept local here to # avoid a worker import cycle from the routes module. DISCOVERED_METRICS_KEY = "__discovered_metrics__" +METRIC_SCORES_META_KEYS = frozenset({DISCOVERED_METRICS_KEY, "_billing"}) + + +def _is_metric_scores_meta_key(key: str) -> bool: + normalized = str(key or "").strip().lower() + if not normalized: + return True + if normalized in {item.lower() for item in METRIC_SCORES_META_KEYS}: + return True + return normalized.endswith("__discovered") # Allowed values for an LLM-suggested top-level metric type. Kept in # sync with ``DiscoveredMetricSuggestedType`` in @@ -8674,7 +8710,8 @@ def _reset_eval_row_for_retry( eval_row.metric_scores = { key: value for key, value in existing.items() - if str(key).lower() not in target_keys + if _is_metric_scores_meta_key(str(key)) + or str(key).lower() not in target_keys } else: eval_row.metric_scores = {} @@ -9181,12 +9218,15 @@ async def retry_call_import_evaluation( payload.metric_ids if payload else None ) if metric_ids is not None: + metric_ids = [ + mid for mid in metric_ids if not _is_metric_scores_meta_key(str(mid)) + ] if not metric_ids: raise HTTPException( status_code=400, detail=( - "metric_ids must be a non-empty list. Omit the " - "field to re-run all metrics." + "metric_ids must be a non-empty list of metric UUIDs. " + "Omit the field to re-run all metrics." ), ) diff --git a/app/api/v1/routes/chat.py b/app/api/v1/routes/chat.py index 6fc026f8..6f023654 100644 --- a/app/api/v1/routes/chat.py +++ b/app/api/v1/routes/chat.py @@ -28,6 +28,7 @@ class ChatRequest(BaseModel): temperature: Optional[float] = 0.7 max_tokens: Optional[int] = None llm_config: Optional[Dict[str, Any]] = None + usage_purpose: Optional[str] = "scenario_description" class ChatResponse(BaseModel): @@ -80,6 +81,7 @@ async def chat_completion( uuid4(), workspace_id=workspace_id, model=result.get("model", request.model), + purpose=request.usage_purpose or "scenario_description", ) return ChatResponse( diff --git a/app/api/v1/routes/evaluator_results.py b/app/api/v1/routes/evaluator_results.py index 962207c6..d5cdc57d 100644 --- a/app/api/v1/routes/evaluator_results.py +++ b/app/api/v1/routes/evaluator_results.py @@ -1,6 +1,6 @@ """Evaluator Results routes.""" -from fastapi import APIRouter, Depends, HTTPException, status, Query +from fastapi import APIRouter, Depends, HTTPException, status, Query, Request from sqlalchemy.orm import Session from sqlalchemy import and_ from uuid import UUID @@ -456,6 +456,7 @@ def get_evaluator_result( "provider_call_id": result.provider_call_id, "provider_platform": result.provider_platform, "call_data": result.call_data, + "synthetic_call_trace_id": result.synthetic_call_trace_id, "created_at": result.created_at, "updated_at": result.updated_at, "created_by": result.created_by, @@ -643,6 +644,31 @@ def get_evaluator_result_metrics( } +@router.get("/{id}/otel-correlation") +def get_evaluator_result_otel_correlation( + id: str, + request: Request, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + """Return OTLP endpoint and correlation env vars for customer Pipecat setup.""" + from app.models.synthetic_trace_schemas import OtelCorrelationInfo + from app.services.synthetic_traces.trace_service import build_otel_correlation + + del api_key + result = _lookup_evaluator_result(db, id, organization_id, workspace_id) + if not result: + raise HTTPException(status_code=404, detail="Evaluator result not found") + info = build_otel_correlation( + db, + result, + api_base_url=str(request.base_url).rstrip("/"), + ) + return OtelCorrelationInfo(**info) + + @router.get("/{id}/audio") async def stream_evaluator_result_audio( id: str, @@ -864,15 +890,17 @@ def re_evaluate_result( detail="Cannot re-evaluate: this result has no transcription. It must be transcribed first." ) - if not result.evaluator_id: + if not result.evaluator_id and not result.agent_id: raise HTTPException( status_code=400, - detail="Cannot re-evaluate: this result is not linked to an evaluator." + detail="Cannot re-evaluate: this result is not linked to an agent or evaluator." ) - evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() - if not evaluator: - raise HTTPException(status_code=404, detail="Linked evaluator no longer exists") + evaluator = None + if result.evaluator_id: + evaluator = db.query(Evaluator).filter(Evaluator.id == result.evaluator_id).first() + if not evaluator: + raise HTTPException(status_code=404, detail="Linked evaluator no longer exists") # ------------------------------------------------------------------ # If no audio in S3 yet, try to download from the voice provider diff --git a/app/api/v1/routes/integrations.py b/app/api/v1/routes/integrations.py index 338f295b..0972c411 100644 --- a/app/api/v1/routes/integrations.py +++ b/app/api/v1/routes/integrations.py @@ -1,473 +1,534 @@ -""" -Integrations API Routes -Manage integrations with external voice AI platforms (Retell, Vapi, etc.) -""" -from fastapi import APIRouter, Depends, HTTPException, status, Query -from fastapi.responses import JSONResponse -from sqlalchemy.orm import Session -from datetime import datetime, timezone -from typing import List -from uuid import UUID -from loguru import logger - -from app.dependencies import get_db, get_organization_id, get_api_key -from app.models.database import Integration, IntegrationPlatform, Agent -from app.models.schemas import ( - IntegrationCreate, IntegrationUpdate, IntegrationResponse, - PreviewIntegrationAgentPromptRequest, PreviewIntegrationAgentPromptResponse, -) -from app.core.encryption import encrypt_api_key, decrypt_api_key -from app.services.credentials.resolver import clear_other_defaults -from app.services.voice_providers import get_voice_provider -from app.services.ai.llm_gateway import get_credential_effective_routing_label - -router = APIRouter(prefix="/integrations", tags=["Integrations"]) - - -def _integration_response( - integration: Integration, - organization_id: UUID, - db: Session, -) -> IntegrationResponse: - effective_routing = get_credential_effective_routing_label( - organization_id, - db, - integration, - ) - response = IntegrationResponse.model_validate(integration) - return response.model_copy(update={"effective_routing": effective_routing}) - - -def _validate_smallest_connection(raw_api_key: str): - """Validate a Smallest key via GET /atoms/v1/user.""" - try: - provider_class = get_voice_provider(IntegrationPlatform.SMALLEST.value) - provider = provider_class(api_key=raw_api_key) - provider.test_connection() - if hasattr(provider, "get_user_details"): - return provider.get_user_details() - return None - except Exception as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Smallest API key validation failed: {str(e)}" - ) - - -@router.post("", response_model=IntegrationResponse, status_code=status.HTTP_201_CREATED, operation_id="createIntegration") -async def create_integration( - integration_data: IntegrationCreate, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """Create a new credential row for a voice AI platform. - - Multiple credentials per platform are now supported. The first row - created for a given (org, platform) automatically becomes the - default; subsequent rows can be promoted via - ``POST /integrations/{id}/set-default``. ``integration_data.is_default`` - can also be set explicitly to mark the new row as the default at - creation time. - Requires at least WRITER role. - """ - from sqlalchemy import func - platform_value = integration_data.platform.value if hasattr(integration_data.platform, 'value') else integration_data.platform - - user_details = None - if platform_value.lower() == IntegrationPlatform.SMALLEST.value: - user_details = _validate_smallest_connection(integration_data.api_key) - - encrypted_api_key = encrypt_api_key(integration_data.api_key) - integration_name = integration_data.name - if not integration_name and isinstance(user_details, dict): - email = user_details.get("email") or user_details.get("userEmail") - if email: - integration_name = f"Smallest ({email})" - - existing_default = db.query(Integration).filter( - Integration.organization_id == organization_id, - func.lower(Integration.platform) == platform_value.lower(), - Integration.is_default.is_(True), - Integration.is_active.is_(True), - ).first() - - requested_default = bool(integration_data.is_default) - will_be_default = requested_default or existing_default is None - insert_as_default = will_be_default and existing_default is None - - integration = Integration( - organization_id=organization_id, - platform=platform_value, - name=integration_name, - api_key=encrypted_api_key, - public_key=integration_data.public_key, - is_active=True, - is_default=insert_as_default, - routing_mode=integration_data.routing_mode.value, - last_tested_at=datetime.now(timezone.utc) if user_details is not None else None, - ) - - db.add(integration) - db.flush() - - if will_be_default: - clear_other_defaults( - Integration, - db, - organization_id, - keep_id=integration.id, - provider_field="platform", - provider_value=platform_value, - ) - integration.is_default = True - - db.commit() - db.refresh(integration) - - return _integration_response(integration, organization_id, db) - - -@router.post( - "/{integration_id}/set-default", - response_model=IntegrationResponse, - operation_id="setDefaultIntegration", -) -async def set_default_integration( - integration_id: UUID, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """Mark this integration as the default for its (org, platform). - - Atomically clears the default flag on every other row for the same - (org, platform) so the partial unique index in migration 028 holds. - """ - integration = db.query(Integration).filter( - Integration.id == integration_id, - Integration.organization_id == organization_id, - ).first() - - if not integration: - raise HTTPException(status_code=404, detail="Integration not found") - - if not integration.is_active: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Cannot mark an inactive integration as default", - ) - - platform_value = ( - integration.platform.value - if hasattr(integration.platform, "value") - else integration.platform - ) - - clear_other_defaults( - Integration, - db, - organization_id, - keep_id=integration.id, - provider_field="platform", - provider_value=platform_value, - ) - integration.is_default = True - db.commit() - db.refresh(integration) - return _integration_response(integration, organization_id, db) - - -@router.get("", response_model=List[IntegrationResponse], operation_id="listIntegrations") -async def list_integrations( - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - List all integrations for the organization. - Requires at least READER role. - """ - integrations = db.query(Integration).filter( - Integration.organization_id == organization_id - ).order_by(Integration.created_at.desc()).all() - - valid_platforms = {p.value for p in IntegrationPlatform} - filtered_integrations: List[Integration] = [] - for integration in integrations: - raw_platform = ( - integration.platform.value - if hasattr(integration.platform, "value") - else str(integration.platform).lower() - ) - if raw_platform in valid_platforms: - filtered_integrations.append(integration) - else: - logger.warning( - "Skipping integration {} with invalid platform '{}'", - integration.id, - integration.platform, - ) - - return [ - _integration_response(integration, organization_id, db) - for integration in filtered_integrations - ] - - -@router.get("/{integration_id}", response_model=IntegrationResponse) -async def get_integration( - integration_id: UUID, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - Get a specific integration. - Requires at least READER role. - """ - integration = db.query(Integration).filter( - Integration.id == integration_id, - Integration.organization_id == organization_id - ).first() - - if not integration: - raise HTTPException(status_code=404, detail="Integration not found") - - raw_platform = ( - integration.platform.value - if hasattr(integration.platform, "value") - else str(integration.platform).lower() - ) - if raw_platform not in {p.value for p in IntegrationPlatform}: - raise HTTPException(status_code=404, detail="Integration not found") - - return _integration_response(integration, organization_id, db) - - -@router.put("/{integration_id}", response_model=IntegrationResponse, operation_id="updateIntegration") -async def update_integration( - integration_id: UUID, - integration_update: IntegrationUpdate, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - Update an integration. - Requires at least WRITER role. - """ - integration = db.query(Integration).filter( - Integration.id == integration_id, - Integration.organization_id == organization_id - ).first() - - if not integration: - raise HTTPException(status_code=404, detail="Integration not found") - - if integration_update.name is not None: - integration.name = integration_update.name - - if integration_update.api_key is not None: - platform_value = integration.platform.value if hasattr(integration.platform, 'value') else integration.platform - if platform_value.lower() == IntegrationPlatform.SMALLEST.value: - _validate_smallest_connection(integration_update.api_key) - integration.last_tested_at = datetime.now(timezone.utc) - integration.api_key = encrypt_api_key(integration_update.api_key) - - if integration_update.public_key is not None: - integration.public_key = integration_update.public_key - - if integration_update.is_active is not None: - integration.is_active = integration_update.is_active - - if integration_update.routing_mode is not None: - integration.routing_mode = integration_update.routing_mode.value - - db.commit() - db.refresh(integration) - - return _integration_response(integration, organization_id, db) - - -@router.delete("/{integration_id}", operation_id="deleteIntegration") -async def delete_integration( - integration_id: UUID, - force: bool = Query(False, description="Force delete and unlink all agents using this integration"), - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - Delete an integration. Returns 409 if agents are using it unless force=true. - Requires at least WRITER role. - """ - integration = db.query(Integration).filter( - Integration.id == integration_id, - Integration.organization_id == organization_id - ).first() - - if not integration: - raise HTTPException(status_code=404, detail="Integration not found") - - agents_count = db.query(Agent).filter( - Agent.voice_ai_integration_id == integration_id, - Agent.organization_id == organization_id, - ).count() - - dependencies = {} - if agents_count > 0: - dependencies["agents"] = agents_count - - if dependencies and not force: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail={ - "message": f"Cannot delete integration. It is used by {agents_count} agent(s).", - "dependencies": dependencies, - "hint": "Use force=true to delete this integration and unlink all agents.", - }, - ) - - if dependencies: - db.query(Agent).filter( - Agent.voice_ai_integration_id == integration_id, - Agent.organization_id == organization_id, - ).update( - {Agent.voice_ai_integration_id: None, Agent.voice_ai_agent_id: None}, - synchronize_session=False, - ) - - was_default = bool(integration.is_default) - platform_value = ( - integration.platform.value - if hasattr(integration.platform, "value") - else integration.platform - ) - db.delete(integration) - db.flush() - - # If we just removed the default credential, promote the next active - # row (most recently updated) so resolution-by-default keeps working. - if was_default: - from sqlalchemy import func, desc - replacement = ( - db.query(Integration) - .filter( - Integration.organization_id == organization_id, - func.lower(Integration.platform) == platform_value.lower(), - Integration.is_active.is_(True), - ) - .order_by(desc(Integration.updated_at), desc(Integration.created_at)) - .first() - ) - if replacement: - replacement.is_default = True - - db.commit() - - if dependencies: - return JSONResponse( - status_code=200, - content={ - "message": "Integration deleted and agents unlinked successfully.", - "deleted": dependencies, - }, - ) - - return JSONResponse(status_code=204, content=None) - -@router.get("/{integration_id}/api-key") -async def get_integration_api_key( - integration_id: UUID, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - Get the decrypted API key for an integration. - This endpoint is used for client-side operations like web calls. - Requires at least READER role. - """ - integration = db.query(Integration).filter( - Integration.id == integration_id, - Integration.organization_id == organization_id, - Integration.is_active == True - ).first() - - if not integration: - raise HTTPException(status_code=404, detail="Integration not found or inactive") - - # Only allow for Retell and Vapi platforms (for web calls) - if integration.platform not in [IntegrationPlatform.RETELL, IntegrationPlatform.VAPI]: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="API key retrieval is only available for Retell and Vapi integrations" - ) - - try: - decrypted_api_key = decrypt_api_key(integration.api_key) - return { - "api_key": decrypted_api_key, - "public_key": integration.public_key - } - except Exception as e: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to decrypt API key: {str(e)}" - ) - - -@router.post( - "/{integration_id}/preview-agent-prompt", - response_model=PreviewIntegrationAgentPromptResponse, - operation_id="previewIntegrationAgentPrompt", -) -async def preview_integration_agent_prompt( - integration_id: UUID, - body: PreviewIntegrationAgentPromptRequest, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """Fetch a provider agent prompt before an EfficientAI agent exists.""" - integration = db.query(Integration).filter( - Integration.id == integration_id, - Integration.organization_id == organization_id, - Integration.is_active == True, - ).first() - - if not integration: - raise HTTPException(status_code=404, detail="Integration not found or inactive") - - if integration.platform not in [ - IntegrationPlatform.RETELL, - IntegrationPlatform.VAPI, - IntegrationPlatform.ELEVENLABS, - IntegrationPlatform.SMALLEST, - ]: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - f"Integration platform {integration.platform.value} is not supported for prompt preview. " - "Only Retell, Vapi, ElevenLabs, and Smallest are supported." - ), - ) - - try: - from app.services.voice_providers.prompt_sync import fetch_provider_prompt - - prompt = fetch_provider_prompt(integration, body.voice_ai_agent_id) - except Exception as e: - raise HTTPException( - status_code=502, - detail=f"Failed to fetch prompt from provider: {str(e)}", - ) - - if not isinstance(prompt, str) or not prompt.strip(): - raise HTTPException( - status_code=422, - detail="Provider returned no prompt. Verify the external agent has a system prompt configured.", - ) - - return PreviewIntegrationAgentPromptResponse(provider_prompt=prompt) +""" +Integrations API Routes +Manage integrations with external voice AI platforms (Retell, Vapi, etc.) +""" +from fastapi import APIRouter, Depends, HTTPException, status, Query +from fastapi.responses import JSONResponse +from sqlalchemy.orm import Session +from datetime import datetime, timezone +from typing import List, Optional +from uuid import UUID +from loguru import logger + +from app.dependencies import get_db, get_organization_id, get_api_key +from app.models.database import Integration, IntegrationPlatform, Agent +from app.models.schemas import ( + IntegrationCreate, IntegrationUpdate, IntegrationResponse, + PreviewIntegrationAgentPromptRequest, PreviewIntegrationAgentPromptResponse, + ListIntegrationVoiceAgentsResponse, IntegrationVoiceAgentListItem, +) +from app.core.encryption import encrypt_api_key, decrypt_api_key +from app.services.credentials.resolver import clear_other_defaults +from app.services.voice_providers import get_voice_provider +from app.services.ai.llm_gateway import get_credential_effective_routing_label + +router = APIRouter(prefix="/integrations", tags=["Integrations"]) + + +def _integration_response( + integration: Integration, + organization_id: UUID, + db: Session, +) -> IntegrationResponse: + effective_routing = get_credential_effective_routing_label( + organization_id, + db, + integration, + ) + response = IntegrationResponse.model_validate(integration) + return response.model_copy(update={"effective_routing": effective_routing}) + + +def _validate_smallest_connection(raw_api_key: str): + """Validate a Smallest key via GET /atoms/v1/user.""" + try: + provider_class = get_voice_provider(IntegrationPlatform.SMALLEST.value) + provider = provider_class(api_key=raw_api_key) + provider.test_connection() + if hasattr(provider, "get_user_details"): + return provider.get_user_details() + return None + except Exception as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Smallest API key validation failed: {str(e)}" + ) + + +@router.post("", response_model=IntegrationResponse, status_code=status.HTTP_201_CREATED, operation_id="createIntegration") +async def create_integration( + integration_data: IntegrationCreate, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """Create a new credential row for a voice AI platform. + + Multiple credentials per platform are now supported. The first row + created for a given (org, platform) automatically becomes the + default; subsequent rows can be promoted via + ``POST /integrations/{id}/set-default``. ``integration_data.is_default`` + can also be set explicitly to mark the new row as the default at + creation time. + Requires at least WRITER role. + """ + from sqlalchemy import func + platform_value = integration_data.platform.value if hasattr(integration_data.platform, 'value') else integration_data.platform + + user_details = None + if platform_value.lower() == IntegrationPlatform.SMALLEST.value: + user_details = _validate_smallest_connection(integration_data.api_key) + + encrypted_api_key = encrypt_api_key(integration_data.api_key) + integration_name = integration_data.name + if not integration_name and isinstance(user_details, dict): + email = user_details.get("email") or user_details.get("userEmail") + if email: + integration_name = f"Smallest ({email})" + + existing_default = db.query(Integration).filter( + Integration.organization_id == organization_id, + func.lower(Integration.platform) == platform_value.lower(), + Integration.is_default.is_(True), + Integration.is_active.is_(True), + ).first() + + requested_default = bool(integration_data.is_default) + will_be_default = requested_default or existing_default is None + insert_as_default = will_be_default and existing_default is None + + integration = Integration( + organization_id=organization_id, + platform=platform_value, + name=integration_name, + api_key=encrypted_api_key, + public_key=integration_data.public_key, + is_active=True, + is_default=insert_as_default, + routing_mode=integration_data.routing_mode.value, + last_tested_at=datetime.now(timezone.utc) if user_details is not None else None, + ) + + db.add(integration) + db.flush() + + if will_be_default: + clear_other_defaults( + Integration, + db, + organization_id, + keep_id=integration.id, + provider_field="platform", + provider_value=platform_value, + ) + integration.is_default = True + + db.commit() + db.refresh(integration) + + return _integration_response(integration, organization_id, db) + + +@router.post( + "/{integration_id}/set-default", + response_model=IntegrationResponse, + operation_id="setDefaultIntegration", +) +async def set_default_integration( + integration_id: UUID, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Mark this integration as the default for its (org, platform). + + Atomically clears the default flag on every other row for the same + (org, platform) so the partial unique index in migration 028 holds. + """ + integration = db.query(Integration).filter( + Integration.id == integration_id, + Integration.organization_id == organization_id, + ).first() + + if not integration: + raise HTTPException(status_code=404, detail="Integration not found") + + if not integration.is_active: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Cannot mark an inactive integration as default", + ) + + platform_value = ( + integration.platform.value + if hasattr(integration.platform, "value") + else integration.platform + ) + + clear_other_defaults( + Integration, + db, + organization_id, + keep_id=integration.id, + provider_field="platform", + provider_value=platform_value, + ) + integration.is_default = True + db.commit() + db.refresh(integration) + return _integration_response(integration, organization_id, db) + + +@router.get("", response_model=List[IntegrationResponse], operation_id="listIntegrations") +async def list_integrations( + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + List all integrations for the organization. + Requires at least READER role. + """ + integrations = db.query(Integration).filter( + Integration.organization_id == organization_id + ).order_by(Integration.created_at.desc()).all() + + valid_platforms = {p.value for p in IntegrationPlatform} + filtered_integrations: List[Integration] = [] + for integration in integrations: + raw_platform = ( + integration.platform.value + if hasattr(integration.platform, "value") + else str(integration.platform).lower() + ) + if raw_platform in valid_platforms: + filtered_integrations.append(integration) + else: + logger.warning( + "Skipping integration {} with invalid platform '{}'", + integration.id, + integration.platform, + ) + + return [ + _integration_response(integration, organization_id, db) + for integration in filtered_integrations + ] + + +@router.get("/{integration_id}", response_model=IntegrationResponse) +async def get_integration( + integration_id: UUID, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + Get a specific integration. + Requires at least READER role. + """ + integration = db.query(Integration).filter( + Integration.id == integration_id, + Integration.organization_id == organization_id + ).first() + + if not integration: + raise HTTPException(status_code=404, detail="Integration not found") + + raw_platform = ( + integration.platform.value + if hasattr(integration.platform, "value") + else str(integration.platform).lower() + ) + if raw_platform not in {p.value for p in IntegrationPlatform}: + raise HTTPException(status_code=404, detail="Integration not found") + + return _integration_response(integration, organization_id, db) + + +@router.put("/{integration_id}", response_model=IntegrationResponse, operation_id="updateIntegration") +async def update_integration( + integration_id: UUID, + integration_update: IntegrationUpdate, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + Update an integration. + Requires at least WRITER role. + """ + integration = db.query(Integration).filter( + Integration.id == integration_id, + Integration.organization_id == organization_id + ).first() + + if not integration: + raise HTTPException(status_code=404, detail="Integration not found") + + if integration_update.name is not None: + integration.name = integration_update.name + + if integration_update.api_key is not None: + platform_value = integration.platform.value if hasattr(integration.platform, 'value') else integration.platform + if platform_value.lower() == IntegrationPlatform.SMALLEST.value: + _validate_smallest_connection(integration_update.api_key) + integration.last_tested_at = datetime.now(timezone.utc) + integration.api_key = encrypt_api_key(integration_update.api_key) + + if integration_update.public_key is not None: + integration.public_key = integration_update.public_key + + if integration_update.is_active is not None: + integration.is_active = integration_update.is_active + + if integration_update.routing_mode is not None: + integration.routing_mode = integration_update.routing_mode.value + + db.commit() + db.refresh(integration) + + return _integration_response(integration, organization_id, db) + + +@router.delete("/{integration_id}", operation_id="deleteIntegration") +async def delete_integration( + integration_id: UUID, + force: bool = Query(False, description="Force delete and unlink all agents using this integration"), + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + Delete an integration. Returns 409 if agents are using it unless force=true. + Requires at least WRITER role. + """ + integration = db.query(Integration).filter( + Integration.id == integration_id, + Integration.organization_id == organization_id + ).first() + + if not integration: + raise HTTPException(status_code=404, detail="Integration not found") + + agents_count = db.query(Agent).filter( + Agent.voice_ai_integration_id == integration_id, + Agent.organization_id == organization_id, + ).count() + + dependencies = {} + if agents_count > 0: + dependencies["agents"] = agents_count + + if dependencies and not force: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={ + "message": f"Cannot delete integration. It is used by {agents_count} agent(s).", + "dependencies": dependencies, + "hint": "Use force=true to delete this integration and unlink all agents.", + }, + ) + + if dependencies: + db.query(Agent).filter( + Agent.voice_ai_integration_id == integration_id, + Agent.organization_id == organization_id, + ).update( + {Agent.voice_ai_integration_id: None, Agent.voice_ai_agent_id: None}, + synchronize_session=False, + ) + + was_default = bool(integration.is_default) + platform_value = ( + integration.platform.value + if hasattr(integration.platform, "value") + else integration.platform + ) + db.delete(integration) + db.flush() + + # If we just removed the default credential, promote the next active + # row (most recently updated) so resolution-by-default keeps working. + if was_default: + from sqlalchemy import func, desc + replacement = ( + db.query(Integration) + .filter( + Integration.organization_id == organization_id, + func.lower(Integration.platform) == platform_value.lower(), + Integration.is_active.is_(True), + ) + .order_by(desc(Integration.updated_at), desc(Integration.created_at)) + .first() + ) + if replacement: + replacement.is_default = True + + db.commit() + + if dependencies: + return JSONResponse( + status_code=200, + content={ + "message": "Integration deleted and agents unlinked successfully.", + "deleted": dependencies, + }, + ) + + return JSONResponse(status_code=204, content=None) + +@router.get("/{integration_id}/api-key") +async def get_integration_api_key( + integration_id: UUID, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + Get the decrypted API key for an integration. + This endpoint is used for client-side operations like web calls. + Requires at least READER role. + """ + integration = db.query(Integration).filter( + Integration.id == integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True + ).first() + + if not integration: + raise HTTPException(status_code=404, detail="Integration not found or inactive") + + # Only allow for Retell and Vapi platforms (for web calls) + if integration.platform not in [IntegrationPlatform.RETELL, IntegrationPlatform.VAPI]: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="API key retrieval is only available for Retell and Vapi integrations" + ) + + try: + decrypted_api_key = decrypt_api_key(integration.api_key) + return { + "api_key": decrypted_api_key, + "public_key": integration.public_key + } + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to decrypt API key: {str(e)}" + ) + + +@router.post( + "/{integration_id}/preview-agent-prompt", + response_model=PreviewIntegrationAgentPromptResponse, + operation_id="previewIntegrationAgentPrompt", +) +async def preview_integration_agent_prompt( + integration_id: UUID, + body: PreviewIntegrationAgentPromptRequest, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Fetch a provider agent prompt before an EfficientAI agent exists.""" + integration = db.query(Integration).filter( + Integration.id == integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ).first() + + if not integration: + raise HTTPException(status_code=404, detail="Integration not found or inactive") + + if integration.platform not in [ + IntegrationPlatform.RETELL, + IntegrationPlatform.VAPI, + IntegrationPlatform.ELEVENLABS, + IntegrationPlatform.SMALLEST, + ]: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Integration platform {integration.platform.value} is not supported for prompt preview. " + "Only Retell, Vapi, ElevenLabs, and Smallest are supported." + ), + ) + + try: + from app.services.voice_providers.prompt_sync import fetch_provider_prompt + + prompt = fetch_provider_prompt(integration, body.voice_ai_agent_id) + except Exception as e: + raise HTTPException( + status_code=502, + detail=f"Failed to fetch prompt from provider: {str(e)}", + ) + + if not isinstance(prompt, str) or not prompt.strip(): + raise HTTPException( + status_code=422, + detail="Provider returned no prompt. Verify the external agent has a system prompt configured.", + ) + + return PreviewIntegrationAgentPromptResponse(provider_prompt=prompt) + + +@router.get( + "/{integration_id}/voice-agents", + response_model=ListIntegrationVoiceAgentsResponse, + operation_id="listIntegrationVoiceAgents", +) +async def list_integration_voice_agents_route( + integration_id: UUID, + refresh: bool = Query(False), + search: Optional[str] = Query(None), + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """List remote voice agents for an org integration (cached ~60s).""" + integration = db.query(Integration).filter( + Integration.id == integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True, + ).first() + + if not integration: + raise HTTPException(status_code=404, detail="Integration not found or inactive") + + if integration.platform not in [ + IntegrationPlatform.RETELL, + IntegrationPlatform.VAPI, + IntegrationPlatform.ELEVENLABS, + IntegrationPlatform.SMALLEST, + ]: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Integration platform {integration.platform.value} is not supported for agent listing." + ), + ) + + try: + from app.services.voice_providers.voice_agent_catalog import list_integration_voice_agents + + result = list_integration_voice_agents( + integration, + refresh=refresh, + search=search, + ) + except Exception as e: + raise HTTPException( + status_code=502, + detail=f"Failed to list agents from provider: {str(e)}", + ) + + return ListIntegrationVoiceAgentsResponse( + agents=[IntegrationVoiceAgentListItem(**row) for row in result.agents], + platform=result.platform, + cached=result.cached, + truncated=result.truncated, + list_supported=result.list_supported, + message=result.message, + ) diff --git a/app/api/v1/routes/metric_studio.py b/app/api/v1/routes/metric_studio.py index aeb155f3..10b407ca 100644 --- a/app/api/v1/routes/metric_studio.py +++ b/app/api/v1/routes/metric_studio.py @@ -25,6 +25,7 @@ MetricStudioRunRetryRequest, ) from app.services.metric_studio.metric_selection import expand_studio_metric_selection +from app.services.metric_studio.run_rollup import rollup_metric_studio_run from app.services.metric_studio.source_resolver import resolve_source router = APIRouter(prefix="/metric-studio", tags=["metric-studio"]) @@ -117,26 +118,7 @@ def _serialize_result( def _rollup_run_status(db: Session, run: MetricStudioRun) -> None: - results = ( - db.query(MetricStudioRunResult) - .filter(MetricStudioRunResult.run_id == run.id) - .all() - ) - completed = sum(1 for r in results if r.status == "completed") - failed = sum(1 for r in results if r.status == "failed") - pending = sum(1 for r in results if r.status in {"pending", "running"}) - run.completed_items = completed - run.failed_items = failed - if pending: - run.status = "running" - elif failed and completed: - run.status = "partial" - elif failed: - run.status = "failed" - else: - run.status = "completed" - run.finished_at = datetime.now(timezone.utc) - db.flush() + rollup_metric_studio_run(db, run, emit_flexprice=True, commit=False) @router.post( @@ -297,6 +279,8 @@ def get_metric_studio_run( ) if not run: raise HTTPException(status_code=404, detail="Studio run not found.") + rollup_metric_studio_run(db, run, emit_flexprice=True, commit=True) + db.refresh(run) return _serialize_run(run) diff --git a/app/api/v1/routes/observability.py b/app/api/v1/routes/observability.py index 4c26a549..611e7866 100644 --- a/app/api/v1/routes/observability.py +++ b/app/api/v1/routes/observability.py @@ -5,15 +5,11 @@ from typing import Any, Dict, List, Optional, Union from uuid import UUID -from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, status +from fastapi import APIRouter, Depends, HTTPException, status from pydantic import BaseModel, ConfigDict from sqlalchemy.orm import Session from app.dependencies import get_api_key, get_db, get_organization_id, get_workspace_id -from app.services.billing.flexprice_service import ( - record_observability_call_evaluated, - record_observability_call_ingested, -) from app.models.database import ( Agent, APIKey, CallRecording, CallRecordingStatus, CallRecordingSource, Evaluator, EvaluatorResult, EvaluatorResultStatus, Scenario, Workspace, @@ -243,13 +239,6 @@ def _upsert_call_recording( response = _serialize_call_recording(call_recording, include_data=True, agent=agent_obj) response["action"] = action - if action == "created": - record_observability_call_ingested( - organization_id, - call_recording.call_short_id, - workspace_id=workspace_id, - provider=provider_platform, - ) return response @@ -765,7 +754,6 @@ def _messages_to_speaker_segments(messages: List[Dict[str, Any]]) -> List[Dict[s async def evaluate_call( call_short_id: str, payload: EvaluateCallPayload, - background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), @@ -877,13 +865,6 @@ async def evaluate_call( except Exception: pass - background_tasks.add_task( - record_observability_call_evaluated, - organization_id, - call_short_id, - workspace_id=workspace_id, - ) - return { "evaluator_result_id": str(evaluator_result.id), "result_id": evaluator_result.result_id, diff --git a/app/api/v1/routes/personas.py b/app/api/v1/routes/personas.py index de07c715..131c7e14 100644 --- a/app/api/v1/routes/personas.py +++ b/app/api/v1/routes/personas.py @@ -3,12 +3,12 @@ CRUD for TTS provider-tied voice personas, voice-options catalog, and custom voice management (ungated). """ -from fastapi import APIRouter, Depends, HTTPException, status, Body, Query +from fastapi import APIRouter, Depends, HTTPException, status, Body, Query, BackgroundTasks from fastapi.responses import JSONResponse from sqlalchemy.orm import Session from sqlalchemy.exc import IntegrityError, SQLAlchemyError from typing import List, Optional, Dict, Any -from uuid import UUID +from uuid import UUID, uuid4 from pydantic import BaseModel from loguru import logger @@ -427,6 +427,7 @@ async def get_agent_prompt_sources( ) async def generate_persona_prompt( data: GeneratePersonaPromptRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), @@ -442,19 +443,38 @@ async def generate_persona_prompt( provider_enum, model_str = _get_llm_provider_and_model( organization_id, db, data.provider, data.model, data.credential_id ) + from app.services.usage.context import ( + llm_usage_context, + usage_context_for_persona_generation, + ) + try: - result = generate_persona_prompt_from_agent( - agent, - source=data.source, - persona_name=data.persona_name, - persona_gender=data.persona_gender, - additional_context=data.additional_context, - llm_provider=provider_enum, - llm_model=model_str, - organization_id=organization_id, - db=db, - llm_config=data.llm_config, - credential_id=data.credential_id, + with llm_usage_context( + usage_context_for_persona_generation(agent, workspace_id=workspace_id) + ): + result = generate_persona_prompt_from_agent( + agent, + source=data.source, + persona_name=data.persona_name, + persona_gender=data.persona_gender, + additional_context=data.additional_context, + llm_provider=provider_enum, + llm_model=model_str, + organization_id=organization_id, + db=db, + llm_config=data.llm_config, + credential_id=data.credential_id, + ) + from app.services.billing.flexprice_service import record_persona_prompt_generated + + background_tasks.add_task( + record_persona_prompt_generated, + organization_id, + uuid4(), + workspace_id=workspace_id, + agent_id=agent.id, + model=result.model, + source=result.source_used, ) return GeneratePersonaPromptResponse( persona_prompt=result.persona_prompt, diff --git a/app/api/v1/routes/platform_admin.py b/app/api/v1/routes/platform_admin.py index f29630ff..433b1e3c 100644 --- a/app/api/v1/routes/platform_admin.py +++ b/app/api/v1/routes/platform_admin.py @@ -6,7 +6,8 @@ from typing import List, Optional from uuid import UUID -from fastapi import APIRouter, Depends, HTTPException, Query, status +from fastapi import APIRouter, Depends, Header, HTTPException, Query, status +from jose import JWTError from pydantic import BaseModel, EmailStr, Field from sqlalchemy import func from sqlalchemy.orm import Session @@ -16,6 +17,7 @@ create_platform_access_token, get_platform_admin, platform_admin_feature_enabled, + revoke_platform_access_token, ) from app.core.auth.refresh_tokens import revoke_all_user_refresh_tokens from app.core.password import hash_password, validate_password_strength, verify_password @@ -184,6 +186,26 @@ def platform_me( return PlatformAdminSummary(id=str(principal.platform_admin_id), email=principal.email) +def _extract_bearer(authorization: Optional[str]) -> Optional[str]: + if not authorization: + return None + scheme, _, token = authorization.partition(" ") + if scheme.lower() != "bearer" or not token.strip(): + return None + return token.strip() + + +@router.post("/auth/logout") +def platform_logout( + authorization: Optional[str] = Header(None, alias="Authorization"), + principal: PlatformAdminPrincipal = Depends(get_platform_admin), +) -> dict: + bearer = _extract_bearer(authorization) + if bearer: + revoke_platform_access_token(bearer) + return {"success": True, "admin_id": str(principal.platform_admin_id)} + + @router.get("/organizations", response_model=OrganizationListResponse) def list_organizations( offset: int = Query(0, ge=0), diff --git a/app/api/v1/routes/playground.py b/app/api/v1/routes/playground.py index 31b1a3c2..9204b296 100644 --- a/app/api/v1/routes/playground.py +++ b/app/api/v1/routes/playground.py @@ -1,1856 +1,2140 @@ -""" -Playground API Routes -API endpoints for testing voice agents in the playground -""" -from fastapi import APIRouter, Depends, HTTPException, status, Body, BackgroundTasks, Form, File, UploadFile -from fastapi.responses import StreamingResponse -from sqlalchemy.orm import Session -from typing import Dict, Any, Optional, List -from uuid import UUID -from pydantic import BaseModel -from loguru import logger -import random -import json -import uuid as _uuid -from datetime import datetime - -from app.services.billing.flexprice_service import ( - record_playground_web_call_started, - record_playground_websocket_session_started, -) -from app.database import get_db -from app.dependencies import get_organization_id, get_workspace_id, get_api_key -from app.models.database import ( - Agent, - Integration, - IntegrationPlatform, - CallRecording, - CallRecordingStatus, - CallRecordingSource, - EvaluatorResult, - EvaluatorResultStatus, - VoiceBundle, - AIProvider, - ModelProvider, -) -from app.core.encryption import decrypt_api_key -from app.services.voice_providers import get_voice_provider -from app.services.evaluators.evaluator_result_call_data import slim_call_data_for_evaluator_result -from app.utils.call_recordings import generate_unique_call_short_id - -from app.services.evaluators.call_data_transcript import extract_transcript_from_call_data - -router = APIRouter(prefix="/playground", tags=["playground"]) -def generate_unique_result_id(db: Session) -> str: - """Generate a unique 6-digit result ID for EvaluatorResult.""" - max_attempts = 100 - for _ in range(max_attempts): - candidate_id = f"{random.randint(100000, 999999)}" - existing = db.query(EvaluatorResult).filter(EvaluatorResult.result_id == candidate_id).first() - if not existing: - return candidate_id - raise ValueError("Failed to generate unique result ID") - - -def poll_call_metrics( - call_recording_id: UUID, - provider_call_id: str, - provider_platform: str, - integration_api_key: str, - max_attempts: int = 60, - poll_interval: int = 5 -): - """ - Background task to poll for call metrics from the provider. - After call is complete, creates an EvaluatorResult and triggers metric evaluation. - - Args: - call_recording_id: The CallRecording database ID - provider_call_id: The provider's call_id (e.g., Retell call_id) - provider_platform: The provider platform (e.g., "retell") - integration_api_key: The decrypted API key for the provider - max_attempts: Maximum number of polling attempts - poll_interval: Seconds between polling attempts - """ - import time - from app.database import SessionLocal - from app.services.voice_providers import get_voice_provider - - db = SessionLocal() - call_complete = False - call_metrics = None - - try: - call_recording = db.query(CallRecording).filter(CallRecording.id == call_recording_id).first() - if not call_recording: - return - - if not provider_call_id or provider_call_id == "None": - return - - # Get the appropriate voice provider - try: - provider_class = get_voice_provider(provider_platform) - provider = provider_class(api_key=integration_api_key) - except ValueError: - return - - # Poll for call metrics - for attempt in range(max_attempts): - try: - # Wait before polling (except first attempt) - if attempt > 0: - time.sleep(poll_interval) - - # Retrieve call metrics - if hasattr(provider, "retrieve_call_metrics"): - call_metrics = provider.retrieve_call_metrics(provider_call_id) - else: - # For other providers, implement similar method - continue - - # Update the call recording with metrics - call_recording.call_data = call_metrics - call_recording.status = CallRecordingStatus.UPDATED - db.commit() - db.refresh(call_recording) - - # Check if call is complete (supports raw + normalized payloads) - call_status = ( - call_metrics.get("call_status") - or call_metrics.get("status") - or "" - ) - call_status = str(call_status).lower() - end_timestamp = call_metrics.get("end_timestamp") or call_metrics.get("endedAt") - - # If call is complete, stop polling - if end_timestamp or call_status in ["ended", "completed", "failed", "end-of-call-report", "done"]: - call_complete = True - break - - except Exception as e: - # Log error but continue polling - logger.warning(f"[Poll Call Metrics] Error on attempt {attempt + 1}: {str(e)}") - # If it's a 404 or similar, the call might not exist yet, continue polling - continue - - # After polling is complete, create EvaluatorResult and trigger evaluation - if call_complete and call_metrics and call_recording.agent_id: - try: - logger.info(f"[Poll Call Metrics] Call complete, creating EvaluatorResult for call {provider_call_id}") - - # Extract transcript and speaker segments from call_data - transcript_text, _ = extract_transcript_from_call_data( - call_metrics, - provider_platform - ) - - if not transcript_text: - logger.warning(f"[Poll Call Metrics] No transcript found in call_data for call {provider_call_id}") - - # Get agent info for naming - agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() - result_name = f"Voice AI Call - {agent.name}" if agent else "Voice AI Call" - - # Calculate duration - duration_seconds = call_metrics.get("duration_seconds", 0) - if not duration_seconds: - # Try to calculate from timestamps - start_ts = call_metrics.get("start_timestamp") or call_metrics.get("startedAt") - end_ts = call_metrics.get("end_timestamp") or call_metrics.get("endedAt") - if start_ts and end_ts: - try: - from dateutil import parser - start_time = parser.parse(start_ts) - end_time = parser.parse(end_ts) - duration_seconds = (end_time - start_time).total_seconds() - except Exception: - pass - - # Download call audio from provider and upload to S3 - audio_s3_key = None - try: - import requests as _http - import uuid as _uuid - from app.services.storage.s3_service import s3_service - - recording_urls = call_metrics.get("recording_urls", {}) - audio_bytes = None - plat = provider_platform.lower() - - if plat == "elevenlabs": - audio_url = recording_urls.get("conversation_audio") - if audio_url: - resp = _http.get(audio_url, headers={"xi-api-key": integration_api_key}, timeout=120) - if resp.status_code == 200: - audio_bytes = resp.content - elif plat == "retell": - audio_url = call_metrics.get("recording_url") - if audio_url: - resp = _http.get(audio_url, timeout=120) - if resp.status_code == 200: - audio_bytes = resp.content - elif plat == "vapi": - artifact = call_metrics.get("artifact", {}) - recording = artifact.get("recording", {}) if isinstance(artifact, dict) else {} - mono_recording = recording.get("mono", {}) if isinstance(recording, dict) else {} - audio_url = ( - call_metrics.get("recordingUrl") - or call_metrics.get("stereoRecordingUrl") - or artifact.get("recordingUrl") - or artifact.get("stereoRecordingUrl") - or mono_recording.get("combinedUrl") - or recording_urls.get("combined_url") - or recording_urls.get("stereo_url") - ) - if audio_url: - vapi_headers = {"Authorization": f"Bearer {integration_api_key}"} - resp = _http.get(audio_url, headers=vapi_headers, timeout=120) - if resp.status_code == 200: - audio_bytes = resp.content - - if audio_bytes: - content_type = getattr(resp, "headers", {}).get("content-type", "audio/mpeg") - ext = "wav" if "wav" in content_type else "mp3" - org_id = str(call_recording.organization_id) - audio_s3_key = f"audio/organizations/{org_id}/agentPlayground/{provider_call_id}/{_uuid.uuid4()}.{ext}" - s3_service.upload_file_by_key(audio_bytes, audio_s3_key, content_type=content_type) - logger.info(f"[Poll Call Metrics] Uploaded call audio to S3: {audio_s3_key} ({len(audio_bytes)} bytes)") - else: - logger.warning(f"[Poll Call Metrics] Could not download audio for call {provider_call_id}") - except Exception as audio_err: - logger.warning(f"[Poll Call Metrics] Audio download/upload failed: {audio_err}") - - # Generate unique result ID - result_id = generate_unique_result_id(db) - - # Create EvaluatorResult. Background-task path: inherit the - # workspace from the call recording rather than the header - # (this task runs without a request context). - evaluator_result = EvaluatorResult( - result_id=result_id, - organization_id=call_recording.organization_id, - workspace_id=call_recording.workspace_id, - evaluator_id=None, - agent_id=call_recording.agent_id, - persona_id=None, - scenario_id=None, - name=result_name, - duration_seconds=duration_seconds, - status=EvaluatorResultStatus.QUEUED.value, - audio_s3_key=audio_s3_key, - transcription=transcript_text, - provider_call_id=provider_call_id, - provider_platform=provider_platform, - call_data=call_metrics, - ) - db.add(evaluator_result) - db.commit() - db.refresh(evaluator_result) - - # Link the EvaluatorResult to CallRecording - call_recording.evaluator_result_id = evaluator_result.id - db.commit() - - logger.info(f"[Poll Call Metrics] Created EvaluatorResult {result_id} for call {provider_call_id}") - - # Trigger Celery task to process evaluator result (run metrics evaluation) - try: - from app.workers.celery_app import process_evaluator_result_task - task = process_evaluator_result_task.delay(str(evaluator_result.id)) - evaluator_result.celery_task_id = task.id - db.commit() - logger.info(f"[Poll Call Metrics] Triggered evaluation task {task.id} for result {result_id}") - except Exception as task_error: - logger.error(f"[Poll Call Metrics] Failed to trigger Celery task: {task_error}") - # Mark the result as failed if we can't trigger the task - # But keep the transcript available for manual review - - except Exception as e: - logger.error(f"[Poll Call Metrics] Error creating EvaluatorResult: {str(e)}", exc_info=True) - - finally: - db.close() - - -class CallRecordingUpdate(BaseModel): - """Schema for updating a call recording.""" - provider_call_id: str - - -@router.put("/call-recordings/{call_short_id}", response_model=Dict[str, Any]) -async def update_call_recording( - call_short_id: str, - update_data: CallRecordingUpdate, - background_tasks: BackgroundTasks, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - Update a call recording within the active workspace, typically to set the provider_call_id. - Triggers polling. - """ - call_recording = db.query(CallRecording).filter( - CallRecording.call_short_id == call_short_id, - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - ).first() - - if not call_recording: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call recording not found" - ) - - call_recording.provider_call_id = update_data.provider_call_id - db.commit() - db.refresh(call_recording) - - # Trigger polling if we have all info - if call_recording.provider_platform: - # Get integration api key - agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() - if agent and agent.voice_ai_integration_id: - integration = db.query(Integration).filter( - Integration.id == agent.voice_ai_integration_id - ).first() - - if integration: - try: - decrypted_api_key = decrypt_api_key(integration.api_key) - background_tasks.add_task( - poll_call_metrics, - call_recording.id, - call_recording.provider_call_id, - call_recording.provider_platform, - decrypted_api_key - ) - except: - pass - - return { - "message": "Call recording updated", - "provider_call_id": call_recording.provider_call_id - } - - -class WebCallCreate(BaseModel): - """Schema for creating a web call.""" - agent_id: str # UUID of the agent in our system - metadata: Optional[Dict[str, Any]] = None - retell_llm_dynamic_variables: Optional[Dict[str, Any]] = None - custom_sip_headers: Optional[Dict[str, str]] = None - - -@router.post("/web-call", response_model=Dict[str, Any]) -async def create_web_call( - web_call_data: WebCallCreate, - background_tasks: BackgroundTasks, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - Create a web call with a voice AI agent within the active workspace. - The agent must belong to the same workspace; the resulting call recording - is stamped with the same workspace_id. - """ - try: - # Get the agent (scoped to the active workspace) - agent_uuid = UUID(web_call_data.agent_id) - agent = db.query(Agent).filter( - Agent.id == agent_uuid, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ).first() - - if not agent: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Agent not found" - ) - - # Check if agent has voice AI integration - if not agent.voice_ai_integration_id or not agent.voice_ai_agent_id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Agent is not configured with a voice AI integration" - ) - - # Check if agent has web call enabled - if agent.call_medium != "web_call": - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Agent is not configured for web calls" - ) - - # Get the integration - integration = db.query(Integration).filter( - Integration.id == agent.voice_ai_integration_id, - Integration.organization_id == organization_id, - Integration.is_active == True - ).first() - - if not integration: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Integration not found or inactive" - ) - - # Decrypt API key - try: - decrypted_api_key = decrypt_api_key(integration.api_key) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to decrypt API key: {str(e)}" - ) - - # Get the appropriate voice provider - try: - provider_class = get_voice_provider(integration.platform) - - platform_value = integration.platform.value if hasattr(integration.platform, 'value') else integration.platform - if platform_value.lower() == "vapi": - provider = provider_class(api_key=decrypted_api_key, public_key=integration.public_key) - else: - provider = provider_class(api_key=decrypted_api_key) - except ValueError as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(e) - ) - - # Create the web call - try: - # Verify agent_id is present - if not agent.voice_ai_agent_id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Agent does not have a voice_ai_agent_id configured" - ) - - print(f"[Playground] Creating web call - Agent ID: {agent.id}, Retell Agent ID: {agent.voice_ai_agent_id}, Platform: {integration.platform}") - - # Build call parameters based on provider - call_params = { - "agent_id": agent.voice_ai_agent_id, - } - - # Add optional parameters if provided - if web_call_data.metadata: - call_params["metadata"] = web_call_data.metadata - if web_call_data.retell_llm_dynamic_variables: - call_params["retell_llm_dynamic_variables"] = web_call_data.retell_llm_dynamic_variables - - # Note: custom_sip_headers is not supported by Retell, but may be supported by other providers - # For now, we'll skip it for Retell. Other providers can handle it in their implementation. - if integration.platform != "retell" and web_call_data.custom_sip_headers: - call_params["custom_sip_headers"] = web_call_data.custom_sip_headers - - platform_value = ( - integration.platform.value - if hasattr(integration.platform, "value") - else integration.platform - ) - plat_lower = str(platform_value).lower() - - # Vapi Web SDK creates the call in the browser; server-side /call/web - # would spawn a second call that never receives the user's microphone. - if plat_lower == "vapi": - from app.services.voice_providers.vapi import VAPI_SAMPLE_RATE - - web_call_response = { - "call_type": "web_call", - "agent_id": agent.voice_ai_agent_id, - "metadata": web_call_data.metadata or {}, - "sample_rate": VAPI_SAMPLE_RATE, - "client_sdk_creates_call": True, - } - provider_call_id = None - else: - web_call_response = provider.create_web_call(**call_params) - provider_call_id = web_call_response.get("call_id") - - call_short_id = generate_unique_call_short_id(db) - call_recording = CallRecording( - organization_id=organization_id, - workspace_id=workspace_id, - call_short_id=call_short_id, - status=CallRecordingStatus.PENDING, - source=CallRecordingSource.PLAYGROUND, - call_data=web_call_response, # Store initial response - provider_call_id=provider_call_id, - provider_platform=integration.platform, - agent_id=agent.id - ) - db.add(call_recording) - db.commit() - db.refresh(call_recording) - - background_tasks.add_task( - record_playground_web_call_started, - organization_id, - call_short_id, - workspace_id=workspace_id, - agent_id=agent.id, - ) - - # Start background task to poll for call metrics - # Note: We need to pass the decrypted API key, but we should be careful with security - # For now, we'll pass it to the background task - # In production, you might want to store it temporarily or use a different approach - if provider_call_id: - background_tasks.add_task( - poll_call_metrics, - call_recording.id, - provider_call_id, - integration.platform, - decrypted_api_key - ) - - # Add call_short_id to response for frontend - response = web_call_response.copy() - response["call_short_id"] = call_short_id - - if plat_lower == "vapi" and integration.public_key: - response["public_key"] = integration.public_key - - if plat_lower == "elevenlabs": - response["signed_url"] = web_call_response.get("signed_url") - - return response - except Exception as e: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to create web call: {str(e)}" - ) - - except ValueError as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Invalid agent ID: {str(e)}" - ) - except Exception as e: - if isinstance(e, HTTPException): - raise e - print(f"[Create Web Call] Error: {str(e)}") - import traceback - traceback.print_exc() - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Internal server error: {str(e)}" - ) - - -@router.get("/call-recordings", response_model=List[Dict[str, Any]]) -async def list_call_recordings( - skip: int = 0, - limit: int = 100, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - List playground call recordings in the active workspace. - Includes evaluator_result_id, evaluation status, and metric_scores if evaluation has been run. - """ - call_recordings = db.query(CallRecording).filter( - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.PLAYGROUND, - ).order_by(CallRecording.created_at.desc()).offset(skip).limit(limit).all() - - # Get evaluator result info for all linked results - result_ids = [cr.evaluator_result_id for cr in call_recordings if cr.evaluator_result_id] - result_info = {} - if result_ids: - results = db.query(EvaluatorResult).filter(EvaluatorResult.id.in_(result_ids)).all() - result_info = { - str(r.id): { - "status": r.status, - "metric_scores": r.metric_scores, - "result_id": r.result_id, - "name": r.name, - } - for r in results - } - - agent_ids = [cr.agent_id for cr in call_recordings if cr.agent_id] - agents_by_id = {} - if agent_ids: - agents = db.query(Agent).filter(Agent.id.in_(agent_ids)).all() - agents_by_id = {a.id: a for a in agents} - - def _display_name(cr: CallRecording) -> str: - linked = result_info.get(str(cr.evaluator_result_id), {}) if cr.evaluator_result_id else {} - if linked.get("name"): - return linked["name"] - agent = agents_by_id.get(cr.agent_id) if cr.agent_id else None - if agent and agent.name: - return agent.name - return cr.call_short_id - - return [ - { - "id": str(cr.id), - "call_short_id": cr.call_short_id, - "display_name": _display_name(cr), - "status": cr.status if cr.status else None, - "provider_platform": cr.provider_platform, - "provider_call_id": cr.provider_call_id, - "agent_id": str(cr.agent_id) if cr.agent_id else None, - "evaluator_result_id": str(cr.evaluator_result_id) if cr.evaluator_result_id else None, - "evaluation_status": result_info.get(str(cr.evaluator_result_id), {}).get("status") if cr.evaluator_result_id else None, - "metric_scores": result_info.get(str(cr.evaluator_result_id), {}).get("metric_scores") if cr.evaluator_result_id else None, - "result_id": result_info.get(str(cr.evaluator_result_id), {}).get("result_id") if cr.evaluator_result_id else None, - "created_at": cr.created_at.isoformat() if cr.created_at else None, - "updated_at": cr.updated_at.isoformat() if cr.updated_at else None, - } - for cr in call_recordings - ] - - -@router.post("/custom-websocket-sessions", response_model=Dict[str, Any]) -async def create_custom_websocket_session( - background_tasks: BackgroundTasks, - agent_id: str = Form(...), - websocket_url: str = Form(...), - transcript_entries: str = Form("[]"), - started_at: Optional[str] = Form(None), - ended_at: Optional[str] = Form(None), - audio_file: Optional[UploadFile] = File(None), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """ - Save a custom websocket test session for later evaluation (scoped to the active workspace). - Stores transcript in call_data and uploads optional audio recording to S3. - """ - try: - agent_uuid = UUID(agent_id) - except ValueError: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid agent_id") - - agent = db.query(Agent).filter( - Agent.id == agent_uuid, - Agent.organization_id == organization_id, - Agent.workspace_id == workspace_id, - ).first() - if not agent: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Agent not found") - - try: - parsed_entries = json.loads(transcript_entries or "[]") - except json.JSONDecodeError: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid transcript_entries payload") - - normalized_entries = [] - for entry in parsed_entries: - if not isinstance(entry, dict): - continue - role = entry.get("role") - content = (entry.get("content") or "").strip() - if role not in {"user", "agent"} or not content: - continue - normalized_entries.append( - { - "role": role, - "content": content, - "timestamp": entry.get("timestamp") or ended_at or started_at, - } - ) - - if not normalized_entries: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No transcript entries provided") - - transcript_text = "\n".join( - [f"{'User' if item['role'] == 'user' else 'Agent'}: {item['content']}" for item in normalized_entries] - ) - - call_short_id = generate_unique_call_short_id(db) - audio_s3_key = None - if audio_file: - from app.services.storage.s3_service import s3_service - - audio_bytes = await audio_file.read() - if audio_bytes: - if not s3_service.is_enabled(): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="S3 storage is not configured. Save without recording or enable storage.", - ) - - filename = audio_file.filename or "session.webm" - extension = filename.split(".")[-1].lower() if "." in filename else "webm" - content_type = audio_file.content_type or "audio/webm" - audio_s3_key = ( - f"audio/organizations/{organization_id}/agentPlayground/customWebsocket/" - f"{call_short_id}/{_uuid.uuid4()}.{extension}" - ) - s3_service.upload_file_by_key(audio_bytes, audio_s3_key, content_type=content_type) - - duration_seconds = 0 - if started_at and ended_at: - try: - from datetime import datetime as _dt - t_start = _dt.fromisoformat(started_at.replace("Z", "+00:00")) - t_end = _dt.fromisoformat(ended_at.replace("Z", "+00:00")) - duration_seconds = max(0, (t_end - t_start).total_seconds()) - except Exception: - duration_seconds = 0 - - speaker_segments = [] - for entry in normalized_entries: - speaker = "user" if entry.get("role") == "user" else "assistant" - speaker_segments.append({ - "speaker": speaker, - "text": entry.get("content", ""), - "start": 0, - "end": 0, - }) - - call_data = { - "source": "custom_websocket", - "websocket_url": websocket_url, - "messages": normalized_entries, - "transcript": transcript_text, - "speaker_segments": speaker_segments, - "recording_s3_key": audio_s3_key, - "started_at": started_at, - "ended_at": ended_at, - "duration_seconds": duration_seconds, - } - - call_recording = CallRecording( - organization_id=organization_id, - workspace_id=workspace_id, - call_short_id=call_short_id, - status=CallRecordingStatus.UPDATED, - source=CallRecordingSource.PLAYGROUND, - call_data=call_data, - provider_call_id=f"custom_{call_short_id}", - provider_platform="custom_websocket", - agent_id=agent.id, - ) - db.add(call_recording) - db.commit() - db.refresh(call_recording) - - background_tasks.add_task( - record_playground_websocket_session_started, - organization_id, - call_short_id, - workspace_id=workspace_id, - ) - - return { - "message": "Custom websocket session saved", - "call_short_id": call_short_id, - "audio_s3_key": audio_s3_key, - "evaluator_result_id": None, - } - - -@router.post("/custom-websocket-sessions/{call_short_id}/evaluate", response_model=Dict[str, Any]) -async def evaluate_custom_websocket_session( - call_short_id: str, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """ - Queue evaluation for a saved custom websocket test session in the active workspace. - """ - call_recording = db.query(CallRecording).filter( - CallRecording.call_short_id == call_short_id, - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.PLAYGROUND, - CallRecording.provider_platform == "custom_websocket", - ).first() - - if not call_recording: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Custom websocket session not found") - - call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} - transcript_text = (call_data.get("transcript") or "").strip() - if not transcript_text: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No transcript found for evaluation") - - existing_result = None - if call_recording.evaluator_result_id: - existing_result = db.query(EvaluatorResult).filter( - EvaluatorResult.id == call_recording.evaluator_result_id, - EvaluatorResult.organization_id == organization_id, - EvaluatorResult.workspace_id == workspace_id, - ).first() - - speaker_segments = call_data.get("speaker_segments") or [] - - if existing_result: - evaluator_result = existing_result - evaluator_result.status = EvaluatorResultStatus.QUEUED.value - evaluator_result.error_message = None - evaluator_result.metric_scores = None - evaluator_result.celery_task_id = None - evaluator_result.transcription = transcript_text - evaluator_result.speaker_segments = speaker_segments - evaluator_result.audio_s3_key = call_data.get("recording_s3_key") - evaluator_result.call_data = slim_call_data_for_evaluator_result(call_data) - evaluator_result.duration_seconds = call_data.get("duration_seconds", 0) - db.commit() - db.refresh(evaluator_result) - else: - agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() - result_id = generate_unique_result_id(db) - result_name = f"Custom WebSocket Test - {agent.name}" if agent else "Custom WebSocket Test" - - evaluator_result = EvaluatorResult( - result_id=result_id, - organization_id=organization_id, - workspace_id=workspace_id, - evaluator_id=None, - agent_id=call_recording.agent_id, - persona_id=None, - scenario_id=None, - name=result_name, - duration_seconds=call_data.get("duration_seconds", 0), - status=EvaluatorResultStatus.QUEUED.value, - audio_s3_key=call_data.get("recording_s3_key"), - transcription=transcript_text, - speaker_segments=speaker_segments, - provider_call_id=call_recording.provider_call_id, - provider_platform="custom_websocket", - call_data=slim_call_data_for_evaluator_result(call_data), - ) - db.add(evaluator_result) - db.commit() - db.refresh(evaluator_result) - - call_recording.evaluator_result_id = evaluator_result.id - db.commit() - - try: - from app.workers.celery_app import process_evaluator_result_task - task = process_evaluator_result_task.delay(str(evaluator_result.id)) - evaluator_result.celery_task_id = task.id - db.commit() - except Exception as e: - logger.error(f"[Custom WebSocket] Failed to trigger evaluation worker: {e}") - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to trigger evaluation worker") - - return { - "message": "Evaluation queued", - "evaluator_result_id": str(evaluator_result.id), - "result_id": evaluator_result.result_id, - "task_id": task.id, - } - - -@router.get("/call-recordings/{call_short_id}", response_model=Dict[str, Any]) -async def get_call_recording( - call_short_id: str, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - Get a specific playground call recording within the active workspace. - Returns the full JSON data stored for the call and evaluation information. - """ - call_recording = db.query(CallRecording).filter( - CallRecording.call_short_id == call_short_id, - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.PLAYGROUND, - ).first() - - if not call_recording: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call recording not found" - ) - - # Get evaluator result info if available - evaluation_info = None - if call_recording.evaluator_result_id: - evaluator_result = db.query(EvaluatorResult).filter( - EvaluatorResult.id == call_recording.evaluator_result_id - ).first() - if evaluator_result: - evaluation_info = { - "id": str(evaluator_result.id), - "result_id": evaluator_result.result_id, - "status": evaluator_result.status, - "metric_scores": evaluator_result.metric_scores, - "transcription": evaluator_result.transcription, - } - - return { - "id": str(call_recording.id), - "call_short_id": call_recording.call_short_id, - "status": call_recording.status if call_recording.status else None, - "provider_platform": call_recording.provider_platform, - "provider_call_id": call_recording.provider_call_id, - "agent_id": str(call_recording.agent_id) if call_recording.agent_id else None, - "evaluator_result_id": str(call_recording.evaluator_result_id) if call_recording.evaluator_result_id else None, - "evaluation": evaluation_info, - "call_data": call_recording.call_data, # Full JSON blob - "created_at": call_recording.created_at.isoformat() if call_recording.created_at else None, - "updated_at": call_recording.updated_at.isoformat() if call_recording.updated_at else None, - } - - -@router.post("/call-recordings/{call_short_id}/refresh", response_model=Dict[str, Any]) -async def refresh_call_recording( - call_short_id: str, - background_tasks: BackgroundTasks, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - Manually trigger a refresh of call metrics for a specific call recording in the active workspace. - """ - call_recording = db.query(CallRecording).filter( - CallRecording.call_short_id == call_short_id, - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.PLAYGROUND, - ).first() - - if not call_recording: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call recording not found" - ) - - if not call_recording.provider_call_id or not call_recording.provider_platform: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Call recording does not have provider information" - ) - - # Get the integration to get the API key - agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() - if not agent or not agent.voice_ai_integration_id: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Agent or integration not found" - ) - - integration = db.query(Integration).filter( - Integration.id == agent.voice_ai_integration_id, - Integration.organization_id == organization_id - ).first() - - if not integration: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Integration not found" - ) - - try: - decrypted_api_key = decrypt_api_key(integration.api_key) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to decrypt API key: {str(e)}" - ) - - # Start background task to poll for call metrics - background_tasks.add_task( - poll_call_metrics, - call_recording.id, - call_recording.provider_call_id, - call_recording.provider_platform, - decrypted_api_key - ) - - return {"message": "Call recording refresh initiated"} - - -@router.delete("/call-recordings/{call_short_id}", response_model=Dict[str, Any]) -async def delete_call_recording( - call_short_id: str, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - Delete a call recording within the active workspace. - """ - call_recording = db.query(CallRecording).filter( - CallRecording.call_short_id == call_short_id, - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - ).first() - - if not call_recording: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Call recording not found" - ) - - db.delete(call_recording) - db.commit() - - return {"message": "Call recording deleted successfully"} - - -@router.post("/call-recordings/{call_short_id}/re-evaluate", response_model=Dict[str, Any]) -async def re_evaluate_call_recording( - call_short_id: str, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """ - Re-evaluate a playground call recording within the active workspace. - - Reuses the S3 audio if it was already downloaded during the first - evaluation. If no audio exists in S3, downloads from the provider, - uploads, then triggers the worker for both conversation quality (LLM) - and audio quality (acoustic / AI voice) metrics. - """ - import requests as http_requests - import uuid as _uuid - from app.services.storage.s3_service import s3_service - - call_recording = db.query(CallRecording).filter( - CallRecording.call_short_id == call_short_id, - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.PLAYGROUND, - ).first() - - if not call_recording: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call recording not found") - - if not call_recording.provider_call_id or not call_recording.provider_platform: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Call recording has no provider data") - - call_data = call_recording.call_data or {} - platform = (call_recording.provider_platform or "").lower() - - # --- Check if S3 audio already exists from a previous evaluation ------- - existing_result = None - if call_recording.evaluator_result_id: - existing_result = db.query(EvaluatorResult).filter( - EvaluatorResult.id == call_recording.evaluator_result_id, - ).first() - - audio_s3_key = existing_result.audio_s3_key if existing_result and existing_result.audio_s3_key else None - - # --- If no S3 audio, download from provider and upload ----------------- - if not audio_s3_key: - agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() - if not agent or not agent.voice_ai_integration_id: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Agent or integration not found") - - integration = db.query(Integration).filter( - Integration.id == agent.voice_ai_integration_id, - Integration.organization_id == organization_id, - ).first() - if not integration: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found") - - decrypted_key = decrypt_api_key(integration.api_key) - - def _download_audio_from_payload(payload: Dict[str, Any]): - payload_urls = payload.get("recording_urls", {}) if isinstance(payload, dict) else {} - artifact = payload.get("artifact", {}) if isinstance(payload, dict) else {} - recording = artifact.get("recording", {}) if isinstance(artifact, dict) else {} - mono_recording = recording.get("mono", {}) if isinstance(recording, dict) else {} - url = None - headers = None - if platform == "elevenlabs": - url = payload_urls.get("conversation_audio") - headers = {"xi-api-key": decrypted_key} - elif platform == "retell": - url = payload.get("recording_url") - elif platform == "vapi": - url = ( - payload.get("recordingUrl") - or payload.get("stereoRecordingUrl") - or artifact.get("recordingUrl") - or artifact.get("stereoRecordingUrl") - or mono_recording.get("combinedUrl") - or payload_urls.get("combined_url") - or payload_urls.get("stereo_url") - ) - elif platform == "smallest": - url = ( - payload.get("recording_url") - or payload.get("recordingUrl") - or payload_urls.get("combined_url") - or payload_urls.get("conversation_audio") - ) - if not url: - return None, None - response = http_requests.get(url, headers=headers, timeout=120) - if response.status_code != 200: - return None, response - return response.content, response - - audio_bytes, resp = _download_audio_from_payload(call_data) - - # Retry once with fresh provider payload (new signed URL) using provider_call_id - if not audio_bytes and call_recording.provider_call_id: - try: - provider_class = get_voice_provider(platform) - provider_kwargs: Dict[str, Any] = {"api_key": decrypted_key} - if platform == "vapi" and integration.public_key: - provider_kwargs["public_key"] = integration.public_key - provider = provider_class(**provider_kwargs) - if hasattr(provider, "retrieve_call_metrics"): - refreshed_call_data = provider.retrieve_call_metrics(call_recording.provider_call_id) - if isinstance(refreshed_call_data, dict) and refreshed_call_data: - call_data = refreshed_call_data - call_recording.call_data = refreshed_call_data - db.commit() - logger.info(f"[Re-evaluate] Refreshed provider call data for call {call_recording.provider_call_id}") - audio_bytes, resp = _download_audio_from_payload(call_data) - except Exception as refresh_err: - logger.warning(f"[Re-evaluate] Provider audio URL refresh failed: {refresh_err}") - - if not audio_bytes: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Could not download audio from provider. The recording may not be available yet.", - ) - - content_type = getattr(resp, "headers", {}).get("content-type", "audio/mpeg") - ext = "wav" if "wav" in content_type else "mp3" - org_id = str(organization_id) - audio_s3_key = f"audio/organizations/{org_id}/agentPlayground/{call_recording.provider_call_id}/{_uuid.uuid4()}.{ext}" - try: - s3_service.upload_file_by_key(audio_bytes, audio_s3_key, content_type=content_type) - logger.info(f"[Re-evaluate] Uploaded audio to S3: {audio_s3_key} ({len(audio_bytes)} bytes)") - except Exception as e: - logger.error(f"[Re-evaluate] S3 upload failed: {e}") - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to store audio: {str(e)}") - else: - logger.info(f"[Re-evaluate] Reusing existing S3 audio: {audio_s3_key}") - - # --- Extract transcript from existing call data ------------------------ - transcript_text, _ = extract_transcript_from_call_data(call_data, platform) - - # --- Create or reset EvaluatorResult ----------------------------------- - if existing_result: - existing_result.status = EvaluatorResultStatus.QUEUED.value - existing_result.audio_s3_key = audio_s3_key - # Metadata only; transcript is on existing_result.transcription. - existing_result.call_data = ( - slim_call_data_for_evaluator_result(call_data) - if isinstance(call_data, dict) - else existing_result.call_data - ) - existing_result.transcription = transcript_text or existing_result.transcription - existing_result.metric_scores = None - existing_result.error_message = None - existing_result.celery_task_id = None - db.commit() - db.refresh(existing_result) - evaluator_result = existing_result - logger.info(f"[Re-evaluate] Reset existing EvaluatorResult {evaluator_result.result_id}") - else: - agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() - result_id = generate_unique_result_id(db) - duration_seconds = call_data.get("duration_seconds", 0) - if not duration_seconds: - start_ts = call_data.get("start_timestamp") or call_data.get("startedAt") - end_ts = call_data.get("end_timestamp") or call_data.get("endedAt") - if start_ts and end_ts: - try: - from dateutil import parser - duration_seconds = (parser.parse(end_ts) - parser.parse(start_ts)).total_seconds() - except Exception: - duration_seconds = 0 - result_name = f"Voice AI Call - {agent.name}" if agent else "Voice AI Call" - - evaluator_result = EvaluatorResult( - result_id=result_id, - organization_id=call_recording.organization_id, - workspace_id=call_recording.workspace_id, - evaluator_id=None, - agent_id=call_recording.agent_id, - persona_id=None, - scenario_id=None, - name=result_name, - duration_seconds=duration_seconds, - status=EvaluatorResultStatus.QUEUED.value, - audio_s3_key=audio_s3_key, - transcription=transcript_text, - provider_call_id=call_recording.provider_call_id, - provider_platform=platform, - call_data=slim_call_data_for_evaluator_result(call_data), - ) - db.add(evaluator_result) - db.commit() - db.refresh(evaluator_result) - - call_recording.evaluator_result_id = evaluator_result.id - db.commit() - logger.info(f"[Re-evaluate] Created new EvaluatorResult {evaluator_result.result_id}") - - # --- Trigger the worker ------------------------------------------------ - try: - from app.workers.celery_app import process_evaluator_result_task - task = process_evaluator_result_task.delay(str(evaluator_result.id)) - evaluator_result.celery_task_id = task.id - db.commit() - logger.info(f"[Re-evaluate] Triggered evaluation task {task.id} for result {evaluator_result.result_id}") - except Exception as e: - logger.error(f"[Re-evaluate] Failed to trigger Celery task: {e}") - raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to trigger evaluation worker") - - return { - "message": "Re-evaluation started", - "evaluator_result_id": str(evaluator_result.id), - "result_id": evaluator_result.result_id, - "audio_s3_key": audio_s3_key, - "task_id": task.id, - } - - -@router.get("/call-recordings/{call_short_id}/audio") -async def stream_call_audio( - call_short_id: str, - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """ - Proxy endpoint to stream playground call recording audio in the active workspace. - Required for providers like ElevenLabs whose audio URLs need auth headers. - """ - import requests as http_requests - - call_recording = db.query(CallRecording).filter( - CallRecording.call_short_id == call_short_id, - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - CallRecording.source == CallRecordingSource.PLAYGROUND, - ).first() - - if not call_recording: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call recording not found") - - call_data = call_recording.call_data or {} - recording_urls = call_data.get("recording_urls", {}) - platform = (call_recording.provider_platform or "").lower() - - # For Retell / Vapi / Smallest the URL is public – redirect directly - if platform in ("retell", "vapi", "smallest"): - artifact = call_data.get("artifact", {}) - recording = artifact.get("recording", {}) if isinstance(artifact, dict) else {} - mono_recording = recording.get("mono", {}) if isinstance(recording, dict) else {} - url = ( - call_data.get("recordingUrl") - or call_data.get("stereoRecordingUrl") - or artifact.get("recordingUrl") - or artifact.get("stereoRecordingUrl") - or mono_recording.get("combinedUrl") - or recording_urls.get("combined_url") - or recording_urls.get("stereo_url") - or call_data.get("recording_url") - or recording_urls.get("conversation_audio") - ) - if not url: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No recording URL available") - from fastapi.responses import RedirectResponse - return RedirectResponse(url) - - # ElevenLabs requires API key header – proxy the stream - if platform == "elevenlabs": - audio_url = recording_urls.get("conversation_audio") - if not audio_url: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No recording URL available") - - agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() - if not agent or not agent.voice_ai_integration_id: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Agent or integration not found") - - integration = db.query(Integration).filter( - Integration.id == agent.voice_ai_integration_id, - Integration.organization_id == organization_id, - ).first() - if not integration: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found") - - decrypted_key = decrypt_api_key(integration.api_key) - - upstream = http_requests.get( - audio_url, - headers={"xi-api-key": decrypted_key}, - stream=True, - timeout=60, - ) - if upstream.status_code != 200: - raise HTTPException( - status_code=upstream.status_code, - detail=f"ElevenLabs audio fetch failed ({upstream.status_code})", - ) - - content_type = upstream.headers.get("content-type", "audio/mpeg") - - return StreamingResponse( - upstream.iter_content(chunk_size=8192), - media_type=content_type, - headers={ - "Content-Disposition": f'inline; filename="call_{call_short_id}.mp3"', - }, - ) - - # Custom WebSocket sessions store audio in S3 - if platform == "custom_websocket": - s3_key = call_data.get("recording_s3_key") - if not s3_key: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No recording available for this session") - - from app.services.storage.s3_service import s3_service - if not s3_service.is_enabled(): - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="S3 storage is not configured") - - try: - audio_bytes = s3_service.download_file_by_key(s3_key) - except Exception: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Audio file not found in storage") - if not audio_bytes: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Audio file not found in storage") - - extension = s3_key.rsplit(".", 1)[-1].lower() if "." in s3_key else "webm" - content_type_map = {"webm": "audio/webm", "mp3": "audio/mpeg", "wav": "audio/wav", "ogg": "audio/ogg"} - content_type = content_type_map.get(extension, "audio/webm") - - from io import BytesIO - return StreamingResponse( - BytesIO(audio_bytes), - media_type=content_type, - headers={ - "Content-Disposition": f'inline; filename="call_{call_short_id}.{extension}"', - }, - ) - - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Audio not supported for platform: {platform}") - - -# --------------------------------------------------------------------------- -# STT config resolution helper -# --------------------------------------------------------------------------- - -_STT_ENV_KEYS = { - "deepgram": "DEEPGRAM_API_KEY", - "openai": "OPENAI_API_KEY", - "elevenlabs": "ELEVENLABS_API_KEY", - "sarvam": "SARVAM_API_KEY", -} - -_STT_DEFAULT_MODELS = { - "deepgram": "nova-2", - "openai": "whisper-1", - "elevenlabs": "scribe_v2", - "sarvam": "saaras:v3", -} - - -def _resolve_agent_stt_config( - agent_id: str, organization_id: UUID, db: Session -) -> tuple: - """Resolve STT provider, model, and API key for an agent. - - Lookup chain: Agent -> VoiceBundle -> stt_provider/stt_model - -> AIProvider (by org + provider name) or Integration or env var fallback. - - Returns (stt_provider, stt_model, api_key). Any element may be None. - """ - import os - from sqlalchemy import func - - agent = db.query(Agent).filter( - Agent.id == agent_id, - Agent.organization_id == organization_id, - ).first() - if not agent or not agent.voice_bundle_id: - return None, None, None - - voice_bundle = db.query(VoiceBundle).filter( - VoiceBundle.id == agent.voice_bundle_id, - VoiceBundle.organization_id == organization_id, - ).first() - if not voice_bundle or not voice_bundle.stt_provider: - return None, None, None - - stt_provider = ( - voice_bundle.stt_provider.value - if hasattr(voice_bundle.stt_provider, "value") - else str(voice_bundle.stt_provider) - ).lower() - stt_model = ( - getattr(voice_bundle, "stt_model", None) - or _STT_DEFAULT_MODELS.get(stt_provider) - ) - - # 1) AIProvider - api_key = None - ai_prov = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - AIProvider.provider == stt_provider, - AIProvider.is_active == True, - ).first() - if not ai_prov: - ai_prov = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - func.lower(AIProvider.provider) == stt_provider, - AIProvider.is_active == True, - ).first() - if ai_prov: - try: - api_key = decrypt_api_key(ai_prov.api_key) - except Exception: - pass - - # 2) Integration fallback - if not api_key: - _platform_map = { - "deepgram": "deepgram", - "elevenlabs": "elevenlabs", - "sarvam": "sarvam", - } - plat_value = _platform_map.get(stt_provider) - if plat_value: - integ = db.query(Integration).filter( - Integration.organization_id == organization_id, - func.lower(Integration.platform) == plat_value, - Integration.is_active == True, - ).first() - if integ: - try: - api_key = decrypt_api_key(integ.api_key) - except Exception: - pass - - # 3) Env var fallback - if not api_key: - env_key = _STT_ENV_KEYS.get(stt_provider) - if env_key: - api_key = os.getenv(env_key) - - return stt_provider, stt_model, api_key - - -# --------------------------------------------------------------------------- -# GET /playground/agents/{agent_id}/stt-config -# --------------------------------------------------------------------------- - -@router.get("/agents/{agent_id}/stt-config", response_model=Dict[str, Any]) -async def get_agent_stt_config( - agent_id: str, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """Check whether an agent has STT configured via its voice bundle.""" - stt_provider, stt_model, stt_api_key = _resolve_agent_stt_config( - agent_id, organization_id, db - ) - if stt_provider and stt_api_key: - return {"available": True, "provider": stt_provider, "model": stt_model} - if stt_provider and not stt_api_key: - return { - "available": False, - "reason": f"STT provider '{stt_provider}' is configured but no API key was found", - } - return {"available": False, "reason": "No voice bundle with STT configured for this agent"} - - -# --------------------------------------------------------------------------- -# POST /playground/transcribe-turn -# --------------------------------------------------------------------------- - -@router.post("/transcribe-turn", response_model=Dict[str, Any]) -async def transcribe_turn( - agent_id: str = Form(...), - channel: str = Form(...), - audio_file: UploadFile = File(...), - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """Transcribe a single conversation turn (user or agent audio).""" - import tempfile - import os - - if channel not in ("user", "agent"): - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="channel must be 'user' or 'agent'", - ) - - stt_provider, stt_model, stt_api_key = _resolve_agent_stt_config( - agent_id, organization_id, db - ) - if not stt_provider or not stt_api_key: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="STT is not configured for this agent (missing provider or API key)", - ) - - audio_bytes = await audio_file.read() - if not audio_bytes or len(audio_bytes) < 100: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Audio file is empty or too small", - ) - - tmp_path = None - try: - suffix = ".wav" - if audio_file.filename and "." in audio_file.filename: - suffix = "." + audio_file.filename.rsplit(".", 1)[-1].lower() - with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: - tmp.write(audio_bytes) - tmp_path = tmp.name - - from app.services.ai.stt_clients import ( - transcribe_openai, - transcribe_deepgram, - transcribe_elevenlabs, - transcribe_sarvam, - ) - - if stt_provider == "deepgram": - result = transcribe_deepgram(tmp_path, stt_model, stt_api_key) - elif stt_provider == "openai": - result = transcribe_openai(tmp_path, stt_model, stt_api_key) - elif stt_provider == "elevenlabs": - result = transcribe_elevenlabs(tmp_path, stt_model, stt_api_key) - elif stt_provider == "sarvam": - result = transcribe_sarvam(tmp_path, stt_model, stt_api_key) - else: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unsupported STT provider: {stt_provider}", - ) - - transcript_text = (result.get("text") or "").strip() - return {"transcript": transcript_text, "channel": channel} - - except HTTPException: - raise - except Exception as e: - logger.error(f"[transcribe-turn] STT failed: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Transcription failed: {str(e)}", - ) - finally: - if tmp_path and os.path.exists(tmp_path): - os.unlink(tmp_path) - - -# --------------------------------------------------------------------------- -# POST /playground/summarize-transcript -# --------------------------------------------------------------------------- - -# Fallback defaults if the caller provides no agent context AND the voice -# bundle doesn't specify a model. The providers referenced here must exist in -# ``ModelProvider`` and be reachable via LiteLLM. -_SUMMARY_LLM_FALLBACK: List[tuple] = [ - (ModelProvider.OPENAI, "gpt-4o-mini"), - (ModelProvider.GOOGLE, "gemini-2.5-flash"), - (ModelProvider.ANTHROPIC, "claude-haiku-4.5"), -] - - -class SummarizeTranscriptRequest(BaseModel): - transcript: Optional[str] = None - entries: Optional[List[Dict[str, Any]]] = None - call_short_id: Optional[str] = None - agent_id: Optional[str] = None - # When true, ignore any cached summary on the CallRecording and regenerate. - force: Optional[bool] = False - - -def _coerce_model_provider(provider_str: str) -> Optional[ModelProvider]: - """Safely convert a string like ``"openai"`` to the matching enum.""" - if not provider_str: - return None - want = provider_str.strip().lower() - for m in ModelProvider: - if m.value.lower() == want: - return m - return None - - -def _resolve_agent_llm_config( - agent_id: Optional[UUID], organization_id: UUID, db: Session -) -> tuple: - """Resolve the LLM provider + model for an agent via its VoiceBundle. - - Returns ``(ModelProvider | None, model_name | None)``. - """ - if not agent_id: - return None, None - - agent = db.query(Agent).filter( - Agent.id == agent_id, - Agent.organization_id == organization_id, - ).first() - if not agent or not agent.voice_bundle_id: - return None, None - - voice_bundle = db.query(VoiceBundle).filter( - VoiceBundle.id == agent.voice_bundle_id, - VoiceBundle.organization_id == organization_id, - ).first() - if not voice_bundle or not voice_bundle.llm_provider: - return None, None - - provider_enum = _coerce_model_provider(voice_bundle.llm_provider) - if not provider_enum: - return None, None - - return provider_enum, (voice_bundle.llm_model or None) - - -def _pick_fallback_llm(organization_id: UUID, db: Session) -> Optional[tuple]: - """Pick any (ModelProvider, default_model) that has an active AIProvider - row for the organization. Used only when the agent / voice bundle doesn't - specify an LLM. - """ - from sqlalchemy import func - - for provider_enum, default_model in _SUMMARY_LLM_FALLBACK: - provider_value = provider_enum.value - ai_prov = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - func.lower(AIProvider.provider) == provider_value.lower(), - AIProvider.is_active == True, - ).first() - if ai_prov: - return provider_enum, default_model - return None - - -def _format_entries_as_transcript(entries: List[Dict[str, Any]]) -> str: - """Turn a list of {role, content} entries into a plain-text transcript.""" - lines = [] - for e in entries or []: - if not isinstance(e, dict): - continue - role = (e.get("role") or "").strip().lower() - text = (e.get("content") or e.get("text") or "").strip() - if not text: - continue - label = "User" if role in ("user", "caller", "speaker 1") else "Agent" - lines.append(f"{label}: {text}") - return "\n".join(lines) - - -@router.post("/summarize-transcript", response_model=Dict[str, Any]) -async def summarize_transcript( - payload: SummarizeTranscriptRequest = Body(...), - organization_id: UUID = Depends(get_organization_id), - workspace_id: UUID = Depends(get_workspace_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - """Generate a short natural-language summary of a transcript using an LLM. - - Provider selection order: - 1. The voice bundle of ``agent_id`` (or the agent on ``call_short_id``). - 2. Any configured AIProvider matching the fallback preference list. - """ - from contextlib import nullcontext - - from app.services.ai.llm_service import llm_service - from app.services.usage.context import llm_usage_context, usage_context_for_agent - - transcript_text = (payload.transcript or "").strip() - if not transcript_text and payload.entries: - transcript_text = _format_entries_as_transcript(payload.entries) - - if not transcript_text: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="Provide either `transcript` text or non-empty `entries`.", - ) - - # Defensive cap on transcript size to protect token budgets. Keep the tail - # so the most recent exchange wins. - MAX_CHARS = 16000 - if len(transcript_text) > MAX_CHARS: - transcript_text = transcript_text[-MAX_CHARS:] - - # --- Resolve CallRecording (for caching) and agent context ------------- - call_rec: Optional[CallRecording] = None - if payload.call_short_id: - call_rec = db.query(CallRecording).filter( - CallRecording.call_short_id == payload.call_short_id, - CallRecording.organization_id == organization_id, - CallRecording.workspace_id == workspace_id, - ).first() - - # Cache hit: serve the previously generated summary without re-calling the LLM. - if call_rec and not payload.force and isinstance(call_rec.call_data, dict): - cached = call_rec.call_data.get("ai_summary") - if isinstance(cached, dict) and (cached.get("text") or "").strip(): - return { - "summary": cached.get("text", ""), - "provider": cached.get("provider", ""), - "model": cached.get("model", ""), - "source": cached.get("source", "voice_bundle"), - "cached": True, - "generated_at": cached.get("generated_at"), - "usage": {}, - } - - agent_uuid: Optional[UUID] = None - if payload.agent_id: - try: - agent_uuid = UUID(payload.agent_id) - except ValueError: - agent_uuid = None - - if not agent_uuid and call_rec and call_rec.agent_id: - agent_uuid = call_rec.agent_id - - # --- Pick LLM: prefer voice-bundle config, else fall back -------------- - llm_provider, llm_model = _resolve_agent_llm_config( - agent_uuid, organization_id, db - ) - source = "voice_bundle" - - if not llm_provider: - picked = _pick_fallback_llm(organization_id, db) - if not picked: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=( - "No LLM is configured. Either set an LLM on the agent's voice bundle, " - "or add an active AIProvider (OpenAI / Google / Anthropic) for this organization." - ), - ) - llm_provider, llm_model = picked - source = "org_fallback" - - if not llm_model: - # Voice bundle had a provider but no model; fill in a safe default. - defaults = { - ModelProvider.OPENAI: "gpt-4o-mini", - ModelProvider.GOOGLE: "gemini-2.5-flash", - ModelProvider.ANTHROPIC: "claude-haiku-4.5", - } - llm_model = defaults.get(llm_provider) or "gpt-4o-mini" - - messages = [ - { - "role": "system", - "content": ( - "You are an expert call analyst. Read the conversation transcript " - "the user provides and write a concise, neutral summary of what happened. " - "Focus on the caller's intent, what the agent did, any outcomes or " - "action items, and the overall tone. Respond in 2-4 plain-text " - "sentences only — no bullet points, no markdown, no preamble." - ), - }, - { - "role": "user", - "content": f"Conversation transcript:\n\n{transcript_text}", - }, - ] - - try: - usage_ctx = nullcontext() - if agent_uuid: - agent_row = db.query(Agent).filter( - Agent.id == agent_uuid, - Agent.organization_id == organization_id, - ).first() - if agent_row: - usage_ctx = llm_usage_context( - usage_context_for_agent(agent_row, workspace_id=workspace_id) - ) - - with usage_ctx: - result = llm_service.generate_response( - messages=messages, - llm_provider=llm_provider, - llm_model=llm_model, - organization_id=organization_id, - db=db, - temperature=0.3, - max_tokens=400, - ) - except Exception as e: - logger.error(f"[summarize-transcript] LLM call failed: {e}") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Summary generation failed: {str(e)}", - ) - - summary = (result.get("text") or "").strip() - if not summary: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="LLM returned an empty summary.", - ) - - generated_at = datetime.utcnow().isoformat() + "Z" - - # Persist on the CallRecording so subsequent loads don't re-invoke the LLM. - if call_rec is not None: - try: - # Reassign the dict so SQLAlchemy notices the JSON column changed. - existing = call_rec.call_data if isinstance(call_rec.call_data, dict) else {} - new_call_data = dict(existing) - new_call_data["ai_summary"] = { - "text": summary, - "provider": llm_provider.value, - "model": llm_model, - "source": source, - "generated_at": generated_at, - } - call_rec.call_data = new_call_data - db.commit() - except Exception as e: - # Non-fatal: return the generated summary even if persistence fails. - logger.warning(f"[summarize-transcript] Failed to cache summary: {e}") - db.rollback() - - return { - "summary": summary, - "provider": llm_provider.value, - "model": llm_model, - "source": source, - "cached": False, - "generated_at": generated_at, - "usage": result.get("usage", {}), - } - - -from app.core.auth.capabilities import SIM_MANAGE, SIM_VIEW -from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities - -apply_workspace_route_capabilities( - router, - view_capability=SIM_VIEW, - manage_capability=SIM_MANAGE, -) - +""" +Playground API Routes +API endpoints for testing voice agents in the playground +""" +from fastapi import APIRouter, Depends, HTTPException, status, Body, BackgroundTasks, Form, File, UploadFile, Query +from fastapi.responses import StreamingResponse +from sqlalchemy.orm import Session +from typing import Dict, Any, Optional, List +from uuid import UUID +from pydantic import BaseModel +from loguru import logger +import random +import json +import uuid as _uuid +from datetime import datetime + +from app.services.billing.flexprice_service import ( + record_playground_web_call_started, + record_playground_websocket_session_started, +) +from app.database import get_db +from app.dependencies import get_organization_id, get_workspace_id, get_api_key +from app.models.database import ( + Agent, + Integration, + IntegrationPlatform, + CallRecording, + CallRecordingStatus, + CallRecordingSource, + EvaluatorResult, + EvaluatorResultStatus, + VoiceBundle, + AIProvider, + ModelProvider, +) +from app.core.encryption import decrypt_api_key +from app.services.voice_providers import get_voice_provider +from app.services.evaluators.evaluator_result_call_data import slim_call_data_for_evaluator_result +from app.utils.call_recordings import generate_unique_call_short_id + +from app.services.evaluators.call_data_transcript import extract_transcript_from_call_data + +router = APIRouter(prefix="/playground", tags=["playground"]) + + +def _resolve_playground_audio_url( + call_data: dict[str, Any], + platform: str, + *, + stereo: bool = False, +) -> Optional[str]: + artifact = call_data.get("artifact", {}) if isinstance(call_data.get("artifact"), dict) else {} + recording = artifact.get("recording", {}) if isinstance(artifact.get("recording"), dict) else {} + mono_recording = recording.get("mono", {}) if isinstance(recording.get("mono"), dict) else {} + recording_urls = call_data.get("recording_urls", {}) if isinstance(call_data.get("recording_urls"), dict) else {} + + plat = (platform or "").lower() + if plat == "vapi": + from app.services.voice_providers.vapi_recording import extract_vapi_recording_url + + return extract_vapi_recording_url(call_data, stereo=stereo) + + return ( + call_data.get("recordingUrl") + or call_data.get("stereoRecordingUrl") + or artifact.get("recordingUrl") + or artifact.get("stereoRecordingUrl") + or mono_recording.get("combinedUrl") + or recording_urls.get("combined_url") + or recording_urls.get("stereo_url") + or call_data.get("recording_url") + or recording_urls.get("conversation_audio") + ) + + +def _refresh_call_recording_from_provider_sync( + db: Session, + call_recording: CallRecording, + organization_id: UUID, +) -> bool: + """Re-fetch provider call payload to refresh expiring presigned recording URLs.""" + from app.services.playground.post_call_processing import persist_provider_call_metrics + + if not call_recording.provider_call_id or not call_recording.provider_platform: + return False + + agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + if not agent or not agent.voice_ai_integration_id: + return False + + integration = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + ).first() + if not integration: + return False + + try: + provider_class = get_voice_provider(call_recording.provider_platform) + provider = provider_class(api_key=decrypt_api_key(integration.api_key)) + if not hasattr(provider, "retrieve_call_metrics"): + return False + metrics = provider.retrieve_call_metrics(call_recording.provider_call_id) + if not isinstance(metrics, dict): + return False + persist_provider_call_metrics(db, call_recording.id, metrics) + db.refresh(call_recording) + return True + except Exception as exc: + logger.warning( + "[Audio Proxy] Failed to refresh provider call data for %s: %s", + call_recording.call_short_id, + exc, + ) + return False + + +def generate_unique_result_id(db: Session) -> str: + """Generate a unique 6-digit result ID for EvaluatorResult.""" + max_attempts = 100 + for _ in range(max_attempts): + candidate_id = f"{random.randint(100000, 999999)}" + existing = db.query(EvaluatorResult).filter(EvaluatorResult.result_id == candidate_id).first() + if not existing: + return candidate_id + raise ValueError("Failed to generate unique result ID") + + +def sync_provider_call_metrics( + call_recording_id: UUID, + provider_call_id: str, + provider_platform: str, + integration_api_key: str, + max_attempts: int = 12, + poll_interval: int = 5, +): + """Re-fetch provider metrics and update stored call_data (no evaluator creation).""" + import time + + from app.database import SessionLocal + from app.services.playground.post_call_processing import ( + call_metrics_indicate_ended, + persist_provider_call_metrics, + provider_metrics_enriched, + ) + + db = SessionLocal() + try: + call_recording = db.query(CallRecording).filter(CallRecording.id == call_recording_id).first() + if not call_recording or not provider_call_id: + return + + try: + provider_class = get_voice_provider(provider_platform) + provider = provider_class(api_key=integration_api_key) + except ValueError: + return + + if not hasattr(provider, "retrieve_call_metrics"): + return + + call_metrics: dict[str, Any] | None = None + for attempt in range(max_attempts): + if attempt > 0: + time.sleep(poll_interval) + try: + call_metrics = provider.retrieve_call_metrics(provider_call_id) + if not isinstance(call_metrics, dict): + continue + persist_provider_call_metrics(db, call_recording_id, call_metrics) + if provider_metrics_enriched(provider_platform, call_metrics): + break + if call_metrics_indicate_ended(call_metrics) and attempt >= 3: + break + except Exception as exc: + logger.warning(f"[Sync Provider Metrics] attempt {attempt + 1} failed: {exc}") + finally: + db.close() + + +def poll_call_metrics( + call_recording_id: UUID, + provider_call_id: str, + provider_platform: str, + integration_api_key: str, + max_attempts: int = 60, + poll_interval: int = 5 +): + """ + Background task to poll for call metrics from the provider. + After call is complete, creates an EvaluatorResult and triggers metric evaluation. + + Args: + call_recording_id: The CallRecording database ID + provider_call_id: The provider's call_id (e.g., Retell call_id) + provider_platform: The provider platform (e.g., "retell") + integration_api_key: The decrypted API key for the provider + max_attempts: Maximum number of polling attempts + poll_interval: Seconds between polling attempts + """ + import time + from app.database import SessionLocal + from app.services.playground.post_call_processing import ( + call_metrics_indicate_ended, + merge_playground_call_data, + provider_metrics_enriched, + ) + from app.services.voice_providers import get_voice_provider + + db = SessionLocal() + call_complete = False + call_metrics = None + + try: + call_recording = db.query(CallRecording).filter(CallRecording.id == call_recording_id).first() + if not call_recording: + return + + if not provider_call_id or provider_call_id == "None": + return + + # Get the appropriate voice provider + try: + provider_class = get_voice_provider(provider_platform) + provider = provider_class(api_key=integration_api_key) + except ValueError: + return + + # Poll for call metrics + for attempt in range(max_attempts): + try: + # Wait before polling (except first attempt) + if attempt > 0: + time.sleep(poll_interval) + + # Retrieve call metrics + if hasattr(provider, "retrieve_call_metrics"): + call_metrics = provider.retrieve_call_metrics(provider_call_id) + else: + # For other providers, implement similar method + continue + + # Update the call recording with metrics (preserve usage-recorded flag) + prev_data = ( + call_recording.call_data + if isinstance(call_recording.call_data, dict) + else {} + ) + if isinstance(call_metrics, dict): + call_metrics = merge_playground_call_data(prev_data, call_metrics) + call_recording.call_data = call_metrics + call_recording.status = CallRecordingStatus.UPDATED + db.commit() + db.refresh(call_recording) + + # Keep polling after hangup until provider enriches analysis/latency payloads. + if call_metrics_indicate_ended(call_metrics): + call_complete = True + if provider_metrics_enriched(provider_platform, call_metrics): + break + + except Exception as e: + # Log error but continue polling + logger.warning(f"[Poll Call Metrics] Error on attempt {attempt + 1}: {str(e)}") + # If it's a 404 or similar, the call might not exist yet, continue polling + continue + + # After polling is complete, create EvaluatorResult and trigger evaluation + if call_complete and call_metrics and call_recording.agent_id: + try: + from app.services.playground.post_call_processing import ( + claim_playground_evaluator_result_slot, + record_playground_post_call_usage_once, + ) + + should_create_evaluator, call_metrics = record_playground_post_call_usage_once( + db, + call_recording_id, + provider_platform=provider_platform, + call_metrics=call_metrics if isinstance(call_metrics, dict) else {}, + ) + if not should_create_evaluator: + return + + logger.info(f"[Poll Call Metrics] Call complete, creating EvaluatorResult for call {provider_call_id}") + + # Extract transcript and speaker segments from call_data + transcript_text, _ = extract_transcript_from_call_data( + call_metrics, + provider_platform + ) + + if not transcript_text: + logger.warning(f"[Poll Call Metrics] No transcript found in call_data for call {provider_call_id}") + + # Get agent info for naming + agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + result_name = f"Voice AI Call - {agent.name}" if agent else "Voice AI Call" + + # Calculate duration + duration_seconds = call_metrics.get("duration_seconds", 0) + if not duration_seconds: + # Try to calculate from timestamps + start_ts = call_metrics.get("start_timestamp") or call_metrics.get("startedAt") + end_ts = call_metrics.get("end_timestamp") or call_metrics.get("endedAt") + if start_ts and end_ts: + try: + from dateutil import parser + start_time = parser.parse(start_ts) + end_time = parser.parse(end_ts) + duration_seconds = (end_time - start_time).total_seconds() + except Exception: + pass + + # Download call audio from provider and upload to S3 + audio_s3_key = None + try: + import requests as _http + import uuid as _uuid + from app.services.storage.s3_service import s3_service + + recording_urls = call_metrics.get("recording_urls", {}) + audio_bytes = None + plat = provider_platform.lower() + + if plat == "elevenlabs": + audio_url = recording_urls.get("conversation_audio") + if audio_url: + resp = _http.get(audio_url, headers={"xi-api-key": integration_api_key}, timeout=120) + if resp.status_code == 200: + audio_bytes = resp.content + elif plat == "retell": + audio_url = call_metrics.get("recording_url") + if audio_url: + resp = _http.get(audio_url, timeout=120) + if resp.status_code == 200: + audio_bytes = resp.content + elif plat == "vapi": + artifact = call_metrics.get("artifact", {}) + recording = artifact.get("recording", {}) if isinstance(artifact, dict) else {} + mono_recording = recording.get("mono", {}) if isinstance(recording, dict) else {} + audio_url = ( + call_metrics.get("recordingUrl") + or call_metrics.get("stereoRecordingUrl") + or artifact.get("recordingUrl") + or artifact.get("stereoRecordingUrl") + or mono_recording.get("combinedUrl") + or recording_urls.get("combined_url") + or recording_urls.get("stereo_url") + ) + if audio_url: + vapi_headers = {"Authorization": f"Bearer {integration_api_key}"} + resp = _http.get(audio_url, headers=vapi_headers, timeout=120) + if resp.status_code == 200: + audio_bytes = resp.content + + if audio_bytes: + content_type = getattr(resp, "headers", {}).get("content-type", "audio/mpeg") + ext = "wav" if "wav" in content_type else "mp3" + org_id = str(call_recording.organization_id) + audio_s3_key = f"audio/organizations/{org_id}/agentPlayground/{provider_call_id}/{_uuid.uuid4()}.{ext}" + s3_service.upload_file_by_key(audio_bytes, audio_s3_key, content_type=content_type) + logger.info(f"[Poll Call Metrics] Uploaded call audio to S3: {audio_s3_key} ({len(audio_bytes)} bytes)") + else: + logger.warning(f"[Poll Call Metrics] Could not download audio for call {provider_call_id}") + except Exception as audio_err: + logger.warning(f"[Poll Call Metrics] Audio download/upload failed: {audio_err}") + + locked_recording = claim_playground_evaluator_result_slot( + db, + call_recording_id, + provider_call_id=provider_call_id, + ) + if not locked_recording: + return + + result_id = generate_unique_result_id(db) + evaluator_result = EvaluatorResult( + result_id=result_id, + organization_id=locked_recording.organization_id, + workspace_id=locked_recording.workspace_id, + evaluator_id=None, + agent_id=locked_recording.agent_id, + persona_id=None, + scenario_id=None, + name=result_name, + duration_seconds=duration_seconds, + status=EvaluatorResultStatus.QUEUED.value, + audio_s3_key=audio_s3_key, + transcription=transcript_text, + provider_call_id=provider_call_id, + provider_platform=provider_platform, + call_data=call_metrics, + ) + db.add(evaluator_result) + db.flush() + locked_recording.evaluator_result_id = evaluator_result.id + db.commit() + db.refresh(evaluator_result) + + logger.info(f"[Poll Call Metrics] Created EvaluatorResult {result_id} for call {provider_call_id}") + + # Trigger Celery task to process evaluator result (run metrics evaluation) + try: + from app.workers.celery_app import process_evaluator_result_task + task = process_evaluator_result_task.delay(str(evaluator_result.id)) + evaluator_result.celery_task_id = task.id + db.commit() + logger.info(f"[Poll Call Metrics] Triggered evaluation task {task.id} for result {result_id}") + except Exception as task_error: + logger.error(f"[Poll Call Metrics] Failed to trigger Celery task: {task_error}") + # Mark the result as failed if we can't trigger the task + # But keep the transcript available for manual review + + except Exception as e: + logger.error(f"[Poll Call Metrics] Error creating EvaluatorResult: {str(e)}", exc_info=True) + + finally: + db.close() + + +class CallRecordingUpdate(BaseModel): + """Schema for updating a call recording.""" + provider_call_id: str + + +@router.put("/call-recordings/{call_short_id}", response_model=Dict[str, Any]) +async def update_call_recording( + call_short_id: str, + update_data: CallRecordingUpdate, + background_tasks: BackgroundTasks, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + Update a call recording within the active workspace, typically to set the provider_call_id. + Triggers polling. + """ + call_recording = db.query(CallRecording).filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + ).first() + + if not call_recording: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call recording not found" + ) + + call_recording.provider_call_id = update_data.provider_call_id + db.commit() + db.refresh(call_recording) + + # Trigger polling if we have all info + if call_recording.provider_platform: + # Get integration api key + agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + if agent and agent.voice_ai_integration_id: + integration = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id + ).first() + + if integration: + try: + decrypted_api_key = decrypt_api_key(integration.api_key) + platform_key = (call_recording.provider_platform or "").lower() + # Vapi polls on call-end refresh only; update fires mid-call too. + if platform_key != "vapi": + background_tasks.add_task( + poll_call_metrics, + call_recording.id, + call_recording.provider_call_id, + call_recording.provider_platform, + decrypted_api_key + ) + except: + pass + + return { + "message": "Call recording updated", + "provider_call_id": call_recording.provider_call_id + } + + +class WebCallCreate(BaseModel): + """Schema for creating a web call.""" + agent_id: str # UUID of the agent in our system + metadata: Optional[Dict[str, Any]] = None + retell_llm_dynamic_variables: Optional[Dict[str, Any]] = None + custom_sip_headers: Optional[Dict[str, str]] = None + ui_surface: Optional[str] = None + + +@router.post("/web-call", response_model=Dict[str, Any]) +async def create_web_call( + web_call_data: WebCallCreate, + background_tasks: BackgroundTasks, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + Create a web call with a voice AI agent within the active workspace. + The agent must belong to the same workspace; the resulting call recording + is stamped with the same workspace_id. + """ + try: + # Get the agent (scoped to the active workspace) + agent_uuid = UUID(web_call_data.agent_id) + agent = db.query(Agent).filter( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ).first() + + if not agent: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Agent not found" + ) + + # Check if agent has voice AI integration + if not agent.voice_ai_integration_id or not agent.voice_ai_agent_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Agent is not configured with a voice AI integration" + ) + + # Check if agent has web call enabled + if agent.call_medium != "web_call": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Agent is not configured for web calls" + ) + + # Get the integration + integration = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + Integration.is_active == True + ).first() + + if not integration: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Integration not found or inactive" + ) + + # Decrypt API key + try: + decrypted_api_key = decrypt_api_key(integration.api_key) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to decrypt API key: {str(e)}" + ) + + # Get the appropriate voice provider + try: + provider_class = get_voice_provider(integration.platform) + + platform_value = integration.platform.value if hasattr(integration.platform, 'value') else integration.platform + if platform_value.lower() == "vapi": + provider = provider_class(api_key=decrypted_api_key, public_key=integration.public_key) + else: + provider = provider_class(api_key=decrypted_api_key) + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(e) + ) + + # Create the web call + try: + # Verify agent_id is present + if not agent.voice_ai_agent_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Agent does not have a voice_ai_agent_id configured" + ) + + print(f"[Playground] Creating web call - Agent ID: {agent.id}, Retell Agent ID: {agent.voice_ai_agent_id}, Platform: {integration.platform}") + + # Build call parameters based on provider + call_params = { + "agent_id": agent.voice_ai_agent_id, + } + + # Add optional parameters if provided + if web_call_data.metadata: + call_params["metadata"] = web_call_data.metadata + if web_call_data.retell_llm_dynamic_variables: + call_params["retell_llm_dynamic_variables"] = web_call_data.retell_llm_dynamic_variables + + # Note: custom_sip_headers is not supported by Retell, but may be supported by other providers + # For now, we'll skip it for Retell. Other providers can handle it in their implementation. + if integration.platform != "retell" and web_call_data.custom_sip_headers: + call_params["custom_sip_headers"] = web_call_data.custom_sip_headers + + platform_value = ( + integration.platform.value + if hasattr(integration.platform, "value") + else integration.platform + ) + plat_lower = str(platform_value).lower() + + # Vapi Web SDK creates the call in the browser; server-side /call/web + # would spawn a second call that never receives the user's microphone. + if plat_lower == "vapi": + from app.services.voice_providers.vapi import VAPI_SAMPLE_RATE + + web_call_response = { + "call_type": "web_call", + "agent_id": agent.voice_ai_agent_id, + "metadata": web_call_data.metadata or {}, + "sample_rate": VAPI_SAMPLE_RATE, + "client_sdk_creates_call": True, + } + provider_call_id = None + else: + web_call_response = provider.create_web_call(**call_params) + provider_call_id = web_call_response.get("call_id") + + call_short_id = generate_unique_call_short_id(db) + stored_call_data = dict(web_call_response) + if web_call_data.ui_surface: + stored_call_data["ui_surface"] = web_call_data.ui_surface + call_recording = CallRecording( + organization_id=organization_id, + workspace_id=workspace_id, + call_short_id=call_short_id, + status=CallRecordingStatus.PENDING, + source=CallRecordingSource.PLAYGROUND, + call_data=stored_call_data, + provider_call_id=provider_call_id, + provider_platform=integration.platform, + agent_id=agent.id + ) + db.add(call_recording) + db.commit() + db.refresh(call_recording) + + background_tasks.add_task( + record_playground_web_call_started, + organization_id, + call_short_id, + workspace_id=workspace_id, + agent_id=agent.id, + ) + + # Start background task to poll for call metrics + # Note: We need to pass the decrypted API key, but we should be careful with security + # For now, we'll pass it to the background task + # In production, you might want to store it temporarily or use a different approach + if provider_call_id and plat_lower != "vapi": + background_tasks.add_task( + poll_call_metrics, + call_recording.id, + provider_call_id, + integration.platform, + decrypted_api_key + ) + + # Add call_short_id to response for frontend + response = web_call_response.copy() + response["call_short_id"] = call_short_id + + if plat_lower == "vapi" and integration.public_key: + response["public_key"] = integration.public_key + + if plat_lower == "elevenlabs": + response["signed_url"] = web_call_response.get("signed_url") + + return response + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to create web call: {str(e)}" + ) + + except ValueError as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Invalid agent ID: {str(e)}" + ) + except Exception as e: + if isinstance(e, HTTPException): + raise e + print(f"[Create Web Call] Error: {str(e)}") + import traceback + traceback.print_exc() + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Internal server error: {str(e)}" + ) + + +@router.get("/call-recordings", response_model=List[Dict[str, Any]]) +async def list_call_recordings( + skip: int = 0, + limit: int = 100, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + List playground call recordings in the active workspace. + Includes evaluator_result_id, evaluation status, and metric_scores if evaluation has been run. + """ + call_recordings = db.query(CallRecording).filter( + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source == CallRecordingSource.PLAYGROUND, + ).order_by(CallRecording.created_at.desc()).offset(skip).limit(limit).all() + + # Get evaluator result info for all linked results + result_ids = [cr.evaluator_result_id for cr in call_recordings if cr.evaluator_result_id] + result_info = {} + if result_ids: + results = db.query(EvaluatorResult).filter(EvaluatorResult.id.in_(result_ids)).all() + result_info = { + str(r.id): { + "status": r.status, + "metric_scores": r.metric_scores, + "result_id": r.result_id, + "name": r.name, + } + for r in results + } + + agent_ids = [cr.agent_id for cr in call_recordings if cr.agent_id] + agents_by_id = {} + if agent_ids: + agents = db.query(Agent).filter(Agent.id.in_(agent_ids)).all() + agents_by_id = {a.id: a for a in agents} + + def _display_name(cr: CallRecording) -> str: + linked = result_info.get(str(cr.evaluator_result_id), {}) if cr.evaluator_result_id else {} + if linked.get("name"): + return linked["name"] + agent = agents_by_id.get(cr.agent_id) if cr.agent_id else None + if agent and agent.name: + return agent.name + return cr.call_short_id + + return [ + { + "id": str(cr.id), + "call_short_id": cr.call_short_id, + "display_name": _display_name(cr), + "status": cr.status if cr.status else None, + "provider_platform": cr.provider_platform, + "provider_call_id": cr.provider_call_id, + "agent_id": str(cr.agent_id) if cr.agent_id else None, + "evaluator_result_id": str(cr.evaluator_result_id) if cr.evaluator_result_id else None, + "evaluation_status": result_info.get(str(cr.evaluator_result_id), {}).get("status") if cr.evaluator_result_id else None, + "metric_scores": result_info.get(str(cr.evaluator_result_id), {}).get("metric_scores") if cr.evaluator_result_id else None, + "result_id": result_info.get(str(cr.evaluator_result_id), {}).get("result_id") if cr.evaluator_result_id else None, + "created_at": cr.created_at.isoformat() if cr.created_at else None, + "updated_at": cr.updated_at.isoformat() if cr.updated_at else None, + } + for cr in call_recordings + ] + + +@router.post("/custom-websocket-sessions", response_model=Dict[str, Any]) +async def create_custom_websocket_session( + background_tasks: BackgroundTasks, + agent_id: str = Form(...), + websocket_url: str = Form(...), + transcript_entries: str = Form("[]"), + started_at: Optional[str] = Form(None), + ended_at: Optional[str] = Form(None), + call_short_id: Optional[str] = Form(None), + audio_file: Optional[UploadFile] = File(None), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """ + Save a custom websocket test session for later evaluation (scoped to the active workspace). + Stores transcript in call_data and uploads optional audio recording to S3. + """ + try: + agent_uuid = UUID(agent_id) + except ValueError: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid agent_id") + + agent = db.query(Agent).filter( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + Agent.workspace_id == workspace_id, + ).first() + if not agent: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Agent not found") + + try: + parsed_entries = json.loads(transcript_entries or "[]") + except json.JSONDecodeError: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid transcript_entries payload") + + normalized_entries = [] + for entry in parsed_entries: + if not isinstance(entry, dict): + continue + role = entry.get("role") + content = (entry.get("content") or "").strip() + if role not in {"user", "agent"} or not content: + continue + normalized_entries.append( + { + "role": role, + "content": content, + "timestamp": entry.get("timestamp") or ended_at or started_at, + } + ) + + if not normalized_entries: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No transcript entries provided") + + transcript_text = "\n".join( + [f"{'User' if item['role'] == 'user' else 'Agent'}: {item['content']}" for item in normalized_entries] + ) + + call_short_id = (call_short_id or "").strip() or generate_unique_call_short_id(db) + audio_s3_key = None + if audio_file: + from app.services.storage.s3_service import s3_service + + audio_bytes = await audio_file.read() + if audio_bytes: + if not s3_service.is_enabled(): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="S3 storage is not configured. Save without recording or enable storage.", + ) + + filename = audio_file.filename or "session.webm" + extension = filename.split(".")[-1].lower() if "." in filename else "webm" + content_type = audio_file.content_type or "audio/webm" + audio_s3_key = ( + f"audio/organizations/{organization_id}/agentPlayground/customWebsocket/" + f"{call_short_id}/{_uuid.uuid4()}.{extension}" + ) + s3_service.upload_file_by_key(audio_bytes, audio_s3_key, content_type=content_type) + + duration_seconds = 0 + if started_at and ended_at: + try: + from datetime import datetime as _dt + t_start = _dt.fromisoformat(started_at.replace("Z", "+00:00")) + t_end = _dt.fromisoformat(ended_at.replace("Z", "+00:00")) + duration_seconds = max(0, (t_end - t_start).total_seconds()) + except Exception: + duration_seconds = 0 + + speaker_segments = [] + for entry in normalized_entries: + speaker = "user" if entry.get("role") == "user" else "assistant" + speaker_segments.append({ + "speaker": speaker, + "text": entry.get("content", ""), + "start": 0, + "end": 0, + }) + + call_data = { + "source": "custom_websocket", + "websocket_url": websocket_url, + "messages": normalized_entries, + "transcript": transcript_text, + "speaker_segments": speaker_segments, + "recording_s3_key": audio_s3_key, + "started_at": started_at, + "ended_at": ended_at, + "duration_seconds": duration_seconds, + } + + call_recording = CallRecording( + organization_id=organization_id, + workspace_id=workspace_id, + call_short_id=call_short_id, + status=CallRecordingStatus.UPDATED, + source=CallRecordingSource.PLAYGROUND, + call_data=call_data, + provider_call_id=f"custom_{call_short_id}", + provider_platform="custom_websocket", + agent_id=agent.id, + ) + db.add(call_recording) + db.commit() + db.refresh(call_recording) + + from app.services.synthetic_traces.trace_service import ( + close_trace_session, + link_trace_to_call_recording, + ) + + link_trace_to_call_recording( + db, + organization_id=organization_id, + call_short_id=call_short_id, + call_recording_id=call_recording.id, + ) + close_trace_session( + db, + organization_id=organization_id, + call_short_id=call_short_id, + workspace_id=workspace_id, + ) + + background_tasks.add_task( + record_playground_websocket_session_started, + organization_id, + call_short_id, + workspace_id=workspace_id, + ) + + return { + "message": "Custom websocket session saved", + "call_short_id": call_short_id, + "audio_s3_key": audio_s3_key, + "evaluator_result_id": None, + } + + +@router.post("/custom-websocket-sessions/{call_short_id}/evaluate", response_model=Dict[str, Any]) +async def evaluate_custom_websocket_session( + call_short_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """ + Queue evaluation for a saved custom websocket test session in the active workspace. + """ + call_recording = db.query(CallRecording).filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source == CallRecordingSource.PLAYGROUND, + CallRecording.provider_platform == "custom_websocket", + ).first() + + if not call_recording: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Custom websocket session not found") + + call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + transcript_text = (call_data.get("transcript") or "").strip() + if not transcript_text: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="No transcript found for evaluation") + + existing_result = None + if call_recording.evaluator_result_id: + existing_result = db.query(EvaluatorResult).filter( + EvaluatorResult.id == call_recording.evaluator_result_id, + EvaluatorResult.organization_id == organization_id, + EvaluatorResult.workspace_id == workspace_id, + ).first() + + speaker_segments = call_data.get("speaker_segments") or [] + + if existing_result: + evaluator_result = existing_result + evaluator_result.status = EvaluatorResultStatus.QUEUED.value + evaluator_result.error_message = None + evaluator_result.metric_scores = None + evaluator_result.celery_task_id = None + evaluator_result.transcription = transcript_text + evaluator_result.speaker_segments = speaker_segments + evaluator_result.audio_s3_key = call_data.get("recording_s3_key") + evaluator_result.call_data = slim_call_data_for_evaluator_result(call_data) + evaluator_result.duration_seconds = call_data.get("duration_seconds", 0) + db.commit() + db.refresh(evaluator_result) + else: + agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + result_id = generate_unique_result_id(db) + result_name = f"Custom WebSocket Test - {agent.name}" if agent else "Custom WebSocket Test" + + evaluator_result = EvaluatorResult( + result_id=result_id, + organization_id=organization_id, + workspace_id=workspace_id, + evaluator_id=None, + agent_id=call_recording.agent_id, + persona_id=None, + scenario_id=None, + name=result_name, + duration_seconds=call_data.get("duration_seconds", 0), + status=EvaluatorResultStatus.QUEUED.value, + audio_s3_key=call_data.get("recording_s3_key"), + transcription=transcript_text, + speaker_segments=speaker_segments, + provider_call_id=call_recording.provider_call_id, + provider_platform="custom_websocket", + call_data=slim_call_data_for_evaluator_result(call_data), + ) + db.add(evaluator_result) + db.commit() + db.refresh(evaluator_result) + + call_recording.evaluator_result_id = evaluator_result.id + db.commit() + + try: + from app.workers.celery_app import process_evaluator_result_task + task = process_evaluator_result_task.delay(str(evaluator_result.id)) + evaluator_result.celery_task_id = task.id + db.commit() + except Exception as e: + logger.error(f"[Custom WebSocket] Failed to trigger evaluation worker: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to trigger evaluation worker") + + return { + "message": "Evaluation queued", + "evaluator_result_id": str(evaluator_result.id), + "result_id": evaluator_result.result_id, + "task_id": task.id, + } + + +@router.get("/call-recordings/{call_short_id}", response_model=Dict[str, Any]) +async def get_call_recording( + call_short_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + Get a specific playground call recording within the active workspace. + Returns the full JSON data stored for the call and evaluation information. + """ + call_recording = db.query(CallRecording).filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source == CallRecordingSource.PLAYGROUND, + ).first() + + if not call_recording: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call recording not found" + ) + + # Get evaluator result info if available + evaluation_info = None + if call_recording.evaluator_result_id: + evaluator_result = db.query(EvaluatorResult).filter( + EvaluatorResult.id == call_recording.evaluator_result_id + ).first() + if evaluator_result: + eval_call_data = evaluator_result.call_data if isinstance(evaluator_result.call_data, dict) else {} + evaluation_info = { + "id": str(evaluator_result.id), + "result_id": evaluator_result.result_id, + "status": evaluator_result.status, + "metric_scores": evaluator_result.metric_scores, + "transcription": evaluator_result.transcription, + "call_analysis": eval_call_data.get("call_analysis"), + } + + return { + "id": str(call_recording.id), + "call_short_id": call_recording.call_short_id, + "status": call_recording.status if call_recording.status else None, + "provider_platform": call_recording.provider_platform, + "provider_call_id": call_recording.provider_call_id, + "agent_id": str(call_recording.agent_id) if call_recording.agent_id else None, + "evaluator_result_id": str(call_recording.evaluator_result_id) if call_recording.evaluator_result_id else None, + "evaluation": evaluation_info, + "call_data": call_recording.call_data, # Full JSON blob + "created_at": call_recording.created_at.isoformat() if call_recording.created_at else None, + "updated_at": call_recording.updated_at.isoformat() if call_recording.updated_at else None, + } + + +@router.post("/call-recordings/{call_short_id}/refresh", response_model=Dict[str, Any]) +async def refresh_call_recording( + call_short_id: str, + background_tasks: BackgroundTasks, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + Manually trigger a refresh of call metrics for a specific call recording in the active workspace. + """ + call_recording = db.query(CallRecording).filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source == CallRecordingSource.PLAYGROUND, + ).first() + + if not call_recording: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call recording not found" + ) + + if not call_recording.provider_call_id or not call_recording.provider_platform: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Call recording does not have provider information" + ) + + # Get the integration to get the API key + agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + if not agent or not agent.voice_ai_integration_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Agent or integration not found" + ) + + integration = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id + ).first() + + if not integration: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Integration not found" + ) + + try: + decrypted_api_key = decrypt_api_key(integration.api_key) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Failed to decrypt API key: {str(e)}" + ) + + if call_recording.evaluator_result_id: + background_tasks.add_task( + sync_provider_call_metrics, + call_recording.id, + call_recording.provider_call_id, + call_recording.provider_platform, + decrypted_api_key, + ) + return {"message": "Provider metrics sync initiated"} + + # Start background task to poll for call metrics + background_tasks.add_task( + poll_call_metrics, + call_recording.id, + call_recording.provider_call_id, + call_recording.provider_platform, + decrypted_api_key + ) + + return {"message": "Call recording refresh initiated"} + + +@router.delete("/call-recordings/{call_short_id}", response_model=Dict[str, Any]) +async def delete_call_recording( + call_short_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db) +): + """ + Delete a call recording within the active workspace. + """ + call_recording = db.query(CallRecording).filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + ).first() + + if not call_recording: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Call recording not found" + ) + + db.delete(call_recording) + db.commit() + + return {"message": "Call recording deleted successfully"} + + +@router.post("/call-recordings/{call_short_id}/re-evaluate", response_model=Dict[str, Any]) +async def re_evaluate_call_recording( + call_short_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """ + Re-evaluate a playground call recording within the active workspace. + + Reuses the S3 audio if it was already downloaded during the first + evaluation. If no audio exists in S3, downloads from the provider, + uploads, then triggers the worker for both conversation quality (LLM) + and audio quality (acoustic / AI voice) metrics. + """ + import requests as http_requests + import uuid as _uuid + from app.services.storage.s3_service import s3_service + + call_recording = db.query(CallRecording).filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source == CallRecordingSource.PLAYGROUND, + ).first() + + if not call_recording: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call recording not found") + + if not call_recording.provider_call_id or not call_recording.provider_platform: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Call recording has no provider data") + + call_data = call_recording.call_data or {} + platform = (call_recording.provider_platform or "").lower() + + # --- Check if S3 audio already exists from a previous evaluation ------- + existing_result = None + if call_recording.evaluator_result_id: + existing_result = db.query(EvaluatorResult).filter( + EvaluatorResult.id == call_recording.evaluator_result_id, + ).first() + + audio_s3_key = existing_result.audio_s3_key if existing_result and existing_result.audio_s3_key else None + + # --- If no S3 audio, download from provider and upload ----------------- + if not audio_s3_key: + agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + if not agent or not agent.voice_ai_integration_id: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Agent or integration not found") + + integration = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + ).first() + if not integration: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found") + + decrypted_key = decrypt_api_key(integration.api_key) + + def _download_audio_from_payload(payload: Dict[str, Any]): + payload_urls = payload.get("recording_urls", {}) if isinstance(payload, dict) else {} + artifact = payload.get("artifact", {}) if isinstance(payload, dict) else {} + recording = artifact.get("recording", {}) if isinstance(artifact, dict) else {} + mono_recording = recording.get("mono", {}) if isinstance(recording, dict) else {} + url = None + headers = None + if platform == "elevenlabs": + url = payload_urls.get("conversation_audio") + headers = {"xi-api-key": decrypted_key} + elif platform == "retell": + url = payload.get("recording_url") + elif platform == "vapi": + url = ( + payload.get("recordingUrl") + or payload.get("stereoRecordingUrl") + or artifact.get("recordingUrl") + or artifact.get("stereoRecordingUrl") + or mono_recording.get("combinedUrl") + or payload_urls.get("combined_url") + or payload_urls.get("stereo_url") + ) + elif platform == "smallest": + url = ( + payload.get("recording_url") + or payload.get("recordingUrl") + or payload_urls.get("combined_url") + or payload_urls.get("conversation_audio") + ) + if not url: + return None, None + response = http_requests.get(url, headers=headers, timeout=120) + if response.status_code != 200: + return None, response + return response.content, response + + audio_bytes, resp = _download_audio_from_payload(call_data) + + # Retry once with fresh provider payload (new signed URL) using provider_call_id + if not audio_bytes and call_recording.provider_call_id: + try: + provider_class = get_voice_provider(platform) + provider_kwargs: Dict[str, Any] = {"api_key": decrypted_key} + if platform == "vapi" and integration.public_key: + provider_kwargs["public_key"] = integration.public_key + provider = provider_class(**provider_kwargs) + if hasattr(provider, "retrieve_call_metrics"): + refreshed_call_data = provider.retrieve_call_metrics(call_recording.provider_call_id) + if isinstance(refreshed_call_data, dict) and refreshed_call_data: + prev_data = ( + call_recording.call_data + if isinstance(call_recording.call_data, dict) + else {} + ) + call_data = merge_playground_call_data(prev_data, refreshed_call_data) + call_recording.call_data = call_data + db.commit() + logger.info(f"[Re-evaluate] Refreshed provider call data for call {call_recording.provider_call_id}") + audio_bytes, resp = _download_audio_from_payload(call_data) + except Exception as refresh_err: + logger.warning(f"[Re-evaluate] Provider audio URL refresh failed: {refresh_err}") + + if not audio_bytes: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Could not download audio from provider. The recording may not be available yet.", + ) + + content_type = getattr(resp, "headers", {}).get("content-type", "audio/mpeg") + ext = "wav" if "wav" in content_type else "mp3" + org_id = str(organization_id) + audio_s3_key = f"audio/organizations/{org_id}/agentPlayground/{call_recording.provider_call_id}/{_uuid.uuid4()}.{ext}" + try: + s3_service.upload_file_by_key(audio_bytes, audio_s3_key, content_type=content_type) + logger.info(f"[Re-evaluate] Uploaded audio to S3: {audio_s3_key} ({len(audio_bytes)} bytes)") + except Exception as e: + logger.error(f"[Re-evaluate] S3 upload failed: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=f"Failed to store audio: {str(e)}") + else: + logger.info(f"[Re-evaluate] Reusing existing S3 audio: {audio_s3_key}") + + # --- Extract transcript from existing call data ------------------------ + transcript_text, _ = extract_transcript_from_call_data(call_data, platform) + + # --- Create or reset EvaluatorResult ----------------------------------- + if existing_result: + existing_result.status = EvaluatorResultStatus.QUEUED.value + existing_result.audio_s3_key = audio_s3_key + # Metadata only; transcript is on existing_result.transcription. + existing_result.call_data = ( + slim_call_data_for_evaluator_result(call_data) + if isinstance(call_data, dict) + else existing_result.call_data + ) + existing_result.transcription = transcript_text or existing_result.transcription + existing_result.metric_scores = None + existing_result.error_message = None + existing_result.celery_task_id = None + db.commit() + db.refresh(existing_result) + evaluator_result = existing_result + logger.info(f"[Re-evaluate] Reset existing EvaluatorResult {evaluator_result.result_id}") + else: + agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + result_id = generate_unique_result_id(db) + duration_seconds = call_data.get("duration_seconds", 0) + if not duration_seconds: + start_ts = call_data.get("start_timestamp") or call_data.get("startedAt") + end_ts = call_data.get("end_timestamp") or call_data.get("endedAt") + if start_ts and end_ts: + try: + from dateutil import parser + duration_seconds = (parser.parse(end_ts) - parser.parse(start_ts)).total_seconds() + except Exception: + duration_seconds = 0 + result_name = f"Voice AI Call - {agent.name}" if agent else "Voice AI Call" + + evaluator_result = EvaluatorResult( + result_id=result_id, + organization_id=call_recording.organization_id, + workspace_id=call_recording.workspace_id, + evaluator_id=None, + agent_id=call_recording.agent_id, + persona_id=None, + scenario_id=None, + name=result_name, + duration_seconds=duration_seconds, + status=EvaluatorResultStatus.QUEUED.value, + audio_s3_key=audio_s3_key, + transcription=transcript_text, + provider_call_id=call_recording.provider_call_id, + provider_platform=platform, + call_data=slim_call_data_for_evaluator_result(call_data), + ) + db.add(evaluator_result) + db.commit() + db.refresh(evaluator_result) + + call_recording.evaluator_result_id = evaluator_result.id + db.commit() + logger.info(f"[Re-evaluate] Created new EvaluatorResult {evaluator_result.result_id}") + + # --- Trigger the worker ------------------------------------------------ + try: + from app.workers.celery_app import process_evaluator_result_task + task = process_evaluator_result_task.delay(str(evaluator_result.id)) + evaluator_result.celery_task_id = task.id + db.commit() + logger.info(f"[Re-evaluate] Triggered evaluation task {task.id} for result {evaluator_result.result_id}") + except Exception as e: + logger.error(f"[Re-evaluate] Failed to trigger Celery task: {e}") + raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to trigger evaluation worker") + + return { + "message": "Re-evaluation started", + "evaluator_result_id": str(evaluator_result.id), + "result_id": evaluator_result.result_id, + "audio_s3_key": audio_s3_key, + "task_id": task.id, + } + + +@router.get("/call-recordings/{call_short_id}/logs", response_model=Dict[str, Any]) +async def get_call_recording_logs( + call_short_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Return normalized provider logs for Vapi and Retell playground calls.""" + del api_key # Dependency enforcement only + from app.services.playground.provider_call_logs import ( + fetch_retell_call_logs, + fetch_vapi_call_logs, + ) + + call_recording = db.query(CallRecording).filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source == CallRecordingSource.PLAYGROUND, + ).first() + + if not call_recording: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call recording not found") + + platform = (call_recording.provider_platform or "").lower() + call_data = call_recording.call_data if isinstance(call_recording.call_data, dict) else {} + + if platform not in ("vapi", "retell"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Provider logs are only available for Vapi and Retell calls", + ) + + try: + if platform == "retell": + entries = fetch_retell_call_logs(call_data) + return {"platform": platform, "entries": entries, "count": len(entries)} + + if not call_recording.provider_call_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Call recording does not have a provider call id", + ) + + agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + if not agent or not agent.voice_ai_integration_id: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Agent or integration not found") + + integration = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + ).first() + if not integration: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found") + + decrypted_api_key = decrypt_api_key(integration.api_key) + entries = fetch_vapi_call_logs( + api_key=decrypted_api_key, + provider_call_id=call_recording.provider_call_id, + call_data=call_data, + ) + return {"platform": platform, "entries": entries, "count": len(entries)} + except HTTPException: + raise + except Exception as exc: + logger.warning("[CallLogs] Failed to fetch logs for %s: %s", call_short_id, exc) + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail="Failed to fetch provider logs. Try Refresh on the call first.", + ) + + +@router.get("/call-recordings/{call_short_id}/audio") +async def stream_call_audio( + call_short_id: str, + proxy: bool = Query(False, description="Stream audio through API (CORS-safe for waveform)"), + stereo: bool = Query(False, description="Prefer stereo recording when available"), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """ + Proxy endpoint to stream playground call recording audio in the active workspace. + Required for providers like ElevenLabs whose audio URLs need auth headers. + """ + import requests as http_requests + + call_recording = db.query(CallRecording).filter( + CallRecording.call_short_id == call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + CallRecording.source == CallRecordingSource.PLAYGROUND, + ).first() + + if not call_recording: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Call recording not found") + + call_data = call_recording.call_data or {} + platform = (call_recording.provider_platform or "").lower() + + if platform in ("retell", "vapi", "smallest"): + url = _resolve_playground_audio_url(call_data, platform, stereo=stereo and platform == "vapi") + if not url: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No recording URL available") + if proxy: + import requests as http_requests + + upstream = http_requests.get(url, stream=True, timeout=90) + if upstream.status_code in (401, 403) and platform == "vapi": + upstream.close() + if not _refresh_call_recording_from_provider_sync(db, call_recording, organization_id): + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=( + "Provider recording URL expired. " + "Use Sync provider data on the call, then try again." + ), + ) + call_data = call_recording.call_data or {} + url = _resolve_playground_audio_url(call_data, platform, stereo=stereo) + if not url: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="No recording URL available after refresh", + ) + upstream = http_requests.get(url, stream=True, timeout=90) + if upstream.status_code != 200: + raise HTTPException( + status_code=status.HTTP_502_BAD_GATEWAY, + detail=( + "Provider recording URL expired or unavailable. " + "Use Sync provider data on the call, then try again." + ), + ) + content_type = upstream.headers.get("content-type", "audio/wav") + return StreamingResponse( + upstream.iter_content(chunk_size=8192), + media_type=content_type, + headers={"Content-Disposition": f'inline; filename="call_{call_short_id}.wav"'}, + ) + from fastapi.responses import RedirectResponse + return RedirectResponse(url) + + recording_urls = call_data.get("recording_urls", {}) + if platform == "elevenlabs": + audio_url = recording_urls.get("conversation_audio") + if not audio_url: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No recording URL available") + + agent = db.query(Agent).filter(Agent.id == call_recording.agent_id).first() + if not agent or not agent.voice_ai_integration_id: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Agent or integration not found") + + integration = db.query(Integration).filter( + Integration.id == agent.voice_ai_integration_id, + Integration.organization_id == organization_id, + ).first() + if not integration: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Integration not found") + + decrypted_key = decrypt_api_key(integration.api_key) + + upstream = http_requests.get( + audio_url, + headers={"xi-api-key": decrypted_key}, + stream=True, + timeout=60, + ) + if upstream.status_code != 200: + raise HTTPException( + status_code=upstream.status_code, + detail=f"ElevenLabs audio fetch failed ({upstream.status_code})", + ) + + content_type = upstream.headers.get("content-type", "audio/mpeg") + + return StreamingResponse( + upstream.iter_content(chunk_size=8192), + media_type=content_type, + headers={ + "Content-Disposition": f'inline; filename="call_{call_short_id}.mp3"', + }, + ) + + # Custom WebSocket sessions store audio in S3 + if platform == "custom_websocket": + s3_key = call_data.get("recording_s3_key") + if not s3_key: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No recording available for this session") + + from app.services.storage.s3_service import s3_service + if not s3_service.is_enabled(): + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="S3 storage is not configured") + + try: + audio_bytes = s3_service.download_file_by_key(s3_key) + except Exception: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Audio file not found in storage") + if not audio_bytes: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Audio file not found in storage") + + extension = s3_key.rsplit(".", 1)[-1].lower() if "." in s3_key else "webm" + content_type_map = {"webm": "audio/webm", "mp3": "audio/mpeg", "wav": "audio/wav", "ogg": "audio/ogg"} + content_type = content_type_map.get(extension, "audio/webm") + + from io import BytesIO + return StreamingResponse( + BytesIO(audio_bytes), + media_type=content_type, + headers={ + "Content-Disposition": f'inline; filename="call_{call_short_id}.{extension}"', + }, + ) + + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"Audio not supported for platform: {platform}") + + +# --------------------------------------------------------------------------- +# STT config resolution helper +# --------------------------------------------------------------------------- + +_STT_ENV_KEYS = { + "deepgram": "DEEPGRAM_API_KEY", + "openai": "OPENAI_API_KEY", + "elevenlabs": "ELEVENLABS_API_KEY", + "sarvam": "SARVAM_API_KEY", +} + +_STT_DEFAULT_MODELS = { + "deepgram": "nova-2", + "openai": "whisper-1", + "elevenlabs": "scribe_v2", + "sarvam": "saaras:v3", +} + + +def _resolve_agent_stt_config( + agent_id: str, organization_id: UUID, db: Session +) -> tuple: + """Resolve STT provider, model, and API key for an agent. + + Lookup chain: Agent -> VoiceBundle -> stt_provider/stt_model + -> AIProvider (by org + provider name) or Integration or env var fallback. + + Returns (stt_provider, stt_model, api_key). Any element may be None. + """ + import os + from sqlalchemy import func + + agent = db.query(Agent).filter( + Agent.id == agent_id, + Agent.organization_id == organization_id, + ).first() + if not agent or not agent.voice_bundle_id: + return None, None, None + + voice_bundle = db.query(VoiceBundle).filter( + VoiceBundle.id == agent.voice_bundle_id, + VoiceBundle.organization_id == organization_id, + ).first() + if not voice_bundle or not voice_bundle.stt_provider: + return None, None, None + + stt_provider = ( + voice_bundle.stt_provider.value + if hasattr(voice_bundle.stt_provider, "value") + else str(voice_bundle.stt_provider) + ).lower() + stt_model = ( + getattr(voice_bundle, "stt_model", None) + or _STT_DEFAULT_MODELS.get(stt_provider) + ) + + # 1) AIProvider + api_key = None + ai_prov = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + AIProvider.provider == stt_provider, + AIProvider.is_active == True, + ).first() + if not ai_prov: + ai_prov = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + func.lower(AIProvider.provider) == stt_provider, + AIProvider.is_active == True, + ).first() + if ai_prov: + try: + api_key = decrypt_api_key(ai_prov.api_key) + except Exception: + pass + + # 2) Integration fallback + if not api_key: + _platform_map = { + "deepgram": "deepgram", + "elevenlabs": "elevenlabs", + "sarvam": "sarvam", + } + plat_value = _platform_map.get(stt_provider) + if plat_value: + integ = db.query(Integration).filter( + Integration.organization_id == organization_id, + func.lower(Integration.platform) == plat_value, + Integration.is_active == True, + ).first() + if integ: + try: + api_key = decrypt_api_key(integ.api_key) + except Exception: + pass + + # 3) Env var fallback + if not api_key: + env_key = _STT_ENV_KEYS.get(stt_provider) + if env_key: + api_key = os.getenv(env_key) + + return stt_provider, stt_model, api_key + + +# --------------------------------------------------------------------------- +# GET /playground/agents/{agent_id}/stt-config +# --------------------------------------------------------------------------- + +@router.get("/agents/{agent_id}/stt-config", response_model=Dict[str, Any]) +async def get_agent_stt_config( + agent_id: str, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Check whether an agent has STT configured via its voice bundle.""" + stt_provider, stt_model, stt_api_key = _resolve_agent_stt_config( + agent_id, organization_id, db + ) + if stt_provider and stt_api_key: + return {"available": True, "provider": stt_provider, "model": stt_model} + if stt_provider and not stt_api_key: + return { + "available": False, + "reason": f"STT provider '{stt_provider}' is configured but no API key was found", + } + return {"available": False, "reason": "No voice bundle with STT configured for this agent"} + + +# --------------------------------------------------------------------------- +# POST /playground/transcribe-turn +# --------------------------------------------------------------------------- + +@router.post("/transcribe-turn", response_model=Dict[str, Any]) +async def transcribe_turn( + agent_id: str = Form(...), + channel: str = Form(...), + audio_file: UploadFile = File(...), + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Transcribe a single conversation turn (user or agent audio).""" + import tempfile + import os + + if channel not in ("user", "agent"): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="channel must be 'user' or 'agent'", + ) + + stt_provider, stt_model, stt_api_key = _resolve_agent_stt_config( + agent_id, organization_id, db + ) + if not stt_provider or not stt_api_key: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="STT is not configured for this agent (missing provider or API key)", + ) + + audio_bytes = await audio_file.read() + if not audio_bytes or len(audio_bytes) < 100: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Audio file is empty or too small", + ) + + tmp_path = None + try: + suffix = ".wav" + if audio_file.filename and "." in audio_file.filename: + suffix = "." + audio_file.filename.rsplit(".", 1)[-1].lower() + with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp: + tmp.write(audio_bytes) + tmp_path = tmp.name + + from app.services.ai.stt_clients import ( + transcribe_openai, + transcribe_deepgram, + transcribe_elevenlabs, + transcribe_sarvam, + ) + + if stt_provider == "deepgram": + result = transcribe_deepgram(tmp_path, stt_model, stt_api_key) + elif stt_provider == "openai": + result = transcribe_openai(tmp_path, stt_model, stt_api_key) + elif stt_provider == "elevenlabs": + result = transcribe_elevenlabs(tmp_path, stt_model, stt_api_key) + elif stt_provider == "sarvam": + result = transcribe_sarvam(tmp_path, stt_model, stt_api_key) + else: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported STT provider: {stt_provider}", + ) + + transcript_text = (result.get("text") or "").strip() + return {"transcript": transcript_text, "channel": channel} + + except HTTPException: + raise + except Exception as e: + logger.error(f"[transcribe-turn] STT failed: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Transcription failed: {str(e)}", + ) + finally: + if tmp_path and os.path.exists(tmp_path): + os.unlink(tmp_path) + + +# --------------------------------------------------------------------------- +# POST /playground/summarize-transcript +# --------------------------------------------------------------------------- + +# Fallback defaults if the caller provides no agent context AND the voice +# bundle doesn't specify a model. The providers referenced here must exist in +# ``ModelProvider`` and be reachable via LiteLLM. +_SUMMARY_LLM_FALLBACK: List[tuple] = [ + (ModelProvider.OPENAI, "gpt-4o-mini"), + (ModelProvider.GOOGLE, "gemini-2.5-flash"), + (ModelProvider.ANTHROPIC, "claude-haiku-4.5"), +] + + +class SummarizeTranscriptRequest(BaseModel): + transcript: Optional[str] = None + entries: Optional[List[Dict[str, Any]]] = None + call_short_id: Optional[str] = None + agent_id: Optional[str] = None + # When true, ignore any cached summary on the CallRecording and regenerate. + force: Optional[bool] = False + + +def _coerce_model_provider(provider_str: str) -> Optional[ModelProvider]: + """Safely convert a string like ``"openai"`` to the matching enum.""" + if not provider_str: + return None + want = provider_str.strip().lower() + for m in ModelProvider: + if m.value.lower() == want: + return m + return None + + +def _resolve_agent_llm_config( + agent_id: Optional[UUID], organization_id: UUID, db: Session +) -> tuple: + """Resolve the LLM provider + model for an agent via its VoiceBundle. + + Returns ``(ModelProvider | None, model_name | None)``. + """ + if not agent_id: + return None, None + + agent = db.query(Agent).filter( + Agent.id == agent_id, + Agent.organization_id == organization_id, + ).first() + if not agent or not agent.voice_bundle_id: + return None, None + + voice_bundle = db.query(VoiceBundle).filter( + VoiceBundle.id == agent.voice_bundle_id, + VoiceBundle.organization_id == organization_id, + ).first() + if not voice_bundle or not voice_bundle.llm_provider: + return None, None + + provider_enum = _coerce_model_provider(voice_bundle.llm_provider) + if not provider_enum: + return None, None + + return provider_enum, (voice_bundle.llm_model or None) + + +def _pick_fallback_llm(organization_id: UUID, db: Session) -> Optional[tuple]: + """Pick any (ModelProvider, default_model) that has an active AIProvider + row for the organization. Used only when the agent / voice bundle doesn't + specify an LLM. + """ + from sqlalchemy import func + + for provider_enum, default_model in _SUMMARY_LLM_FALLBACK: + provider_value = provider_enum.value + ai_prov = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + func.lower(AIProvider.provider) == provider_value.lower(), + AIProvider.is_active == True, + ).first() + if ai_prov: + return provider_enum, default_model + return None + + +def _format_entries_as_transcript(entries: List[Dict[str, Any]]) -> str: + """Turn a list of {role, content} entries into a plain-text transcript.""" + lines = [] + for e in entries or []: + if not isinstance(e, dict): + continue + role = (e.get("role") or "").strip().lower() + text = (e.get("content") or e.get("text") or "").strip() + if not text: + continue + label = "User" if role in ("user", "caller", "speaker 1") else "Agent" + lines.append(f"{label}: {text}") + return "\n".join(lines) + + +@router.post("/summarize-transcript", response_model=Dict[str, Any]) +async def summarize_transcript( + payload: SummarizeTranscriptRequest = Body(...), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + """Generate a short natural-language summary of a transcript using an LLM. + + Provider selection order: + 1. The voice bundle of ``agent_id`` (or the agent on ``call_short_id``). + 2. Any configured AIProvider matching the fallback preference list. + """ + from contextlib import nullcontext + + from app.services.ai.llm_service import llm_service + from app.services.usage.context import llm_usage_context, usage_context_for_agent + + transcript_text = (payload.transcript or "").strip() + if not transcript_text and payload.entries: + transcript_text = _format_entries_as_transcript(payload.entries) + + if not transcript_text: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Provide either `transcript` text or non-empty `entries`.", + ) + + # Defensive cap on transcript size to protect token budgets. Keep the tail + # so the most recent exchange wins. + MAX_CHARS = 16000 + if len(transcript_text) > MAX_CHARS: + transcript_text = transcript_text[-MAX_CHARS:] + + # --- Resolve CallRecording (for caching) and agent context ------------- + call_rec: Optional[CallRecording] = None + if payload.call_short_id: + call_rec = db.query(CallRecording).filter( + CallRecording.call_short_id == payload.call_short_id, + CallRecording.organization_id == organization_id, + CallRecording.workspace_id == workspace_id, + ).first() + + # Cache hit: serve the previously generated summary without re-calling the LLM. + if call_rec and not payload.force and isinstance(call_rec.call_data, dict): + cached = call_rec.call_data.get("ai_summary") + if isinstance(cached, dict) and (cached.get("text") or "").strip(): + return { + "summary": cached.get("text", ""), + "provider": cached.get("provider", ""), + "model": cached.get("model", ""), + "source": cached.get("source", "voice_bundle"), + "cached": True, + "generated_at": cached.get("generated_at"), + "usage": {}, + } + + agent_uuid: Optional[UUID] = None + if payload.agent_id: + try: + agent_uuid = UUID(payload.agent_id) + except ValueError: + agent_uuid = None + + if not agent_uuid and call_rec and call_rec.agent_id: + agent_uuid = call_rec.agent_id + + # --- Pick LLM: prefer voice-bundle config, else fall back -------------- + llm_provider, llm_model = _resolve_agent_llm_config( + agent_uuid, organization_id, db + ) + source = "voice_bundle" + + if not llm_provider: + picked = _pick_fallback_llm(organization_id, db) + if not picked: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "No LLM is configured. Either set an LLM on the agent's voice bundle, " + "or add an active AIProvider (OpenAI / Google / Anthropic) for this organization." + ), + ) + llm_provider, llm_model = picked + source = "org_fallback" + + if not llm_model: + # Voice bundle had a provider but no model; fill in a safe default. + defaults = { + ModelProvider.OPENAI: "gpt-4o-mini", + ModelProvider.GOOGLE: "gemini-2.5-flash", + ModelProvider.ANTHROPIC: "claude-haiku-4.5", + } + llm_model = defaults.get(llm_provider) or "gpt-4o-mini" + + messages = [ + { + "role": "system", + "content": ( + "You are an expert call analyst. Read the conversation transcript " + "the user provides and write a concise, neutral summary of what happened. " + "Focus on the caller's intent, what the agent did, any outcomes or " + "action items, and the overall tone. Respond in 2-4 plain-text " + "sentences only — no bullet points, no markdown, no preamble." + ), + }, + { + "role": "user", + "content": f"Conversation transcript:\n\n{transcript_text}", + }, + ] + + try: + usage_ctx = nullcontext() + if agent_uuid: + agent_row = db.query(Agent).filter( + Agent.id == agent_uuid, + Agent.organization_id == organization_id, + ).first() + if agent_row: + usage_ctx = llm_usage_context( + usage_context_for_agent(agent_row, workspace_id=workspace_id) + ) + + with usage_ctx: + result = llm_service.generate_response( + messages=messages, + llm_provider=llm_provider, + llm_model=llm_model, + organization_id=organization_id, + db=db, + temperature=0.3, + max_tokens=400, + ) + except Exception as e: + logger.error(f"[summarize-transcript] LLM call failed: {e}") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Summary generation failed: {str(e)}", + ) + + summary = (result.get("text") or "").strip() + if not summary: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="LLM returned an empty summary.", + ) + + generated_at = datetime.utcnow().isoformat() + "Z" + + # Persist on the CallRecording so subsequent loads don't re-invoke the LLM. + if call_rec is not None: + try: + # Reassign the dict so SQLAlchemy notices the JSON column changed. + existing = call_rec.call_data if isinstance(call_rec.call_data, dict) else {} + new_call_data = dict(existing) + new_call_data["ai_summary"] = { + "text": summary, + "provider": llm_provider.value, + "model": llm_model, + "source": source, + "generated_at": generated_at, + } + call_rec.call_data = new_call_data + db.commit() + except Exception as e: + # Non-fatal: return the generated summary even if persistence fails. + logger.warning(f"[summarize-transcript] Failed to cache summary: {e}") + db.rollback() + + return { + "summary": summary, + "provider": llm_provider.value, + "model": llm_model, + "source": source, + "cached": False, + "generated_at": generated_at, + "usage": result.get("usage", {}), + } + + +from app.core.auth.capabilities import SIM_MANAGE, SIM_VIEW +from app.core.auth.workspace_route_capabilities import apply_workspace_route_capabilities + +apply_workspace_route_capabilities( + router, + view_capability=SIM_VIEW, + manage_capability=SIM_MANAGE, +) + diff --git a/app/api/v1/routes/prompt_partials.py b/app/api/v1/routes/prompt_partials.py index f65395c0..3380e7b9 100644 --- a/app/api/v1/routes/prompt_partials.py +++ b/app/api/v1/routes/prompt_partials.py @@ -2,12 +2,12 @@ Prompt Partials API Routes CRUD operations with version history for reusable prompt templates. """ -from fastapi import APIRouter, Depends, HTTPException, status, Query +from fastapi import APIRouter, Depends, HTTPException, status, Query, BackgroundTasks from fastapi.responses import JSONResponse, Response from sqlalchemy.orm import Session from sqlalchemy.exc import IntegrityError, SQLAlchemyError from typing import List, Optional, Dict, Any -from uuid import UUID +from uuid import UUID, uuid4 from pydantic import BaseModel from loguru import logger @@ -187,12 +187,15 @@ def _apply_prompt_partial_kind_filter( @router.post("/generate") async def generate_prompt_with_ai( data: GeneratePromptRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), db: Session = Depends(get_db), ): """Generate a new prompt using AI from a description.""" from app.services.ai.llm_service import llm_service + from app.services.billing.flexprice_service import record_prompt_partial_ai_assisted if not data.description.strip(): raise HTTPException(400, "Description is required") @@ -225,6 +228,15 @@ async def generate_prompt_with_ai( task_defaults={"temperature": 0.7, "max_tokens": 4000}, credential_id=data.credential_id, ) + request_id = uuid4() + background_tasks.add_task( + record_prompt_partial_ai_assisted, + organization_id, + request_id, + workspace_id=workspace_id, + mode="generate", + model=model_str, + ) return {"content": result["text"], "provider": provider_enum.value, "model": model_str} except Exception as e: logger.error(f"[PromptPartials] AI generation failed: {repr(e)}") @@ -234,12 +246,15 @@ async def generate_prompt_with_ai( @router.post("/improve") async def improve_prompt_with_ai( data: ImprovePromptRequest, + background_tasks: BackgroundTasks, organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), api_key: str = Depends(get_api_key), db: Session = Depends(get_db), ): """Improve/reformat existing prompt content using AI.""" from app.services.ai.llm_service import llm_service + from app.services.billing.flexprice_service import record_prompt_partial_ai_assisted if not data.content.strip(): raise HTTPException(400, "Content is required") @@ -268,6 +283,15 @@ async def improve_prompt_with_ai( task_defaults={"temperature": 0.3, "max_tokens": 4000}, credential_id=data.credential_id, ) + request_id = uuid4() + background_tasks.add_task( + record_prompt_partial_ai_assisted, + organization_id, + request_id, + workspace_id=workspace_id, + mode="improve", + model=model_str, + ) return {"content": result["text"], "provider": provider_enum.value, "model": model_str} except Exception as e: logger.error(f"[PromptPartials] AI improve failed: {repr(e)}") diff --git a/app/api/v1/routes/synthetic_traces.py b/app/api/v1/routes/synthetic_traces.py new file mode 100644 index 00000000..ac7524d7 --- /dev/null +++ b/app/api/v1/routes/synthetic_traces.py @@ -0,0 +1,380 @@ +"""Live call trace observability API (OTLP ingest + read).""" + +from __future__ import annotations + +from typing import Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, status +from sqlalchemy.orm import Session + +from app.dependencies import get_api_key, get_db, get_organization_id, get_workspace_id +from app.models.database import EvaluatorResult +from app.models.synthetic_trace_schemas import ( + JsonTraceIngestRequest, + JsonTraceIngestResponse, + OtlpIngestResponse, + OtlpSetupInfo, + SyntheticCallTraceDetail, + SyntheticCallTraceListResponse, + SyntheticCallTraceSummary, + TraceSessionCloseResponse, + TraceSessionCreateRequest, + TraceSessionOtelCorrelation, + TraceSessionResponse, + VALID_TRACE_TRANSPORTS, +) +from app.services.synthetic_traces.otlp_ingest import parse_otlp_body +from app.services.synthetic_traces.trace_service import ( + backfill_missing_traces_from_call_recordings, + build_otlp_setup_info, + build_session_otel_correlation, + close_trace_session, + get_trace_by_call_short_id, + get_trace_by_id, + get_trace_for_result, + ingest_json_spans, + ingest_otlp_spans, + list_traces, + load_trace_detail, + open_trace_session, +) + +router = APIRouter(prefix="/observability/traces", tags=["observability-traces"]) + + +def _lookup_evaluator_result( + db: Session, + id: str, + organization_id: UUID, + workspace_id: UUID, +) -> EvaluatorResult | None: + try: + result_uuid = UUID(id) + return ( + db.query(EvaluatorResult) + .filter( + EvaluatorResult.id == result_uuid, + EvaluatorResult.organization_id == organization_id, + EvaluatorResult.workspace_id == workspace_id, + ) + .first() + ) + except ValueError: + return ( + db.query(EvaluatorResult) + .filter( + EvaluatorResult.result_id == id, + EvaluatorResult.organization_id == organization_id, + EvaluatorResult.workspace_id == workspace_id, + ) + .first() + ) + + +def _api_base_url(request: Request) -> str: + return str(request.base_url).rstrip("/") + + +def _build_trace_detail_response(db: Session, trace) -> SyntheticCallTraceDetail: + detail = load_trace_detail(db, trace) + summary = detail.get("latency_summary") or {} + base = SyntheticCallTraceSummary.model_validate(trace).model_dump() + if summary: + base["turn_count"] = summary.get("turn_count", base.get("turn_count")) + base["response_latency_p50_ms"] = summary.get("response_latency_p50_ms") + base["response_latency_p90_ms"] = summary.get("response_latency_p90_ms") + base["response_latency_p95_ms"] = summary.get("response_latency_p95_ms") + base["component_aggregates"] = summary.get("component_aggregates") + return SyntheticCallTraceDetail( + **base, + turns=detail["turns"], + otel_spans=detail["otel_spans"], + otel_trace_ids=detail["otel_trace_ids"], + pipeline_models=detail.get("pipeline_models") or {}, + ) + + +async def _ingest_otlp_traces_handler( + request: Request, + db: Session, + organization_id: UUID, + workspace_id: UUID, + x_efficientai_run_id: Optional[str], + x_efficientai_agent_id: Optional[str], + x_efficientai_call_short_id: Optional[str], +) -> OtlpIngestResponse: + body = await request.body() + if not body: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Empty OTLP body") + + content_type = request.headers.get("content-type", "") + try: + spans, _fmt = parse_otlp_body(body, content_type) + except Exception as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Failed to parse OTLP payload: {exc}", + ) from exc + + trace, accepted, correlated = ingest_otlp_spans( + db, + organization_id=organization_id, + spans=spans, + header_evaluator_result_id=x_efficientai_run_id, + header_agent_id=x_efficientai_agent_id, + header_call_short_id=x_efficientai_call_short_id, + workspace_id=workspace_id, + ) + return OtlpIngestResponse( + accepted_spans=accepted, + synthetic_call_trace_id=trace.id if trace else None, + correlated=correlated, + ) + + +@router.post("", response_model=OtlpIngestResponse) +async def ingest_observability_traces( + request: Request, + db: Session = Depends(get_db), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + api_key: str = Depends(get_api_key), + x_efficientai_run_id: Optional[str] = Header(None, alias="X-EfficientAI-Run-Id"), + x_efficientai_agent_id: Optional[str] = Header(None, alias="X-EfficientAI-Agent-Id"), + x_efficientai_call_short_id: Optional[str] = Header(None, alias="X-EfficientAI-Call-Short-Id"), +): + """Ingest OTLP spans for a live call (primary export endpoint).""" + _ = api_key + return await _ingest_otlp_traces_handler( + request, + db, + organization_id, + workspace_id, + x_efficientai_run_id, + x_efficientai_agent_id, + x_efficientai_call_short_id, + ) + + +@router.post("/sessions", response_model=TraceSessionResponse) +def create_trace_session( + payload: TraceSessionCreateRequest, + request: Request, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + """Mint call_short_id and open a trace before live audio / OTLP export.""" + _ = api_key + if payload.transport not in VALID_TRACE_TRANSPORTS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"transport must be one of: {', '.join(VALID_TRACE_TRANSPORTS)}", + ) + + if payload.evaluator_result_id: + result = _lookup_evaluator_result( + db, + str(payload.evaluator_result_id), + organization_id, + workspace_id, + ) + if not result: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Evaluator result not found", + ) + + try: + trace = open_trace_session( + db, + organization_id=organization_id, + workspace_id=workspace_id, + evaluator_result_id=payload.evaluator_result_id, + agent_id=payload.agent_id, + transport=payload.transport, + ) + except ValueError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) from exc + + otel = build_session_otel_correlation( + api_base_url=_api_base_url(request), + call_short_id=trace.call_short_id or "", + workspace_id=workspace_id, + evaluator_result_id=payload.evaluator_result_id, + ) + return TraceSessionResponse( + trace_id=trace.id, + call_short_id=trace.call_short_id or "", + workspace_id=workspace_id, + transport=trace.transport, + status=trace.status, + otel_correlation=TraceSessionOtelCorrelation(**otel), + ) + + +@router.post("/sessions/{call_short_id}/close", response_model=TraceSessionCloseResponse) +def close_trace_session_route( + call_short_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + _ = api_key + trace = close_trace_session( + db, + organization_id=organization_id, + call_short_id=call_short_id, + workspace_id=workspace_id, + ) + if not trace: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Trace session not found") + return TraceSessionCloseResponse( + trace_id=trace.id, + call_short_id=call_short_id, + status=trace.status, + ) + + +@router.post("/ingest", response_model=JsonTraceIngestResponse, deprecated=True) +def ingest_json_traces( + payload: JsonTraceIngestRequest, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + """Deprecated: prefer OTLP export. Simple JSON shim for non-OTel prototypes only.""" + _ = api_key + if not payload.spans: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="spans required") + + span_dicts = [s.model_dump() for s in payload.spans] + trace, accepted, correlated = ingest_json_spans( + db, + organization_id=organization_id, + workspace_id=workspace_id, + call_short_id=payload.call_short_id, + spans=span_dicts, + ) + return JsonTraceIngestResponse( + accepted_spans=accepted, + synthetic_call_trace_id=trace.id if trace else None, + correlated=correlated, + ) + + +@router.get("/setup", response_model=OtlpSetupInfo) +def get_otlp_setup( + request: Request, + api_key: str = Depends(get_api_key), +): + """One-time OTLP endpoint + Pipecat config.""" + _ = api_key + return OtlpSetupInfo(**build_otlp_setup_info(api_base_url=_api_base_url(request))) + + +@router.get("", response_model=SyntheticCallTraceListResponse) +def list_observability_traces( + skip: int = Query(0, ge=0), + limit: int = Query(50, ge=1, le=200), + status: Optional[str] = Query(None, description="open or closed"), + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + _ = api_key + rows, total = list_traces( + db, + organization_id=organization_id, + workspace_id=workspace_id, + skip=skip, + limit=limit, + status=status, + ) + return SyntheticCallTraceListResponse( + items=[SyntheticCallTraceSummary.model_validate(r) for r in rows], + total=total, + ) + + +@router.get("/results/{evaluator_result_id}", response_model=SyntheticCallTraceDetail) +def get_trace_for_evaluator_result( + evaluator_result_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + _ = api_key + result = _lookup_evaluator_result( + db, evaluator_result_id, organization_id, workspace_id + ) + if not result: + raise HTTPException(status_code=404, detail="Evaluator result not found") + + backfill_missing_traces_from_call_recordings( + db, organization_id=organization_id, limit=5 + ) + + trace = get_trace_for_result( + db, + organization_id=organization_id, + evaluator_result_id=result.id, + workspace_id=workspace_id, + ) + if not trace and result.synthetic_call_trace_id: + trace = get_trace_by_id( + db, + organization_id=organization_id, + trace_id=result.synthetic_call_trace_id, + workspace_id=workspace_id, + ) + if not trace: + raise HTTPException(status_code=404, detail="Call trace not found") + + return _build_trace_detail_response(db, trace) + + +@router.get("/by-call-short-id/{call_short_id}", response_model=SyntheticCallTraceDetail) +def get_trace_by_call_short_id_route( + call_short_id: str, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + _ = api_key + trace = get_trace_by_call_short_id( + db, + organization_id=organization_id, + call_short_id=call_short_id, + workspace_id=workspace_id, + ) + if not trace: + raise HTTPException(status_code=404, detail="Call trace not found") + return _build_trace_detail_response(db, trace) + + +@router.get("/{trace_id}", response_model=SyntheticCallTraceDetail) +def get_observability_trace( + trace_id: UUID, + organization_id: UUID = Depends(get_organization_id), + workspace_id: UUID = Depends(get_workspace_id), + db: Session = Depends(get_db), + api_key: str = Depends(get_api_key), +): + _ = api_key + trace = get_trace_by_id( + db, + organization_id=organization_id, + trace_id=trace_id, + workspace_id=workspace_id, + ) + if not trace: + raise HTTPException(status_code=404, detail="Call trace not found") + return _build_trace_detail_response(db, trace) diff --git a/app/api/v1/routes/test_agents.py b/app/api/v1/routes/test_agents.py index f6b6ea1b..6ce1f7a9 100644 --- a/app/api/v1/routes/test_agents.py +++ b/app/api/v1/routes/test_agents.py @@ -16,10 +16,7 @@ TestAgentConversationUpdate, TestAgentConversationResponse ) -from app.services.billing.flexprice_service import ( - record_test_agent_conversation_ended, - record_test_agent_conversation_started, -) +from app.services.billing.flexprice_service import record_test_agent_conversation_ended from app.services.testing.test_agent_service import test_agent_service router = APIRouter(prefix="/test-agents", tags=["test-agents"]) @@ -102,12 +99,6 @@ async def start_conversation( organization_id=organization_id, db=db ) - background_tasks.add_task( - record_test_agent_conversation_started, - organization_id, - conversation_id, - workspace_id=workspace_id, - ) return conversation except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) diff --git a/app/api/v1/routes/vobiz_telephony.py b/app/api/v1/routes/vobiz_telephony.py index 052d03b9..66a2875b 100644 --- a/app/api/v1/routes/vobiz_telephony.py +++ b/app/api/v1/routes/vobiz_telephony.py @@ -1,757 +1,810 @@ -"""Vobiz telephony API routes (per-org BYO credentials + platform pool fallback).""" - -from __future__ import annotations - -import random -import string -from typing import Any, Dict, List, Optional -from uuid import UUID - -from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket, WebSocketDisconnect, status -from loguru import logger -from pydantic import BaseModel -from sqlalchemy.orm import Session - -from app.config import settings -from app.dependencies import get_api_key, get_db, get_organization_id -from app.models.database import Agent, CallRecording, CallRecordingSource, Evaluator, TelephonyPhoneNumber -from app.models.enums import CallRecordingStatus -from app.services.telephony.phone_routing import resolve_inbound_agent_for_number -from app.services.telephony.plivo_client import normalize_e164 -from app.services.telephony.vobiz_agent_context import ( - build_vobiz_ws_url, - extract_webhook_params, - resolve_vobiz_agent_context, - vobiz_webhook_base_url, -) -from app.services.telephony.call_recording_lifecycle import ( - create_inbound_call_recording, - finalize_call_on_media_disconnect, - find_call_recording, - ingest_carrier_recording_url, - link_provider_call_id, - mark_call_in_progress, - update_call_from_vobiz_event, -) -from app.services.telephony.vobiz_client import build_vobiz_client_for_org -from app.services.telephony.vobiz_number_service import ( - deactivate_imported_number, - import_vobiz_numbers, - list_available_vobiz_numbers, -) -from app.services.telephony.webhook_auth import verify_vobiz_webhook -from app.services.telephony.vobiz_outbound_pool import ( - outbound_pool_api_payload, - release_pool_slot, - resolve_outbound_from_number, -) -from app.services.telephony.vobiz_session import create_call_session, delete_call_session, get_call_session -from app.services.telephony.vobiz_xml import reject_call, speak_and_hangup, stream_to_agent -from app.services.voice_agent.bot_fast_api import run_bot -from app.services.voice_agent.voice_bundle import run_voice_bundle_fastapi -from efficientai.runner.utils import parse_telephony_websocket -from efficientai.serializers.vobiz import VobizFrameSerializer - -# Exposed at module scope so tests can patch `.delay` without importing Celery tasks. -initiate_vobiz_outbound_call_task = None - -router = APIRouter(prefix="/telephony/vobiz", tags=["Vobiz Telephony"]) -webhook_router = APIRouter(prefix="/telephony/vobiz", tags=["Vobiz Telephony Webhooks"]) -ws_router = APIRouter(prefix="/telephony/vobiz", tags=["Vobiz Telephony Media"]) - - -class VobizOutboundCallRequest(BaseModel): - to_number: str - agent_id: UUID - from_number: Optional[str] = None - persona_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - evaluator_id: Optional[UUID] = None - - -class VobizOutboundCallResponse(BaseModel): - provider_request_uuid: str - call_status: str - from_number: str - to_number: str - call_ref: str - call_short_id: str = "" - message: str = "Outbound call initiated" - - -class VobizAvailableNumberResponse(BaseModel): - e164: str - provider_number_id: Optional[str] = None - country: Optional[str] = None - region: Optional[str] = None - capabilities: Optional[Dict[str, Any]] = None - status: Optional[str] = None - application_id: Optional[str] = None - already_imported: bool = False - imported_number_id: Optional[str] = None - - -class VobizImportNumbersRequest(BaseModel): - numbers: List[str] - agent_id: Optional[UUID] = None - - -class VobizImportNumberResult(BaseModel): - number: str - success: bool - message: str - answer_url: str - webhook_configured: Optional[bool] = None - imported_number_id: Optional[str] = None - application_id: Optional[str] = None - - -class VobizImportNumbersResponse(BaseModel): - provider: str = "vobiz" - results: List[VobizImportNumberResult] - answer_url: str - - -class VobizOutboundPoolNumberResponse(BaseModel): - phone_number: str - provider: str - - -class VobizOutboundPoolResponse(BaseModel): - numbers: List[VobizOutboundPoolNumberResponse] - max_concurrent_per_org: int - shared_across_orgs: bool = True - - -def _parse_request_payload(raw: Any) -> Dict[str, Any]: - if isinstance(raw, dict): - return raw - if hasattr(raw, "items"): - return dict(raw) - return {} - - -async def _read_webhook_payload(request: Request) -> Dict[str, Any]: - if not getattr(request.state, "webhook_raw_body", None): - request.state.webhook_raw_body = await request.body() - params: Dict[str, Any] = dict(request.query_params) - content_type = (request.headers.get("content-type") or "").lower() - if "application/json" in content_type: - try: - body = await request.json() - if isinstance(body, dict): - params.update(body) - return params - except Exception: - pass - try: - form = await request.form() - params.update(_parse_request_payload(form)) - except Exception: - pass - return params - - -def _resolve_agent_for_answer( - db: Session, - params: Dict[str, Any], - *, - call_ref: Optional[str] = None, -) -> tuple[Optional[UUID], Optional[UUID], Optional[str]]: - """Return (agent_id, organization_id, session_token).""" - if call_ref: - session = get_call_session(call_ref) - if session and session.agent_id and session.organization_id: - return UUID(session.agent_id), UUID(session.organization_id), call_ref - - agent_id, organization_id = resolve_inbound_agent_for_number(db, params.get("to")) - if not agent_id or not organization_id: - return None, None, None - return agent_id, organization_id, None - - -@router.get("/numbers/available", response_model=List[VobizAvailableNumberResponse]) -async def list_vobiz_available_numbers( - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - del api_key - try: - return list_available_vobiz_numbers(db, organization_id) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - - -@router.post("/numbers/import", response_model=VobizImportNumbersResponse) -async def import_vobiz_numbers_route( - payload: VobizImportNumbersRequest, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - del api_key - try: - result = import_vobiz_numbers( - db, - organization_id, - numbers=payload.numbers, - agent_id=payload.agent_id, - ) - return VobizImportNumbersResponse(**result) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - - -@router.delete("/numbers/{number_id}", status_code=status.HTTP_204_NO_CONTENT) -async def delete_imported_vobiz_number( - number_id: UUID, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - del api_key - try: - deactivate_imported_number(db, organization_id, number_id) - except ValueError as e: - message = str(e) - if message == "Imported number not found": - raise HTTPException(status_code=404, detail=message) from e - status_code = ( - status.HTTP_409_CONFLICT - if "active number-masking sessions" in message - else status.HTTP_400_BAD_REQUEST - ) - raise HTTPException(status_code=status_code, detail=message) from e - return Response(status_code=status.HTTP_204_NO_CONTENT) - - -@router.get("/outbound-pool", response_model=VobizOutboundPoolResponse) -async def get_vobiz_outbound_pool( - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), -): - del organization_id, api_key - return VobizOutboundPoolResponse(**outbound_pool_api_payload()) - - -@router.post("/calls/outbound", response_model=VobizOutboundCallResponse) -async def create_vobiz_outbound_call( - payload: VobizOutboundCallRequest, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db), -): - del api_key - try: - build_vobiz_client_for_org(db, organization_id) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - - agent = db.query(Agent).filter( - Agent.id == payload.agent_id, - Agent.organization_id == organization_id, - ).first() - if not agent: - raise HTTPException(status_code=404, detail="Agent not found for organization") - - persona_id = payload.persona_id - scenario_id = payload.scenario_id - evaluator_id = payload.evaluator_id - - if payload.evaluator_id: - evaluator = db.query(Evaluator).filter( - Evaluator.id == payload.evaluator_id, - Evaluator.organization_id == organization_id, - ).first() - if not evaluator: - raise HTTPException(status_code=404, detail="Evaluator not found for organization") - if evaluator.agent_id: - agent = db.query(Agent).filter( - Agent.id == evaluator.agent_id, - Agent.organization_id == organization_id, - ).first() - if not agent: - raise HTTPException(status_code=404, detail="Agent not found for evaluator") - payload = payload.model_copy(update={"agent_id": agent.id}) - persona_id = persona_id or evaluator.persona_id - scenario_id = scenario_id or evaluator.scenario_id - - try: - from_number, used_pool, provider = resolve_outbound_from_number( - db, - organization_id, - explicit_from_number=payload.from_number, - ) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - - if provider != "vobiz": - if used_pool: - release_pool_slot(organization_id) - raise HTTPException( - status_code=400, - detail=( - f"Outbound via the Vobiz API requires a Vobiz caller ID; " - f"resolved provider is {provider}. Use org-owned numbers or " - f"configure Vobiz entries in telephony.outbound_pool." - ), - ) - - to_number = normalize_e164(payload.to_number) - - session = create_call_session( - agent_id=str(agent.id), - organization_id=str(organization_id), - direction="outbound", - from_number=from_number, - to_number=to_number, - used_pool=used_pool, - persona_id=str(persona_id) if persona_id else None, - scenario_id=str(scenario_id) if scenario_id else None, - evaluator_id=str(evaluator_id) if evaluator_id else None, - ) - - base = vobiz_webhook_base_url() - answer_url = ( - f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/answer" - f"?call_ref={session.call_ref}" - ) - events_url = ( - f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/events" - f"?call_ref={session.call_ref}" - ) - recording_url = f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/recording-ready" - - call_short_id = "".join(random.choices(string.digits, k=6)) - recording = CallRecording( - organization_id=organization_id, - workspace_id=agent.workspace_id, - call_short_id=call_short_id, - status=CallRecordingStatus.PENDING, - source=CallRecordingSource.WEBHOOK, - call_event="outbound_initiated", - call_data={ - "call_ref": session.call_ref, - "call_short_id": call_short_id, - "recording_callback": recording_url, - "used_pool": used_pool, - "evaluator_id": str(evaluator_id) if evaluator_id else None, - "direction": "outbound", - "from_number": from_number, - "to_number": to_number, - "live_transcript": [], - }, - provider_call_id=None, - provider_platform="vobiz", - agent_id=agent.id, - ) - db.add(recording) - db.commit() - db.refresh(recording) - - task = initiate_vobiz_outbound_call_task - if task is None: - from app.workers.tasks.initiate_vobiz_outbound import ( - initiate_vobiz_outbound_call_task as task, - ) - - task.delay( - organization_id=str(organization_id), - call_ref=session.call_ref, - from_number=from_number, - to_number=to_number, - answer_url=answer_url, - events_url=events_url, - used_pool=used_pool, - call_recording_id=str(recording.id), - ) - - return VobizOutboundCallResponse( - provider_request_uuid="", - call_status="queued", - from_number=from_number, - to_number=to_number, - call_ref=session.call_ref, - call_short_id=call_short_id, - ) - - -@webhook_router.post("/webhooks/answer") -@webhook_router.get("/webhooks/answer") -async def vobiz_answer_webhook( - request: Request, - call_ref: Optional[str] = None, - db: Session = Depends(get_db), -): - payload = await _read_webhook_payload(request) - params = extract_webhook_params(payload) - if not call_ref: - call_ref = request.query_params.get("call_ref") - verify_vobiz_webhook(request, payload, "answer", db, call_ref=call_ref) - logger.info( - "Vobiz answer webhook To={} From={} call_ref={}", - params.get("to"), - params.get("from"), - call_ref or request.query_params.get("call_ref"), - ) - agent_id, organization_id, session_token = _resolve_agent_for_answer( - db, params, call_ref=call_ref - ) - if not agent_id or not organization_id: - return Response(content=reject_call("No active routing found for this number."), media_type="application/xml") - - if not session_token: - agent = db.query(Agent).filter(Agent.id == agent_id).first() - inbound_evaluator_id: Optional[UUID] = None - inbound_evaluator_result_id: Optional[UUID] = None - inbound_persona_id: Optional[UUID] = None - inbound_scenario_id: Optional[UUID] = None - if agent and agent.workspace_id: - from app.services.evaluators.evaluator_inbound_service import ( - consume_inbound_evaluator_combination, - create_inbound_evaluator_result, - find_inbound_suite_for_agent, - ) - - suite = find_inbound_suite_for_agent( - db, agent, organization_id, agent.workspace_id - ) - if suite: - selected, _idx, _next_idx = consume_inbound_evaluator_combination(db, suite) - inbound_evaluator_id = selected.id - inbound_persona_id = selected.persona_id - inbound_scenario_id = selected.scenario_id - result_row = create_inbound_evaluator_result( - db, - organization_id, - agent.workspace_id, - selected, - ) - inbound_evaluator_result_id = result_row.id - - session = create_call_session( - agent_id=str(agent_id), - organization_id=str(organization_id), - direction="inbound", - from_number=params.get("from"), - to_number=params.get("to"), - persona_id=str(inbound_persona_id) if inbound_persona_id else None, - scenario_id=str(inbound_scenario_id) if inbound_scenario_id else None, - evaluator_id=str(inbound_evaluator_id) if inbound_evaluator_id else None, - ) - session_token = session.call_ref - persona_id = session.persona_id - scenario_id = session.scenario_id - if agent: - inbound_row = create_inbound_call_recording( - db, - agent=agent, - organization_id=organization_id, - call_ref=session_token, - from_number=params.get("from"), - to_number=params.get("to"), - provider_call_id=params.get("call_uuid"), - evaluator_id=inbound_evaluator_id, - evaluator_result_id=inbound_evaluator_result_id, - ) - # region agent log - from app.utils.debug_agent_log import agent_debug_log - - agent_debug_log( - "vobiz_telephony.py:answer_webhook", - "inbound CallRecording created", - { - "call_ref": session_token, - "call_short_id": inbound_row.call_short_id, - "provider_call_id": params.get("call_uuid"), - }, - "H1", - ) - # endregion - else: - existing_session = get_call_session(session_token) - persona_id = existing_session.persona_id if existing_session else None - scenario_id = existing_session.scenario_id if existing_session else None - - call_uuid = params.get("call_uuid") - if session_token and call_uuid: - link_provider_call_id(db, call_ref=session_token, provider_call_id=call_uuid) - - ws_url = build_vobiz_ws_url( - agent_id=str(agent_id), - session=session_token, - persona_id=persona_id, - scenario_id=scenario_id, - ) - record_action_url = None - if settings.VOBIZ_CARRIER_SESSION_RECORDING: - record_action_url = ( - f"{vobiz_webhook_base_url()}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/recording-ready" - f"?call_ref={session_token}" - ) - xml = stream_to_agent(ws_url, record_action_url=record_action_url) - return Response(content=xml, media_type="application/xml") - - -@webhook_router.post("/webhooks/events") -async def vobiz_events_webhook( - request: Request, - db: Session = Depends(get_db), -): - payload = await _read_webhook_payload(request) - params = extract_webhook_params(payload) - call_ref = request.query_params.get("call_ref") or payload.get("call_ref") - verify_vobiz_webhook(request, payload, "events", db, call_ref=call_ref) - call_uuid = params.get("call_uuid") - if not call_uuid: - return {"status": "ignored"} - - update_call_from_vobiz_event( - db, - provider_call_id=call_uuid, - call_status=params.get("call_status"), - payload=payload, - call_ref=call_ref, - ) - - row = find_call_recording( - db, - provider_call_id=call_uuid, - call_ref=call_ref, - ) - terminal_statuses = { - "completed", - "hangup", - "failed", - "busy", - "no-answer", - "canceled", - } - if call_ref and (params.get("call_status") or "").lower() in terminal_statuses: - session = get_call_session(call_ref) - if session and session.used_pool: - release_pool_slot(UUID(session.organization_id)) - delete_call_session(call_ref) - elif row and (params.get("call_status") or "").lower() in terminal_statuses: - call_data = row.call_data if isinstance(row.call_data, dict) else {} - if call_data.get("used_pool"): - release_pool_slot(row.organization_id) - - return {"status": "ok"} - - -@webhook_router.post("/webhooks/recording-ready") -async def vobiz_recording_ready_webhook( - request: Request, - db: Session = Depends(get_db), -): - payload = await _read_webhook_payload(request) - params = extract_webhook_params(payload) - call_ref = request.query_params.get("call_ref") or payload.get("call_ref") - verify_vobiz_webhook(request, payload, "recording", db, call_ref=call_ref) - recording_url = params.get("recording_url") - call_uuid = params.get("call_uuid") - logger.info( - "Vobiz recording ready call_uuid={} recording_id={} url={}", - call_uuid, - params.get("recording_id"), - recording_url, - ) - if call_uuid: - call_ref = request.query_params.get("call_ref") or payload.get("call_ref") - row = find_call_recording( - db, - provider_call_id=call_uuid, - call_ref=call_ref, - ) - if row: - from sqlalchemy.orm.attributes import flag_modified - - current = dict(row.call_data) if isinstance(row.call_data, dict) else {} - current["recording"] = payload - if recording_url: - current["recording_url"] = recording_url - row.call_data = current - flag_modified(row, "call_data") - db.commit() - if recording_url and settings.VOBIZ_CARRIER_SESSION_RECORDING: - ingest_carrier_recording_url(db, row, recording_url) - elif recording_url and not settings.VOBIZ_CARRIER_SESSION_RECORDING: - logger.debug( - "Skipping Vobiz carrier recording ingest (VOBIZ_CARRIER_SESSION_RECORDING=false) " - "call_short_id={}", - row.call_short_id, - ) - # region agent log - from app.utils.debug_agent_log import agent_debug_log - - agent_debug_log( - "vobiz_telephony.py:recording_ready", - "recording webhook persisted", - { - "call_short_id": row.call_short_id, - "call_ref": call_ref, - "call_uuid": call_uuid, - "has_recording_url": bool(recording_url), - }, - "H4", - ) - # endregion - else: - # region agent log - from app.utils.debug_agent_log import agent_debug_log - - agent_debug_log( - "vobiz_telephony.py:recording_ready", - "recording webhook: no CallRecording match", - {"call_uuid": call_uuid, "call_ref": call_ref}, - "H4", - ) - # endregion - return {"status": "ok"} - - -@ws_router.websocket("/ws") -async def vobiz_media_websocket(websocket: WebSocket): - agent_id = websocket.query_params.get("agent_id") - session_token = websocket.query_params.get("session") - persona_id = websocket.query_params.get("persona_id") - scenario_id = websocket.query_params.get("scenario_id") - - if not agent_id or not session_token: - await websocket.close(code=1008, reason="agent_id and session are required") - return - - session = get_call_session(session_token) - if not session: - await websocket.close(code=1008, reason="Invalid or expired call session") - return - if session.agent_id != agent_id: - await websocket.close(code=1008, reason="Session does not match agent") - return - - persona_id = persona_id or session.persona_id - scenario_id = scenario_id or session.scenario_id - - await websocket.accept() - db = next(get_db()) - call_row = find_call_recording(db, call_ref=session_token, provider_call_id=None) - call_short_id = call_row.call_short_id if call_row else None - # region agent log - from app.utils.debug_agent_log import agent_debug_log - - agent_debug_log( - "vobiz_telephony.py:media_websocket", - "CallRecording lookup for media session", - { - "call_ref": session_token, - "call_short_id": call_short_id, - "found_row": call_row is not None, - "provider_call_id": call_row.provider_call_id if call_row else None, - }, - "H1", - ) - # endregion - if not call_short_id: - logger.warning( - "No CallRecording for Vobiz session {}; live transcript and recording will not be linked", - session_token, - ) - try: - mark_call_in_progress(db, call_ref=session_token) - try: - transport_type, call_data = await parse_telephony_websocket(websocket) - if transport_type not in {"plivo", "unknown"}: - logger.warning("Unexpected telephony transport type for Vobiz: {}", transport_type) - stream_id = call_data.get("stream_id") or "" - call_id = call_data.get("call_id") - if call_id: - link_provider_call_id(db, call_ref=session_token, provider_call_id=str(call_id)) - if not stream_id: - await websocket.close(code=1011, reason="Missing stream id from Vobiz") - return - - context = resolve_vobiz_agent_context( - db, - agent_id=UUID(agent_id), - organization_id=UUID(session.organization_id), - persona_id=persona_id, - scenario_id=scenario_id, - ) - serializer = VobizFrameSerializer( - stream_id=stream_id, - call_id=call_id, - auth_id=settings.VOBIZ_AUTH_ID, - auth_token=settings.VOBIZ_AUTH_TOKEN, - params=VobizFrameSerializer.InputParams( - sample_rate=8000, - api_base=settings.VOBIZ_API_BASE, - ), - ) - - if context.use_voice_bundle_pipeline: - from app.services.voice_agent.call_silence_hangup import resolve_agent_silence_hangup_secs - - hangup_secs = resolve_agent_silence_hangup_secs(context.agent) - await run_voice_bundle_fastapi( - websocket, - context.system_instruction, - str(context.organization_id), - str(context.workspace_id) if context.workspace_id else None, - agent_id, - persona_id, - scenario_id, - voice_bundle=context.voice_bundle, - persona=context.persona, - stt_api_key=context.stt_api_key, - tts_api_key=context.tts_api_key, - llm_api_key=context.llm_api_key, - serializer=serializer, - telephony_mode=True, - call_short_id=call_short_id, - silence_hangup_secs=hangup_secs, - ) - else: - if not context.google_api_key: - await websocket.close(code=1011, reason="Google API key not configured for agent") - return - from app.services.voice_agent.call_silence_hangup import resolve_agent_silence_hangup_secs - - hangup_secs = resolve_agent_silence_hangup_secs(context.agent) - await run_bot( - websocket, - context.google_api_key, - context.system_instruction, - str(context.organization_id), - agent_id, - persona_id, - scenario_id, - model_name=context.model_name, - serializer=serializer, - telephony_mode=True, - call_short_id=call_short_id, - silence_hangup_secs=hangup_secs, - ) - except ValueError as e: - logger.error("Vobiz media websocket setup failed: {}", e) - await websocket.close(code=1011, reason=str(e)) - except WebSocketDisconnect: - logger.info("Vobiz media websocket disconnected") - except Exception as e: - logger.error("Vobiz media websocket error: {}", e, exc_info=True) - try: - await websocket.close(code=1011, reason="Server error") - except Exception: - pass - finally: - from app.database import SessionLocal - - finalize_db = SessionLocal() - try: - finalize_call_on_media_disconnect(finalize_db, call_ref=session_token) - finally: - finalize_db.close() - delete_call_session(session_token) - finally: - db.close() +"""Vobiz telephony API routes (per-org BYO credentials + platform pool fallback).""" + +from __future__ import annotations + +import random +import string +from typing import Any, Dict, List, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket, WebSocketDisconnect, status +from loguru import logger +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from app.config import settings +from app.dependencies import get_api_key, get_db, get_organization_id +from app.models.database import Agent, CallRecording, CallRecordingSource, Evaluator, TelephonyPhoneNumber +from app.models.enums import CallRecordingStatus +from app.services.telephony.phone_routing import resolve_inbound_agent_for_number +from app.services.telephony.plivo_client import normalize_e164 +from app.services.telephony.vobiz_agent_context import ( + build_vobiz_ws_url, + extract_webhook_params, + resolve_vobiz_agent_context, + vobiz_webhook_base_url, +) +from app.services.telephony.call_recording_lifecycle import ( + create_inbound_call_recording, + finalize_call_on_media_disconnect, + find_call_recording, + ingest_carrier_recording_url, + link_provider_call_id, + mark_call_in_progress, + update_call_from_vobiz_event, +) +from app.services.telephony.vobiz_client import build_vobiz_client_for_org +from app.services.telephony.vobiz_number_service import ( + deactivate_imported_number, + import_vobiz_numbers, + list_available_vobiz_numbers, +) +from app.services.telephony.webhook_auth import verify_vobiz_webhook +from app.services.telephony.vobiz_outbound_pool import ( + outbound_pool_api_payload, + release_pool_slot, + resolve_outbound_from_number, +) +from app.services.telephony.vobiz_session import create_call_session, delete_call_session, get_call_session +from app.services.telephony.vobiz_xml import reject_call, speak_and_hangup, stream_to_agent +from app.services.voice_agent.bot_fast_api import run_bot +from app.services.voice_agent.voice_bundle import run_voice_bundle_fastapi +from efficientai.runner.utils import parse_telephony_websocket +from efficientai.serializers.vobiz import VobizFrameSerializer + +# Exposed at module scope so tests can patch `.delay` without importing Celery tasks. +initiate_vobiz_outbound_call_task = None + +router = APIRouter(prefix="/telephony/vobiz", tags=["Vobiz Telephony"]) +webhook_router = APIRouter(prefix="/telephony/vobiz", tags=["Vobiz Telephony Webhooks"]) +ws_router = APIRouter(prefix="/telephony/vobiz", tags=["Vobiz Telephony Media"]) + + +class VobizOutboundCallRequest(BaseModel): + to_number: str + agent_id: UUID + from_number: Optional[str] = None + persona_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + evaluator_id: Optional[UUID] = None + + +class VobizOutboundCallResponse(BaseModel): + provider_request_uuid: str + call_status: str + from_number: str + to_number: str + call_ref: str + call_short_id: str = "" + evaluator_result_id: Optional[UUID] = None + result_id: Optional[str] = None + otel_correlation: Optional[Dict[str, Any]] = None + message: str = "Outbound call initiated" + + +class VobizAvailableNumberResponse(BaseModel): + e164: str + provider_number_id: Optional[str] = None + country: Optional[str] = None + region: Optional[str] = None + capabilities: Optional[Dict[str, Any]] = None + status: Optional[str] = None + application_id: Optional[str] = None + already_imported: bool = False + imported_number_id: Optional[str] = None + + +class VobizImportNumbersRequest(BaseModel): + numbers: List[str] + agent_id: Optional[UUID] = None + + +class VobizImportNumberResult(BaseModel): + number: str + success: bool + message: str + answer_url: str + webhook_configured: Optional[bool] = None + imported_number_id: Optional[str] = None + application_id: Optional[str] = None + + +class VobizImportNumbersResponse(BaseModel): + provider: str = "vobiz" + results: List[VobizImportNumberResult] + answer_url: str + + +class VobizOutboundPoolNumberResponse(BaseModel): + phone_number: str + provider: str + + +class VobizOutboundPoolResponse(BaseModel): + numbers: List[VobizOutboundPoolNumberResponse] + max_concurrent_per_org: int + shared_across_orgs: bool = True + + +def _parse_request_payload(raw: Any) -> Dict[str, Any]: + if isinstance(raw, dict): + return raw + if hasattr(raw, "items"): + return dict(raw) + return {} + + +async def _read_webhook_payload(request: Request) -> Dict[str, Any]: + if not getattr(request.state, "webhook_raw_body", None): + request.state.webhook_raw_body = await request.body() + params: Dict[str, Any] = dict(request.query_params) + content_type = (request.headers.get("content-type") or "").lower() + if "application/json" in content_type: + try: + body = await request.json() + if isinstance(body, dict): + params.update(body) + return params + except Exception: + pass + try: + form = await request.form() + params.update(_parse_request_payload(form)) + except Exception: + pass + return params + + +def _resolve_agent_for_answer( + db: Session, + params: Dict[str, Any], + *, + call_ref: Optional[str] = None, +) -> tuple[Optional[UUID], Optional[UUID], Optional[str]]: + """Return (agent_id, organization_id, session_token).""" + if call_ref: + session = get_call_session(call_ref) + if session and session.agent_id and session.organization_id: + return UUID(session.agent_id), UUID(session.organization_id), call_ref + + agent_id, organization_id = resolve_inbound_agent_for_number(db, params.get("to")) + if not agent_id or not organization_id: + return None, None, None + return agent_id, organization_id, None + + +@router.get("/numbers/available", response_model=List[VobizAvailableNumberResponse]) +async def list_vobiz_available_numbers( + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + del api_key + try: + return list_available_vobiz_numbers(db, organization_id) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + +@router.post("/numbers/import", response_model=VobizImportNumbersResponse) +async def import_vobiz_numbers_route( + payload: VobizImportNumbersRequest, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + del api_key + try: + result = import_vobiz_numbers( + db, + organization_id, + numbers=payload.numbers, + agent_id=payload.agent_id, + ) + return VobizImportNumbersResponse(**result) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + +@router.delete("/numbers/{number_id}", status_code=status.HTTP_204_NO_CONTENT) +async def delete_imported_vobiz_number( + number_id: UUID, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + del api_key + try: + deactivate_imported_number(db, organization_id, number_id) + except ValueError as e: + message = str(e) + if message == "Imported number not found": + raise HTTPException(status_code=404, detail=message) from e + status_code = ( + status.HTTP_409_CONFLICT + if "active number-masking sessions" in message + else status.HTTP_400_BAD_REQUEST + ) + raise HTTPException(status_code=status_code, detail=message) from e + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.get("/outbound-pool", response_model=VobizOutboundPoolResponse) +async def get_vobiz_outbound_pool( + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), +): + del organization_id, api_key + return VobizOutboundPoolResponse(**outbound_pool_api_payload()) + + +@router.post("/calls/outbound", response_model=VobizOutboundCallResponse) +async def create_vobiz_outbound_call( + payload: VobizOutboundCallRequest, + request: Request, + organization_id: UUID = Depends(get_organization_id), + api_key: str = Depends(get_api_key), + db: Session = Depends(get_db), +): + del api_key + try: + build_vobiz_client_for_org(db, organization_id) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + agent = db.query(Agent).filter( + Agent.id == payload.agent_id, + Agent.organization_id == organization_id, + ).first() + if not agent: + raise HTTPException(status_code=404, detail="Agent not found for organization") + + persona_id = payload.persona_id + scenario_id = payload.scenario_id + evaluator_id = payload.evaluator_id + evaluator: Evaluator | None = None + + if payload.evaluator_id: + evaluator = db.query(Evaluator).filter( + Evaluator.id == payload.evaluator_id, + Evaluator.organization_id == organization_id, + ).first() + if not evaluator: + raise HTTPException(status_code=404, detail="Evaluator not found for organization") + if evaluator.agent_id: + agent = db.query(Agent).filter( + Agent.id == evaluator.agent_id, + Agent.organization_id == organization_id, + ).first() + if not agent: + raise HTTPException(status_code=404, detail="Agent not found for evaluator") + persona_id = persona_id or evaluator.persona_id + scenario_id = scenario_id or evaluator.scenario_id + + to_number = normalize_e164(payload.to_number) + + if evaluator and agent.workspace_id: + from app.services.evaluators.evaluator_phone_run_service import initiate_phone_evaluator_call + + call_ref, call_short_id, result_response = initiate_phone_evaluator_call( + db, + organization_id, + agent.workspace_id, + evaluator, + agent, + to_number, + from_number=payload.from_number, + ) + recording = ( + db.query(CallRecording) + .filter(CallRecording.call_short_id == call_short_id) + .first() + ) + from_number = "" + if recording and isinstance(recording.call_data, dict): + from_number = recording.call_data.get("from_number") or "" + from app.services.synthetic_traces.trace_service import build_session_otel_correlation + + otel_correlation = None + if call_short_id and agent.workspace_id: + otel_correlation = build_session_otel_correlation( + api_base_url=str(request.base_url).rstrip("/"), + call_short_id=call_short_id, + workspace_id=agent.workspace_id, + evaluator_result_id=result_response.id if result_response else None, + ) + return VobizOutboundCallResponse( + provider_request_uuid="", + call_status="queued", + from_number=from_number, + to_number=to_number, + call_ref=call_ref, + call_short_id=call_short_id, + evaluator_result_id=result_response.id if result_response else None, + result_id=result_response.result_id if result_response else None, + otel_correlation=otel_correlation, + ) + + try: + from_number, used_pool, provider = resolve_outbound_from_number( + db, + organization_id, + explicit_from_number=payload.from_number, + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + if provider != "vobiz": + if used_pool: + release_pool_slot(organization_id) + raise HTTPException( + status_code=400, + detail=( + f"Outbound via the Vobiz API requires a Vobiz caller ID; " + f"resolved provider is {provider}. Use org-owned numbers or " + f"configure Vobiz entries in telephony.outbound_pool." + ), + ) + + to_number = normalize_e164(payload.to_number) + + session = create_call_session( + agent_id=str(agent.id), + organization_id=str(organization_id), + direction="outbound", + from_number=from_number, + to_number=to_number, + used_pool=used_pool, + persona_id=str(persona_id) if persona_id else None, + scenario_id=str(scenario_id) if scenario_id else None, + evaluator_id=str(evaluator_id) if evaluator_id else None, + ) + + base = vobiz_webhook_base_url() + answer_url = ( + f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/answer" + f"?call_ref={session.call_ref}" + ) + events_url = ( + f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/events" + f"?call_ref={session.call_ref}" + ) + recording_url = f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/recording-ready" + + call_short_id = "".join(random.choices(string.digits, k=6)) + recording = CallRecording( + organization_id=organization_id, + workspace_id=agent.workspace_id, + call_short_id=call_short_id, + status=CallRecordingStatus.PENDING, + source=CallRecordingSource.WEBHOOK, + call_event="outbound_initiated", + call_data={ + "call_ref": session.call_ref, + "call_short_id": call_short_id, + "recording_callback": recording_url, + "used_pool": used_pool, + "evaluator_id": str(evaluator_id) if evaluator_id else None, + "direction": "outbound", + "from_number": from_number, + "to_number": to_number, + "live_transcript": [], + }, + provider_call_id=None, + provider_platform="vobiz", + agent_id=agent.id, + ) + db.add(recording) + db.commit() + db.refresh(recording) + + task = initiate_vobiz_outbound_call_task + if task is None: + from app.workers.tasks.initiate_vobiz_outbound import ( + initiate_vobiz_outbound_call_task as task, + ) + + task.delay( + organization_id=str(organization_id), + call_ref=session.call_ref, + from_number=from_number, + to_number=to_number, + answer_url=answer_url, + events_url=events_url, + used_pool=used_pool, + call_recording_id=str(recording.id), + ) + + return VobizOutboundCallResponse( + provider_request_uuid="", + call_status="queued", + from_number=from_number, + to_number=to_number, + call_ref=session.call_ref, + call_short_id=call_short_id, + ) + + +@webhook_router.post("/webhooks/answer") +@webhook_router.get("/webhooks/answer") +async def vobiz_answer_webhook( + request: Request, + call_ref: Optional[str] = None, + db: Session = Depends(get_db), +): + payload = await _read_webhook_payload(request) + params = extract_webhook_params(payload) + if not call_ref: + call_ref = request.query_params.get("call_ref") + verify_vobiz_webhook(request, payload, "answer", db, call_ref=call_ref) + logger.info( + "Vobiz answer webhook To={} From={} call_ref={}", + params.get("to"), + params.get("from"), + call_ref or request.query_params.get("call_ref"), + ) + agent_id, organization_id, session_token = _resolve_agent_for_answer( + db, params, call_ref=call_ref + ) + if not agent_id or not organization_id: + return Response(content=reject_call("No active routing found for this number."), media_type="application/xml") + + if not session_token: + agent = db.query(Agent).filter(Agent.id == agent_id).first() + inbound_evaluator_id: Optional[UUID] = None + inbound_evaluator_result_id: Optional[UUID] = None + inbound_persona_id: Optional[UUID] = None + inbound_scenario_id: Optional[UUID] = None + if agent and agent.workspace_id: + from app.services.evaluators.evaluator_inbound_service import ( + consume_inbound_evaluator_combination, + create_inbound_evaluator_result, + find_inbound_suite_for_agent, + ) + + suite = find_inbound_suite_for_agent( + db, agent, organization_id, agent.workspace_id + ) + if suite: + selected, _idx, _next_idx = consume_inbound_evaluator_combination(db, suite) + inbound_evaluator_id = selected.id + inbound_persona_id = selected.persona_id + inbound_scenario_id = selected.scenario_id + result_row = create_inbound_evaluator_result( + db, + organization_id, + agent.workspace_id, + selected, + ) + inbound_evaluator_result_id = result_row.id + + session = create_call_session( + agent_id=str(agent_id), + organization_id=str(organization_id), + direction="inbound", + from_number=params.get("from"), + to_number=params.get("to"), + persona_id=str(inbound_persona_id) if inbound_persona_id else None, + scenario_id=str(inbound_scenario_id) if inbound_scenario_id else None, + evaluator_id=str(inbound_evaluator_id) if inbound_evaluator_id else None, + ) + session_token = session.call_ref + persona_id = session.persona_id + scenario_id = session.scenario_id + if agent: + inbound_row = create_inbound_call_recording( + db, + agent=agent, + organization_id=organization_id, + call_ref=session_token, + from_number=params.get("from"), + to_number=params.get("to"), + provider_call_id=params.get("call_uuid"), + evaluator_id=inbound_evaluator_id, + evaluator_result_id=inbound_evaluator_result_id, + ) + # region agent log + from app.utils.debug_agent_log import agent_debug_log + + agent_debug_log( + "vobiz_telephony.py:answer_webhook", + "inbound CallRecording created", + { + "call_ref": session_token, + "call_short_id": inbound_row.call_short_id, + "provider_call_id": params.get("call_uuid"), + }, + "H1", + ) + # endregion + else: + existing_session = get_call_session(session_token) + persona_id = existing_session.persona_id if existing_session else None + scenario_id = existing_session.scenario_id if existing_session else None + + call_uuid = params.get("call_uuid") + if session_token and call_uuid: + link_provider_call_id(db, call_ref=session_token, provider_call_id=call_uuid) + + ws_url = build_vobiz_ws_url( + agent_id=str(agent_id), + session=session_token, + persona_id=persona_id, + scenario_id=scenario_id, + ) + record_action_url = None + if settings.VOBIZ_CARRIER_SESSION_RECORDING: + record_action_url = ( + f"{vobiz_webhook_base_url()}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/recording-ready" + f"?call_ref={session_token}" + ) + xml = stream_to_agent(ws_url, record_action_url=record_action_url) + return Response(content=xml, media_type="application/xml") + + +@webhook_router.post("/webhooks/events") +async def vobiz_events_webhook( + request: Request, + db: Session = Depends(get_db), +): + payload = await _read_webhook_payload(request) + params = extract_webhook_params(payload) + call_ref = request.query_params.get("call_ref") or payload.get("call_ref") + verify_vobiz_webhook(request, payload, "events", db, call_ref=call_ref) + call_uuid = params.get("call_uuid") + if not call_uuid: + return {"status": "ignored"} + + update_call_from_vobiz_event( + db, + provider_call_id=call_uuid, + call_status=params.get("call_status"), + payload=payload, + call_ref=call_ref, + ) + + row = find_call_recording( + db, + provider_call_id=call_uuid, + call_ref=call_ref, + ) + terminal_statuses = { + "completed", + "hangup", + "failed", + "busy", + "no-answer", + "canceled", + } + if call_ref and (params.get("call_status") or "").lower() in terminal_statuses: + session = get_call_session(call_ref) + if session and session.used_pool: + release_pool_slot(UUID(session.organization_id)) + delete_call_session(call_ref) + elif row and (params.get("call_status") or "").lower() in terminal_statuses: + call_data = row.call_data if isinstance(row.call_data, dict) else {} + if call_data.get("used_pool"): + release_pool_slot(row.organization_id) + + return {"status": "ok"} + + +@webhook_router.post("/webhooks/recording-ready") +async def vobiz_recording_ready_webhook( + request: Request, + db: Session = Depends(get_db), +): + payload = await _read_webhook_payload(request) + params = extract_webhook_params(payload) + call_ref = request.query_params.get("call_ref") or payload.get("call_ref") + verify_vobiz_webhook(request, payload, "recording", db, call_ref=call_ref) + recording_url = params.get("recording_url") + call_uuid = params.get("call_uuid") + logger.info( + "Vobiz recording ready call_uuid={} recording_id={} url={}", + call_uuid, + params.get("recording_id"), + recording_url, + ) + if call_uuid: + call_ref = request.query_params.get("call_ref") or payload.get("call_ref") + row = find_call_recording( + db, + provider_call_id=call_uuid, + call_ref=call_ref, + ) + if row: + from sqlalchemy.orm.attributes import flag_modified + + current = dict(row.call_data) if isinstance(row.call_data, dict) else {} + current["recording"] = payload + if recording_url: + current["recording_url"] = recording_url + row.call_data = current + flag_modified(row, "call_data") + db.commit() + if recording_url and settings.VOBIZ_CARRIER_SESSION_RECORDING: + ingest_carrier_recording_url(db, row, recording_url) + elif recording_url and not settings.VOBIZ_CARRIER_SESSION_RECORDING: + logger.debug( + "Skipping Vobiz carrier recording ingest (VOBIZ_CARRIER_SESSION_RECORDING=false) " + "call_short_id={}", + row.call_short_id, + ) + # region agent log + from app.utils.debug_agent_log import agent_debug_log + + agent_debug_log( + "vobiz_telephony.py:recording_ready", + "recording webhook persisted", + { + "call_short_id": row.call_short_id, + "call_ref": call_ref, + "call_uuid": call_uuid, + "has_recording_url": bool(recording_url), + }, + "H4", + ) + # endregion + else: + # region agent log + from app.utils.debug_agent_log import agent_debug_log + + agent_debug_log( + "vobiz_telephony.py:recording_ready", + "recording webhook: no CallRecording match", + {"call_uuid": call_uuid, "call_ref": call_ref}, + "H4", + ) + # endregion + return {"status": "ok"} + + +@ws_router.websocket("/ws") +async def vobiz_media_websocket(websocket: WebSocket): + agent_id = websocket.query_params.get("agent_id") + session_token = websocket.query_params.get("session") + persona_id = websocket.query_params.get("persona_id") + scenario_id = websocket.query_params.get("scenario_id") + + if not agent_id or not session_token: + await websocket.close(code=1008, reason="agent_id and session are required") + return + + session = get_call_session(session_token) + if not session: + await websocket.close(code=1008, reason="Invalid or expired call session") + return + if session.agent_id != agent_id: + await websocket.close(code=1008, reason="Session does not match agent") + return + + persona_id = persona_id or session.persona_id + scenario_id = scenario_id or session.scenario_id + + await websocket.accept() + db = next(get_db()) + call_row = find_call_recording(db, call_ref=session_token, provider_call_id=None) + call_short_id = call_row.call_short_id if call_row else None + evaluator_result_id = ( + str(call_row.evaluator_result_id) if call_row and call_row.evaluator_result_id else None + ) + # region agent log + from app.utils.debug_agent_log import agent_debug_log + + agent_debug_log( + "vobiz_telephony.py:media_websocket", + "CallRecording lookup for media session", + { + "call_ref": session_token, + "call_short_id": call_short_id, + "found_row": call_row is not None, + "provider_call_id": call_row.provider_call_id if call_row else None, + }, + "H1", + ) + # endregion + if not call_short_id: + logger.warning( + "No CallRecording for Vobiz session {}; live transcript and recording will not be linked", + session_token, + ) + try: + mark_call_in_progress(db, call_ref=session_token) + try: + transport_type, call_data = await parse_telephony_websocket(websocket) + if transport_type not in {"plivo", "unknown"}: + logger.warning("Unexpected telephony transport type for Vobiz: {}", transport_type) + stream_id = call_data.get("stream_id") or "" + call_id = call_data.get("call_id") + if call_id: + link_provider_call_id(db, call_ref=session_token, provider_call_id=str(call_id)) + if not stream_id: + await websocket.close(code=1011, reason="Missing stream id from Vobiz") + return + + context = resolve_vobiz_agent_context( + db, + agent_id=UUID(agent_id), + organization_id=UUID(session.organization_id), + persona_id=persona_id, + scenario_id=scenario_id, + ) + serializer = VobizFrameSerializer( + stream_id=stream_id, + call_id=call_id, + auth_id=settings.VOBIZ_AUTH_ID, + auth_token=settings.VOBIZ_AUTH_TOKEN, + params=VobizFrameSerializer.InputParams( + sample_rate=8000, + api_base=settings.VOBIZ_API_BASE, + ), + ) + + if context.use_voice_bundle_pipeline: + from app.services.voice_agent.call_silence_hangup import resolve_agent_silence_hangup_secs + + hangup_secs = resolve_agent_silence_hangup_secs(context.agent) + await run_voice_bundle_fastapi( + websocket, + context.system_instruction, + str(context.organization_id), + str(context.workspace_id) if context.workspace_id else None, + agent_id, + persona_id, + scenario_id, + evaluator_id=session.evaluator_id, + result_id=evaluator_result_id, + voice_bundle=context.voice_bundle, + persona=context.persona, + stt_api_key=context.stt_api_key, + tts_api_key=context.tts_api_key, + llm_api_key=context.llm_api_key, + serializer=serializer, + telephony_mode=True, + call_short_id=call_short_id, + silence_hangup_secs=hangup_secs, + ) + else: + if not context.google_api_key: + await websocket.close(code=1011, reason="Google API key not configured for agent") + return + from app.services.voice_agent.call_silence_hangup import resolve_agent_silence_hangup_secs + + hangup_secs = resolve_agent_silence_hangup_secs(context.agent) + await run_bot( + websocket, + context.google_api_key, + context.system_instruction, + str(context.organization_id), + agent_id, + persona_id, + scenario_id, + model_name=context.model_name, + serializer=serializer, + telephony_mode=True, + call_short_id=call_short_id, + silence_hangup_secs=hangup_secs, + ) + except ValueError as e: + logger.error("Vobiz media websocket setup failed: {}", e) + await websocket.close(code=1011, reason=str(e)) + except WebSocketDisconnect: + logger.info("Vobiz media websocket disconnected") + except Exception as e: + logger.error("Vobiz media websocket error: {}", e, exc_info=True) + try: + await websocket.close(code=1011, reason="Server error") + except Exception: + pass + finally: + from app.database import SessionLocal + + finalize_db = SessionLocal() + try: + finalize_call_on_media_disconnect(finalize_db, call_ref=session_token) + finally: + finalize_db.close() + delete_call_session(session_token) + finally: + db.close() diff --git a/app/api/v1/routes/voice_agent.py b/app/api/v1/routes/voice_agent.py index e6d553b5..ef3de23b 100644 --- a/app/api/v1/routes/voice_agent.py +++ b/app/api/v1/routes/voice_agent.py @@ -1,1018 +1,1120 @@ -""" -Voice Agent API Routes -API endpoints for managing voice agent WebSocket connections. -""" - -from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect, Request, status -from sqlalchemy.orm import Session -from uuid import UUID -from typing import Dict, Any, Optional, List -from loguru import logger - -from app.database import get_db -from app.dependencies import get_organization_id, get_api_key -from app.models.database import AIProvider, ModelProvider, Integration, IntegrationPlatform, Workspace -from app.core.encryption import decrypt_api_key -from app.services.voice_agent.bot_fast_api import run_bot -from app.services.ai.llm_service import _resolve_azure_endpoint_from_provider -from app.services.voice_agent.voice_bundle import run_voice_bundle_fastapi -from app.services.storage.s3_service import s3_service - -router = APIRouter(prefix="/voice-agent", tags=["voice-agent"]) -ws_router = APIRouter(prefix="/voice-agent", tags=["voice-agent-media"]) - - -@ws_router.websocket("/ws") -async def websocket_endpoint( - websocket: WebSocket, -): - """ - WebSocket endpoint for voice agent connection. - - Authentication precedence (since browsers can't set custom headers on - WebSockets reliably, everything goes on query params): - - 1. ?token= -> local-password / SSO session token - 2. ?X-API-Key= -> API key (legacy header name) - 3. ?api_key= -> API key - - Under the hood we reuse the same pluggable auth registry used by the - HTTP routes so the authorization rules stay consistent. - """ - from app.core.auth.providers import AuthError, RawCredential, get_provider_registry - - bearer_token = ( - websocket.query_params.get("token") - or websocket.query_params.get("access_token") - ) - api_key = ( - websocket.query_params.get("X-API-Key") - or websocket.query_params.get("api_key") - ) - - if not bearer_token and not api_key: - from urllib.parse import parse_qs - - raw_qs = websocket.scope.get("query_string", b"") - if isinstance(raw_qs, bytes): - raw_qs = raw_qs.decode("utf-8", errors="replace") - parsed = parse_qs(raw_qs) - bearer_token = bearer_token or (parsed.get("token") or parsed.get("access_token") or [None])[0] - api_key = api_key or (parsed.get("X-API-Key") or parsed.get("api_key") or [None])[0] - - if not bearer_token and not api_key: - print( - f"[MEDIA-WS] rejected: no credentials (query_string_len=" - f"{len(websocket.scope.get('query_string') or b'')})", - flush=True, - ) - await websocket.close(code=1008, reason="Authentication required") - return - - await websocket.accept() - print("[MEDIA-WS] WebSocket connection accepted", flush=True) - - try: - db = next(get_db()) - - cred = RawCredential(bearer_token=bearer_token, api_key=api_key) - registry = get_provider_registry() - provider = registry.find(cred) - if provider is None: - print("WebSocket connection rejected: No provider accepts the credential") - await websocket.close(code=1008, reason="Invalid credentials") - db.close() - return - - try: - principal = provider.authenticate(cred, db) - except AuthError as e: - print(f"WebSocket connection rejected: {e}") - await websocket.close(code=1008, reason=str(e)) - db.close() - return - - organization_id = principal.organization_id - if not organization_id: - print("WebSocket connection rejected: Principal has no organization") - await websocket.close(code=1008, reason="Invalid credentials") - db.close() - return - - # Get agent_id, persona_id and scenario_id from query params - agent_id = websocket.query_params.get("agent_id") - persona_id = websocket.query_params.get("persona_id") - scenario_id = websocket.query_params.get("scenario_id") - - # Fetch agent and voice bundle once for routing and instructions - agent = None - voice_bundle = None - try: - if agent_id: - from app.models.database import Agent, VoiceBundle - agent_uuid = UUID(agent_id) - agent = db.query(Agent).filter( - Agent.id == agent_uuid, - Agent.organization_id == organization_id - ).first() - if agent and agent.voice_bundle_id: - voice_bundle = db.query(VoiceBundle).filter( - VoiceBundle.id == agent.voice_bundle_id, - VoiceBundle.organization_id == organization_id - ).first() - except ValueError: - pass - - # Resolve workspace_id: derive from agent when available, otherwise fall - # back to the organization's default workspace so created records remain - # workspace-scoped. - workspace_id: Optional[UUID] = None - if agent and getattr(agent, "workspace_id", None): - workspace_id = agent.workspace_id - else: - default_ws = db.query(Workspace).filter( - Workspace.organization_id == organization_id, - Workspace.is_default == True, # noqa: E712 - ).first() - if default_ws: - workspace_id = default_ws.id - - use_voice_bundle_pipeline = bool(voice_bundle and voice_bundle.bundle_type == "stt_llm_tts") - - def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: - """Resolve API key from AIProvider (preferred) or Integration for given provider.""" - from sqlalchemy import func - # 1) AIProvider (handle both string and enum comparisons) - provider_value = provider.value if hasattr(provider, 'value') else provider - - ai_provider_rec = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - AIProvider.provider == provider_value, - AIProvider.is_active == True, - ).first() - - # If not found, try case-insensitive match - if not ai_provider_rec: - ai_provider_rec = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - func.lower(AIProvider.provider) == provider_value.lower(), - AIProvider.is_active == True, - ).first() - if ai_provider_rec: - try: - key = decrypt_api_key(ai_provider_rec.api_key) - logger.debug( - f"[resolve_api_key] Found AIProvider key for '{provider_value}': " - f"starts={key[:6]}... ends=...{key[-4:]}, len={len(key)}" - ) - return key - except Exception as e: - logger.error(f"Failed to decrypt AIProvider key for {provider}: {e}", exc_info=True) - else: - logger.debug(f"[resolve_api_key] No AIProvider record found for '{provider_value}'") - - # 2) Integration mapping (only for platforms that exist in IntegrationPlatform) - platform_map = { - ModelProvider.DEEPGRAM: IntegrationPlatform.DEEPGRAM, - ModelProvider.CARTESIA: IntegrationPlatform.CARTESIA, - ModelProvider.ELEVENLABS: IntegrationPlatform.ELEVENLABS, - ModelProvider.MURF: IntegrationPlatform.MURF, - ModelProvider.SARVAM: IntegrationPlatform.SARVAM, - ModelProvider.VOICEMAKER: IntegrationPlatform.VOICEMAKER, - ModelProvider.SMALLEST: IntegrationPlatform.SMALLEST, - } - plat = platform_map.get(provider) - if plat: - # Handle both string and enum comparisons for platform - plat_value = plat.value if hasattr(plat, 'value') else plat - integ = db.query(Integration).filter( - Integration.organization_id == organization_id, - Integration.platform == plat_value, - Integration.is_active == True, - ).first() - - # If not found, try case-insensitive match - if not integ: - integ = db.query(Integration).filter( - Integration.organization_id == organization_id, - func.lower(Integration.platform) == plat_value.lower(), - Integration.is_active == True, - ).first() - - if integ: - try: - key = decrypt_api_key(integ.api_key) - logger.debug( - f"[resolve_api_key] Found Integration key for '{provider_value}' (platform={plat_value}): " - f"starts={key[:6]}... ends=...{key[-4:]}, len={len(key)}" - ) - return key - except Exception as e: - logger.error(f"Failed to decrypt Integration key for {provider}: {e}", exc_info=True) - else: - logger.debug(f"[resolve_api_key] No Integration record found for platform '{plat_value}'") - else: - logger.debug(f"[resolve_api_key] No platform mapping for provider '{provider_value}'") - - logger.warning(f"[resolve_api_key] Could not resolve any API key for provider '{provider_value}'") - return None - - def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: - """Resolve Azure OpenAI endpoint URL from the org's AIProvider credential.""" - from sqlalchemy import func - - provider_value = provider.value if hasattr(provider, "value") else provider - ai_provider_rec = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - AIProvider.provider == provider_value, - AIProvider.is_active == True, - ).first() - if not ai_provider_rec: - ai_provider_rec = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - func.lower(AIProvider.provider) == provider_value.lower(), - AIProvider.is_active == True, - ).first() - if not ai_provider_rec: - return None - return _resolve_azure_endpoint_from_provider(ai_provider_rec, None) - - # Determine which AI Provider to use (only needed for S2S/Gemini path) - # Priority: 1) Agent's ai_provider_id, 2) Default Google - ai_provider = None - google_api_key = None - if not use_voice_bundle_pipeline: - if agent and agent.ai_provider_id: - ai_provider = db.query(AIProvider).filter( - AIProvider.id == agent.ai_provider_id, - AIProvider.organization_id == organization_id, - AIProvider.is_active == True - ).first() - - # Fall back to Google AI Provider if no agent-specific provider found - if not ai_provider: - from sqlalchemy import func - google_value = ModelProvider.GOOGLE.value - ai_provider = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - AIProvider.provider == google_value, - AIProvider.is_active == True - ).first() - # If not found, try case-insensitive match - if not ai_provider: - ai_provider = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - func.lower(AIProvider.provider) == google_value.lower(), - AIProvider.is_active == True - ).first() - - if not ai_provider: - await websocket.close( - code=1008, - reason="AI Provider not configured. Please configure an AI Provider in AI Providers settings or select one when creating the agent." - ) - return - - # Decrypt API key - try: - google_api_key = decrypt_api_key(ai_provider.api_key) - if not google_api_key or not google_api_key.strip(): - logger.error("Decrypted API key is empty") - await websocket.close( - code=1008, - reason="API key is empty. Please configure a valid API key in AI Providers settings." - ) - return - except Exception as e: - logger.error(f"Failed to decrypt API key: {e}", exc_info=True) - await websocket.close( - code=1008, - reason=f"Failed to decrypt API key: {str(e)}" - ) - return - - system_instruction = None - instruction_parts = [] - - # Build system instruction as a bundle: Agent + Persona + Scenario - from app.models.database import Agent, Persona, Scenario - - # 1. Add Agent description (base instruction) and get voice bundle for model - model_name = None - if agent: - if agent.description: - instruction_parts.append(agent.description) - if voice_bundle and voice_bundle.bundle_type == "s2s" and voice_bundle.s2s_model: - model_name = voice_bundle.s2s_model - - # 2. Add Persona information (characteristics) - persona = None - if persona_id: - try: - persona_uuid = UUID(persona_id) - persona_query = db.query(Persona).filter( - Persona.id == persona_uuid, - Persona.organization_id == organization_id - ) - if workspace_id is not None: - persona_query = persona_query.filter(Persona.workspace_id == workspace_id) - persona = persona_query.first() - if persona: - persona_parts = [] - persona_parts.append(f"\n\nPersona: {persona.name}") - if persona.gender: - gender_val = persona.gender.value if hasattr(persona.gender, "value") else persona.gender - persona_parts.append(f"Gender: {gender_val}") - if getattr(persona, "tts_provider", None): - persona_parts.append(f"Voice provider: {persona.tts_provider}") - if getattr(persona, "tts_voice_name", None): - persona_parts.append(f"Voice: {persona.tts_voice_name}") - - if persona_parts: - instruction_parts.append("\n".join(persona_parts)) - except ValueError: - pass - - # 3. Add Scenario information (context and goals) - if scenario_id: - try: - scenario_uuid = UUID(scenario_id) - scenario_query = db.query(Scenario).filter( - Scenario.id == scenario_uuid, - Scenario.organization_id == organization_id - ) - if workspace_id is not None: - scenario_query = scenario_query.filter(Scenario.workspace_id == workspace_id) - scenario = scenario_query.first() - if scenario: - scenario_parts = [] - scenario_parts.append(f"\n\nScenario: {scenario.name}") - if scenario.description: - scenario_parts.append(f"Description: {scenario.description}") - if scenario.required_info: - required_info_str = ", ".join([f"{k}: {v}" for k, v in scenario.required_info.items()]) if isinstance(scenario.required_info, dict) else str(scenario.required_info) - if required_info_str: - scenario_parts.append(f"Required information to collect: {required_info_str}") - - if scenario_parts: - instruction_parts.append("\n".join(scenario_parts)) - except ValueError: - pass - - # Combine all parts into final system instruction - if instruction_parts: - system_instruction = "\n".join(instruction_parts) - - # Generate result_id BEFORE running bot (for meaningful S3 path) - # Evaluator is only created if persona_id and scenario_id are provided - # But evaluator results can be created even without persona/scenario - evaluator = None - result_id = None - scenario_name = "Test Call" # Default name for calls without scenario - - # Generate result_id for all test calls (with or without persona/scenario) - if agent_id: - try: - from app.models.database import EvaluatorResult - import random - - # Generate unique 6-digit result ID - max_attempts = 100 - for _ in range(max_attempts): - candidate_id = f"{random.randint(100000, 999999)}" - existing = db.query(EvaluatorResult).filter(EvaluatorResult.result_id == candidate_id).first() - if not existing: - result_id = candidate_id - break - - if not result_id: - logger.warning("Failed to generate unique result ID, will use UUID in S3 path") - except Exception as e: - logger.warning(f"Error generating result_id: {e}") - - # Find or create evaluator only if persona_id and scenario_id are provided - if agent_id and persona_id and scenario_id: - try: - from app.models.database import Evaluator, EvaluatorResult, EvaluatorResultStatus, Scenario - from app.api.v1.routes.evaluators import generate_unique_evaluator_id - import random - - # Find evaluator by agent, persona, scenario - evaluator_query = db.query(Evaluator).filter( - Evaluator.agent_id == UUID(agent_id), - Evaluator.persona_id == UUID(persona_id), - Evaluator.scenario_id == UUID(scenario_id), - Evaluator.organization_id == organization_id - ) - if workspace_id is not None: - evaluator_query = evaluator_query.filter(Evaluator.workspace_id == workspace_id) - evaluator = evaluator_query.first() - - # If no evaluator exists, create one automatically for test voice agent calls - if not evaluator: - logger.info(f"Creating evaluator automatically for test voice agent: agent={agent_id}, persona={persona_id}, scenario={scenario_id}") - evaluator_id = generate_unique_evaluator_id(db) - evaluator = Evaluator( - evaluator_id=evaluator_id, - organization_id=organization_id, - workspace_id=workspace_id, - agent_id=UUID(agent_id), - persona_id=UUID(persona_id), - scenario_id=UUID(scenario_id), - tags=["auto-created", "test-voice-agent"] - ) - db.add(evaluator) - db.commit() - db.refresh(evaluator) - logger.info(f"✅ Created evaluator {evaluator_id} for test voice agent") - - if evaluator: - # Get scenario name - scenario_name_query = db.query(Scenario).filter(Scenario.id == UUID(scenario_id)) - if workspace_id is not None: - scenario_name_query = scenario_name_query.filter(Scenario.workspace_id == workspace_id) - scenario = scenario_name_query.first() - scenario_name = scenario.name if scenario else "Unknown Scenario" - except Exception as e: - logger.warning(f"Error finding/creating evaluator or generating result_id: {e}") - - # Check if this is a test agent that should bridge to Voice AI agent - # This happens when evaluator is run via the run_evaluator_task - test_agent_bridge_mode = False - retell_call_id = None - retell_access_token = None - retell_sample_rate = None - - if evaluator and agent and agent.voice_bundle_id and agent.voice_ai_integration_id: - # Check if there's an active call for this evaluator - # The call info is stored in the EvaluatorResult's error_message field temporarily - try: - from app.models.database import EvaluatorResult - # Find the most recent result for this evaluator that has call info - active_result = db.query(EvaluatorResult).filter( - EvaluatorResult.evaluator_id == evaluator.id, - EvaluatorResult.status == EvaluatorResultStatus.QUEUED.value, - EvaluatorResult.error_message.isnot(None), - EvaluatorResult.error_message.like("call_id:%") - ).order_by(EvaluatorResult.timestamp.desc()).first() - - if active_result and active_result.error_message: - # Parse call info from error_message: "call_id:xxx|access_token:yyy|sample_rate:zzz" - call_info_parts = active_result.error_message.split("|") - for part in call_info_parts: - if part.startswith("call_id:"): - retell_call_id = part.split(":", 1)[1] - elif part.startswith("access_token:"): - retell_access_token = part.split(":", 1)[1] - elif part.startswith("sample_rate:"): - retell_sample_rate = int(part.split(":", 1)[1]) - - if retell_call_id and retell_access_token: - test_agent_bridge_mode = True - logger.info( - f"[VoiceAgent] Test agent bridge mode detected: " - f"evaluator={evaluator.evaluator_id}, call_id={retell_call_id}" - ) - except Exception as e: - logger.warning(f"[VoiceAgent] Error checking for bridge mode: {e}", exc_info=True) - - # Run the bot with the appropriate pipeline - call_metadata = None - from app.services.voice_agent.call_silence_hangup import resolve_agent_silence_hangup_secs - - agent_silence_hangup_secs = resolve_agent_silence_hangup_secs(agent) - try: - if use_voice_bundle_pipeline: - # Resolve per-provider keys for voice bundle - stt_provider = voice_bundle.stt_provider if voice_bundle else None - tts_provider = voice_bundle.tts_provider if voice_bundle else None - llm_provider = voice_bundle.llm_provider if voice_bundle else None - - stt_api_key = resolve_api_key_for_provider(stt_provider) if stt_provider else None - tts_api_key = resolve_api_key_for_provider(tts_provider) if tts_provider else None - llm_api_key = resolve_api_key_for_provider(llm_provider) if llm_provider else None - llm_endpoint_url = ( - resolve_azure_endpoint_for_provider(llm_provider) - if llm_provider and ( - llm_provider.value if hasattr(llm_provider, "value") else str(llm_provider) - ).lower() == "azure" - else None - ) - - # If in bridge mode, we need to bridge test agent to Retell call - # For now, we'll run the voice bundle normally and note that bridging - # requires WebRTC implementation on the backend - if test_agent_bridge_mode: - logger.info( - f"[VoiceAgent] Bridge mode: Test agent should connect to Retell call {retell_call_id}. " - f"Full WebRTC bridging requires additional implementation." - ) - # TODO: Implement WebRTC bridging between test agent WebSocket and Retell call - # This would require: - # 1. Joining Retell call using WebRTC (aiortc or similar) - # 2. Bridging audio streams bidirectionally - # 3. Recording the bridged conversation - - call_metadata = await run_voice_bundle_fastapi( - websocket, - system_instruction, - str(organization_id), - str(workspace_id) if workspace_id else None, - agent_id, - persona_id, - scenario_id, - evaluator_id=str(evaluator.id) if evaluator else None, - result_id=result_id, - voice_bundle=voice_bundle, - persona=persona, - stt_api_key=stt_api_key, - tts_api_key=tts_api_key, - llm_api_key=llm_api_key, - llm_endpoint_url=llm_endpoint_url, - silence_hangup_secs=agent_silence_hangup_secs, - ) - else: - call_metadata = await run_bot( - websocket, - google_api_key, - system_instruction, - str(organization_id), - agent_id, - persona_id, - scenario_id, - evaluator_id=str(evaluator.id) if evaluator else None, - result_id=result_id, - model_name=model_name, # Pass model name from voice bundle - silence_hangup_secs=agent_silence_hangup_secs, - ) - except Exception as bot_error: - logger.error(f"Error in run_bot: {bot_error}", exc_info=True) - # Continue to try creating evaluator result if we have metadata - - # Create evaluator result if we have the required data (only if no error) - # Also create if we have a live transcript even without S3 audio - has_audio = call_metadata and call_metadata.get("s3_key") - has_transcript = call_metadata and call_metadata.get("transcription") - has_usable_data = has_audio or has_transcript - if call_metadata and has_usable_data and not call_metadata.get("error") and agent_id and result_id: - # If we don't have evaluator but have persona/scenario, try to create one - if not evaluator and agent_id and persona_id and scenario_id: - try: - from app.models.database import Evaluator, Scenario - from app.api.v1.routes.evaluators import generate_unique_evaluator_id - import random - - fallback_eval_query = db.query(Evaluator).filter( - Evaluator.agent_id == UUID(agent_id), - Evaluator.persona_id == UUID(persona_id), - Evaluator.scenario_id == UUID(scenario_id), - Evaluator.organization_id == organization_id - ) - if workspace_id is not None: - fallback_eval_query = fallback_eval_query.filter(Evaluator.workspace_id == workspace_id) - evaluator = fallback_eval_query.first() - - if not evaluator: - evaluator_id = generate_unique_evaluator_id(db) - evaluator = Evaluator( - evaluator_id=evaluator_id, - organization_id=organization_id, - workspace_id=workspace_id, - agent_id=UUID(agent_id), - persona_id=UUID(persona_id), - scenario_id=UUID(scenario_id), - tags=["auto-created", "test-voice-agent"] - ) - db.add(evaluator) - db.commit() - db.refresh(evaluator) - - fallback_scenario_query = db.query(Scenario).filter(Scenario.id == UUID(scenario_id)) - if workspace_id is not None: - fallback_scenario_query = fallback_scenario_query.filter(Scenario.workspace_id == workspace_id) - scenario = fallback_scenario_query.first() - scenario_name = scenario.name if scenario else "Unknown Scenario" - - # Generate result_id - max_attempts = 100 - for _ in range(max_attempts): - candidate_id = f"{random.randint(100000, 999999)}" - existing = db.query(EvaluatorResult).filter(EvaluatorResult.result_id == candidate_id).first() - if not existing: - result_id = candidate_id - break - except Exception as e: - logger.error(f"Error creating evaluator/result_id after bot run: {e}") - - # Create evaluator result for all test calls (with or without persona/scenario) - # evaluator_id is optional - can be None if no persona/scenario - if result_id and agent_id: - try: - from app.models.database import EvaluatorResult, EvaluatorResultStatus - from app.workers.celery_app import process_evaluator_result_task - - # Determine name for the result - if scenario_name and scenario_name != "Test Call": - result_name = scenario_name - elif agent: - result_name = f"Test Call - {agent.name}" - else: - result_name = "Test Call" - - logger.info(f"Creating evaluator result: result_id={result_id}, agent_id={agent_id}, persona_id={persona_id}, scenario_id={scenario_id}, s3_key={call_metadata.get('s3_key')}") - - # Create evaluator result with QUEUED status - # persona_id and scenario_id can be None for test calls without persona/scenario - evaluator_result = EvaluatorResult( - result_id=result_id, - organization_id=organization_id, - workspace_id=workspace_id, - evaluator_id=evaluator.id if evaluator else None, # Optional - agent_id=UUID(agent_id), - persona_id=UUID(persona_id) if persona_id else None, # Optional - scenario_id=UUID(scenario_id) if scenario_id else None, # Optional - name=result_name, - duration_seconds=call_metadata.get("duration"), - status=EvaluatorResultStatus.QUEUED.value, # Use .value to get the string - audio_s3_key=call_metadata.get("s3_key"), - transcription=call_metadata.get("transcription"), - speaker_segments=call_metadata.get("speaker_segments"), - ) - db.add(evaluator_result) - db.commit() - db.refresh(evaluator_result) - - logger.info(f"✅ Evaluator result created in database: id={evaluator_result.id}, result_id={result_id}") - - # Trigger Celery task - try: - logger.info(f"Triggering Celery task for evaluator result: {evaluator_result.id}") - - # Check if Celery app is properly configured - from app.workers.celery_app import celery_app - logger.info(f"Celery broker URL: {celery_app.conf.broker_url}") - logger.info(f"Celery result backend: {celery_app.conf.result_backend}") - - # Verify task is registered - if 'process_evaluator_result' not in celery_app.tasks: - logger.error("❌ Task 'process_evaluator_result' is not registered in Celery app!") - logger.error(f"Available tasks: {list(celery_app.tasks.keys())}") - else: - logger.info("✅ Task 'process_evaluator_result' is registered") - - task = process_evaluator_result_task.delay(str(evaluator_result.id)) - logger.info(f"✅ Celery task triggered: task_id={task.id}, task_state={task.state}") - - # Try to get task info to verify it was queued - try: - task_info = task.info - logger.info(f"Task info: {task_info}") - except Exception as info_error: - logger.warning(f"Could not get task info (this is normal for async tasks): {info_error}") - - evaluator_result.celery_task_id = task.id - db.commit() - logger.info(f"✅ Updated evaluator result with celery_task_id: {task.id}") - except Exception as task_error: - logger.error(f"❌ Failed to trigger Celery task: {task_error}", exc_info=True) - # Still log that we created the result even if task trigger failed - logger.warning(f"Evaluator result {result_id} created but Celery task was not triggered. Task may need to be triggered manually.") - logger.warning(f"Please ensure Celery worker is running: celery -A app.workers.celery_app worker --loglevel=info") - - logger.info(f"✅ Created evaluator result {result_id} and triggered processing task") - except Exception as e: - logger.error(f"❌ Error creating evaluator result: {e}", exc_info=True) - - except WebSocketDisconnect: - print("WebSocket disconnected by client") - except Exception as e: - print(f"Exception in voice agent WebSocket: {e}") - import traceback - traceback.print_exc() - try: - await websocket.close(code=1011, reason=f"Server error: {str(e)}") - except: - pass - finally: - try: - if 'db' in locals(): - db.close() - except: - pass - - -@router.options("/connect") -async def bot_connect_options(): - """Handle CORS preflight requests.""" - from fastapi.responses import Response - return Response( - status_code=200, - headers={ - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET, POST, OPTIONS", - "Access-Control-Allow-Headers": "*", - "Access-Control-Allow-Credentials": "true", - } - ) - -@router.post("/connect", response_model=Dict[str, Any]) -@router.get("/connect", response_model=Dict[str, Any]) -async def bot_connect( - request: Request, - db: Session = Depends(get_db), -) -> Dict[str, Any]: - """ - Get WebSocket connection URL for voice agent. - Returns the WebSocket URL that the client should connect to. - - Accepts either a Bearer access token (email/password / SSO login) or an - API key (legacy / machine access). Credentials may be supplied via the - `Authorization` header, `X-API-Key` header, cookies (`access_token` / - `api_key`), or query parameters (`token` / `X-API-Key` / `api_key`). - - Supports both GET and POST requests for compatibility with different client - implementations. Pipecat's `startBotAndConnect` issues an HTTP request to - this endpoint; the WebSocket URL returned here embeds the same credential - so the subsequent /ws connection can authenticate without re-prompting. - """ - from app.core.auth.providers import ( - AuthError, - RawCredential, - get_provider_registry, - ) - - print("=" * 80) - print(f"[BACKEND] /connect endpoint called at {__import__('datetime').datetime.now()}") - print(f"[BACKEND] Request method: {request.method}") - print(f"[BACKEND] Request URL: {request.url}") - print(f"[BACKEND] Request cookies present: {list(request.cookies.keys())}") - print(f"[BACKEND] Query params: {dict(request.query_params)}") - - def _extract_bearer(value: Optional[str]) -> Optional[str]: - if not value: - return None - scheme, _, token = value.partition(" ") - if scheme.lower() != "bearer" or not token.strip(): - return None - return token.strip() - - # Bearer / access token: header, query param, then cookie. - bearer_token = ( - _extract_bearer(request.headers.get("Authorization")) - or request.query_params.get("token") - or request.query_params.get("access_token") - or request.cookies.get("access_token") - ) - - # API key: header, query param, then cookie. - api_key = ( - request.headers.get("X-API-Key") - or request.headers.get("X-EFFICIENTAI-API-KEY") - or request.query_params.get("X-API-Key") - or request.query_params.get("api_key") - or request.cookies.get("api_key") - ) - - print( - f"[BACKEND] Bearer token: {'found' if bearer_token else 'not found'}, " - f"API key: {'found' if api_key else 'not found'}" - ) - - from app.config import settings - - if not bearer_token and not api_key: - print("[BACKEND] ❌ No credentials found, returning 401") - raise HTTPException( - status_code=401, - detail=( - "Authentication required. Send an Authorization: Bearer " - "header, X-API-Key header, or matching cookie/query param." - ), - ) - - cred = RawCredential(bearer_token=bearer_token, api_key=api_key) - registry = get_provider_registry() - provider = registry.find(cred) - if provider is None: - print("[BACKEND] ❌ No auth provider accepted the credential") - raise HTTPException(status_code=401, detail="Invalid credentials") - - try: - principal = provider.authenticate(cred, db) - except AuthError as e: - print(f"[BACKEND] ❌ Auth provider rejected credential: {e}") - raise HTTPException(status_code=e.status_code, detail=str(e)) - - organization_id = principal.organization_id - if not organization_id: - print("[BACKEND] ❌ Principal has no organization") - raise HTTPException(status_code=401, detail="Invalid credentials") - - print(f"[BACKEND] ✅ Authenticated via {provider.name} (org={organization_id})") - - # Get agent_id, persona_id and scenario_id from query params first - agent_id = request.query_params.get("agent_id") - persona_id = request.query_params.get("persona_id") - scenario_id = request.query_params.get("scenario_id") - - # Determine which AI Provider to use based on agent configuration - ai_provider = None - agent = None - voice_bundle = None - use_voice_bundle_pipeline = False - - if agent_id: - try: - from app.models.database import Agent, VoiceBundle - agent_uuid = UUID(agent_id) - agent = db.query(Agent).filter( - Agent.id == agent_uuid, - Agent.organization_id == organization_id - ).first() - - # Check if agent has a voice bundle (stt_llm_tts type) - if agent and agent.voice_bundle_id: - voice_bundle = db.query(VoiceBundle).filter( - VoiceBundle.id == agent.voice_bundle_id, - VoiceBundle.organization_id == organization_id - ).first() - if voice_bundle and voice_bundle.bundle_type == "stt_llm_tts": - use_voice_bundle_pipeline = True - print(f"[BACKEND] ✅ Using custom voice bundle: {voice_bundle.name}") - - if agent and agent.ai_provider_id: - ai_provider = db.query(AIProvider).filter( - AIProvider.id == agent.ai_provider_id, - AIProvider.organization_id == organization_id, - AIProvider.is_active == True - ).first() - except ValueError: - pass - - # For custom voice bundles (stt_llm_tts), we don't need a Google provider - # The voice bundle has its own STT/LLM/TTS providers configured - if not use_voice_bundle_pipeline: - # Fall back to Google AI Provider if no agent-specific provider found (needed for S2S/Gemini path) - if not ai_provider: - from sqlalchemy import func - google_value = ModelProvider.GOOGLE.value - ai_provider = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - AIProvider.provider == google_value, - AIProvider.is_active == True - ).first() - # If not found, try case-insensitive match - if not ai_provider: - ai_provider = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - func.lower(AIProvider.provider) == google_value.lower(), - AIProvider.is_active == True - ).first() - - if not ai_provider: - print("[BACKEND] ❌ AI Provider not configured") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="AI Provider not configured. Please configure an AI Provider in AI Providers settings or select one when creating the agent." - ) - - provider_val = ai_provider.provider.value if hasattr(ai_provider.provider, 'value') else ai_provider.provider - print(f"[BACKEND] ✅ AI Provider found: {provider_val}") - else: - # For voice bundle pipeline, verify the required providers are configured - from sqlalchemy import func - missing_providers = [] - - def check_provider(provider_enum): - """Check if a provider is configured (either as AIProvider or Integration).""" - if not provider_enum: - return True # Not required - - provider_value = provider_enum.value if hasattr(provider_enum, 'value') else str(provider_enum) - - # Check AIProvider - found = db.query(AIProvider).filter( - AIProvider.organization_id == organization_id, - AIProvider.is_active == True, - ).filter( - (AIProvider.provider == provider_value) | - (func.lower(AIProvider.provider) == provider_value.lower()) - ).first() - - if found: - return True - - # Check Integration for Deepgram, Cartesia, and ElevenLabs - platform_map = { - 'deepgram': IntegrationPlatform.DEEPGRAM, - 'cartesia': IntegrationPlatform.CARTESIA, - 'elevenlabs': IntegrationPlatform.ELEVENLABS, - 'murf': IntegrationPlatform.MURF, - 'sarvam': IntegrationPlatform.SARVAM, - 'voicemaker': IntegrationPlatform.VOICEMAKER, - 'smallest': IntegrationPlatform.SMALLEST, - } - plat = platform_map.get(provider_value.lower()) - if plat: - plat_value = plat.value if hasattr(plat, 'value') else plat - found = db.query(Integration).filter( - Integration.organization_id == organization_id, - Integration.is_active == True, - ).filter( - (Integration.platform == plat_value) | - (func.lower(Integration.platform) == plat_value.lower()) - ).first() - if found: - return True - - return False - - # Check required providers for the voice bundle - if voice_bundle.stt_provider and not check_provider(voice_bundle.stt_provider): - missing_providers.append(f"STT: {voice_bundle.stt_provider}") - if voice_bundle.llm_provider and not check_provider(voice_bundle.llm_provider): - missing_providers.append(f"LLM: {voice_bundle.llm_provider}") - if voice_bundle.tts_provider and not check_provider(voice_bundle.tts_provider): - missing_providers.append(f"TTS: {voice_bundle.tts_provider}") - - if missing_providers: - print(f"[BACKEND] ❌ Missing providers for voice bundle: {missing_providers}") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Missing AI providers for voice bundle: {', '.join(missing_providers)}. Please configure them in Integrations settings." - ) - - print(f"[BACKEND] ✅ Voice bundle providers configured") - - # Determine WebSocket URL — prefer dedicated media server when configured. - from urllib.parse import quote - - from app.services.media_urls import build_voice_agent_ws_url - - if bearer_token: - ws_auth_query = f"token={quote(bearer_token, safe='')}" - else: - ws_auth_query = f"X-API-Key={quote(api_key or '', safe='')}" - - ws_url = build_voice_agent_ws_url( - auth_query=ws_auth_query, - agent_id=agent_id, - persona_id=persona_id, - scenario_id=scenario_id, - fallback_host=request.headers.get("host", f"localhost:{settings.PORT}"), - fallback_scheme=( - request.headers.get("x-forwarded-proto") - or getattr(request.url, "scheme", "http") - or "http" - ), - ) - - # Return the response in the format Pipecat expects - # Pipecat expects a JSON response with ws_url field - # The response should be simple and match exactly what Pipecat expects - from fastapi.responses import JSONResponse - response_data = { - "ws_url": ws_url - } - print(f"[BACKEND] ✅ Returning WebSocket URL: {ws_url}") - print(f"[BACKEND] Response data: {response_data}") - print("=" * 80) - - # Return JSON response - CORS is handled by middleware - return JSONResponse( - content=response_data, - status_code=200, - headers={ - "Content-Type": "application/json", - } - ) - - -@router.get("/audio", response_model=List[Dict[str, Any]]) -async def list_voice_agent_audio_files( - organization_id: UUID = Depends(get_organization_id), - max_keys: int = 1000, -): - """ - List audio files for voice agent conversations for the current organization. - - Args: - organization_id: Organization ID from API key - max_keys: Maximum number of files to return - - Returns: - List of audio file metadata with keys: key, size, last_modified, filename - """ - try: - files = s3_service.list_audio_files( - organization_id=str(organization_id), - max_keys=max_keys - ) - return files - except Exception as e: - logger.error(f"Error listing voice agent audio files: {e}") - raise HTTPException( - status_code=500, - detail=f"Failed to list audio files: {str(e)}" - ) - +""" +Voice Agent API Routes +API endpoints for managing voice agent WebSocket connections. +""" + +from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect, Request, status +from sqlalchemy.orm import Session +from uuid import UUID, uuid4 +from typing import Dict, Any, Optional, List +from loguru import logger + +from app.database import get_db +from app.dependencies import get_organization_id, get_api_key +from app.models.database import AIProvider, ModelProvider, Integration, IntegrationPlatform, Workspace +from app.core.encryption import decrypt_api_key +from app.services.voice_agent.bot_fast_api import run_bot +from app.services.ai.llm_service import _resolve_azure_endpoint_from_provider +from app.services.voice_agent.voice_bundle import run_voice_bundle_fastapi +from app.services.storage.s3_service import s3_service + +router = APIRouter(prefix="/voice-agent", tags=["voice-agent"]) +ws_router = APIRouter(prefix="/voice-agent", tags=["voice-agent-media"]) + + +@ws_router.websocket("/ws") +async def websocket_endpoint( + websocket: WebSocket, +): + """ + WebSocket endpoint for voice agent connection. + + Authentication precedence (since browsers can't set custom headers on + WebSockets reliably, everything goes on query params): + + 1. ?token= -> local-password / SSO session token + 2. ?X-API-Key= -> API key (legacy header name) + 3. ?api_key= -> API key + + Under the hood we reuse the same pluggable auth registry used by the + HTTP routes so the authorization rules stay consistent. + """ + from app.core.auth.providers import AuthError, RawCredential, get_provider_registry + + bearer_token = ( + websocket.query_params.get("token") + or websocket.query_params.get("access_token") + ) + api_key = ( + websocket.query_params.get("X-API-Key") + or websocket.query_params.get("api_key") + ) + + if not bearer_token and not api_key: + from urllib.parse import parse_qs + + raw_qs = websocket.scope.get("query_string", b"") + if isinstance(raw_qs, bytes): + raw_qs = raw_qs.decode("utf-8", errors="replace") + parsed = parse_qs(raw_qs) + bearer_token = bearer_token or (parsed.get("token") or parsed.get("access_token") or [None])[0] + api_key = api_key or (parsed.get("X-API-Key") or parsed.get("api_key") or [None])[0] + + if not bearer_token and not api_key: + print( + f"[MEDIA-WS] rejected: no credentials (query_string_len=" + f"{len(websocket.scope.get('query_string') or b'')})", + flush=True, + ) + await websocket.close(code=1008, reason="Authentication required") + return + + await websocket.accept() + print("[MEDIA-WS] WebSocket connection accepted", flush=True) + + trace_call_short_id: Optional[str] = None + organization_id: Optional[UUID] = None + workspace_id: Optional[UUID] = None + + try: + db = next(get_db()) + + cred = RawCredential(bearer_token=bearer_token, api_key=api_key) + registry = get_provider_registry() + provider = registry.find(cred) + if provider is None: + print("WebSocket connection rejected: No provider accepts the credential") + await websocket.close(code=1008, reason="Invalid credentials") + db.close() + return + + try: + principal = provider.authenticate(cred, db) + except AuthError as e: + print(f"WebSocket connection rejected: {e}") + await websocket.close(code=1008, reason=str(e)) + db.close() + return + + organization_id = principal.organization_id + if not organization_id: + print("WebSocket connection rejected: Principal has no organization") + await websocket.close(code=1008, reason="Invalid credentials") + db.close() + return + + # Get agent_id, persona_id and scenario_id from query params + agent_id = websocket.query_params.get("agent_id") + persona_id = websocket.query_params.get("persona_id") + scenario_id = websocket.query_params.get("scenario_id") + ui_surface = websocket.query_params.get("ui_surface") + trace_call_short_id = (websocket.query_params.get("call_short_id") or "").strip() or None + trace_api_key = api_key + + # Fetch agent and voice bundle once for routing and instructions + agent = None + voice_bundle = None + try: + if agent_id: + from app.models.database import Agent, VoiceBundle + agent_uuid = UUID(agent_id) + agent = db.query(Agent).filter( + Agent.id == agent_uuid, + Agent.organization_id == organization_id + ).first() + if agent and agent.voice_bundle_id: + voice_bundle = db.query(VoiceBundle).filter( + VoiceBundle.id == agent.voice_bundle_id, + VoiceBundle.organization_id == organization_id + ).first() + except ValueError: + pass + + # Resolve workspace_id: derive from agent when available, otherwise fall + # back to the organization's default workspace so created records remain + # workspace-scoped. + workspace_id = None + if agent and getattr(agent, "workspace_id", None): + workspace_id = agent.workspace_id + else: + default_ws = db.query(Workspace).filter( + Workspace.organization_id == organization_id, + Workspace.is_default == True, # noqa: E712 + ).first() + if default_ws: + workspace_id = default_ws.id + + use_voice_bundle_pipeline = bool(voice_bundle and voice_bundle.bundle_type == "stt_llm_tts") + + def resolve_api_key_for_provider(provider: ModelProvider) -> str | None: + """Resolve API key from AIProvider (preferred) or Integration for given provider.""" + from sqlalchemy import func + # 1) AIProvider (handle both string and enum comparisons) + provider_value = provider.value if hasattr(provider, 'value') else provider + + ai_provider_rec = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + AIProvider.provider == provider_value, + AIProvider.is_active == True, + ).first() + + # If not found, try case-insensitive match + if not ai_provider_rec: + ai_provider_rec = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + func.lower(AIProvider.provider) == provider_value.lower(), + AIProvider.is_active == True, + ).first() + if ai_provider_rec: + try: + key = decrypt_api_key(ai_provider_rec.api_key) + logger.debug( + f"[resolve_api_key] Found AIProvider key for '{provider_value}': " + f"starts={key[:6]}... ends=...{key[-4:]}, len={len(key)}" + ) + return key + except Exception as e: + logger.error(f"Failed to decrypt AIProvider key for {provider}: {e}", exc_info=True) + else: + logger.debug(f"[resolve_api_key] No AIProvider record found for '{provider_value}'") + + # 2) Integration mapping (only for platforms that exist in IntegrationPlatform) + platform_map = { + ModelProvider.DEEPGRAM: IntegrationPlatform.DEEPGRAM, + ModelProvider.CARTESIA: IntegrationPlatform.CARTESIA, + ModelProvider.ELEVENLABS: IntegrationPlatform.ELEVENLABS, + ModelProvider.MURF: IntegrationPlatform.MURF, + ModelProvider.SARVAM: IntegrationPlatform.SARVAM, + ModelProvider.VOICEMAKER: IntegrationPlatform.VOICEMAKER, + ModelProvider.SMALLEST: IntegrationPlatform.SMALLEST, + } + plat = platform_map.get(provider) + if plat: + # Handle both string and enum comparisons for platform + plat_value = plat.value if hasattr(plat, 'value') else plat + integ = db.query(Integration).filter( + Integration.organization_id == organization_id, + Integration.platform == plat_value, + Integration.is_active == True, + ).first() + + # If not found, try case-insensitive match + if not integ: + integ = db.query(Integration).filter( + Integration.organization_id == organization_id, + func.lower(Integration.platform) == plat_value.lower(), + Integration.is_active == True, + ).first() + + if integ: + try: + key = decrypt_api_key(integ.api_key) + logger.debug( + f"[resolve_api_key] Found Integration key for '{provider_value}' (platform={plat_value}): " + f"starts={key[:6]}... ends=...{key[-4:]}, len={len(key)}" + ) + return key + except Exception as e: + logger.error(f"Failed to decrypt Integration key for {provider}: {e}", exc_info=True) + else: + logger.debug(f"[resolve_api_key] No Integration record found for platform '{plat_value}'") + else: + logger.debug(f"[resolve_api_key] No platform mapping for provider '{provider_value}'") + + logger.warning(f"[resolve_api_key] Could not resolve any API key for provider '{provider_value}'") + return None + + def resolve_azure_endpoint_for_provider(provider: ModelProvider) -> str | None: + """Resolve Azure OpenAI endpoint URL from the org's AIProvider credential.""" + from sqlalchemy import func + + provider_value = provider.value if hasattr(provider, "value") else provider + ai_provider_rec = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + AIProvider.provider == provider_value, + AIProvider.is_active == True, + ).first() + if not ai_provider_rec: + ai_provider_rec = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + func.lower(AIProvider.provider) == provider_value.lower(), + AIProvider.is_active == True, + ).first() + if not ai_provider_rec: + return None + return _resolve_azure_endpoint_from_provider(ai_provider_rec, None) + + # Determine which AI Provider to use (only needed for S2S/Gemini path) + # Priority: 1) Agent's ai_provider_id, 2) Default Google + ai_provider = None + google_api_key = None + if not use_voice_bundle_pipeline: + if agent and agent.ai_provider_id: + ai_provider = db.query(AIProvider).filter( + AIProvider.id == agent.ai_provider_id, + AIProvider.organization_id == organization_id, + AIProvider.is_active == True + ).first() + + # Fall back to Google AI Provider if no agent-specific provider found + if not ai_provider: + from sqlalchemy import func + google_value = ModelProvider.GOOGLE.value + ai_provider = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + AIProvider.provider == google_value, + AIProvider.is_active == True + ).first() + # If not found, try case-insensitive match + if not ai_provider: + ai_provider = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + func.lower(AIProvider.provider) == google_value.lower(), + AIProvider.is_active == True + ).first() + + if not ai_provider: + await websocket.close( + code=1008, + reason="AI Provider not configured. Please configure an AI Provider in AI Providers settings or select one when creating the agent." + ) + return + + # Decrypt API key + try: + google_api_key = decrypt_api_key(ai_provider.api_key) + if not google_api_key or not google_api_key.strip(): + logger.error("Decrypted API key is empty") + await websocket.close( + code=1008, + reason="API key is empty. Please configure a valid API key in AI Providers settings." + ) + return + except Exception as e: + logger.error(f"Failed to decrypt API key: {e}", exc_info=True) + await websocket.close( + code=1008, + reason=f"Failed to decrypt API key: {str(e)}" + ) + return + + system_instruction = None + instruction_parts = [] + + # Build system instruction as a bundle: Agent + Persona + Scenario + from app.models.database import Agent, Persona, Scenario + + # 1. Add Agent description (base instruction) and get voice bundle for model + model_name = None + if agent: + if agent.description: + instruction_parts.append(agent.description) + if voice_bundle and voice_bundle.bundle_type == "s2s" and voice_bundle.s2s_model: + model_name = voice_bundle.s2s_model + + # 2. Add Persona information (characteristics) + persona = None + if persona_id: + try: + persona_uuid = UUID(persona_id) + persona_query = db.query(Persona).filter( + Persona.id == persona_uuid, + Persona.organization_id == organization_id + ) + if workspace_id is not None: + persona_query = persona_query.filter(Persona.workspace_id == workspace_id) + persona = persona_query.first() + if persona: + persona_parts = [] + persona_parts.append(f"\n\nPersona: {persona.name}") + if persona.gender: + gender_val = persona.gender.value if hasattr(persona.gender, "value") else persona.gender + persona_parts.append(f"Gender: {gender_val}") + if getattr(persona, "tts_provider", None): + persona_parts.append(f"Voice provider: {persona.tts_provider}") + if getattr(persona, "tts_voice_name", None): + persona_parts.append(f"Voice: {persona.tts_voice_name}") + + if persona_parts: + instruction_parts.append("\n".join(persona_parts)) + except ValueError: + pass + + # 3. Add Scenario information (context and goals) + if scenario_id: + try: + scenario_uuid = UUID(scenario_id) + scenario_query = db.query(Scenario).filter( + Scenario.id == scenario_uuid, + Scenario.organization_id == organization_id + ) + if workspace_id is not None: + scenario_query = scenario_query.filter(Scenario.workspace_id == workspace_id) + scenario = scenario_query.first() + if scenario: + scenario_parts = [] + scenario_parts.append(f"\n\nScenario: {scenario.name}") + if scenario.description: + scenario_parts.append(f"Description: {scenario.description}") + if scenario.required_info: + required_info_str = ", ".join([f"{k}: {v}" for k, v in scenario.required_info.items()]) if isinstance(scenario.required_info, dict) else str(scenario.required_info) + if required_info_str: + scenario_parts.append(f"Required information to collect: {required_info_str}") + + if scenario_parts: + instruction_parts.append("\n".join(scenario_parts)) + except ValueError: + pass + + # Combine all parts into final system instruction + if instruction_parts: + system_instruction = "\n".join(instruction_parts) + + # Generate result_id BEFORE running bot (for meaningful S3 path) + # Evaluator is only created if persona_id and scenario_id are provided + # But evaluator results can be created even without persona/scenario + evaluator = None + result_id = None + scenario_name = "Test Call" # Default name for calls without scenario + + # Generate result_id for all test calls (with or without persona/scenario) + if agent_id: + try: + from app.models.database import EvaluatorResult + import random + + # Generate unique 6-digit result ID + max_attempts = 100 + for _ in range(max_attempts): + candidate_id = f"{random.randint(100000, 999999)}" + existing = db.query(EvaluatorResult).filter(EvaluatorResult.result_id == candidate_id).first() + if not existing: + result_id = candidate_id + break + + if not result_id: + logger.warning("Failed to generate unique result ID, will use UUID in S3 path") + except Exception as e: + logger.warning(f"Error generating result_id: {e}") + + # Find or create evaluator only if persona_id and scenario_id are provided + if agent_id and persona_id and scenario_id: + try: + from app.models.database import Evaluator, EvaluatorResult, EvaluatorResultStatus, Scenario + from app.api.v1.routes.evaluators import generate_unique_evaluator_id + import random + + # Find evaluator by agent, persona, scenario + evaluator_query = db.query(Evaluator).filter( + Evaluator.agent_id == UUID(agent_id), + Evaluator.persona_id == UUID(persona_id), + Evaluator.scenario_id == UUID(scenario_id), + Evaluator.organization_id == organization_id + ) + if workspace_id is not None: + evaluator_query = evaluator_query.filter(Evaluator.workspace_id == workspace_id) + evaluator = evaluator_query.first() + + # If no evaluator exists, create one automatically for test voice agent calls + if not evaluator: + logger.info(f"Creating evaluator automatically for test voice agent: agent={agent_id}, persona={persona_id}, scenario={scenario_id}") + evaluator_id = generate_unique_evaluator_id(db) + evaluator = Evaluator( + evaluator_id=evaluator_id, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=UUID(agent_id), + persona_id=UUID(persona_id), + scenario_id=UUID(scenario_id), + tags=["auto-created", "test-voice-agent"] + ) + db.add(evaluator) + db.commit() + db.refresh(evaluator) + logger.info(f"✅ Created evaluator {evaluator_id} for test voice agent") + + if evaluator: + # Get scenario name + scenario_name_query = db.query(Scenario).filter(Scenario.id == UUID(scenario_id)) + if workspace_id is not None: + scenario_name_query = scenario_name_query.filter(Scenario.workspace_id == workspace_id) + scenario = scenario_name_query.first() + scenario_name = scenario.name if scenario else "Unknown Scenario" + except Exception as e: + logger.warning(f"Error finding/creating evaluator or generating result_id: {e}") + + # Check if this is a test agent that should bridge to Voice AI agent + # This happens when evaluator is run via the run_evaluator_task + test_agent_bridge_mode = False + retell_call_id = None + retell_access_token = None + retell_sample_rate = None + + if evaluator and agent and agent.voice_bundle_id and agent.voice_ai_integration_id: + # Check if there's an active call for this evaluator + # The call info is stored in the EvaluatorResult's error_message field temporarily + try: + from app.models.database import EvaluatorResult + # Find the most recent result for this evaluator that has call info + active_result = db.query(EvaluatorResult).filter( + EvaluatorResult.evaluator_id == evaluator.id, + EvaluatorResult.status == EvaluatorResultStatus.QUEUED.value, + EvaluatorResult.error_message.isnot(None), + EvaluatorResult.error_message.like("call_id:%") + ).order_by(EvaluatorResult.timestamp.desc()).first() + + if active_result and active_result.error_message: + # Parse call info from error_message: "call_id:xxx|access_token:yyy|sample_rate:zzz" + call_info_parts = active_result.error_message.split("|") + for part in call_info_parts: + if part.startswith("call_id:"): + retell_call_id = part.split(":", 1)[1] + elif part.startswith("access_token:"): + retell_access_token = part.split(":", 1)[1] + elif part.startswith("sample_rate:"): + retell_sample_rate = int(part.split(":", 1)[1]) + + if retell_call_id and retell_access_token: + test_agent_bridge_mode = True + logger.info( + f"[VoiceAgent] Test agent bridge mode detected: " + f"evaluator={evaluator.evaluator_id}, call_id={retell_call_id}" + ) + except Exception as e: + logger.warning(f"[VoiceAgent] Error checking for bridge mode: {e}", exc_info=True) + + # Run the bot with the appropriate pipeline + call_metadata = None + session_uuid: Optional[UUID] = None + from app.services.voice_agent.call_silence_hangup import resolve_agent_silence_hangup_secs + + if agent_id and workspace_id and not test_agent_bridge_mode: + session_uuid = uuid4() + + agent_silence_hangup_secs = resolve_agent_silence_hangup_secs(agent) + tracing_task_kwargs: Dict[str, Any] = {} + if trace_call_short_id and workspace_id and organization_id: + from app.services.voice_agent.playground_tracing import build_pipeline_tracing_kwargs + + tracing_task_kwargs = build_pipeline_tracing_kwargs( + call_short_id=trace_call_short_id, + workspace_id=str(workspace_id), + organization_id=str(organization_id), + agent_id=agent_id, + api_key=trace_api_key, + ) + if not tracing_task_kwargs: + logger.warning( + "Playground OTLP tracing disabled for call_short_id={} " + "(install efficientai[otel] on the media/API process)", + trace_call_short_id, + ) + try: + if use_voice_bundle_pipeline: + # Resolve per-provider keys for voice bundle + stt_provider = voice_bundle.stt_provider if voice_bundle else None + tts_provider = voice_bundle.tts_provider if voice_bundle else None + llm_provider = voice_bundle.llm_provider if voice_bundle else None + + stt_api_key = resolve_api_key_for_provider(stt_provider) if stt_provider else None + tts_api_key = resolve_api_key_for_provider(tts_provider) if tts_provider else None + llm_api_key = resolve_api_key_for_provider(llm_provider) if llm_provider else None + llm_endpoint_url = ( + resolve_azure_endpoint_for_provider(llm_provider) + if llm_provider and ( + llm_provider.value if hasattr(llm_provider, "value") else str(llm_provider) + ).lower() == "azure" + else None + ) + + # If in bridge mode, we need to bridge test agent to Retell call + # For now, we'll run the voice bundle normally and note that bridging + # requires WebRTC implementation on the backend + if test_agent_bridge_mode: + logger.info( + f"[VoiceAgent] Bridge mode: Test agent should connect to Retell call {retell_call_id}. " + f"Full WebRTC bridging requires additional implementation." + ) + # TODO: Implement WebRTC bridging between test agent WebSocket and Retell call + # This would require: + # 1. Joining Retell call using WebRTC (aiortc or similar) + # 2. Bridging audio streams bidirectionally + # 3. Recording the bridged conversation + + call_metadata = await run_voice_bundle_fastapi( + websocket, + system_instruction, + str(organization_id), + str(workspace_id) if workspace_id else None, + agent_id, + persona_id, + scenario_id, + evaluator_id=str(evaluator.id) if evaluator else None, + result_id=result_id, + voice_bundle=voice_bundle, + persona=persona, + stt_api_key=stt_api_key, + tts_api_key=tts_api_key, + llm_api_key=llm_api_key, + llm_endpoint_url=llm_endpoint_url, + silence_hangup_secs=agent_silence_hangup_secs, + tracing_task_kwargs=tracing_task_kwargs, + ) + else: + call_metadata = await run_bot( + websocket, + google_api_key, + system_instruction, + str(organization_id), + agent_id, + persona_id, + scenario_id, + evaluator_id=str(evaluator.id) if evaluator else None, + result_id=result_id, + model_name=model_name, # Pass model name from voice bundle + silence_hangup_secs=agent_silence_hangup_secs, + workspace_id=str(workspace_id) if workspace_id else None, + tracing_task_kwargs=tracing_task_kwargs, + ) + except Exception as bot_error: + logger.error(f"Error in run_bot: {bot_error}", exc_info=True) + # Continue to try creating evaluator result if we have metadata + + # Create evaluator result if we have the required data (only if no error) + # Also create if we have a live transcript even without S3 audio + has_audio = call_metadata and call_metadata.get("s3_key") + has_transcript = call_metadata and call_metadata.get("transcription") + has_usable_data = has_audio or has_transcript + if call_metadata and has_usable_data and not call_metadata.get("error") and agent_id and result_id: + # If we don't have evaluator but have persona/scenario, try to create one + if not evaluator and agent_id and persona_id and scenario_id: + try: + from app.models.database import Evaluator, Scenario + from app.api.v1.routes.evaluators import generate_unique_evaluator_id + import random + + fallback_eval_query = db.query(Evaluator).filter( + Evaluator.agent_id == UUID(agent_id), + Evaluator.persona_id == UUID(persona_id), + Evaluator.scenario_id == UUID(scenario_id), + Evaluator.organization_id == organization_id + ) + if workspace_id is not None: + fallback_eval_query = fallback_eval_query.filter(Evaluator.workspace_id == workspace_id) + evaluator = fallback_eval_query.first() + + if not evaluator: + evaluator_id = generate_unique_evaluator_id(db) + evaluator = Evaluator( + evaluator_id=evaluator_id, + organization_id=organization_id, + workspace_id=workspace_id, + agent_id=UUID(agent_id), + persona_id=UUID(persona_id), + scenario_id=UUID(scenario_id), + tags=["auto-created", "test-voice-agent"] + ) + db.add(evaluator) + db.commit() + db.refresh(evaluator) + + fallback_scenario_query = db.query(Scenario).filter(Scenario.id == UUID(scenario_id)) + if workspace_id is not None: + fallback_scenario_query = fallback_scenario_query.filter(Scenario.workspace_id == workspace_id) + scenario = fallback_scenario_query.first() + scenario_name = scenario.name if scenario else "Unknown Scenario" + + # Generate result_id + max_attempts = 100 + for _ in range(max_attempts): + candidate_id = f"{random.randint(100000, 999999)}" + existing = db.query(EvaluatorResult).filter(EvaluatorResult.result_id == candidate_id).first() + if not existing: + result_id = candidate_id + break + except Exception as e: + logger.error(f"Error creating evaluator/result_id after bot run: {e}") + + # Create evaluator result for all test calls (with or without persona/scenario) + # evaluator_id is optional - can be None if no persona/scenario + if result_id and agent_id: + try: + from app.models.database import ( + CallRecording, + CallRecordingSource, + EvaluatorResult, + EvaluatorResultStatus, + ) + from app.models.enums import CallRecordingStatus + from app.utils.call_recordings import generate_unique_call_short_id + from app.workers.celery_app import process_evaluator_result_task + + # Determine name for the result + if scenario_name and scenario_name != "Test Call": + result_name = scenario_name + elif agent: + result_name = f"Test Call - {agent.name}" + else: + result_name = "Test Call" + + logger.info(f"Creating evaluator result: result_id={result_id}, agent_id={agent_id}, persona_id={persona_id}, scenario_id={scenario_id}, s3_key={call_metadata.get('s3_key')}") + + call_short_id = trace_call_short_id or generate_unique_call_short_id(db) + speaker_segments = call_metadata.get("speaker_segments") or [] + if not isinstance(speaker_segments, list): + speaker_segments = [] + playground_call_data = { + "source": "voice_bundle", + "result_id": result_id, + "transcript": call_metadata.get("transcription"), + "speaker_segments": speaker_segments, + "recording_s3_key": call_metadata.get("s3_key"), + "duration_seconds": call_metadata.get("duration"), + } + if ui_surface: + playground_call_data["ui_surface"] = ui_surface + + # Create evaluator result with QUEUED status + # persona_id and scenario_id can be None for test calls without persona/scenario + evaluator_result = EvaluatorResult( + result_id=result_id, + organization_id=organization_id, + workspace_id=workspace_id, + evaluator_id=evaluator.id if evaluator else None, # Optional + agent_id=UUID(agent_id), + persona_id=UUID(persona_id) if persona_id else None, # Optional + scenario_id=UUID(scenario_id) if scenario_id else None, # Optional + name=result_name, + duration_seconds=call_metadata.get("duration"), + status=EvaluatorResultStatus.QUEUED.value, # Use .value to get the string + audio_s3_key=call_metadata.get("s3_key"), + transcription=call_metadata.get("transcription"), + speaker_segments=speaker_segments or None, + ) + db.add(evaluator_result) + db.flush() + + call_recording = CallRecording( + organization_id=organization_id, + workspace_id=workspace_id, + call_short_id=call_short_id, + status=CallRecordingStatus.UPDATED, + source=CallRecordingSource.PLAYGROUND, + call_data=playground_call_data, + provider_call_id=f"voice_bundle_{result_id}", + provider_platform="voice_bundle", + agent_id=UUID(agent_id), + evaluator_result_id=evaluator_result.id, + ) + db.add(call_recording) + db.flush() + + if trace_call_short_id: + from app.services.synthetic_traces.trace_service import link_trace_to_evaluator_result + + link_trace_to_evaluator_result( + db, + organization_id=organization_id, + call_short_id=trace_call_short_id, + evaluator_result_id=evaluator_result.id, + call_recording_id=call_recording.id, + ) + + db.commit() + db.refresh(evaluator_result) + + # Playground scoring bills via playground.evaluation_completed. + + logger.info(f"✅ Evaluator result created in database: id={evaluator_result.id}, result_id={result_id}") + + # Trigger Celery task + try: + logger.info(f"Triggering Celery task for evaluator result: {evaluator_result.id}") + + # Check if Celery app is properly configured + from app.workers.celery_app import celery_app + logger.info(f"Celery broker URL: {celery_app.conf.broker_url}") + logger.info(f"Celery result backend: {celery_app.conf.result_backend}") + + # Verify task is registered + if 'process_evaluator_result' not in celery_app.tasks: + logger.error("❌ Task 'process_evaluator_result' is not registered in Celery app!") + logger.error(f"Available tasks: {list(celery_app.tasks.keys())}") + else: + logger.info("✅ Task 'process_evaluator_result' is registered") + + task = process_evaluator_result_task.delay(str(evaluator_result.id)) + logger.info(f"✅ Celery task triggered: task_id={task.id}, task_state={task.state}") + + # Try to get task info to verify it was queued + try: + task_info = task.info + logger.info(f"Task info: {task_info}") + except Exception as info_error: + logger.warning(f"Could not get task info (this is normal for async tasks): {info_error}") + + evaluator_result.celery_task_id = task.id + db.commit() + logger.info(f"✅ Updated evaluator result with celery_task_id: {task.id}") + except Exception as task_error: + logger.error(f"❌ Failed to trigger Celery task: {task_error}", exc_info=True) + # Still log that we created the result even if task trigger failed + logger.warning(f"Evaluator result {result_id} created but Celery task was not triggered. Task may need to be triggered manually.") + logger.warning(f"Please ensure Celery worker is running: celery -A app.workers.celery_app worker --loglevel=info") + + logger.info(f"✅ Created evaluator result {result_id} and triggered processing task") + except Exception as e: + logger.error(f"❌ Error creating evaluator result: {e}", exc_info=True) + + except WebSocketDisconnect: + print("WebSocket disconnected by client") + except Exception as e: + print(f"Exception in voice agent WebSocket: {e}") + import traceback + traceback.print_exc() + try: + await websocket.close(code=1011, reason=f"Server error: {str(e)}") + except: + pass + finally: + try: + if trace_call_short_id and organization_id: + from app.services.voice_agent.playground_tracing import flush_playground_tracing + from app.services.synthetic_traces.trace_service import close_trace_session + + flush_playground_tracing() + if 'db' in locals(): + close_trace_session( + db, + organization_id=organization_id, + call_short_id=trace_call_short_id, + workspace_id=workspace_id, + ) + except Exception as trace_close_error: + logger.warning(f"Failed to close playground trace session: {trace_close_error}") + try: + if 'db' in locals(): + db.close() + except: + pass + + +@router.options("/connect") +async def bot_connect_options(): + """Handle CORS preflight requests.""" + from fastapi.responses import Response + return Response( + status_code=200, + headers={ + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "*", + "Access-Control-Allow-Credentials": "true", + } + ) + +@router.post("/connect", response_model=Dict[str, Any]) +@router.get("/connect", response_model=Dict[str, Any]) +async def bot_connect( + request: Request, + db: Session = Depends(get_db), +) -> Dict[str, Any]: + """ + Get WebSocket connection URL for voice agent. + Returns the WebSocket URL that the client should connect to. + + Accepts either a Bearer access token (email/password / SSO login) or an + API key (legacy / machine access). Credentials may be supplied via the + `Authorization` header, `X-API-Key` header, cookies (`access_token` / + `api_key`), or query parameters (`token` / `X-API-Key` / `api_key`). + + Supports both GET and POST requests for compatibility with different client + implementations. Pipecat's `startBotAndConnect` issues an HTTP request to + this endpoint; the WebSocket URL returned here embeds the same credential + so the subsequent /ws connection can authenticate without re-prompting. + """ + from app.core.auth.providers import ( + AuthError, + RawCredential, + get_provider_registry, + ) + + print("=" * 80) + print(f"[BACKEND] /connect endpoint called at {__import__('datetime').datetime.now()}") + print(f"[BACKEND] Request method: {request.method}") + print(f"[BACKEND] Request URL: {request.url}") + print(f"[BACKEND] Request cookies present: {list(request.cookies.keys())}") + print(f"[BACKEND] Query params: {dict(request.query_params)}") + + def _extract_bearer(value: Optional[str]) -> Optional[str]: + if not value: + return None + scheme, _, token = value.partition(" ") + if scheme.lower() != "bearer" or not token.strip(): + return None + return token.strip() + + # Bearer / access token: header, query param, then cookie. + bearer_token = ( + _extract_bearer(request.headers.get("Authorization")) + or request.query_params.get("token") + or request.query_params.get("access_token") + or request.cookies.get("access_token") + ) + + # API key: header, query param, then cookie. + api_key = ( + request.headers.get("X-API-Key") + or request.headers.get("X-EFFICIENTAI-API-KEY") + or request.query_params.get("X-API-Key") + or request.query_params.get("api_key") + or request.cookies.get("api_key") + ) + + print( + f"[BACKEND] Bearer token: {'found' if bearer_token else 'not found'}, " + f"API key: {'found' if api_key else 'not found'}" + ) + + from app.config import settings + + if not bearer_token and not api_key: + print("[BACKEND] ❌ No credentials found, returning 401") + raise HTTPException( + status_code=401, + detail=( + "Authentication required. Send an Authorization: Bearer " + "header, X-API-Key header, or matching cookie/query param." + ), + ) + + cred = RawCredential(bearer_token=bearer_token, api_key=api_key) + registry = get_provider_registry() + provider = registry.find(cred) + if provider is None: + print("[BACKEND] ❌ No auth provider accepted the credential") + raise HTTPException(status_code=401, detail="Invalid credentials") + + try: + principal = provider.authenticate(cred, db) + except AuthError as e: + print(f"[BACKEND] ❌ Auth provider rejected credential: {e}") + raise HTTPException(status_code=e.status_code, detail=str(e)) + + organization_id = principal.organization_id + if not organization_id: + print("[BACKEND] ❌ Principal has no organization") + raise HTTPException(status_code=401, detail="Invalid credentials") + + print(f"[BACKEND] ✅ Authenticated via {provider.name} (org={organization_id})") + + # Get agent_id, persona_id and scenario_id from query params first + agent_id = request.query_params.get("agent_id") + persona_id = request.query_params.get("persona_id") + scenario_id = request.query_params.get("scenario_id") + ui_surface = request.query_params.get("ui_surface") + call_short_id = request.query_params.get("call_short_id") + + # Determine which AI Provider to use based on agent configuration + ai_provider = None + agent = None + voice_bundle = None + use_voice_bundle_pipeline = False + + if agent_id: + try: + from app.models.database import Agent, VoiceBundle + agent_uuid = UUID(agent_id) + agent = db.query(Agent).filter( + Agent.id == agent_uuid, + Agent.organization_id == organization_id + ).first() + + # Check if agent has a voice bundle (stt_llm_tts type) + if agent and agent.voice_bundle_id: + voice_bundle = db.query(VoiceBundle).filter( + VoiceBundle.id == agent.voice_bundle_id, + VoiceBundle.organization_id == organization_id + ).first() + if voice_bundle and voice_bundle.bundle_type == "stt_llm_tts": + use_voice_bundle_pipeline = True + print(f"[BACKEND] ✅ Using custom voice bundle: {voice_bundle.name}") + + if agent and agent.ai_provider_id: + ai_provider = db.query(AIProvider).filter( + AIProvider.id == agent.ai_provider_id, + AIProvider.organization_id == organization_id, + AIProvider.is_active == True + ).first() + except ValueError: + pass + + # For custom voice bundles (stt_llm_tts), we don't need a Google provider + # The voice bundle has its own STT/LLM/TTS providers configured + if not use_voice_bundle_pipeline: + # Fall back to Google AI Provider if no agent-specific provider found (needed for S2S/Gemini path) + if not ai_provider: + from sqlalchemy import func + google_value = ModelProvider.GOOGLE.value + ai_provider = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + AIProvider.provider == google_value, + AIProvider.is_active == True + ).first() + # If not found, try case-insensitive match + if not ai_provider: + ai_provider = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + func.lower(AIProvider.provider) == google_value.lower(), + AIProvider.is_active == True + ).first() + + if not ai_provider: + print("[BACKEND] ❌ AI Provider not configured") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="AI Provider not configured. Please configure an AI Provider in AI Providers settings or select one when creating the agent." + ) + + provider_val = ai_provider.provider.value if hasattr(ai_provider.provider, 'value') else ai_provider.provider + print(f"[BACKEND] ✅ AI Provider found: {provider_val}") + else: + # For voice bundle pipeline, verify the required providers are configured + from sqlalchemy import func + missing_providers = [] + + def check_provider(provider_enum): + """Check if a provider is configured (either as AIProvider or Integration).""" + if not provider_enum: + return True # Not required + + provider_value = provider_enum.value if hasattr(provider_enum, 'value') else str(provider_enum) + + # Check AIProvider + found = db.query(AIProvider).filter( + AIProvider.organization_id == organization_id, + AIProvider.is_active == True, + ).filter( + (AIProvider.provider == provider_value) | + (func.lower(AIProvider.provider) == provider_value.lower()) + ).first() + + if found: + return True + + # Check Integration for Deepgram, Cartesia, and ElevenLabs + platform_map = { + 'deepgram': IntegrationPlatform.DEEPGRAM, + 'cartesia': IntegrationPlatform.CARTESIA, + 'elevenlabs': IntegrationPlatform.ELEVENLABS, + 'murf': IntegrationPlatform.MURF, + 'sarvam': IntegrationPlatform.SARVAM, + 'voicemaker': IntegrationPlatform.VOICEMAKER, + 'smallest': IntegrationPlatform.SMALLEST, + } + plat = platform_map.get(provider_value.lower()) + if plat: + plat_value = plat.value if hasattr(plat, 'value') else plat + found = db.query(Integration).filter( + Integration.organization_id == organization_id, + Integration.is_active == True, + ).filter( + (Integration.platform == plat_value) | + (func.lower(Integration.platform) == plat_value.lower()) + ).first() + if found: + return True + + return False + + # Check required providers for the voice bundle + if voice_bundle.stt_provider and not check_provider(voice_bundle.stt_provider): + missing_providers.append(f"STT: {voice_bundle.stt_provider}") + if voice_bundle.llm_provider and not check_provider(voice_bundle.llm_provider): + missing_providers.append(f"LLM: {voice_bundle.llm_provider}") + if voice_bundle.tts_provider and not check_provider(voice_bundle.tts_provider): + missing_providers.append(f"TTS: {voice_bundle.tts_provider}") + + if missing_providers: + print(f"[BACKEND] ❌ Missing providers for voice bundle: {missing_providers}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Missing AI providers for voice bundle: {', '.join(missing_providers)}. Please configure them in Integrations settings." + ) + + print(f"[BACKEND] ✅ Voice bundle providers configured") + + # Determine WebSocket URL — prefer dedicated media server when configured. + from urllib.parse import quote + + from app.services.media_urls import build_voice_agent_ws_url + + if bearer_token: + ws_auth_query = f"token={quote(bearer_token, safe='')}" + else: + ws_auth_query = f"X-API-Key={quote(api_key or '', safe='')}" + + ws_url = build_voice_agent_ws_url( + auth_query=ws_auth_query, + agent_id=agent_id, + persona_id=persona_id, + scenario_id=scenario_id, + ui_surface=ui_surface, + call_short_id=call_short_id, + fallback_host=request.headers.get("host", f"localhost:{settings.PORT}"), + fallback_scheme=( + request.headers.get("x-forwarded-proto") + or getattr(request.url, "scheme", "http") + or "http" + ), + ) + + # Return the response in the format Pipecat expects + # Pipecat expects a JSON response with ws_url field + # The response should be simple and match exactly what Pipecat expects + from fastapi.responses import JSONResponse + response_data = { + "ws_url": ws_url + } + print(f"[BACKEND] ✅ Returning WebSocket URL: {ws_url}") + print(f"[BACKEND] Response data: {response_data}") + print("=" * 80) + + # Return JSON response - CORS is handled by middleware + return JSONResponse( + content=response_data, + status_code=200, + headers={ + "Content-Type": "application/json", + } + ) + + +@router.get("/audio", response_model=List[Dict[str, Any]]) +async def list_voice_agent_audio_files( + organization_id: UUID = Depends(get_organization_id), + max_keys: int = 1000, +): + """ + List audio files for voice agent conversations for the current organization. + + Args: + organization_id: Organization ID from API key + max_keys: Maximum number of files to return + + Returns: + List of audio file metadata with keys: key, size, last_modified, filename + """ + try: + files = s3_service.list_audio_files( + organization_id=str(organization_id), + max_keys=max_keys + ) + return files + except Exception as e: + logger.error(f"Error listing voice agent audio files: {e}") + raise HTTPException( + status_code=500, + detail=f"Failed to list audio files: {str(e)}" + ) + diff --git a/app/app_factory.py b/app/app_factory.py index c2af3e09..2b4e6ff2 100644 --- a/app/app_factory.py +++ b/app/app_factory.py @@ -1,249 +1,263 @@ -"""Application factory for API and media service modes.""" - -from __future__ import annotations - -import logging -from contextlib import asynccontextmanager -from pathlib import Path - -from fastapi import Depends, FastAPI -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import FileResponse, JSONResponse -from fastapi.staticfiles import StaticFiles - -from app.config import settings, validate_auth_configuration -from app.core.auth.rbac import require_admin -from app.core.health import build_health_status -from app.core.migration_middleware import MigrationCheckMiddleware -from app.core.migrations import check_migrations_status, ensure_migrations_directory, run_migrations -from app.core.operational_access_middleware import OperationalAccessMiddleware -from app.core.rbac_middleware import ReaderReadOnlyMiddleware -from app.core.security_headers_middleware import SecurityHeadersMiddleware -from app.core.usage_context_middleware import LLMUsageContextMiddleware -from app.database import init_db - -logger = logging.getLogger(__name__) - - -def _service_mode() -> str: - """Prefer os.environ so subprocess / reload cannot leave stale settings.""" - import os - - return (os.environ.get("SERVICE_MODE") or settings.SERVICE_MODE or "api").strip().lower() - - -def _includes_http_routes() -> bool: - return _service_mode() in ("api", "all") - - -def _includes_media_routes() -> bool: - mode = _service_mode() - if mode in ("media", "all"): - return True - if mode == "api": - # Single-process dev: mount voice-agent / Vobiz WS on API when no - # dedicated media URL is configured (e.g. bare ``eai start``). - from app.services.media_urls import separate_media_server_configured - - return not separate_media_server_configured() - return False - - -@asynccontextmanager -async def _api_lifespan(app: FastAPI): - logger.info("=" * 60) - logger.info("Starting EfficientAI Application (mode=%s)", settings.SERVICE_MODE) - logger.info("=" * 60) - - ensure_migrations_directory() - - if _includes_http_routes(): - try: - validate_auth_configuration() - logger.info("Authentication configuration validated") - except Exception as e: - logger.error("CRITICAL: Authentication configuration is invalid: %s", e) - raise - - try: - init_db() - logger.info("Database tables initialized") - except Exception as e: - logger.error("Error initializing database: %s", e) - raise - - if _includes_http_routes(): - try: - run_migrations() - except Exception as e: - logger.error("CRITICAL: Database migrations failed: %s", e) - raise - - is_up_to_date, pending = check_migrations_status() - if not is_up_to_date: - logger.warning("Warning: %d migration(s) still pending: %s", len(pending), ", ".join(pending)) - else: - logger.info("All migrations are up to date") - - from app.services.billing.flexprice_service import log_startup_status - - log_startup_status(component="api") - - logger.info("Application startup complete - Ready to serve requests") - logger.info("=" * 60) - yield - logger.info("Shutting down EfficientAI Application...") - - -def _add_common_middleware(app: FastAPI) -> None: - if _includes_http_routes(): - app.add_middleware(MigrationCheckMiddleware) - app.add_middleware(ReaderReadOnlyMiddleware) - app.add_middleware(LLMUsageContextMiddleware) - - if settings.OBSERVABILITY_ENABLED and settings.LOKI_ENABLED and settings.LOKI_MULTI_TENANT: - from app.core.observability_middleware import OrgLoggingMiddleware - - app.add_middleware(OrgLoggingMiddleware) - - app.add_middleware( - CORSMiddleware, - allow_origins=settings.CORS_ORIGINS, - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ) - app.add_middleware(SecurityHeadersMiddleware) - if _includes_http_routes(): - app.add_middleware(OperationalAccessMiddleware) - - if settings.OBSERVABILITY_ENABLED and _includes_http_routes(): - from prometheus_fastapi_instrumentator import Instrumentator - - Instrumentator( - should_group_status_codes=True, - should_ignore_untemplated=True, - should_group_untemplated=True, - excluded_handlers=["/health", "/metrics"], - ).instrument(app).expose(app, endpoint="/metrics", include_in_schema=False) - - -def _mount_frontend(app: FastAPI) -> None: - if not _includes_http_routes(): - return - - frontend_dist = Path(settings.FRONTEND_DIR) - if not frontend_dist.exists() or not frontend_dist.is_dir(): - return - - static_dir = frontend_dist / "assets" - if static_dir.exists(): - app.mount("/assets", StaticFiles(directory=str(static_dir)), name="assets") - - @app.get("/{full_path:path}") - async def serve_frontend(full_path: str): - if ( - full_path.startswith("api/") - or full_path.startswith("docs") - or full_path.startswith("redoc") - or full_path.startswith("assets/") - or full_path == "health" - or full_path == "health/detail" - or full_path == "metrics" - ): - return {"detail": "Not found"} - - file_path = frontend_dist / full_path - if file_path.exists() and file_path.is_file() and file_path.parent == frontend_dist: - return FileResponse(str(file_path)) - - index_path = frontend_dist / "index.html" - if index_path.exists(): - return FileResponse(str(index_path)) - return {"detail": "Frontend not found"} - - -def create_app() -> FastAPI: - """Create API, media, or combined app based on SERVICE_MODE.""" - import os - - from app.config import apply_service_mode - - env_mode = os.environ.get("SERVICE_MODE") - if env_mode: - apply_service_mode(env_mode) - - mode = _service_mode() - print( - f"[{mode.upper()}] create_app: http_routes={_includes_http_routes()} " - f"media_routes={_includes_media_routes()} " - f"MEDIA_WS_BASE_URL={settings.MEDIA_WS_BASE_URL!r}", - flush=True, - ) - - title_suffix = { - "api": " API", - "media": " Media", - "all": "", - }.get(settings.SERVICE_MODE, "") - - app = FastAPI( - title=f"{settings.APP_NAME}{title_suffix}", - version=settings.APP_VERSION, - description="EfficientAI Voice AI Evaluation Platform", - docs_url="/docs" if settings.DEBUG and _includes_http_routes() else None, - redoc_url="/redoc" if settings.DEBUG and _includes_http_routes() else None, - openapi_url="/openapi.json" if settings.DEBUG and _includes_http_routes() else None, - lifespan=_api_lifespan, - ) - - _add_common_middleware(app) - - if _includes_http_routes(): - from app.api.v1.api import api_router - - app.include_router(api_router, prefix=settings.API_V1_PREFIX) - - if _includes_media_routes(): - from app.api.v1.media import media_router - - app.include_router(media_router, prefix=settings.API_V1_PREFIX) - logger.info( - "Live voice WebSockets mounted (SERVICE_MODE=%s, MEDIA_WS_BASE_URL=%r)", - settings.SERVICE_MODE, - settings.MEDIA_WS_BASE_URL, - ) - if _service_mode() == "media": - logger.info( - "Vobiz telephony edge: /telephony/vobiz/webhooks/* and /telephony/vobiz/ws" - ) - if settings.SERVICE_MODE == "api": - logger.info( - "Voice WebSockets co-located on API (unset MEDIA_WS_BASE_URL to " - "keep this mode; set it to ws://host:8001 when running eai telephony-worker)" - ) - - @app.get("/health") - async def health_check(): - payload, status_code = build_health_status(detailed=False) - return JSONResponse(content=payload, status_code=status_code) - - if _includes_http_routes(): - - @app.get("/health/detail") - async def health_detail(_admin=Depends(require_admin)): - from app.database import SessionLocal - from app.services.observability.catalog_storage_stats import collect_catalog_storage_stats - - payload, status_code = build_health_status(detailed=True) - db = SessionLocal() - try: - payload["catalog_storage"] = collect_catalog_storage_stats(db) - except Exception as exc: - payload["catalog_storage"] = {"error": str(exc)} - finally: - db.close() - return JSONResponse(content=payload, status_code=status_code) - - _mount_frontend(app) - return app +"""Application factory for API and media service modes.""" + +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager +from pathlib import Path + +from fastapi import Depends, FastAPI +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, JSONResponse + +from app.config import settings, validate_auth_configuration +from app.core.auth.rbac import require_admin +from app.core.health import build_health_status +from app.core.migration_middleware import MigrationCheckMiddleware +from app.core.migrations import check_migrations_status, ensure_migrations_directory, run_migrations +from app.core.operational_access_middleware import OperationalAccessMiddleware +from app.core.rbac_middleware import ReaderReadOnlyMiddleware +from app.core.security_headers_middleware import SecurityHeadersMiddleware +from app.core.usage_context_middleware import LLMUsageContextMiddleware +from app.database import init_db + +logger = logging.getLogger(__name__) + + +def _service_mode() -> str: + """Prefer os.environ so subprocess / reload cannot leave stale settings.""" + import os + + return (os.environ.get("SERVICE_MODE") or settings.SERVICE_MODE or "api").strip().lower() + + +def _includes_http_routes() -> bool: + return _service_mode() in ("api", "all") + + +def _includes_media_routes() -> bool: + mode = _service_mode() + if mode in ("media", "all"): + return True + if mode == "api": + # Single-process dev: mount voice-agent / Vobiz WS on API when no + # dedicated media URL is configured (e.g. bare ``eai start``). + from app.services.media_urls import separate_media_server_configured + + return not separate_media_server_configured() + return False + + +@asynccontextmanager +async def _api_lifespan(app: FastAPI): + logger.info("=" * 60) + logger.info("Starting EfficientAI Application (mode=%s)", settings.SERVICE_MODE) + logger.info("=" * 60) + + ensure_migrations_directory() + + if _includes_http_routes(): + try: + validate_auth_configuration() + logger.info("Authentication configuration validated") + except Exception as e: + logger.error("CRITICAL: Authentication configuration is invalid: %s", e) + raise + + try: + init_db() + logger.info("Database tables initialized") + except Exception as e: + logger.error("Error initializing database: %s", e) + raise + + if _includes_http_routes(): + try: + run_migrations() + except Exception as e: + logger.error("CRITICAL: Database migrations failed: %s", e) + raise + + is_up_to_date, pending = check_migrations_status() + if not is_up_to_date: + logger.warning("Warning: %d migration(s) still pending: %s", len(pending), ", ".join(pending)) + else: + logger.info("All migrations are up to date") + + from app.services.billing.flexprice_service import log_startup_status + + log_startup_status(component="api") + + logger.info("Application startup complete - Ready to serve requests") + logger.info("=" * 60) + yield + logger.info("Shutting down EfficientAI Application...") + + +def _add_common_middleware(app: FastAPI) -> None: + if _includes_http_routes(): + app.add_middleware(MigrationCheckMiddleware) + app.add_middleware(ReaderReadOnlyMiddleware) + app.add_middleware(LLMUsageContextMiddleware) + + if settings.OBSERVABILITY_ENABLED and settings.LOKI_ENABLED and settings.LOKI_MULTI_TENANT: + from app.core.observability_middleware import OrgLoggingMiddleware + + app.add_middleware(OrgLoggingMiddleware) + + app.add_middleware( + CORSMiddleware, + allow_origins=settings.CORS_ORIGINS, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + app.add_middleware(SecurityHeadersMiddleware) + if _includes_http_routes(): + app.add_middleware(OperationalAccessMiddleware) + + if settings.OBSERVABILITY_ENABLED and _includes_http_routes(): + from prometheus_fastapi_instrumentator import Instrumentator + + Instrumentator( + should_group_status_codes=True, + should_ignore_untemplated=True, + should_group_untemplated=True, + excluded_handlers=["/health", "/metrics"], + ).instrument(app).expose(app, endpoint="/metrics", include_in_schema=False) + + +def _mount_frontend(app: FastAPI) -> None: + if not _includes_http_routes(): + return + + frontend_dist = Path(settings.FRONTEND_DIR).resolve() + if not frontend_dist.exists() or not frontend_dist.is_dir(): + return + + assets_dir = frontend_dist / "assets" + + @app.get("/assets/{asset_path:path}", include_in_schema=False) + async def serve_frontend_asset(asset_path: str): + from fastapi import HTTPException + + if not assets_dir.is_dir(): + raise HTTPException(status_code=404, detail="Asset not found") + candidate = (assets_dir / asset_path).resolve() + try: + candidate.relative_to(assets_dir.resolve()) + except ValueError: + raise HTTPException(status_code=404, detail="Asset not found") + if not candidate.is_file(): + raise HTTPException(status_code=404, detail="Asset not found") + return FileResponse(str(candidate)) + + @app.get("/{full_path:path}", include_in_schema=False) + async def serve_frontend(full_path: str): + from fastapi import HTTPException + + if ( + full_path.startswith("api/") + or full_path.startswith("docs") + or full_path.startswith("redoc") + or full_path.startswith("assets/") + or full_path == "health" + or full_path == "health/detail" + or full_path == "metrics" + ): + raise HTTPException(status_code=404, detail="Not found") + + file_path = frontend_dist / full_path + if file_path.exists() and file_path.is_file() and file_path.parent == frontend_dist: + return FileResponse(str(file_path)) + + index_path = frontend_dist / "index.html" + if index_path.exists(): + return FileResponse(str(index_path)) + raise HTTPException(status_code=404, detail="Frontend not found") + + +def create_app() -> FastAPI: + """Create API, media, or combined app based on SERVICE_MODE.""" + import os + + from app.config import apply_service_mode + + env_mode = os.environ.get("SERVICE_MODE") + if env_mode: + apply_service_mode(env_mode) + + mode = _service_mode() + print( + f"[{mode.upper()}] create_app: http_routes={_includes_http_routes()} " + f"media_routes={_includes_media_routes()} " + f"MEDIA_WS_BASE_URL={settings.MEDIA_WS_BASE_URL!r}", + flush=True, + ) + + title_suffix = { + "api": " API", + "media": " Media", + "all": "", + }.get(settings.SERVICE_MODE, "") + + app = FastAPI( + title=f"{settings.APP_NAME}{title_suffix}", + version=settings.APP_VERSION, + description="EfficientAI Voice AI Evaluation Platform", + docs_url="/docs" if settings.DEBUG and _includes_http_routes() else None, + redoc_url="/redoc" if settings.DEBUG and _includes_http_routes() else None, + openapi_url="/openapi.json" if settings.DEBUG and _includes_http_routes() else None, + lifespan=_api_lifespan, + ) + + _add_common_middleware(app) + + if _includes_http_routes(): + from app.api.v1.api import api_router + + app.include_router(api_router, prefix=settings.API_V1_PREFIX) + + if _includes_media_routes(): + from app.api.v1.media import media_router + + app.include_router(media_router, prefix=settings.API_V1_PREFIX) + logger.info( + "Live voice WebSockets mounted (SERVICE_MODE=%s, MEDIA_WS_BASE_URL=%r)", + settings.SERVICE_MODE, + settings.MEDIA_WS_BASE_URL, + ) + if _service_mode() == "media": + logger.info( + "Vobiz telephony edge: /telephony/vobiz/webhooks/* and /telephony/vobiz/ws" + ) + if settings.SERVICE_MODE == "api": + logger.info( + "Voice WebSockets co-located on API (unset MEDIA_WS_BASE_URL to " + "keep this mode; set it to ws://host:8001 when running eai telephony-worker)" + ) + + @app.get("/health") + async def health_check(): + payload, status_code = build_health_status(detailed=False) + return JSONResponse(content=payload, status_code=status_code) + + if _includes_http_routes(): + + @app.get("/health/detail") + async def health_detail(_admin=Depends(require_admin)): + from app.database import SessionLocal + from app.services.observability.catalog_storage_stats import collect_catalog_storage_stats + + payload, status_code = build_health_status(detailed=True) + db = SessionLocal() + try: + payload["catalog_storage"] = collect_catalog_storage_stats(db) + except Exception as exc: + payload["catalog_storage"] = {"error": str(exc)} + finally: + db.close() + return JSONResponse(content=payload, status_code=status_code) + + _mount_frontend(app) + return app diff --git a/app/cli.py b/app/cli.py index 7fbefbff..2e07fbd8 100644 --- a/app/cli.py +++ b/app/cli.py @@ -1,1621 +1,1633 @@ -"""CLI for EfficientAI platform.""" - -import click -import yaml -import os -import sys -import subprocess -import threading -import time -from pathlib import Path -from typing import Optional - - -@click.group() -def main(): - """EfficientAI - Voice AI Evaluation Platform CLI.""" - pass - - -class FrontendWatcher: - """Watch frontend files and rebuild on changes.""" - - def __init__(self, frontend_dir: Path): - self.frontend_dir = frontend_dir - self.watching = False - self.thread = None - self.last_build_time = 0 - self.build_lock = threading.Lock() - - def should_rebuild(self) -> bool: - """Check if frontend files have changed.""" - src_dir = self.frontend_dir / "src" - if not src_dir.exists(): - return False - - # Check modification time of source files - max_mtime = 0 - for ext in [".tsx", ".ts", ".css", ".jsx", ".js"]: - for file_path in src_dir.rglob(f"*{ext}"): - if file_path.is_file(): - max_mtime = max(max_mtime, file_path.stat().st_mtime) - - # Also check config files - config_files = [ - self.frontend_dir / "vite.config.ts", - self.frontend_dir / "tailwind.config.js", - self.frontend_dir / "tsconfig.json", - self.frontend_dir / "package.json", - ] - for config_file in config_files: - if config_file.exists(): - max_mtime = max(max_mtime, config_file.stat().st_mtime) - - if max_mtime > self.last_build_time: - self.last_build_time = max_mtime - return True - return False - - def build_frontend(self): - """Rebuild the frontend.""" - with self.build_lock: - try: - click.echo("\n🔄 Frontend files changed, rebuilding...") - result = subprocess.run( - ["npm", "run", "build"], - cwd=self.frontend_dir, - check=False, - capture_output=True, - text=True, - ) - if result.returncode == 0: - click.echo("✅ Frontend rebuilt successfully") - else: - click.echo(f"⚠️ Frontend build had warnings (check logs)", err=True) - if result.stderr: - click.echo(result.stderr[:500], err=True) # Show first 500 chars - except Exception as e: - click.echo(f"❌ Frontend build error: {e}", err=True) - - def watch_loop(self): - """Watch loop that runs in background thread.""" - while self.watching: - try: - if self.should_rebuild(): - self.build_frontend() - time.sleep(1) # Check every second - except Exception as e: - click.echo(f"❌ Watcher error: {e}", err=True) - time.sleep(5) # Wait longer on error - - def start(self): - """Start the watcher in a background thread.""" - if self.watching: - return - self.watching = True - # Set initial build time to avoid rebuilding immediately - self.last_build_time = time.time() - self.thread = threading.Thread(target=self.watch_loop, daemon=True) - self.thread.start() - - def stop(self): - """Stop the watcher.""" - self.watching = False - if self.thread: - self.thread.join(timeout=1) - - -def start_frontend_watcher(frontend_dir: Path) -> FrontendWatcher: - """Start a frontend file watcher.""" - watcher = FrontendWatcher(frontend_dir) - watcher.start() - return watcher - - -def _read_config_media_ws_base_url(config_path: Path) -> Optional[str]: - """Return vobiz.media_ws_base_url from YAML when explicitly configured.""" - try: - with open(config_path, encoding="utf-8") as handle: - data = yaml.safe_load(handle) or {} - value = ((data.get("vobiz") or {}).get("media_ws_base_url") or "").strip() - return value or None - except Exception: - return None - - -@main.command() -@click.option( - "--verbose", - "-v", - is_flag=True, - help="Show detailed migration output", -) -def migrate(verbose: bool): - """Run pending database migrations.""" - import logging - from app.core.migrations import run_migrations, ensure_migrations_directory - - if verbose: - logging.basicConfig(level=logging.INFO) - - click.echo("🔄 Running database migrations...") - ensure_migrations_directory() - - try: - run_migrations() - click.echo("✅ All migrations completed successfully!") - except Exception as e: - click.echo(f"❌ Migration failed: {e}", err=True) - sys.exit(1) - - -@main.command() -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--host", - default=None, - help="Host to bind to (overrides config)", -) -@click.option( - "--port", - default=None, - type=int, - help="Port to bind to (overrides config)", -) -@click.option( - "--build-frontend/--no-build-frontend", - default=True, - help="Build frontend before starting (default: True)", -) -@click.option( - "--reload/--no-reload", - default=True, - help="Enable auto-reload for development (default: True)", -) -@click.option( - "--watch-frontend/--no-watch-frontend", - default=False, - help="Watch frontend files and rebuild automatically (default: False)", -) -@click.option( - "--force-rebuild", - is_flag=True, - default=False, - help="Force rebuild of frontend without prompting", -) -@click.option( - "--skip-migrations", - is_flag=True, - default=False, - help="Skip running migrations before starting (not recommended)", -) -def start(config: str, host: Optional[str], port: Optional[int], build_frontend: bool, reload: bool, watch_frontend: bool, force_rebuild: bool, skip_migrations: bool): - """Start the EfficientAI application server.""" - from app.config import apply_service_mode, load_config_from_file, settings - - # Load configuration from YAML file - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) - sys.exit(1) - - explicit_media_ws = _read_config_media_ws_base_url(config_path) - os.environ["SERVICE_MODE"] = "api" - if not explicit_media_ws: - # Single-process dev: co-locate voice WebSockets on the API port unless - # config.yml sets vobiz.media_ws_base_url (used with eai telephony-worker). - os.environ.pop("MEDIA_WS_BASE_URL", None) - - try: - load_config_from_file(str(config_path)) - apply_service_mode("api") - if not explicit_media_ws: - settings.MEDIA_WS_BASE_URL = "" - click.echo(f"✅ Loaded configuration from {config_path}") - except Exception as e: - click.echo(f"❌ Error loading config: {e}", err=True) - sys.exit(1) - - # Override with CLI options if provided - if host: - settings.HOST = host - if port: - settings.PORT = port - - # Build frontend if requested - # Check if frontend is already built - frontend_dist = Path(__file__).parent.parent / "frontend" / "dist" - if build_frontend and frontend_dist.exists() and any(frontend_dist.iterdir()): - # If watching, we always want to rebuild to catch latest changes at start - if not force_rebuild and not watch_frontend and not click.confirm("Frontend dist directory already exists. Rebuild anyway?"): - build_frontend = False - - if build_frontend: - click.echo("🔨 Building frontend...") - frontend_dir = Path(__file__).parent.parent / "frontend" - if not frontend_dir.exists(): - click.echo(f"❌ Frontend directory not found: {frontend_dir}", err=True) - sys.exit(1) - - try: - # Check if node_modules exists, if not, install dependencies - if not (frontend_dir / "node_modules").exists(): - click.echo("📦 Installing frontend dependencies...") - subprocess.run( - ["npm", "install", "--legacy-peer-deps"], - cwd=frontend_dir, - check=True, - capture_output=True, - ) - - # Build frontend (show output in real-time for better debugging) - click.echo(" Running TypeScript check and Vite build...") - result = subprocess.run( - ["npm", "run", "build"], - cwd=frontend_dir, - check=False, # Don't fail immediately, we'll check return code - capture_output=True, - text=True, - ) - if result.returncode != 0: - click.echo(f"❌ Frontend build failed with return code {result.returncode}", err=True) - if result.stdout: - click.echo(f"\nSTDOUT:\n{result.stdout}", err=True) - if result.stderr: - click.echo(f"\nSTDERR:\n{result.stderr}", err=True) - click.echo("\n💡 Try running 'npm run build' manually in the frontend directory to see full error details.", err=True) - sys.exit(1) - - click.echo("✅ Frontend built successfully") - except subprocess.CalledProcessError as e: - click.echo(f"❌ Error building frontend:", err=True) - if e.stdout: - click.echo(f"STDOUT:\n{e.stdout}", err=True) - if e.stderr: - click.echo(f"STDERR:\n{e.stderr}", err=True) - click.echo(f"\nReturn code: {e.returncode}", err=True) - sys.exit(1) - except FileNotFoundError: - click.echo("❌ npm not found. Please install Node.js and npm.", err=True) - sys.exit(1) - - # Initialize DB tables and run migrations before starting (unless explicitly skipped) - if not skip_migrations: - click.echo("🔄 Initializing database and running migrations...") - from app.database import init_db - from app.core.migrations import run_migrations, ensure_migrations_directory - try: - init_db() - ensure_migrations_directory() - run_migrations() - click.echo("✅ Database initialized and migrations completed") - except Exception as e: - click.echo(f"❌ Migration failed: {e}", err=True) - click.echo("💡 You can skip migrations with --skip-migrations (not recommended)", err=True) - sys.exit(1) - else: - click.echo("⚠️ Skipping migrations (not recommended - migrations will run on startup)") - - # Start frontend watcher if requested - frontend_watcher = None - if watch_frontend: - click.echo("👀 Starting frontend file watcher...") - frontend_dir = Path(__file__).parent.parent / "frontend" - frontend_watcher = start_frontend_watcher(frontend_dir) - - # Start the server - import uvicorn - - click.echo(f"🚀 Starting EfficientAI server...") - click.echo(f" Host: {settings.HOST}") - click.echo(f" Port: {settings.PORT}") - click.echo(f" API: http://{settings.HOST}:{settings.PORT}{settings.API_V1_PREFIX}") - click.echo(f" Frontend: http://{settings.HOST}:{settings.PORT}/") - click.echo(f" Docs: http://{settings.HOST}:{settings.PORT}/docs") - if watch_frontend: - click.echo(f" Frontend watcher: Active (rebuilding on file changes)") - - # Use import string for reload to work properly - try: - uvicorn.run( - "app.main:app", - host=settings.HOST, - port=settings.PORT, - reload=reload, - ) - finally: - # Clean up watcher on exit - if frontend_watcher: - frontend_watcher.stop() - - -@main.command() -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--loglevel", - "-l", - default="info", - type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), - help="Log level for Celery worker", -) -@click.option( - "--queues", - "-Q", - "queues", - default=None, - help=( - "Comma-separated list of Celery queues this worker should consume " - "(forwarded to celery's -Q flag). Defaults to the default queue." - ), -) -@click.option( - "--concurrency", - default=None, - type=int, - help="Number of concurrent worker processes/threads (Celery --concurrency).", -) -@click.option( - "--pool", - "-P", - "pool", - default=None, - type=click.Choice(["prefork", "threads", "solo", "eventlet", "gevent"], case_sensitive=False), - help="Celery worker pool implementation (forwarded to celery's -P flag).", -) -def worker(config: str, loglevel: str, queues: Optional[str], concurrency: Optional[int], pool: Optional[str]): - """Start the Celery worker for background task processing.""" - from app.config import load_config_from_file - - # Load configuration from YAML file - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) - sys.exit(1) - - try: - load_config_from_file(str(config_path)) - click.echo(f"✅ Loaded configuration from {config_path}") - except Exception as e: - click.echo(f"❌ Error loading config: {e}", err=True) - sys.exit(1) - - click.echo(f"🚀 Starting Celery worker...") - click.echo(f" Log level: {loglevel}") - if queues: - click.echo(f" Queues: {queues}") - if concurrency is not None: - click.echo(f" Concurrency: {concurrency}") - if pool: - click.echo(f" Pool: {pool}") - - # Start Celery worker - try: - import subprocess - cmd = ["celery", "-A", "app.workers.celery_app", "worker", f"--loglevel={loglevel}"] - if queues: - cmd.append(f"--queues={queues}") - if concurrency is not None: - cmd.append(f"--concurrency={concurrency}") - if pool: - cmd.append(f"--pool={pool}") - subprocess.run(cmd, check=True) - except KeyboardInterrupt: - click.echo("\n👋 Celery worker stopped") - except subprocess.CalledProcessError as e: - click.echo(f"❌ Celery worker failed: {e}", err=True) - sys.exit(1) - except FileNotFoundError: - click.echo("❌ Celery not found. Please install it: pip install celery", err=True) - sys.exit(1) - - -@main.command("beat") -@click.option( - "--config", - "-c", - default="config.yml", - help="Path to configuration file", -) -@click.option( - "--loglevel", - "-l", - default="info", - type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), - help="Log level for Celery beat", -) -@click.option( - "--platform-worker-concurrency", - default=2, - type=int, - help="Concurrency for the co-located platform task worker (default: 2; thread pool).", -) -def beat(config: str, loglevel: str, platform_worker_concurrency: int): - """Start Celery Beat + platform task worker (alerts, FX, prune) — single replica only.""" - from app.config import load_config_from_file - - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - sys.exit(1) - - try: - load_config_from_file(str(config_path)) - click.echo(f"✅ Loaded configuration from {config_path}") - except Exception as e: - click.echo(f"❌ Error loading config: {e}", err=True) - sys.exit(1) - - from app.workers.config import PLATFORM_WORKER_QUEUE - - click.echo( - "🚀 Starting Celery Beat + platform worker " - f"(queue={PLATFORM_WORKER_QUEUE}, concurrency={platform_worker_concurrency})" - ) - platform_proc = None - try: - import subprocess - - platform_proc = subprocess.Popen( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--queues={PLATFORM_WORKER_QUEUE}", - "--pool=threads", - f"--concurrency={platform_worker_concurrency}", - f"--loglevel={loglevel}", - ], - ) - subprocess.run( - [ - "celery", - "-A", - "app.workers.celery_app", - "beat", - f"--loglevel={loglevel}", - ], - check=True, - ) - except KeyboardInterrupt: - click.echo("\n👋 Celery Beat stopped") - except subprocess.CalledProcessError as e: - click.echo(f"❌ Celery Beat failed: {e}", err=True) - sys.exit(1) - except FileNotFoundError: - click.echo("❌ Celery not found. Please install it: pip install celery", err=True) - sys.exit(1) - finally: - if platform_proc is not None and platform_proc.poll() is None: - platform_proc.terminate() - try: - platform_proc.wait(timeout=5) - except subprocess.TimeoutExpired: - platform_proc.kill() - - -@main.command("telephony-worker") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option("--host", default=None, help="Host to bind (default from config)") -@click.option("--port", default=None, type=int, help="Media server port (default 8001)") -def telephony_worker(config: str, host: Optional[str], port: Optional[int]): - """Start the media server for live voice WebSocket connections.""" - os.environ["SERVICE_MODE"] = "media" - from app.config import apply_service_mode, load_config_from_file, settings - - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - sys.exit(1) - - try: - load_config_from_file(str(config_path)) - apply_service_mode("media") - except Exception as e: - click.echo(f"❌ Error loading config: {e}", err=True) - sys.exit(1) - bind_host = host or settings.HOST - bind_port = port or settings.MEDIA_PORT - - from app.app_factory import create_app - - app = create_app() - print( - f"[MEDIA] telephony-worker ready on {bind_host}:{bind_port} " - f"(SERVICE_MODE={settings.SERVICE_MODE}, media_routes mounted)", - flush=True, - ) - - import uvicorn - - click.echo(f"🎙️ Starting EfficientAI media server on {bind_host}:{bind_port}") - uvicorn.run( - app, - host=bind_host, - port=bind_port, - reload=False, - ) - - -@main.command("start-worker-all") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option("--loglevel", "-l", default="info", help="Celery log level") -@click.option("--media-port", default=None, type=int, help="Media server port (default 8001)") -def start_worker_all(config: str, loglevel: str, media_port: Optional[int]): - """Start Celery worker and media server together. - - Deprecated: prefer separate ``eai telephony-worker`` and ``eai worker`` services - (see docker-compose ``media`` + ``worker``). - """ - import signal - import atexit - - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - sys.exit(1) - - from app.config import load_config_from_file, settings - - load_config_from_file(str(config_path)) - - media_proc = None - celery_proc = None - - def cleanup(): - nonlocal media_proc, celery_proc - for proc, label in ((media_proc, "media server"), (celery_proc, "Celery worker")): - if proc is None or proc.poll() is not None: - continue - try: - proc.terminate() - proc.wait(timeout=5) - click.echo(f"✅ {label} stopped") - except Exception: - proc.kill() - - atexit.register(cleanup) - - def _handle_signal(sig, frame): - cleanup() - sys.exit(0) - - signal.signal(signal.SIGINT, _handle_signal) - signal.signal(signal.SIGTERM, _handle_signal) - - port = media_port or settings.MEDIA_PORT - telephony_env = os.environ.copy() - telephony_env["SERVICE_MODE"] = "media" - media_proc = subprocess.Popen( - [sys.executable, "-m", "app.cli", "telephony-worker", "--config", str(config_path), "--port", str(port)], - env=telephony_env, - ) - celery_proc = subprocess.Popen( - ["celery", "-A", "app.workers.celery_app", "worker", f"--loglevel={loglevel}"], - ) - click.echo(f"🚀 Started media server (pid={media_proc.pid}) and Celery worker (pid={celery_proc.pid})") - - try: - while True: - if media_proc.poll() is not None: - click.echo("❌ Media server exited", err=True) - break - if celery_proc.poll() is not None: - click.echo("❌ Celery worker exited", err=True) - break - time.sleep(1) - except KeyboardInterrupt: - pass - finally: - cleanup() - - -@main.command() -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--host", - default=None, - help="Host to bind to (overrides config)", -) -@click.option( - "--port", - default=None, - type=int, - help="Port to bind to (overrides config)", -) -@click.option( - "--build-frontend/--no-build-frontend", - default=True, - help="Build frontend before starting (default: True)", -) -@click.option( - "--reload/--no-reload", - default=True, - help="Enable auto-reload for development (default: True)", -) -@click.option( - "--watch-frontend/--no-watch-frontend", - default=False, - help="Watch frontend files and rebuild automatically (default: False)", -) -@click.option( - "--force-rebuild", - is_flag=True, - default=False, - help="Force rebuild of frontend without prompting", -) -@click.option( - "--skip-migrations", - is_flag=True, - default=False, - help="Skip running migrations before starting (not recommended)", -) -@click.option( - "--worker-loglevel", - "-l", - default="info", - type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), - help="Log level for Celery worker", -) -@click.option( - "--imports-worker/--no-imports-worker", - default=True, - help=( - "Also start a dedicated worker for the `imports` queue used by call " - "import CSV processing (default: True). Disable to keep the previous " - "single-worker behavior." - ), -) -@click.option( - "--imports-worker-concurrency", - default=12, - type=int, - help=( - "Concurrency for the imports+diarization+evaluations worker " - "(default: 12; thread pool). Use lower values (8–12) when DB sharding " - "is enabled; 32 threads can exhaust per-shard SQLAlchemy pools." - ), -) -@click.option( - "--usage-worker/--no-usage-worker", - default=True, - help=( - "Also start a dedicated worker for the `usage` queue (flush + cost recompute; " - "default: True)." - ), -) -@click.option( - "--usage-worker-concurrency", - default=4, - type=int, - help="Concurrency for the usage worker (default: 4; thread pool).", -) -@click.option( - "--beat/--no-beat", - default=True, - help=( - "Start Celery Beat for platform periodic tasks (usage flush, alerts, etc.; " - "default: True — single replica only in production)." - ), -) -@click.option( - "--beat-loglevel", - default=None, - type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), - help="Log level for Celery Beat (defaults to --worker-loglevel).", -) -@click.option( - "--telephony-worker/--no-telephony-worker", - default=True, - help="Spawn telephony media server for live voice WebSockets (default: on).", -) -@click.option( - "--media-port", - default=None, - type=int, - help="Telephony media server port (default: MEDIA_PORT / 8001).", -) -def start_all( - config: str, - host: Optional[str], - port: Optional[int], - build_frontend: bool, - reload: bool, - watch_frontend: bool, - force_rebuild: bool, - skip_migrations: bool, - worker_loglevel: str, - imports_worker: bool, - imports_worker_concurrency: int, - usage_worker: bool, - usage_worker_concurrency: int, - beat: bool, - beat_loglevel: Optional[str], - telephony_worker: bool, - media_port: Optional[int], -): - """Start the application server and Celery worker(s) together. - - By default this also spawns a telephony media server (``eai telephony-worker``) - and a second Celery worker that consumes the ``imports`` queue (call-import CSV - fan-out). Use --no-telephony-worker or --no-imports-worker to skip either. - """ - import signal - import atexit - - click.echo("🚀 Starting EfficientAI (App + Worker)...") - if telephony_worker: - click.echo(" Telephony media server will run on a separate port (SERVICE_MODE=media).") - if imports_worker: - click.echo( - " This will start the API server, the default Celery worker, " - "and a dedicated worker for the `imports` queue." - ) - else: - click.echo(" This will start both the API server and Celery worker") - click.echo(" Press Ctrl+C to stop all services\n") - - # Load configuration - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) - sys.exit(1) - - try: - from app.config import load_config_from_file, settings - load_config_from_file(str(config_path)) - click.echo(f"✅ Loaded configuration from {config_path}") - except Exception as e: - click.echo(f"❌ Error loading config: {e}", err=True) - sys.exit(1) - - bind_media_port = media_port or settings.MEDIA_PORT - - os.environ["SERVICE_MODE"] = "api" - - # Store worker processes for cleanup. - worker_process = None - worker_imports_process = None - worker_usage_process = None - beat_process = None - platform_worker_process = None - telephony_process = None - - def _terminate(proc, label: str): - """Best-effort terminate -> wait -> kill for a worker subprocess.""" - if proc is None or proc.poll() is not None: - return - try: - click.echo(f"\n👋 Stopping {label}...") - proc.terminate() - proc.wait(timeout=5) - click.echo(f"✅ {label} stopped") - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() - except Exception: - pass - - def cleanup_processes(): - """Clean up spawned processes.""" - nonlocal worker_process, worker_imports_process, worker_usage_process, beat_process, platform_worker_process, telephony_process - _terminate(telephony_process, "Telephony media server") - _terminate(beat_process, "Celery Beat") - _terminate(platform_worker_process, "Celery platform worker") - _terminate(worker_process, "Celery worker (default)") - _terminate(worker_imports_process, "Celery worker (imports)") - _terminate(worker_usage_process, "Celery worker (usage)") - - # Register cleanup on exit - atexit.register(cleanup_processes) - - def signal_handler(sig, frame): - """Handle Ctrl+C gracefully.""" - cleanup_processes() - sys.exit(0) - - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - # Initialize DB tables and run migrations before starting (unless explicitly skipped) - if not skip_migrations: - click.echo("🔄 Initializing database and running migrations...") - from app.database import init_db - from app.core.migrations import run_migrations, ensure_migrations_directory - try: - init_db() - ensure_migrations_directory() - run_migrations() - click.echo("✅ Database initialized and migrations completed") - except Exception as e: - click.echo(f"❌ Migration failed: {e}", err=True) - click.echo("💡 You can skip migrations with --skip-migrations (not recommended)", err=True) - sys.exit(1) - - # Build frontend if needed - if build_frontend: - frontend_dir = Path(__file__).parent.parent / "frontend" - if frontend_dir.exists(): - click.echo("🔨 Building frontend...") - try: - if not (frontend_dir / "node_modules").exists(): - click.echo(" Installing frontend dependencies...") - subprocess.run(["npm", "install", "--legacy-peer-deps"], cwd=frontend_dir, check=True, capture_output=True) - subprocess.run(["npm", "run", "build"], cwd=frontend_dir, check=True, capture_output=True) - click.echo("✅ Frontend built successfully") - except subprocess.CalledProcessError as e: - click.echo(f"❌ Frontend build failed: {e}", err=True) - sys.exit(1) - - # Start frontend watcher if requested - frontend_watcher = None - if watch_frontend: - click.echo("👀 Starting frontend file watcher...") - frontend_dir = Path(__file__).parent.parent / "frontend" - frontend_watcher = start_frontend_watcher(frontend_dir) - - def _spawn_worker(args: list[str], label: str, prefix: str) -> subprocess.Popen: - """Spawn a Celery worker subprocess and stream its stdout with a prefix.""" - proc = subprocess.Popen( - args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - ) - click.echo(f"✅ {label} started") - - def _stream(): - if proc.stdout: - for line in iter(proc.stdout.readline, ""): - if line: - click.echo(f"{prefix} {line.rstrip()}", err=False) - proc.stdout.close() - - threading.Thread(target=_stream, daemon=True).start() - return proc - - if telephony_worker: - try: - telephony_env = os.environ.copy() - telephony_env["SERVICE_MODE"] = "media" - telephony_process = subprocess.Popen( - [ - sys.executable, - "-m", - "app.cli", - "telephony-worker", - "--config", - str(config_path), - "--port", - str(bind_media_port), - ], - env=telephony_env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - bufsize=1, - ) - click.echo( - f"✅ Telephony media server started on port {bind_media_port} " - f"(pid={telephony_process.pid})" - ) - - def _stream_telephony(): - if telephony_process.stdout: - for line in iter(telephony_process.stdout.readline, ""): - if line: - click.echo(f"[MEDIA] {line.rstrip()}", err=False) - telephony_process.stdout.close() - - threading.Thread(target=_stream_telephony, daemon=True).start() - except FileNotFoundError: - click.echo("❌ Python interpreter not found for telephony-worker subprocess", err=True) - sys.exit(1) - except Exception as e: - click.echo(f"❌ Failed to start telephony media server: {e}", err=True) - sys.exit(1) - - # Start Celery workers as subprocess(es) with output streaming - try: - worker_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--loglevel={worker_loglevel}", - "-Q", - "celery,audio-metrics", - "-c", - "8", - ], - label="Celery worker (celery + audio-metrics queues, concurrency=8)", - prefix="[WORKER]", - ) - - if imports_worker: - from app.workers.config import IMPORTS_WORKER_QUEUES - - worker_imports_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--loglevel={worker_loglevel}", - "-Q", - IMPORTS_WORKER_QUEUES, - "-P", - "threads", - "-c", - str(imports_worker_concurrency), - ], - label=( - f"Celery worker ({IMPORTS_WORKER_QUEUES} queues, " - f"pool=threads, concurrency={imports_worker_concurrency})" - ), - prefix="[WORKER-IMPORTS]", - ) - - if usage_worker: - from app.workers.config import USAGE_WORKER_QUEUE - - worker_usage_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--loglevel={worker_loglevel}", - "-Q", - USAGE_WORKER_QUEUE, - "-P", - "threads", - "-c", - str(usage_worker_concurrency), - ], - label=( - f"Celery worker ({USAGE_WORKER_QUEUE} queue, " - f"pool=threads, concurrency={usage_worker_concurrency})" - ), - prefix="[WORKER-USAGE]", - ) - - if beat: - from app.workers.config import PLATFORM_WORKER_QUEUE - - beat_level = beat_loglevel or worker_loglevel - platform_worker_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "worker", - f"--loglevel={beat_level}", - "-Q", - PLATFORM_WORKER_QUEUE, - "-P", - "threads", - "-c", - "2", - ], - label=f"Celery platform worker ({PLATFORM_WORKER_QUEUE} queue)", - prefix="[BEAT-WORKER]", - ) - beat_process = _spawn_worker( - [ - "celery", - "-A", - "app.workers.celery_app", - "beat", - f"--loglevel={beat_level}", - ], - label=f"Celery Beat (scheduler, loglevel={beat_level})", - prefix="[BEAT]", - ) - - except FileNotFoundError: - click.echo("❌ Celery not found. Please install it: pip install celery", err=True) - sys.exit(1) - except Exception as e: - click.echo(f"❌ Failed to start worker: {e}", err=True) - sys.exit(1) - - # Small delay to let worker start - time.sleep(1) - - # Start the application server in the main process - # This allows uvicorn's reload to work properly (it needs to spawn child processes) - try: - from app.config import settings - import uvicorn - - # Override with CLI options if provided - if host: - settings.HOST = host - if port: - settings.PORT = port - - click.echo("✅ Application server starting...") - click.echo(f" Host: {settings.HOST}") - click.echo(f" Port: {settings.PORT}") - click.echo(f" API: http://{settings.HOST}:{settings.PORT}{settings.API_V1_PREFIX}") - click.echo(f" Frontend: http://{settings.HOST}:{settings.PORT}/") - click.echo(f" Docs: http://{settings.HOST}:{settings.PORT}/docs") - if watch_frontend: - click.echo(f" Frontend watcher: Active (rebuilding on file changes)") - if imports_worker: - from app.workers.config import IMPORTS_WORKER_QUEUES - - click.echo( - f" Workers: default queue + {IMPORTS_WORKER_QUEUES} " - f"(concurrency={imports_worker_concurrency}; imports preferred)" - ) - else: - click.echo(" Workers: default queue only (--no-imports-worker)") - if usage_worker: - from app.workers.config import USAGE_WORKER_QUEUE - - click.echo( - f" Usage worker: {USAGE_WORKER_QUEUE} queue " - f"(concurrency={usage_worker_concurrency})" - ) - else: - click.echo(" Usage worker: disabled (--no-usage-worker)") - if beat: - click.echo(" Celery Beat: scheduler + platform worker (alerts, FX, prune); flush on worker-usage") - else: - click.echo(" Celery Beat: disabled (--no-beat)") - if telephony_worker: - telephony_public = (settings.VOBIZ_WEBHOOK_BASE_URL or "").strip() - click.echo(f" Telephony edge: http://localhost:{bind_media_port} (local)") - if telephony_public: - click.echo(f" Vobiz webhook_base_url: {telephony_public}") - else: - click.echo( - " Set vobiz.webhook_base_url to your telephony public URL " - f"(e.g. ngrok http {bind_media_port}) for inbound PSTN" - ) - if (settings.MEDIA_WS_BASE_URL or "").strip(): - click.echo(f" Browser voice-agent WS: {settings.MEDIA_WS_BASE_URL}") - click.echo("\n📝 All services are running. Press Ctrl+C to stop.\n") - - # Run uvicorn in the main process (allows reload to work) - uvicorn.run( - "app.main:app", - host=settings.HOST, - port=settings.PORT, - reload=reload, - ) - except KeyboardInterrupt: - pass - except Exception as e: - click.echo(f"❌ App error: {e}", err=True) - finally: - cleanup_processes() - if frontend_watcher: - frontend_watcher.stop() - - -@main.command() -@click.option( - "--output", - "-o", - type=click.Path(), - default="config.yml", - help="Output file path for example config", -) -def init_config(output: str): - """Generate an example configuration file.""" - example_config = """# EfficientAI Configuration File - -# Application Settings -app: - name: "Voice AI Evaluation Platform" - version: "0.1.0" - debug: true - secret_key: "your-secret-key-here-change-in-production" - -# Server Settings -server: - host: "0.0.0.0" - port: 8000 - -# Database Configuration -database: - url: "postgresql://efficientai:password@localhost:5432/efficientai" - # Alternative: specify individual components - # user: "efficientai" - # password: "password" - # host: "localhost" - # port: 5432 - # db: "efficientai" - -# Redis Configuration -redis: - url: "redis://localhost:6379/0" - # Alternative: specify individual components - # host: "localhost" - # port: 6379 - # db: 0 - -# Celery Configuration -celery: - broker_url: "redis://localhost:6379/0" - result_backend: "redis://localhost:6379/0" - -# File Storage -storage: - upload_dir: "./uploads" - max_file_size_mb: 500 - allowed_audio_formats: - - "wav" - - "mp3" - - "flac" - - "m4a" - -# CORS Settings -cors: - origins: - - "http://localhost:3000" - - "http://localhost:8000" - -# API Settings -api: - prefix: "/api/v1" - key_header: "X-API-Key" - rate_limit_per_minute: 60 -""" - - output_path = Path(output) - if output_path.exists(): - if not click.confirm(f"File {output} already exists. Overwrite?"): - click.echo("Cancelled.") - return - - try: - output_path.write_text(example_config) - click.echo(f"✅ Created example configuration file: {output}") - click.echo(f"💡 Edit {output} with your settings, then run: eai start --config {output}") - except Exception as e: - click.echo(f"❌ Error creating config file: {e}", err=True) - sys.exit(1) - - -def _bootstrap_sharding_config(config_path: str) -> None: - from app.config import load_config_from_file - from app.db_sharding.pool_manager import db_pool_manager - - load_config_from_file(config_path) - db_pool_manager.reset() - - -@click.group() -def sharding(): - """Call-import data-plane sharding admin (rebalance / registry).""" - pass - - -main.add_command(sharding) - - -@sharding.command("list-slices") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--call-import-id", - required=True, - help="Call import UUID whose slice registry to inspect", -) -def sharding_list_slices(config: str, call_import_id: str): - """Show catalog slice registry rows for a call import.""" - from uuid import UUID - - from app.db_sharding.pool_manager import open_catalog_session - from app.db_sharding.rebalance import RebalanceError, list_shard_slices, require_sharding_enabled - - try: - _bootstrap_sharding_config(config) - require_sharding_enabled() - cid = UUID(call_import_id) - except (ValueError, RebalanceError) as exc: - click.echo(f"❌ {exc}", err=True) - sys.exit(1) - - catalog = open_catalog_session() - try: - slices = list_shard_slices(catalog, cid) - if not slices: - click.echo(f"No registry slices for call_import {cid}") - return - click.echo(f"call_import_id: {cid}") - click.echo(f"slices: {len(slices)}") - for item in slices: - click.echo( - f" slice {item.slice_id}: shard={item.shard_id} " - f"rows [{item.row_index_min}, {item.row_index_max}] " - f"(count={item.row_count})" - ) - finally: - catalog.close() - - -@sharding.command("rebalance-slices") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option("--call-import-id", required=True, help="Call import UUID to rebalance") -@click.option("--from-shard", required=True, help="Source shard id (e.g. data-shard-01)") -@click.option("--to-shard", required=True, help="Target shard id (e.g. data-shard-02)") -@click.option( - "--slice-id", - "slice_ids", - multiple=True, - type=int, - help="Move only these slice ids (default: all slices on --from-shard)", -) -@click.option( - "--dry-run", - is_flag=True, - help="Plan only: print row counts without copying or updating registry", -) -@click.option( - "--force", - is_flag=True, - help="Skip quiescence checks (use only after pausing workers)", -) -def sharding_rebalance_slices( - config: str, - call_import_id: str, - from_shard: str, - to_shard: str, - slice_ids: tuple[int, ...], - dry_run: bool, - force: bool, -): - """Copy call-import slice rows (and eval rows) from one shard to another.""" - from uuid import UUID - - from app.db_sharding.pool_manager import open_catalog_session - from app.db_sharding.rebalance import ( - RebalanceError, - require_sharding_enabled, - build_rebalance_plan, - execute_rebalance_slices, - ) - - try: - _bootstrap_sharding_config(config) - require_sharding_enabled() - cid = UUID(call_import_id) - except (ValueError, RebalanceError) as exc: - click.echo(f"❌ {exc}", err=True) - sys.exit(1) - - catalog = open_catalog_session() - try: - plan = build_rebalance_plan( - catalog, - cid, - from_shard_id=from_shard, - to_shard_id=to_shard, - slice_ids=slice_ids or None, - ) - click.echo("Rebalance plan:") - click.echo(f" call_import_id: {plan.call_import_id}") - click.echo(f" from_shard: {plan.from_shard_id}") - click.echo(f" to_shard: {plan.to_shard_id}") - click.echo(f" slices: {len(plan.slices)}") - for item in plan.slices: - click.echo( - f" slice {item.slice_id}: rows [{item.row_index_min}, {item.row_index_max}]" - ) - click.echo(f" import_rows: {plan.import_row_count}") - click.echo(f" eval_rows: {plan.eval_row_count}") - - result = execute_rebalance_slices( - catalog, - plan, - dry_run=dry_run, - force=force, - ) - if result.dry_run: - click.echo("✅ Dry run complete (no changes made)") - else: - click.echo( - "✅ Rebalance complete: " - f"slices={result.slices_moved}, " - f"import_rows={result.import_rows_moved}, " - f"eval_rows={result.eval_rows_moved}" - ) - except RebalanceError as exc: - click.echo(f"❌ {exc}", err=True) - sys.exit(1) - finally: - catalog.close() - - -@click.group() -def usage(): - """Usage pricing ops (seed rates, diff catalog, recompute costs).""" - pass - - -main.add_command(usage) - - -def _load_cli_config(config: str) -> Path: - config_path = Path(config) - if not config_path.exists(): - click.echo(f"❌ Config file not found: {config}", err=True) - sys.exit(1) - from app.config import load_config_from_file - - load_config_from_file(str(config_path)) - return config_path - - -@usage.command("seed-rates") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--effective-from", - type=click.DateTime(formats=["%Y-%m-%d"]), - default=None, - help="Effective date for seeded rates (default: 2020-01-01)", -) -def usage_seed_rates(config: str, effective_from): - """Upsert model_pricing_rates from models.json pricing blocks.""" - _load_cli_config(config) - from app.database import SessionLocal - from app.services.usage.pricing_ops import seed_rates_from_models_json - - day = effective_from.date() if effective_from else None - db = SessionLocal() - try: - count = seed_rates_from_models_json(db, effective_from=day) - db.commit() - click.echo(f"✅ Seeded/updated {count} pricing rate row(s)") - finally: - db.close() - - -@usage.command("diff-rates") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option( - "--effective-from", - type=click.DateTime(formats=["%Y-%m-%d"]), - default=None, - help="Compare rates at this effective_from date (default: 2020-01-01)", -) -@click.option("--json", "as_json", is_flag=True, help="Print machine-readable JSON") -def usage_diff_rates(config: str, effective_from, as_json: bool): - """Diff models.json pricing blocks vs model_pricing_rates in Postgres.""" - import json as json_module - - _load_cli_config(config) - from app.database import SessionLocal - from app.services.usage.pricing_ops import diff_models_json_vs_db - - day = effective_from.date() if effective_from else None - db = SessionLocal() - try: - report = diff_models_json_vs_db(db, effective_from=day) - finally: - db.close() - - if as_json: - click.echo(json_module.dumps(report, indent=2, default=str)) - return - - click.echo(f"effective_from: {report['effective_from']}") - click.echo( - f"models.json priced: {report['models_json_count']} | " - f"database rows: {report['database_count']} | " - f"in_sync: {report['in_sync']}" - ) - if report["only_in_models_json"]: - click.echo(f"\nOnly in models.json ({len(report['only_in_models_json'])}):") - for item in report["only_in_models_json"][:20]: - click.echo(f" - {item['model']} ({item['usage_kind']})") - if report["only_in_database"]: - click.echo(f"\nOnly in database ({len(report['only_in_database'])}):") - for item in report["only_in_database"][:20]: - click.echo(f" - {item['model']} ({item['usage_kind']})") - if report["mismatches"]: - click.echo(f"\nMismatched rates ({len(report['mismatches'])}):") - for item in report["mismatches"][:20]: - click.echo(f" - {item['model']} ({item['usage_kind']})") - for field, values in item["fields"].items(): - click.echo( - f" {field}: json={values['models_json']} db={values['database']}" - ) - missing = report["missing_pricing_blocks"] - if missing: - click.echo(f"\nmodels.json entries missing pricing blocks ({len(missing)}):") - for model in missing[:20]: - click.echo(f" - {model}") - unresolved = report["litellm_unresolved"] - if unresolved: - click.echo(f"\nLiteLLM unresolved ({len(unresolved)}):") - for item in unresolved[:20]: - click.echo(f" - {item.get('model')} ({item.get('reason', 'unresolved')})") - - -@usage.command("recompute") -@click.option( - "--config", - "-c", - type=click.Path(exists=True, readable=True), - default="config.yml", - help="Path to configuration YAML file", -) -@click.option("--organization-id", default=None, help="Scope recompute to one org UUID") -@click.option("--model", default=None, help="Scope recompute to one model") -@click.option("--usage-kind", default=None, help="Scope recompute to llm/stt/tts") -@click.option("--start-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None) -@click.option("--end-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None) -@click.option( - "--async/--sync", - "run_async", - default=True, - help="Enqueue Celery task (default) or run synchronously in this process", -) -def usage_recompute( - config: str, - organization_id: Optional[str], - model: Optional[str], - usage_kind: Optional[str], - start_date, - end_date, - run_async: bool, -): - """Backfill or recompute stored usage costs on llm_usage_daily rollups.""" - from uuid import UUID - - _load_cli_config(config) - start = start_date.date() if start_date else None - end = end_date.date() if end_date else None - org_uuid = UUID(organization_id) if organization_id else None - - if run_async: - if org_uuid is None: - click.echo( - "❌ --organization-id is required for async recompute (creates a tracked job).", - err=True, - ) - click.echo( - "💡 Use --sync to recompute in this process without an org scope, or pass --organization-id.", - err=True, - ) - sys.exit(1) - - from app.database import SessionLocal - from app.services.usage.pricing_jobs import ( - create_recompute_job, - enqueue_recompute_job, - job_to_dict, - ) - - db = SessionLocal() - try: - job = create_recompute_job( - db, - organization_id=org_uuid, - model=model, - usage_kind=usage_kind, - start_date=start, - end_date=end, - ) - enqueue_recompute_job(db, job) - db.refresh(job) - payload = job_to_dict(job) - click.echo(f"✅ Enqueued recompute job {payload['id']} (status={payload['status']})") - if payload.get("celery_task_id"): - click.echo(f" Celery task: {payload['celery_task_id']}") - except Exception as exc: - click.echo(f"❌ Failed to enqueue recompute job: {exc}", err=True) - sys.exit(1) - finally: - db.close() - return - - from app.database import SessionLocal - from app.services.usage.pricing import recompute_usage_costs - - db = SessionLocal() - try: - updated = recompute_usage_costs( - db, - organization_id=org_uuid, - model=model, - usage_kind=usage_kind, - start_date=start, - end_date=end, - ) - click.echo(f"✅ Recomputed costs for {updated} rollup row(s)") - finally: - db.close() - - -@usage.command("sync-litellm") -@click.option("--local", is_flag=True, help="Use bundled LiteLLM model_cost JSON") -@click.option( - "--write-models", - is_flag=True, - help="Merge generated pricing into app/config/models.json", -) -@click.option("--stdout", is_flag=True, help="Print pricing_catalog.json to stdout") -def usage_sync_litellm(local: bool, write_models: bool, stdout: bool): - """Fetch LiteLLM prices and regenerate pricing_catalog.json.""" - import subprocess - import sys as sys_module - - script = Path(__file__).resolve().parent.parent / "scripts" / "sync_pricing_catalog_from_litellm.py" - cmd = [sys_module.executable, str(script)] - if local: - cmd.append("--local") - if write_models: - cmd.append("--write-models") - if stdout: - cmd.append("--stdout") - subprocess.run(cmd, check=True) - - -if __name__ == "__main__": - main() - +"""CLI for EfficientAI platform.""" + +import click +import yaml +import os +import sys +import subprocess +import threading +import time +from pathlib import Path +from typing import Optional + + +@click.group() +def main(): + """EfficientAI - Voice AI Evaluation Platform CLI.""" + pass + + +class FrontendWatcher: + """Watch frontend files and rebuild on changes.""" + + def __init__(self, frontend_dir: Path): + self.frontend_dir = frontend_dir + self.watching = False + self.thread = None + self.last_build_time = 0 + self.build_lock = threading.Lock() + + def should_rebuild(self) -> bool: + """Check if frontend files have changed.""" + src_dir = self.frontend_dir / "src" + if not src_dir.exists(): + return False + + # Check modification time of source files + max_mtime = 0 + for ext in [".tsx", ".ts", ".css", ".jsx", ".js"]: + for file_path in src_dir.rglob(f"*{ext}"): + if file_path.is_file(): + max_mtime = max(max_mtime, file_path.stat().st_mtime) + + # Also check config files + config_files = [ + self.frontend_dir / "vite.config.ts", + self.frontend_dir / "tailwind.config.js", + self.frontend_dir / "tsconfig.json", + self.frontend_dir / "package.json", + ] + for config_file in config_files: + if config_file.exists(): + max_mtime = max(max_mtime, config_file.stat().st_mtime) + + if max_mtime > self.last_build_time: + self.last_build_time = max_mtime + return True + return False + + def build_frontend(self): + """Rebuild the frontend atomically to avoid serving half-written dist.""" + import shutil + + with self.build_lock: + try: + click.echo("\n🔄 Frontend files changed, rebuilding...") + staging_dir = self.frontend_dir / "dist.staging" + dist_dir = self.frontend_dir / "dist" + shutil.rmtree(staging_dir, ignore_errors=True) + result = subprocess.run( + ["npm", "run", "build", "--", "--outDir", "dist.staging"], + cwd=self.frontend_dir, + check=False, + capture_output=True, + text=True, + ) + if result.returncode == 0: + backup_dir = self.frontend_dir / "dist.prev" + shutil.rmtree(backup_dir, ignore_errors=True) + if dist_dir.exists(): + dist_dir.rename(backup_dir) + staging_dir.rename(dist_dir) + shutil.rmtree(backup_dir, ignore_errors=True) + click.echo("✅ Frontend rebuilt successfully") + else: + shutil.rmtree(staging_dir, ignore_errors=True) + click.echo("⚠️ Frontend build had warnings (check logs)", err=True) + if result.stderr: + click.echo(result.stderr[:500], err=True) + except Exception as e: + click.echo(f"❌ Frontend build error: {e}", err=True) + + def watch_loop(self): + """Watch loop that runs in background thread.""" + while self.watching: + try: + if self.should_rebuild(): + self.build_frontend() + time.sleep(1) # Check every second + except Exception as e: + click.echo(f"❌ Watcher error: {e}", err=True) + time.sleep(5) # Wait longer on error + + def start(self): + """Start the watcher in a background thread.""" + if self.watching: + return + self.watching = True + # Set initial build time to avoid rebuilding immediately + self.last_build_time = time.time() + self.thread = threading.Thread(target=self.watch_loop, daemon=True) + self.thread.start() + + def stop(self): + """Stop the watcher.""" + self.watching = False + if self.thread: + self.thread.join(timeout=1) + + +def start_frontend_watcher(frontend_dir: Path) -> FrontendWatcher: + """Start a frontend file watcher.""" + watcher = FrontendWatcher(frontend_dir) + watcher.start() + return watcher + + +def _read_config_media_ws_base_url(config_path: Path) -> Optional[str]: + """Return vobiz.media_ws_base_url from YAML when explicitly configured.""" + try: + with open(config_path, encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + value = ((data.get("vobiz") or {}).get("media_ws_base_url") or "").strip() + return value or None + except Exception: + return None + + +@main.command() +@click.option( + "--verbose", + "-v", + is_flag=True, + help="Show detailed migration output", +) +def migrate(verbose: bool): + """Run pending database migrations.""" + import logging + from app.core.migrations import run_migrations, ensure_migrations_directory + + if verbose: + logging.basicConfig(level=logging.INFO) + + click.echo("🔄 Running database migrations...") + ensure_migrations_directory() + + try: + run_migrations() + click.echo("✅ All migrations completed successfully!") + except Exception as e: + click.echo(f"❌ Migration failed: {e}", err=True) + sys.exit(1) + + +@main.command() +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--host", + default=None, + help="Host to bind to (overrides config)", +) +@click.option( + "--port", + default=None, + type=int, + help="Port to bind to (overrides config)", +) +@click.option( + "--build-frontend/--no-build-frontend", + default=True, + help="Build frontend before starting (default: True)", +) +@click.option( + "--reload/--no-reload", + default=True, + help="Enable auto-reload for development (default: True)", +) +@click.option( + "--watch-frontend/--no-watch-frontend", + default=False, + help="Watch frontend files and rebuild automatically (default: False)", +) +@click.option( + "--force-rebuild", + is_flag=True, + default=False, + help="Force rebuild of frontend without prompting", +) +@click.option( + "--skip-migrations", + is_flag=True, + default=False, + help="Skip running migrations before starting (not recommended)", +) +def start(config: str, host: Optional[str], port: Optional[int], build_frontend: bool, reload: bool, watch_frontend: bool, force_rebuild: bool, skip_migrations: bool): + """Start the EfficientAI application server.""" + from app.config import apply_service_mode, load_config_from_file, settings + + # Load configuration from YAML file + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) + sys.exit(1) + + explicit_media_ws = _read_config_media_ws_base_url(config_path) + os.environ["SERVICE_MODE"] = "api" + if not explicit_media_ws: + # Single-process dev: co-locate voice WebSockets on the API port unless + # config.yml sets vobiz.media_ws_base_url (used with eai telephony-worker). + os.environ.pop("MEDIA_WS_BASE_URL", None) + + try: + load_config_from_file(str(config_path)) + apply_service_mode("api") + if not explicit_media_ws: + settings.MEDIA_WS_BASE_URL = "" + click.echo(f"✅ Loaded configuration from {config_path}") + except Exception as e: + click.echo(f"❌ Error loading config: {e}", err=True) + sys.exit(1) + + # Override with CLI options if provided + if host: + settings.HOST = host + if port: + settings.PORT = port + + # Build frontend if requested + # Check if frontend is already built + frontend_dist = Path(__file__).parent.parent / "frontend" / "dist" + if build_frontend and frontend_dist.exists() and any(frontend_dist.iterdir()): + # If watching, we always want to rebuild to catch latest changes at start + if not force_rebuild and not watch_frontend and not click.confirm("Frontend dist directory already exists. Rebuild anyway?"): + build_frontend = False + + if build_frontend: + click.echo("🔨 Building frontend...") + frontend_dir = Path(__file__).parent.parent / "frontend" + if not frontend_dir.exists(): + click.echo(f"❌ Frontend directory not found: {frontend_dir}", err=True) + sys.exit(1) + + try: + # Check if node_modules exists, if not, install dependencies + if not (frontend_dir / "node_modules").exists(): + click.echo("📦 Installing frontend dependencies...") + subprocess.run( + ["npm", "install", "--legacy-peer-deps"], + cwd=frontend_dir, + check=True, + capture_output=True, + ) + + # Build frontend (show output in real-time for better debugging) + click.echo(" Running TypeScript check and Vite build...") + result = subprocess.run( + ["npm", "run", "build"], + cwd=frontend_dir, + check=False, # Don't fail immediately, we'll check return code + capture_output=True, + text=True, + ) + if result.returncode != 0: + click.echo(f"❌ Frontend build failed with return code {result.returncode}", err=True) + if result.stdout: + click.echo(f"\nSTDOUT:\n{result.stdout}", err=True) + if result.stderr: + click.echo(f"\nSTDERR:\n{result.stderr}", err=True) + click.echo("\n💡 Try running 'npm run build' manually in the frontend directory to see full error details.", err=True) + sys.exit(1) + + click.echo("✅ Frontend built successfully") + except subprocess.CalledProcessError as e: + click.echo(f"❌ Error building frontend:", err=True) + if e.stdout: + click.echo(f"STDOUT:\n{e.stdout}", err=True) + if e.stderr: + click.echo(f"STDERR:\n{e.stderr}", err=True) + click.echo(f"\nReturn code: {e.returncode}", err=True) + sys.exit(1) + except FileNotFoundError: + click.echo("❌ npm not found. Please install Node.js and npm.", err=True) + sys.exit(1) + + # Initialize DB tables and run migrations before starting (unless explicitly skipped) + if not skip_migrations: + click.echo("🔄 Initializing database and running migrations...") + from app.database import init_db + from app.core.migrations import run_migrations, ensure_migrations_directory + try: + init_db() + ensure_migrations_directory() + run_migrations() + click.echo("✅ Database initialized and migrations completed") + except Exception as e: + click.echo(f"❌ Migration failed: {e}", err=True) + click.echo("💡 You can skip migrations with --skip-migrations (not recommended)", err=True) + sys.exit(1) + else: + click.echo("⚠️ Skipping migrations (not recommended - migrations will run on startup)") + + # Start frontend watcher if requested + frontend_watcher = None + if watch_frontend: + click.echo("👀 Starting frontend file watcher...") + frontend_dir = Path(__file__).parent.parent / "frontend" + frontend_watcher = start_frontend_watcher(frontend_dir) + + # Start the server + import uvicorn + + click.echo(f"🚀 Starting EfficientAI server...") + click.echo(f" Host: {settings.HOST}") + click.echo(f" Port: {settings.PORT}") + click.echo(f" API: http://{settings.HOST}:{settings.PORT}{settings.API_V1_PREFIX}") + click.echo(f" Frontend: http://{settings.HOST}:{settings.PORT}/") + click.echo(f" Docs: http://{settings.HOST}:{settings.PORT}/docs") + if watch_frontend: + click.echo(f" Frontend watcher: Active (rebuilding on file changes)") + + # Use import string for reload to work properly + try: + uvicorn.run( + "app.main:app", + host=settings.HOST, + port=settings.PORT, + reload=reload, + ) + finally: + # Clean up watcher on exit + if frontend_watcher: + frontend_watcher.stop() + + +@main.command() +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--loglevel", + "-l", + default="info", + type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), + help="Log level for Celery worker", +) +@click.option( + "--queues", + "-Q", + "queues", + default=None, + help=( + "Comma-separated list of Celery queues this worker should consume " + "(forwarded to celery's -Q flag). Defaults to the default queue." + ), +) +@click.option( + "--concurrency", + default=None, + type=int, + help="Number of concurrent worker processes/threads (Celery --concurrency).", +) +@click.option( + "--pool", + "-P", + "pool", + default=None, + type=click.Choice(["prefork", "threads", "solo", "eventlet", "gevent"], case_sensitive=False), + help="Celery worker pool implementation (forwarded to celery's -P flag).", +) +def worker(config: str, loglevel: str, queues: Optional[str], concurrency: Optional[int], pool: Optional[str]): + """Start the Celery worker for background task processing.""" + from app.config import load_config_from_file + + # Load configuration from YAML file + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) + sys.exit(1) + + try: + load_config_from_file(str(config_path)) + click.echo(f"✅ Loaded configuration from {config_path}") + except Exception as e: + click.echo(f"❌ Error loading config: {e}", err=True) + sys.exit(1) + + click.echo(f"🚀 Starting Celery worker...") + click.echo(f" Log level: {loglevel}") + if queues: + click.echo(f" Queues: {queues}") + if concurrency is not None: + click.echo(f" Concurrency: {concurrency}") + if pool: + click.echo(f" Pool: {pool}") + + # Start Celery worker + try: + import subprocess + cmd = ["celery", "-A", "app.workers.celery_app", "worker", f"--loglevel={loglevel}"] + if queues: + cmd.append(f"--queues={queues}") + if concurrency is not None: + cmd.append(f"--concurrency={concurrency}") + if pool: + cmd.append(f"--pool={pool}") + subprocess.run(cmd, check=True) + except KeyboardInterrupt: + click.echo("\n👋 Celery worker stopped") + except subprocess.CalledProcessError as e: + click.echo(f"❌ Celery worker failed: {e}", err=True) + sys.exit(1) + except FileNotFoundError: + click.echo("❌ Celery not found. Please install it: pip install celery", err=True) + sys.exit(1) + + +@main.command("beat") +@click.option( + "--config", + "-c", + default="config.yml", + help="Path to configuration file", +) +@click.option( + "--loglevel", + "-l", + default="info", + type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), + help="Log level for Celery beat", +) +@click.option( + "--platform-worker-concurrency", + default=2, + type=int, + help="Concurrency for the co-located platform task worker (default: 2; thread pool).", +) +def beat(config: str, loglevel: str, platform_worker_concurrency: int): + """Start Celery Beat + platform task worker (alerts, FX, prune) — single replica only.""" + from app.config import load_config_from_file + + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + sys.exit(1) + + try: + load_config_from_file(str(config_path)) + click.echo(f"✅ Loaded configuration from {config_path}") + except Exception as e: + click.echo(f"❌ Error loading config: {e}", err=True) + sys.exit(1) + + from app.workers.config import PLATFORM_WORKER_QUEUE + + click.echo( + "🚀 Starting Celery Beat + platform worker " + f"(queue={PLATFORM_WORKER_QUEUE}, concurrency={platform_worker_concurrency})" + ) + platform_proc = None + try: + import subprocess + + platform_proc = subprocess.Popen( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--queues={PLATFORM_WORKER_QUEUE}", + "--pool=threads", + f"--concurrency={platform_worker_concurrency}", + f"--loglevel={loglevel}", + ], + ) + subprocess.run( + [ + "celery", + "-A", + "app.workers.celery_app", + "beat", + f"--loglevel={loglevel}", + ], + check=True, + ) + except KeyboardInterrupt: + click.echo("\n👋 Celery Beat stopped") + except subprocess.CalledProcessError as e: + click.echo(f"❌ Celery Beat failed: {e}", err=True) + sys.exit(1) + except FileNotFoundError: + click.echo("❌ Celery not found. Please install it: pip install celery", err=True) + sys.exit(1) + finally: + if platform_proc is not None and platform_proc.poll() is None: + platform_proc.terminate() + try: + platform_proc.wait(timeout=5) + except subprocess.TimeoutExpired: + platform_proc.kill() + + +@main.command("telephony-worker") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option("--host", default=None, help="Host to bind (default from config)") +@click.option("--port", default=None, type=int, help="Media server port (default 8001)") +def telephony_worker(config: str, host: Optional[str], port: Optional[int]): + """Start the media server for live voice WebSocket connections.""" + os.environ["SERVICE_MODE"] = "media" + from app.config import apply_service_mode, load_config_from_file, settings + + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + sys.exit(1) + + try: + load_config_from_file(str(config_path)) + apply_service_mode("media") + except Exception as e: + click.echo(f"❌ Error loading config: {e}", err=True) + sys.exit(1) + bind_host = host or settings.HOST + bind_port = port or settings.MEDIA_PORT + + from app.app_factory import create_app + + app = create_app() + print( + f"[MEDIA] telephony-worker ready on {bind_host}:{bind_port} " + f"(SERVICE_MODE={settings.SERVICE_MODE}, media_routes mounted)", + flush=True, + ) + + import uvicorn + + click.echo(f"🎙️ Starting EfficientAI media server on {bind_host}:{bind_port}") + uvicorn.run( + app, + host=bind_host, + port=bind_port, + reload=False, + ) + + +@main.command("start-worker-all") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option("--loglevel", "-l", default="info", help="Celery log level") +@click.option("--media-port", default=None, type=int, help="Media server port (default 8001)") +def start_worker_all(config: str, loglevel: str, media_port: Optional[int]): + """Start Celery worker and media server together. + + Deprecated: prefer separate ``eai telephony-worker`` and ``eai worker`` services + (see docker-compose ``media`` + ``worker``). + """ + import signal + import atexit + + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + sys.exit(1) + + from app.config import load_config_from_file, settings + + load_config_from_file(str(config_path)) + + media_proc = None + celery_proc = None + + def cleanup(): + nonlocal media_proc, celery_proc + for proc, label in ((media_proc, "media server"), (celery_proc, "Celery worker")): + if proc is None or proc.poll() is not None: + continue + try: + proc.terminate() + proc.wait(timeout=5) + click.echo(f"✅ {label} stopped") + except Exception: + proc.kill() + + atexit.register(cleanup) + + def _handle_signal(sig, frame): + cleanup() + sys.exit(0) + + signal.signal(signal.SIGINT, _handle_signal) + signal.signal(signal.SIGTERM, _handle_signal) + + port = media_port or settings.MEDIA_PORT + telephony_env = os.environ.copy() + telephony_env["SERVICE_MODE"] = "media" + media_proc = subprocess.Popen( + [sys.executable, "-m", "app.cli", "telephony-worker", "--config", str(config_path), "--port", str(port)], + env=telephony_env, + ) + celery_proc = subprocess.Popen( + ["celery", "-A", "app.workers.celery_app", "worker", f"--loglevel={loglevel}"], + ) + click.echo(f"🚀 Started media server (pid={media_proc.pid}) and Celery worker (pid={celery_proc.pid})") + + try: + while True: + if media_proc.poll() is not None: + click.echo("❌ Media server exited", err=True) + break + if celery_proc.poll() is not None: + click.echo("❌ Celery worker exited", err=True) + break + time.sleep(1) + except KeyboardInterrupt: + pass + finally: + cleanup() + + +@main.command() +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--host", + default=None, + help="Host to bind to (overrides config)", +) +@click.option( + "--port", + default=None, + type=int, + help="Port to bind to (overrides config)", +) +@click.option( + "--build-frontend/--no-build-frontend", + default=True, + help="Build frontend before starting (default: True)", +) +@click.option( + "--reload/--no-reload", + default=True, + help="Enable auto-reload for development (default: True)", +) +@click.option( + "--watch-frontend/--no-watch-frontend", + default=False, + help="Watch frontend files and rebuild automatically (default: False)", +) +@click.option( + "--force-rebuild", + is_flag=True, + default=False, + help="Force rebuild of frontend without prompting", +) +@click.option( + "--skip-migrations", + is_flag=True, + default=False, + help="Skip running migrations before starting (not recommended)", +) +@click.option( + "--worker-loglevel", + "-l", + default="info", + type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), + help="Log level for Celery worker", +) +@click.option( + "--imports-worker/--no-imports-worker", + default=True, + help=( + "Also start a dedicated worker for the `imports` queue used by call " + "import CSV processing (default: True). Disable to keep the previous " + "single-worker behavior." + ), +) +@click.option( + "--imports-worker-concurrency", + default=12, + type=int, + help=( + "Concurrency for the imports+diarization+evaluations worker " + "(default: 12; thread pool). Use lower values (8–12) when DB sharding " + "is enabled; 32 threads can exhaust per-shard SQLAlchemy pools." + ), +) +@click.option( + "--usage-worker/--no-usage-worker", + default=True, + help=( + "Also start a dedicated worker for the `usage` queue (flush + cost recompute; " + "default: True)." + ), +) +@click.option( + "--usage-worker-concurrency", + default=4, + type=int, + help="Concurrency for the usage worker (default: 4; thread pool).", +) +@click.option( + "--beat/--no-beat", + default=True, + help=( + "Start Celery Beat for platform periodic tasks (usage flush, alerts, etc.; " + "default: True — single replica only in production)." + ), +) +@click.option( + "--beat-loglevel", + default=None, + type=click.Choice(["debug", "info", "warning", "error", "critical"], case_sensitive=False), + help="Log level for Celery Beat (defaults to --worker-loglevel).", +) +@click.option( + "--telephony-worker/--no-telephony-worker", + default=True, + help="Spawn telephony media server for live voice WebSockets (default: on).", +) +@click.option( + "--media-port", + default=None, + type=int, + help="Telephony media server port (default: MEDIA_PORT / 8001).", +) +def start_all( + config: str, + host: Optional[str], + port: Optional[int], + build_frontend: bool, + reload: bool, + watch_frontend: bool, + force_rebuild: bool, + skip_migrations: bool, + worker_loglevel: str, + imports_worker: bool, + imports_worker_concurrency: int, + usage_worker: bool, + usage_worker_concurrency: int, + beat: bool, + beat_loglevel: Optional[str], + telephony_worker: bool, + media_port: Optional[int], +): + """Start the application server and Celery worker(s) together. + + By default this also spawns a telephony media server (``eai telephony-worker``) + and a second Celery worker that consumes the ``imports`` queue (call-import CSV + fan-out). Use --no-telephony-worker or --no-imports-worker to skip either. + """ + import signal + import atexit + + click.echo("🚀 Starting EfficientAI (App + Worker)...") + if telephony_worker: + click.echo(" Telephony media server will run on a separate port (SERVICE_MODE=media).") + if imports_worker: + click.echo( + " This will start the API server, the default Celery worker, " + "and a dedicated worker for the `imports` queue." + ) + else: + click.echo(" This will start both the API server and Celery worker") + click.echo(" Press Ctrl+C to stop all services\n") + + # Load configuration + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + click.echo(f"💡 Create a config.yml file or use --config to specify a different path.", err=True) + sys.exit(1) + + try: + from app.config import load_config_from_file, settings + load_config_from_file(str(config_path)) + click.echo(f"✅ Loaded configuration from {config_path}") + except Exception as e: + click.echo(f"❌ Error loading config: {e}", err=True) + sys.exit(1) + + bind_media_port = media_port or settings.MEDIA_PORT + + os.environ["SERVICE_MODE"] = "api" + + # Store worker processes for cleanup. + worker_process = None + worker_imports_process = None + worker_usage_process = None + beat_process = None + platform_worker_process = None + telephony_process = None + + def _terminate(proc, label: str): + """Best-effort terminate -> wait -> kill for a worker subprocess.""" + if proc is None or proc.poll() is not None: + return + try: + click.echo(f"\n👋 Stopping {label}...") + proc.terminate() + proc.wait(timeout=5) + click.echo(f"✅ {label} stopped") + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + except Exception: + pass + + def cleanup_processes(): + """Clean up spawned processes.""" + nonlocal worker_process, worker_imports_process, worker_usage_process, beat_process, platform_worker_process, telephony_process + _terminate(telephony_process, "Telephony media server") + _terminate(beat_process, "Celery Beat") + _terminate(platform_worker_process, "Celery platform worker") + _terminate(worker_process, "Celery worker (default)") + _terminate(worker_imports_process, "Celery worker (imports)") + _terminate(worker_usage_process, "Celery worker (usage)") + + # Register cleanup on exit + atexit.register(cleanup_processes) + + def signal_handler(sig, frame): + """Handle Ctrl+C gracefully.""" + cleanup_processes() + sys.exit(0) + + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + + # Initialize DB tables and run migrations before starting (unless explicitly skipped) + if not skip_migrations: + click.echo("🔄 Initializing database and running migrations...") + from app.database import init_db + from app.core.migrations import run_migrations, ensure_migrations_directory + try: + init_db() + ensure_migrations_directory() + run_migrations() + click.echo("✅ Database initialized and migrations completed") + except Exception as e: + click.echo(f"❌ Migration failed: {e}", err=True) + click.echo("💡 You can skip migrations with --skip-migrations (not recommended)", err=True) + sys.exit(1) + + # Build frontend if needed + if build_frontend: + frontend_dir = Path(__file__).parent.parent / "frontend" + if frontend_dir.exists(): + click.echo("🔨 Building frontend...") + try: + if not (frontend_dir / "node_modules").exists(): + click.echo(" Installing frontend dependencies...") + subprocess.run(["npm", "install", "--legacy-peer-deps"], cwd=frontend_dir, check=True, capture_output=True) + subprocess.run(["npm", "run", "build"], cwd=frontend_dir, check=True, capture_output=True) + click.echo("✅ Frontend built successfully") + except subprocess.CalledProcessError as e: + click.echo(f"❌ Frontend build failed: {e}", err=True) + sys.exit(1) + + # Start frontend watcher if requested + frontend_watcher = None + if watch_frontend: + click.echo("👀 Starting frontend file watcher...") + frontend_dir = Path(__file__).parent.parent / "frontend" + frontend_watcher = start_frontend_watcher(frontend_dir) + + def _spawn_worker(args: list[str], label: str, prefix: str) -> subprocess.Popen: + """Spawn a Celery worker subprocess and stream its stdout with a prefix.""" + proc = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + click.echo(f"✅ {label} started") + + def _stream(): + if proc.stdout: + for line in iter(proc.stdout.readline, ""): + if line: + click.echo(f"{prefix} {line.rstrip()}", err=False) + proc.stdout.close() + + threading.Thread(target=_stream, daemon=True).start() + return proc + + if telephony_worker: + try: + telephony_env = os.environ.copy() + telephony_env["SERVICE_MODE"] = "media" + telephony_process = subprocess.Popen( + [ + sys.executable, + "-m", + "app.cli", + "telephony-worker", + "--config", + str(config_path), + "--port", + str(bind_media_port), + ], + env=telephony_env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + click.echo( + f"✅ Telephony media server started on port {bind_media_port} " + f"(pid={telephony_process.pid})" + ) + + def _stream_telephony(): + if telephony_process.stdout: + for line in iter(telephony_process.stdout.readline, ""): + if line: + click.echo(f"[MEDIA] {line.rstrip()}", err=False) + telephony_process.stdout.close() + + threading.Thread(target=_stream_telephony, daemon=True).start() + except FileNotFoundError: + click.echo("❌ Python interpreter not found for telephony-worker subprocess", err=True) + sys.exit(1) + except Exception as e: + click.echo(f"❌ Failed to start telephony media server: {e}", err=True) + sys.exit(1) + + # Start Celery workers as subprocess(es) with output streaming + try: + worker_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={worker_loglevel}", + "-Q", + "celery,audio-metrics", + "-c", + "8", + ], + label="Celery worker (celery + audio-metrics queues, concurrency=8)", + prefix="[WORKER]", + ) + + if imports_worker: + from app.workers.config import IMPORTS_WORKER_QUEUES + + worker_imports_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={worker_loglevel}", + "-Q", + IMPORTS_WORKER_QUEUES, + "-P", + "threads", + "-c", + str(imports_worker_concurrency), + ], + label=( + f"Celery worker ({IMPORTS_WORKER_QUEUES} queues, " + f"pool=threads, concurrency={imports_worker_concurrency})" + ), + prefix="[WORKER-IMPORTS]", + ) + + if usage_worker: + from app.workers.config import USAGE_WORKER_QUEUE + + worker_usage_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={worker_loglevel}", + "-Q", + USAGE_WORKER_QUEUE, + "-P", + "threads", + "-c", + str(usage_worker_concurrency), + ], + label=( + f"Celery worker ({USAGE_WORKER_QUEUE} queue, " + f"pool=threads, concurrency={usage_worker_concurrency})" + ), + prefix="[WORKER-USAGE]", + ) + + if beat: + from app.workers.config import PLATFORM_WORKER_QUEUE + + beat_level = beat_loglevel or worker_loglevel + platform_worker_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "worker", + f"--loglevel={beat_level}", + "-Q", + PLATFORM_WORKER_QUEUE, + "-P", + "threads", + "-c", + "2", + ], + label=f"Celery platform worker ({PLATFORM_WORKER_QUEUE} queue)", + prefix="[BEAT-WORKER]", + ) + beat_process = _spawn_worker( + [ + "celery", + "-A", + "app.workers.celery_app", + "beat", + f"--loglevel={beat_level}", + ], + label=f"Celery Beat (scheduler, loglevel={beat_level})", + prefix="[BEAT]", + ) + + except FileNotFoundError: + click.echo("❌ Celery not found. Please install it: pip install celery", err=True) + sys.exit(1) + except Exception as e: + click.echo(f"❌ Failed to start worker: {e}", err=True) + sys.exit(1) + + # Small delay to let worker start + time.sleep(1) + + # Start the application server in the main process + # This allows uvicorn's reload to work properly (it needs to spawn child processes) + try: + from app.config import settings + import uvicorn + + # Override with CLI options if provided + if host: + settings.HOST = host + if port: + settings.PORT = port + + click.echo("✅ Application server starting...") + click.echo(f" Host: {settings.HOST}") + click.echo(f" Port: {settings.PORT}") + click.echo(f" API: http://{settings.HOST}:{settings.PORT}{settings.API_V1_PREFIX}") + click.echo(f" Frontend: http://{settings.HOST}:{settings.PORT}/") + click.echo(f" Docs: http://{settings.HOST}:{settings.PORT}/docs") + if watch_frontend: + click.echo(f" Frontend watcher: Active (rebuilding on file changes)") + if imports_worker: + from app.workers.config import IMPORTS_WORKER_QUEUES + + click.echo( + f" Workers: default queue + {IMPORTS_WORKER_QUEUES} " + f"(concurrency={imports_worker_concurrency}; imports preferred)" + ) + else: + click.echo(" Workers: default queue only (--no-imports-worker)") + if usage_worker: + from app.workers.config import USAGE_WORKER_QUEUE + + click.echo( + f" Usage worker: {USAGE_WORKER_QUEUE} queue " + f"(concurrency={usage_worker_concurrency})" + ) + else: + click.echo(" Usage worker: disabled (--no-usage-worker)") + if beat: + click.echo(" Celery Beat: scheduler + platform worker (alerts, FX, prune); flush on worker-usage") + else: + click.echo(" Celery Beat: disabled (--no-beat)") + if telephony_worker: + telephony_public = (settings.VOBIZ_WEBHOOK_BASE_URL or "").strip() + click.echo(f" Telephony edge: http://localhost:{bind_media_port} (local)") + if telephony_public: + click.echo(f" Vobiz webhook_base_url: {telephony_public}") + else: + click.echo( + " Set vobiz.webhook_base_url to your telephony public URL " + f"(e.g. ngrok http {bind_media_port}) for inbound PSTN" + ) + if (settings.MEDIA_WS_BASE_URL or "").strip(): + click.echo(f" Browser voice-agent WS: {settings.MEDIA_WS_BASE_URL}") + click.echo("\n📝 All services are running. Press Ctrl+C to stop.\n") + + # Run uvicorn in the main process (allows reload to work) + uvicorn.run( + "app.main:app", + host=settings.HOST, + port=settings.PORT, + reload=reload, + ) + except KeyboardInterrupt: + pass + except Exception as e: + click.echo(f"❌ App error: {e}", err=True) + finally: + cleanup_processes() + if frontend_watcher: + frontend_watcher.stop() + + +@main.command() +@click.option( + "--output", + "-o", + type=click.Path(), + default="config.yml", + help="Output file path for example config", +) +def init_config(output: str): + """Generate an example configuration file.""" + example_config = """# EfficientAI Configuration File + +# Application Settings +app: + name: "Voice AI Evaluation Platform" + version: "0.1.0" + debug: true + secret_key: "your-secret-key-here-change-in-production" + +# Server Settings +server: + host: "0.0.0.0" + port: 8000 + +# Database Configuration +database: + url: "postgresql://efficientai:password@localhost:5432/efficientai" + # Alternative: specify individual components + # user: "efficientai" + # password: "password" + # host: "localhost" + # port: 5432 + # db: "efficientai" + +# Redis Configuration +redis: + url: "redis://localhost:6379/0" + # Alternative: specify individual components + # host: "localhost" + # port: 6379 + # db: 0 + +# Celery Configuration +celery: + broker_url: "redis://localhost:6379/0" + result_backend: "redis://localhost:6379/0" + +# File Storage +storage: + upload_dir: "./uploads" + max_file_size_mb: 500 + allowed_audio_formats: + - "wav" + - "mp3" + - "flac" + - "m4a" + +# CORS Settings +cors: + origins: + - "http://localhost:3000" + - "http://localhost:8000" + +# API Settings +api: + prefix: "/api/v1" + key_header: "X-API-Key" + rate_limit_per_minute: 60 +""" + + output_path = Path(output) + if output_path.exists(): + if not click.confirm(f"File {output} already exists. Overwrite?"): + click.echo("Cancelled.") + return + + try: + output_path.write_text(example_config) + click.echo(f"✅ Created example configuration file: {output}") + click.echo(f"💡 Edit {output} with your settings, then run: eai start --config {output}") + except Exception as e: + click.echo(f"❌ Error creating config file: {e}", err=True) + sys.exit(1) + + +def _bootstrap_sharding_config(config_path: str) -> None: + from app.config import load_config_from_file + from app.db_sharding.pool_manager import db_pool_manager + + load_config_from_file(config_path) + db_pool_manager.reset() + + +@click.group() +def sharding(): + """Call-import data-plane sharding admin (rebalance / registry).""" + pass + + +main.add_command(sharding) + + +@sharding.command("list-slices") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--call-import-id", + required=True, + help="Call import UUID whose slice registry to inspect", +) +def sharding_list_slices(config: str, call_import_id: str): + """Show catalog slice registry rows for a call import.""" + from uuid import UUID + + from app.db_sharding.pool_manager import open_catalog_session + from app.db_sharding.rebalance import RebalanceError, list_shard_slices, require_sharding_enabled + + try: + _bootstrap_sharding_config(config) + require_sharding_enabled() + cid = UUID(call_import_id) + except (ValueError, RebalanceError) as exc: + click.echo(f"❌ {exc}", err=True) + sys.exit(1) + + catalog = open_catalog_session() + try: + slices = list_shard_slices(catalog, cid) + if not slices: + click.echo(f"No registry slices for call_import {cid}") + return + click.echo(f"call_import_id: {cid}") + click.echo(f"slices: {len(slices)}") + for item in slices: + click.echo( + f" slice {item.slice_id}: shard={item.shard_id} " + f"rows [{item.row_index_min}, {item.row_index_max}] " + f"(count={item.row_count})" + ) + finally: + catalog.close() + + +@sharding.command("rebalance-slices") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option("--call-import-id", required=True, help="Call import UUID to rebalance") +@click.option("--from-shard", required=True, help="Source shard id (e.g. data-shard-01)") +@click.option("--to-shard", required=True, help="Target shard id (e.g. data-shard-02)") +@click.option( + "--slice-id", + "slice_ids", + multiple=True, + type=int, + help="Move only these slice ids (default: all slices on --from-shard)", +) +@click.option( + "--dry-run", + is_flag=True, + help="Plan only: print row counts without copying or updating registry", +) +@click.option( + "--force", + is_flag=True, + help="Skip quiescence checks (use only after pausing workers)", +) +def sharding_rebalance_slices( + config: str, + call_import_id: str, + from_shard: str, + to_shard: str, + slice_ids: tuple[int, ...], + dry_run: bool, + force: bool, +): + """Copy call-import slice rows (and eval rows) from one shard to another.""" + from uuid import UUID + + from app.db_sharding.pool_manager import open_catalog_session + from app.db_sharding.rebalance import ( + RebalanceError, + require_sharding_enabled, + build_rebalance_plan, + execute_rebalance_slices, + ) + + try: + _bootstrap_sharding_config(config) + require_sharding_enabled() + cid = UUID(call_import_id) + except (ValueError, RebalanceError) as exc: + click.echo(f"❌ {exc}", err=True) + sys.exit(1) + + catalog = open_catalog_session() + try: + plan = build_rebalance_plan( + catalog, + cid, + from_shard_id=from_shard, + to_shard_id=to_shard, + slice_ids=slice_ids or None, + ) + click.echo("Rebalance plan:") + click.echo(f" call_import_id: {plan.call_import_id}") + click.echo(f" from_shard: {plan.from_shard_id}") + click.echo(f" to_shard: {plan.to_shard_id}") + click.echo(f" slices: {len(plan.slices)}") + for item in plan.slices: + click.echo( + f" slice {item.slice_id}: rows [{item.row_index_min}, {item.row_index_max}]" + ) + click.echo(f" import_rows: {plan.import_row_count}") + click.echo(f" eval_rows: {plan.eval_row_count}") + + result = execute_rebalance_slices( + catalog, + plan, + dry_run=dry_run, + force=force, + ) + if result.dry_run: + click.echo("✅ Dry run complete (no changes made)") + else: + click.echo( + "✅ Rebalance complete: " + f"slices={result.slices_moved}, " + f"import_rows={result.import_rows_moved}, " + f"eval_rows={result.eval_rows_moved}" + ) + except RebalanceError as exc: + click.echo(f"❌ {exc}", err=True) + sys.exit(1) + finally: + catalog.close() + + +@click.group() +def usage(): + """Usage pricing ops (seed rates, diff catalog, recompute costs).""" + pass + + +main.add_command(usage) + + +def _load_cli_config(config: str) -> Path: + config_path = Path(config) + if not config_path.exists(): + click.echo(f"❌ Config file not found: {config}", err=True) + sys.exit(1) + from app.config import load_config_from_file + + load_config_from_file(str(config_path)) + return config_path + + +@usage.command("seed-rates") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--effective-from", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=None, + help="Effective date for seeded rates (default: 2020-01-01)", +) +def usage_seed_rates(config: str, effective_from): + """Upsert model_pricing_rates from models.json pricing blocks.""" + _load_cli_config(config) + from app.database import SessionLocal + from app.services.usage.pricing_ops import seed_rates_from_models_json + + day = effective_from.date() if effective_from else None + db = SessionLocal() + try: + count = seed_rates_from_models_json(db, effective_from=day) + db.commit() + click.echo(f"✅ Seeded/updated {count} pricing rate row(s)") + finally: + db.close() + + +@usage.command("diff-rates") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option( + "--effective-from", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=None, + help="Compare rates at this effective_from date (default: 2020-01-01)", +) +@click.option("--json", "as_json", is_flag=True, help="Print machine-readable JSON") +def usage_diff_rates(config: str, effective_from, as_json: bool): + """Diff models.json pricing blocks vs model_pricing_rates in Postgres.""" + import json as json_module + + _load_cli_config(config) + from app.database import SessionLocal + from app.services.usage.pricing_ops import diff_models_json_vs_db + + day = effective_from.date() if effective_from else None + db = SessionLocal() + try: + report = diff_models_json_vs_db(db, effective_from=day) + finally: + db.close() + + if as_json: + click.echo(json_module.dumps(report, indent=2, default=str)) + return + + click.echo(f"effective_from: {report['effective_from']}") + click.echo( + f"models.json priced: {report['models_json_count']} | " + f"database rows: {report['database_count']} | " + f"in_sync: {report['in_sync']}" + ) + if report["only_in_models_json"]: + click.echo(f"\nOnly in models.json ({len(report['only_in_models_json'])}):") + for item in report["only_in_models_json"][:20]: + click.echo(f" - {item['model']} ({item['usage_kind']})") + if report["only_in_database"]: + click.echo(f"\nOnly in database ({len(report['only_in_database'])}):") + for item in report["only_in_database"][:20]: + click.echo(f" - {item['model']} ({item['usage_kind']})") + if report["mismatches"]: + click.echo(f"\nMismatched rates ({len(report['mismatches'])}):") + for item in report["mismatches"][:20]: + click.echo(f" - {item['model']} ({item['usage_kind']})") + for field, values in item["fields"].items(): + click.echo( + f" {field}: json={values['models_json']} db={values['database']}" + ) + missing = report["missing_pricing_blocks"] + if missing: + click.echo(f"\nmodels.json entries missing pricing blocks ({len(missing)}):") + for model in missing[:20]: + click.echo(f" - {model}") + unresolved = report["litellm_unresolved"] + if unresolved: + click.echo(f"\nLiteLLM unresolved ({len(unresolved)}):") + for item in unresolved[:20]: + click.echo(f" - {item.get('model')} ({item.get('reason', 'unresolved')})") + + +@usage.command("recompute") +@click.option( + "--config", + "-c", + type=click.Path(exists=True, readable=True), + default="config.yml", + help="Path to configuration YAML file", +) +@click.option("--organization-id", default=None, help="Scope recompute to one org UUID") +@click.option("--model", default=None, help="Scope recompute to one model") +@click.option("--usage-kind", default=None, help="Scope recompute to llm/stt/tts") +@click.option("--start-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None) +@click.option("--end-date", type=click.DateTime(formats=["%Y-%m-%d"]), default=None) +@click.option( + "--async/--sync", + "run_async", + default=True, + help="Enqueue Celery task (default) or run synchronously in this process", +) +def usage_recompute( + config: str, + organization_id: Optional[str], + model: Optional[str], + usage_kind: Optional[str], + start_date, + end_date, + run_async: bool, +): + """Backfill or recompute stored usage costs on llm_usage_daily rollups.""" + from uuid import UUID + + _load_cli_config(config) + start = start_date.date() if start_date else None + end = end_date.date() if end_date else None + org_uuid = UUID(organization_id) if organization_id else None + + if run_async: + if org_uuid is None: + click.echo( + "❌ --organization-id is required for async recompute (creates a tracked job).", + err=True, + ) + click.echo( + "💡 Use --sync to recompute in this process without an org scope, or pass --organization-id.", + err=True, + ) + sys.exit(1) + + from app.database import SessionLocal + from app.services.usage.pricing_jobs import ( + create_recompute_job, + enqueue_recompute_job, + job_to_dict, + ) + + db = SessionLocal() + try: + job = create_recompute_job( + db, + organization_id=org_uuid, + model=model, + usage_kind=usage_kind, + start_date=start, + end_date=end, + ) + enqueue_recompute_job(db, job) + db.refresh(job) + payload = job_to_dict(job) + click.echo(f"✅ Enqueued recompute job {payload['id']} (status={payload['status']})") + if payload.get("celery_task_id"): + click.echo(f" Celery task: {payload['celery_task_id']}") + except Exception as exc: + click.echo(f"❌ Failed to enqueue recompute job: {exc}", err=True) + sys.exit(1) + finally: + db.close() + return + + from app.database import SessionLocal + from app.services.usage.pricing import recompute_usage_costs + + db = SessionLocal() + try: + updated = recompute_usage_costs( + db, + organization_id=org_uuid, + model=model, + usage_kind=usage_kind, + start_date=start, + end_date=end, + ) + click.echo(f"✅ Recomputed costs for {updated} rollup row(s)") + finally: + db.close() + + +@usage.command("sync-litellm") +@click.option("--local", is_flag=True, help="Use bundled LiteLLM model_cost JSON") +@click.option( + "--write-models", + is_flag=True, + help="Merge generated pricing into app/config/models.json", +) +@click.option("--stdout", is_flag=True, help="Print pricing_catalog.json to stdout") +def usage_sync_litellm(local: bool, write_models: bool, stdout: bool): + """Fetch LiteLLM prices and regenerate pricing_catalog.json.""" + import subprocess + import sys as sys_module + + script = Path(__file__).resolve().parent.parent / "scripts" / "sync_pricing_catalog_from_litellm.py" + cmd = [sys_module.executable, str(script)] + if local: + cmd.append("--local") + if write_models: + cmd.append("--write-models") + if stdout: + cmd.append("--stdout") + subprocess.run(cmd, check=True) + + +if __name__ == "__main__": + main() + diff --git a/app/config.py b/app/config.py index 8242409c..95c98134 100644 --- a/app/config.py +++ b/app/config.py @@ -125,18 +125,43 @@ class Settings(BaseSettings): FRONTEND_DIR: str = "./frontend/dist" FRONTEND_BASE_URL: str = "" - # Content Security Policy (Report-Only by default; set CSP_REPORT_ONLY=false to enforce) + # Content Security Policy (enforcing by default; set CSP_REPORT_ONLY=true for local report-only mode) CSP_ENABLED: bool = True - CSP_REPORT_ONLY: bool = True + CSP_REPORT_ONLY: bool = False + # Browser voice SDKs (Vapi/Daily, Retell/LiveKit, ElevenLabs convai) and their telemetry. + _CSP_VOICE_CONNECT_SRC: str = ( + "https://api.vapi.ai " + "https://*.vapi.ai " + "https://*.daily.co " + "wss://*.daily.co " + "wss://*.livekit.cloud " + "https://api.elevenlabs.io " + "wss://api.elevenlabs.io " + "https://api.retellai.com " + "wss://api.retellai.com " + "https://*.ingest.sentry.io " + "https://*.ingest.us.sentry.io" + ) + _CSP_FRAME_SRC: str = ( + "https://*.daily.co " + "https://*.s3.amazonaws.com " + "https://*.amazonaws.com " + "https://*.cloudfront.net " + "https://storage.googleapis.com " + "https://*.blob.core.windows.net" + ) + # Vapi → Daily.co call-machine bundle requires eval + blob worklets for audio + _CSP_DAILY_SCRIPT_SRC: str = "'unsafe-eval' blob: https://c.daily.co https://*.daily.co" CSP_POLICY: str = ( "default-src 'self'; " - "script-src 'self'; " + f"script-src 'self' {_CSP_DAILY_SCRIPT_SRC}; " "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " "font-src 'self' https://fonts.gstatic.com; " "img-src 'self' data: blob: https:; " - "connect-src 'self' wss: ws:; " + f"connect-src 'self' wss: ws: {_CSP_VOICE_CONNECT_SRC}; " "media-src 'self' blob: https:; " - "frame-src 'self' blob:; " + f"frame-src 'self' blob: {_CSP_FRAME_SRC}; " + "worker-src 'self' blob:; " "object-src 'none'; " "base-uri 'self'; " "form-action 'self'; " @@ -225,6 +250,10 @@ class Settings(BaseSettings): FLEXPRICE_ENABLED: bool = False FLEXPRICE_API_KEY: Optional[str] = None FLEXPRICE_API_HOST: str = "https://us.api.flexprice.io/v1" + FLEXPRICE_AUTO_SUBSCRIBE: bool = False + FLEXPRICE_DEFAULT_PLAN_ID: Optional[str] = None + FLEXPRICE_DEFAULT_CURRENCY: str = "usd" + FLEXPRICE_DEFAULT_BILLING_PERIOD: str = "MONTHLY" # LLM gateway (optional platform-wide proxy for batch LLM calls). LLM_GATEWAY_ENABLED: bool = False LLM_GATEWAY_TYPE: str = "bifrost" # bifrost | litellm_proxy @@ -859,6 +888,15 @@ def _apply_llm_gateway_settings(gateway_cfg: dict, *, gateway_type: str) -> None if "trusted_ips" in operational_config: settings.OPERATIONAL_TRUSTED_IPS = operational_config["trusted_ips"] + if "security" in config_data: + security_config = config_data["security"] + if "csp_enabled" in security_config: + settings.CSP_ENABLED = bool(security_config["csp_enabled"]) + if "csp_report_only" in security_config: + settings.CSP_REPORT_ONLY = bool(security_config["csp_report_only"]) + if security_config.get("csp_policy"): + settings.CSP_POLICY = security_config["csp_policy"] + if "flexprice" in config_data: flexprice_config = config_data["flexprice"] if "enabled" in flexprice_config: @@ -867,6 +905,19 @@ def _apply_llm_gateway_settings(gateway_cfg: dict, *, gateway_type: str) -> None settings.FLEXPRICE_API_KEY = flexprice_config["api_key"] if flexprice_config.get("api_host"): settings.FLEXPRICE_API_HOST = flexprice_config["api_host"] + if "auto_subscribe" in flexprice_config: + settings.FLEXPRICE_AUTO_SUBSCRIBE = bool(flexprice_config["auto_subscribe"]) + if flexprice_config.get("default_plan_id"): + settings.FLEXPRICE_DEFAULT_PLAN_ID = flexprice_config["default_plan_id"] + if flexprice_config.get("default_currency"): + settings.FLEXPRICE_DEFAULT_CURRENCY = flexprice_config["default_currency"] + if flexprice_config.get("default_billing_period"): + settings.FLEXPRICE_DEFAULT_BILLING_PERIOD = flexprice_config["default_billing_period"] + if ( + os.environ.get("EFFICIENTAI_PYTEST") == "1" + and os.environ.get("FLEXPRICE_TEST_ALLOW") != "1" + ): + settings.FLEXPRICE_ENABLED = False # Update Celery URLs if they weren't explicitly set if not settings.CELERY_BROKER_URL: diff --git a/app/core/auth/platform_admin.py b/app/core/auth/platform_admin.py index 920c0164..6e468385 100644 --- a/app/core/auth/platform_admin.py +++ b/app/core/auth/platform_admin.py @@ -12,6 +12,7 @@ from sqlalchemy.orm import Session from app.config import settings +from app.core.auth.token_revocation import is_access_jti_revoked, revoke_access_jti from app.database import get_db from app.models.database import PlatformAdmin @@ -58,6 +59,18 @@ def decode_platform_access_token(token: str) -> Dict[str, Any]: ) +def revoke_platform_access_token(token: str) -> None: + try: + claims = decode_platform_access_token(token) + jti = claims.get("jti") + exp = claims.get("exp") + if jti and exp: + ttl = max(int(exp) - int(datetime.now(timezone.utc).timestamp()), 1) + revoke_access_jti(jti, ttl) + except JWTError: + pass + + def _extract_bearer(authorization: Optional[str]) -> Optional[str]: if not authorization: return None @@ -107,6 +120,13 @@ def get_platform_admin( detail="Invalid platform admin token scope.", ) + jti = claims.get("jti") + if jti and is_access_jti_revoked(jti): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Token has been revoked.", + ) + try: admin_id = UUID(claims["sub"]) except (KeyError, ValueError) as exc: diff --git a/app/core/operational_access_middleware.py b/app/core/operational_access_middleware.py index 7bf4d8d3..7a0db428 100644 --- a/app/core/operational_access_middleware.py +++ b/app/core/operational_access_middleware.py @@ -1,4 +1,8 @@ -"""Restrict /metrics from the public internet (/health stays open for load balancers).""" +"""Restrict /metrics from the public internet (/health stays open for load balancers). + +Not Spring Boot Actuator: this FastAPI app exposes /health (LB probes) and /metrics +(Prometheus scrape). /metrics is gated by trusted IPs or OPERATIONAL_PUBLIC only. +""" from __future__ import annotations @@ -6,14 +10,11 @@ import logging from typing import Iterable -from fastapi import HTTPException from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request from starlette.responses import JSONResponse, Response from app.config import settings -from app.core.auth.dependency import _resolve -from app.database import SessionLocal logger = logging.getLogger(__name__) @@ -75,22 +76,6 @@ def _resolved_trusted_ip(request: Request) -> str | None: return hops[-1] -def _has_authenticated_caller(request: Request) -> bool: - db = SessionLocal() - try: - principal = _resolve( - request.headers.get("authorization"), - request.headers.get("x-api-key"), - request.headers.get("x-efficientai-api-key"), - db, - ) - return principal is not None - except HTTPException: - return False - finally: - db.close() - - def is_operational_access_allowed(request: Request) -> bool: """Return True when the caller may access a protected operational endpoint.""" if settings.OPERATIONAL_PUBLIC: @@ -100,9 +85,6 @@ def is_operational_access_allowed(request: Request) -> bool: if resolved_ip and _ip_in_trusted(resolved_ip, settings.OPERATIONAL_TRUSTED_IPS): return True - if _has_authenticated_caller(request): - return True - return False diff --git a/app/core/security_headers_middleware.py b/app/core/security_headers_middleware.py index caa726a9..9206c55f 100644 --- a/app/core/security_headers_middleware.py +++ b/app/core/security_headers_middleware.py @@ -1,55 +1,66 @@ -"""HTTP security response headers.""" - -from __future__ import annotations - -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.requests import Request -from starlette.responses import Response - -from app.config import settings - -_NO_STORE_CACHE = "no-cache, no-store, must-revalidate" -_ASSET_CACHE = "public, max-age=31536000, immutable" - - -def _apply_cache_control(request: Request, response: Response) -> None: - if "cache-control" in response.headers: - return - - path = request.url.path - if path.startswith("/assets/"): - response.headers["Cache-Control"] = _ASSET_CACHE - return - - cache_value = _NO_STORE_CACHE - if path.startswith("/api/"): - cache_value = f"{_NO_STORE_CACHE}, private" - - response.headers["Cache-Control"] = cache_value - response.headers["Pragma"] = "no-cache" - response.headers["Expires"] = "0" - - -def _apply_csp(response: Response) -> None: - if not settings.CSP_ENABLED: - return - - header_name = ( - "Content-Security-Policy-Report-Only" - if settings.CSP_REPORT_ONLY - else "Content-Security-Policy" - ) - response.headers[header_name] = settings.CSP_POLICY - - -class SecurityHeadersMiddleware(BaseHTTPMiddleware): - """Add baseline security headers to every response.""" - - async def dispatch(self, request: Request, call_next) -> Response: - response = await call_next(request) - response.headers["X-Frame-Options"] = "SAMEORIGIN" - response.headers["X-Content-Type-Options"] = "nosniff" - response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" - _apply_cache_control(request, response) - _apply_csp(response) - return response +"""HTTP security response headers.""" + +from __future__ import annotations + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response + +from app.config import settings + +_NO_STORE_CACHE = "no-cache, no-store, must-revalidate" +_ASSET_CACHE = "public, max-age=31536000, immutable" +_API_DOCS_PREFIXES = ("/docs", "/redoc") +_API_DOCS_EXACT = frozenset({"/openapi.json"}) + + +def _is_api_docs_path(path: str) -> bool: + """FastAPI Swagger/ReDoc need CDN + inline scripts; skip CSP on those routes.""" + if path in _API_DOCS_EXACT: + return True + return any(path == prefix or path.startswith(f"{prefix}/") for prefix in _API_DOCS_PREFIXES) + + +def _apply_cache_control(request: Request, response: Response) -> None: + if "cache-control" in response.headers: + return + + path = request.url.path + if path.startswith("/assets/"): + response.headers["Cache-Control"] = _ASSET_CACHE + return + + cache_value = _NO_STORE_CACHE + if path.startswith("/api/"): + cache_value = f"{_NO_STORE_CACHE}, private" + + response.headers["Cache-Control"] = cache_value + response.headers["Pragma"] = "no-cache" + response.headers["Expires"] = "0" + + +def _apply_csp(request: Request, response: Response) -> None: + if not settings.CSP_ENABLED: + return + if _is_api_docs_path(request.url.path): + return + + header_name = ( + "Content-Security-Policy-Report-Only" + if settings.CSP_REPORT_ONLY + else "Content-Security-Policy" + ) + response.headers[header_name] = settings.CSP_POLICY + + +class SecurityHeadersMiddleware(BaseHTTPMiddleware): + """Add baseline security headers to every response.""" + + async def dispatch(self, request: Request, call_next) -> Response: + response = await call_next(request) + response.headers["X-Frame-Options"] = "SAMEORIGIN" + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" + _apply_cache_control(request, response) + _apply_csp(request, response) + return response diff --git a/app/migrations/078_synthetic_call_traces.py b/app/migrations/078_synthetic_call_traces.py new file mode 100644 index 00000000..9463b43a --- /dev/null +++ b/app/migrations/078_synthetic_call_traces.py @@ -0,0 +1,170 @@ +"""Add synthetic call trace tables for Pipecat phone + OTLP observability.""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add synthetic_call_traces, payload tables, and evaluator_results link" + + +def _table_exists(db: Session, table_name: str) -> bool: + result = db.execute( + text( + """ + SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_name = :table_name + ) + """ + ), + {"table_name": table_name}, + ) + return bool(result.scalar()) + + +def _column_exists(db: Session, table_name: str, column_name: str) -> bool: + result = db.execute( + text( + """ + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = :table_name AND column_name = :column_name + ) + """ + ), + {"table_name": table_name, "column_name": column_name}, + ) + return bool(result.scalar()) + + +def upgrade(db: Session) -> None: + if not _table_exists(db, "synthetic_call_traces"): + db.execute( + text( + """ + CREATE TABLE synthetic_call_traces ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + organization_id UUID NOT NULL REFERENCES organizations(id), + workspace_id UUID NOT NULL REFERENCES workspaces(id), + evaluator_result_id UUID REFERENCES evaluator_results(id) ON DELETE SET NULL, + agent_id UUID REFERENCES agents(id) ON DELETE SET NULL, + persona_id UUID REFERENCES personas(id) ON DELETE SET NULL, + scenario_id UUID REFERENCES scenarios(id) ON DELETE SET NULL, + evaluator_id UUID REFERENCES evaluators(id) ON DELETE SET NULL, + call_recording_id UUID REFERENCES call_recordings(id) ON DELETE SET NULL, + call_short_id VARCHAR(6), + environment VARCHAR(32) NOT NULL DEFAULT 'pre_prod', + provider_platform VARCHAR(64), + transport VARCHAR(32) NOT NULL DEFAULT 'phone', + tier VARCHAR(32) NOT NULL DEFAULT 'black_box', + status VARCHAR(32) NOT NULL DEFAULT 'open', + started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + ended_at TIMESTAMP WITH TIME ZONE, + turn_count INTEGER NOT NULL DEFAULT 0, + response_latency_p50_ms DOUBLE PRECISION, + response_latency_p90_ms DOUBLE PRECISION, + response_latency_p95_ms DOUBLE PRECISION, + component_aggregates JSONB, + failure_flags JSONB, + trace_version INTEGER NOT NULL DEFAULT 1, + shard_id VARCHAR(64), + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ) + """ + ) + ) + db.execute( + text( + "CREATE INDEX ix_synthetic_call_traces_org_agent_started " + "ON synthetic_call_traces(organization_id, agent_id, started_at)" + ) + ) + db.execute( + text( + "CREATE INDEX ix_synthetic_call_traces_evaluator_result " + "ON synthetic_call_traces(evaluator_result_id)" + ) + ) + db.execute( + text( + "CREATE INDEX ix_synthetic_call_traces_call_short_id " + "ON synthetic_call_traces(call_short_id)" + ) + ) + + if not _table_exists(db, "synthetic_trace_payloads"): + db.execute( + text( + """ + CREATE TABLE synthetic_trace_payloads ( + synthetic_call_trace_id UUID PRIMARY KEY + REFERENCES synthetic_call_traces(id) ON DELETE CASCADE, + workspace_id UUID NOT NULL, + turns JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ) + """ + ) + ) + db.execute( + text( + "CREATE INDEX ix_synthetic_trace_payloads_workspace " + "ON synthetic_trace_payloads(workspace_id)" + ) + ) + + if not _table_exists(db, "synthetic_trace_otel_payloads"): + db.execute( + text( + """ + CREATE TABLE synthetic_trace_otel_payloads ( + synthetic_call_trace_id UUID PRIMARY KEY + REFERENCES synthetic_call_traces(id) ON DELETE CASCADE, + workspace_id UUID NOT NULL, + spans JSONB NOT NULL DEFAULT '[]'::jsonb, + trace_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() + ) + """ + ) + ) + db.execute( + text( + "CREATE INDEX ix_synthetic_trace_otel_payloads_workspace " + "ON synthetic_trace_otel_payloads(workspace_id)" + ) + ) + + if not _column_exists(db, "evaluator_results", "synthetic_call_trace_id"): + db.execute( + text( + """ + ALTER TABLE evaluator_results + ADD COLUMN synthetic_call_trace_id UUID + REFERENCES synthetic_call_traces(id) ON DELETE SET NULL + """ + ) + ) + db.execute( + text( + "CREATE INDEX ix_evaluator_results_synthetic_call_trace_id " + "ON evaluator_results(synthetic_call_trace_id)" + ) + ) + + db.commit() + + +def downgrade(db: Session) -> None: + if _column_exists(db, "evaluator_results", "synthetic_call_trace_id"): + db.execute(text("ALTER TABLE evaluator_results DROP COLUMN synthetic_call_trace_id")) + for table in ( + "synthetic_trace_otel_payloads", + "synthetic_trace_payloads", + "synthetic_call_traces", + ): + if _table_exists(db, table): + db.execute(text(f"DROP TABLE {table}")) + db.commit() diff --git a/app/models/database.py b/app/models/database.py index 92350139..6aa2c896 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -1,3022 +1,3109 @@ -"""SQLAlchemy database models.""" - -from sqlalchemy import ( - BigInteger, - Boolean, - Column, - Date, - DateTime, - DDL, - Enum, - event, - Float, - ForeignKey, - Integer, - JSON, - String, - Text, - UniqueConstraint, - select, - text, -) -from sqlalchemy.dialects.postgresql import JSONB, UUID -from sqlalchemy.orm import relationship -from sqlalchemy.sql import func -import uuid -import enum -from app.models.enums import ( - EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, - LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, - IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, - MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, - AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, - PromptOptimizationStatus, CallImportStatus, CallImportRowStatus, -) - -def get_enum_values(enum_class): - """Helper to get values from enum class for SQLAlchemy.""" - return [e.value for e in enum_class] - -from app.database import Base - - -# Enums moved to enums.py - - -class Organization(Base): - """Organization model for multi-tenancy.""" - - __tablename__ = "organizations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - name = Column(String(255), nullable=False) - voice_playground_threshold_overrides = Column(JSON, nullable=True) - # AlignEval-style judge alignment thresholds. - # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int} - # Falls back to system defaults (20 / 50) when null. - judge_alignment_settings = Column(JSON, nullable=True) - # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). - llm_gateway_settings = Column(JSON, nullable=True) - is_active = Column(Boolean, default=True, nullable=False, server_default=text("true"), index=True) - disabled_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - api_keys = relationship("APIKey", back_populates="organization") - members = relationship("OrganizationMember", back_populates="organization", cascade="all, delete-orphan") - invitations = relationship("Invitation", back_populates="organization", cascade="all, delete-orphan") - workspaces = relationship( - "Workspace", - back_populates="organization", - cascade="all, delete-orphan", - ) - workspace_roles = relationship( - "WorkspaceRole", - back_populates="organization", - cascade="all, delete-orphan", - ) - - -class Workspace(Base): - """Workspace - in-org isolation boundary for call imports and metrics. - - Every organization has at least one workspace (``is_default = True``, - seeded by migration 033). Users pick an "active workspace" in the UI; - list endpoints filter by it so users only see calls/metrics from the - project they're currently working in. Access is governed by - ``workspace_members`` and org-scoped ``workspace_roles`` (capability - bundles); org admins implicitly access all workspaces. - """ - - __tablename__ = "workspaces" - __table_args__ = ( - UniqueConstraint("organization_id", "slug", name="uq_workspaces_org_slug"), - ) - - # ``server_default`` is required so that raw-SQL INSERTs (e.g. the - # per-org Default seed in migration 033) can omit ``id`` and let the - # database fill it in. Without it, ``create_all`` produces a column - # with NOT NULL but no DEFAULT, and the migration crashes with - # ``null value in column "id"``. - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - slug = Column(String(255), nullable=False) - # At most one default per org. Enforced on Postgres by the partial - # unique index attached via the after_create event below; on - # SQLite (test runs) we rely on the route-level _check_slug_unique - # check + the Default-workspace conftest fixture instead, because - # SQLite doesn't support partial indexes the same way. - is_default = Column(Boolean, nullable=False, default=False, server_default="false") - is_active = Column(Boolean, nullable=False, default=True, server_default="true") - # Reusable PDF/report branding metadata scoped to this workspace. Images - # live in S3. Shape: {"heading": str|null, "images": [{id, s3_key, - # content_type, filename, size_bytes, updated_at}, ...]}. - report_branding = Column(JSON, nullable=True) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - organization = relationship("Organization", back_populates="workspaces") - members = relationship( - "WorkspaceMember", - back_populates="workspace", - cascade="all, delete-orphan", - ) - - -class WorkspaceRole(Base): - """Org-scoped workspace role (system or custom) as a capability bundle.""" - - __tablename__ = "workspace_roles" - __table_args__ = ( - UniqueConstraint("organization_id", "name", name="uq_workspace_roles_org_name"), - ) - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - capabilities = Column(JSON, nullable=False, default=list) - is_system = Column(Boolean, nullable=False, default=False, server_default="false") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - organization = relationship("Organization", back_populates="workspace_roles") - members = relationship("WorkspaceMember", back_populates="role") - - -class WorkspaceMember(Base): - """User membership in a workspace with an assigned workspace role.""" - - __tablename__ = "workspace_members" - __table_args__ = ( - UniqueConstraint("workspace_id", "user_id", name="uq_workspace_members_ws_user"), - ) - - id = Column( - UUID(as_uuid=True), - primary_key=True, - default=uuid.uuid4, - server_default=text("gen_random_uuid()"), - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - role_id = Column( - UUID(as_uuid=True), - ForeignKey("workspace_roles.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - added_by_user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="SET NULL"), - nullable=True, - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - workspace = relationship("Workspace", back_populates="members") - user = relationship("User", foreign_keys=[user_id]) - role = relationship("WorkspaceRole", back_populates="members") - added_by = relationship("User", foreign_keys=[added_by_user_id]) - - -# Partial unique index: "at most one default workspace per org". This -# is attached as an after_create event (rather than declared in -# ``__table_args__``) because SQLAlchemy's ``Index(..., -# postgresql_where=...)`` silently degrades to a *full* unique index on -# SQLite - which then forbids any second workspace per org and breaks -# the test suite. ``execute_if(dialect="postgresql")`` makes this DDL -# a no-op on SQLite while still emitting it on Postgres (prod, CI). -event.listen( - Workspace.__table__, - "after_create", - DDL( - "CREATE UNIQUE INDEX IF NOT EXISTS uq_workspaces_org_default " - "ON workspaces (organization_id) WHERE is_default" - ).execute_if(dialect="postgresql"), -) - - -class User(Base): - """User model for authentication and profile management.""" - - __tablename__ = "users" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - email = Column(String(255), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=True) - first_name = Column(String(255), nullable=True) - last_name = Column(String(255), nullable=True) - password_hash = Column(String(255), nullable=True) # Nullable for users created via invitation - external_id = Column(String(255), unique=True, nullable=True, index=True) - auth_provider = Column(String(50), nullable=True) - mfa_enabled = Column(Boolean, default=False, nullable=False) - last_login_at = Column(DateTime(timezone=True), nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - organization_memberships = relationship("OrganizationMember", back_populates="user", cascade="all, delete-orphan") - api_keys = relationship("APIKey", back_populates="user") - invitations = relationship("Invitation", back_populates="invited_user", foreign_keys="Invitation.invited_user_id") - refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") - - -class PlatformAdmin(Base): - """Platform-level administrator (separate from org-scoped users).""" - - __tablename__ = "platform_admins" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - email = Column(String(255), unique=True, nullable=False, index=True) - password_hash = Column(String(255), nullable=False) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - last_login_at = Column(DateTime(timezone=True), nullable=True) - - signup_reference_codes = relationship( - "SignupReferenceCode", - back_populates="created_by_admin", - foreign_keys="SignupReferenceCode.created_by", - ) - - -class SignupReferenceCode(Base): - """Single- or multi-use reference code required for gated self-service signup.""" - - __tablename__ = "signup_reference_codes" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - code_hash = Column(String(64), unique=True, nullable=False) - label = Column(String(255), nullable=True) - max_uses = Column(Integer, nullable=True) - use_count = Column(Integer, default=0, nullable=False) - expires_at = Column(DateTime(timezone=True), nullable=True) - is_active = Column(Boolean, default=True, nullable=False, index=True) - created_by = Column(UUID(as_uuid=True), ForeignKey("platform_admins.id"), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - created_by_admin = relationship( - "PlatformAdmin", - back_populates="signup_reference_codes", - foreign_keys=[created_by], - ) - - -class RefreshToken(Base): - """Opaque refresh token for extending local-password sessions.""" - - __tablename__ = "refresh_tokens" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True) - token_hash = Column(String(64), unique=True, nullable=False, index=True) - expires_at = Column(DateTime(timezone=True), nullable=False) - revoked_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - user = relationship("User", back_populates="refresh_tokens") - - -class OrganizationMember(Base): - """Organization membership with role.""" - - __tablename__ = "organization_members" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True) - role = Column(String, nullable=False, default=RoleEnum.READER.value) - - # User preferences for this organization - default_agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) - - joined_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Unique constraint: one membership per user per organization - __table_args__ = ( - UniqueConstraint('organization_id', 'user_id', name='uq_org_user'), - ) - - # Relationships - organization = relationship("Organization", back_populates="members") - user = relationship("User", back_populates="organization_memberships") - default_agent = relationship("Agent", foreign_keys=[default_agent_id]) - - -class Invitation(Base): - """Invitation model for inviting users to organizations.""" - - __tablename__ = "invitations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - invited_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Null if user doesn't exist yet - invited_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) - email = Column(String(255), nullable=False) # Email of invited user - role = Column(String, nullable=False, default=RoleEnum.READER.value) - status = Column(String, nullable=False, default=InvitationStatus.PENDING.value) - - - - token = Column(String(255), unique=True, nullable=False, index=True) # Invitation token - expires_at = Column(DateTime(timezone=True), nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - accepted_at = Column(DateTime(timezone=True), nullable=True) - - # Relationships - organization = relationship("Organization", back_populates="invitations") - invited_user = relationship("User", foreign_keys=[invited_user_id], back_populates="invitations") - invited_by = relationship("User", foreign_keys=[invited_by_id]) - - -class APIKey(Base): - """API Key model for authentication.""" - - __tablename__ = "api_keys" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - key = Column(String(255), unique=True, nullable=False, index=True) - name = Column(String(255), nullable=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Optional: link to user - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - last_used = Column(DateTime(timezone=True), nullable=True) - - # Relationships - organization = relationship("Organization", back_populates="api_keys") - user = relationship("User", back_populates="api_keys") - - -class AudioFile(Base): - """Audio file model.""" - - __tablename__ = "audio_files" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - filename = Column(String(255), nullable=False) - file_path = Column(String(512), nullable=False) - file_size = Column(Integer, nullable=False) # Size in bytes - duration = Column(Float, nullable=True) # Duration in seconds - sample_rate = Column(Integer, nullable=True) - channels = Column(Integer, nullable=True) - format = Column(String(10), nullable=False) # wav, mp3, flac, etc. - uploaded_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - evaluations = relationship("Evaluation", back_populates="audio_file") - - -class Evaluation(Base): - """Evaluation job model.""" - - __tablename__ = "evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every legacy audio evaluation belongs to a - # workspace within its org. Stamped from the X-Workspace-Id header - # (falling back to the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - audio_id = Column(UUID(as_uuid=True), ForeignKey("audio_files.id"), nullable=False) - reference_text = Column(String, nullable=True) # For WER calculation - evaluation_type = Column(String, nullable=False) - model_name = Column(String(100), nullable=True) - status = Column(String, default=EvaluationStatus.PENDING.value, nullable=False) - - - - metrics_requested = Column(JSON, nullable=True) # List of requested metrics - created_at = Column(DateTime(timezone=True), server_default=func.now()) - started_at = Column(DateTime(timezone=True), nullable=True) - completed_at = Column(DateTime(timezone=True), nullable=True) - error_message = Column(String, nullable=True) - - # Relationships - audio_file = relationship("AudioFile", back_populates="evaluations") - result = relationship("EvaluationResult", back_populates="evaluation", uselist=False) - - -class EvaluationResult(Base): - """Evaluation result model.""" - - __tablename__ = "evaluation_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column(UUID(as_uuid=True), ForeignKey("evaluations.id"), nullable=False, unique=True) - # Workspace isolation: mirrors the parent Evaluation's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - transcript = Column(String, nullable=True) - metrics = Column(JSON, nullable=False) # {"wer": 0.05, "latency_ms": 1250, ...} - raw_output = Column(JSON, nullable=True) # Full model output - processing_time = Column(Float, nullable=True) # Processing time in seconds - model_used = Column(String(100), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - # Relationships - evaluation = relationship("Evaluation", back_populates="result") - - -# ============================================ -# VAIOPS MODELS - Voice AI Ops -# ============================================ - -# Enums moved to enums.py - - -class Agent(Base): - """Test Agent - The voice AI agent being evaluated""" - __tablename__ = "agents" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - agent_id = Column(String(6), unique=True, nullable=True, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every agent belongs to a workspace within its - # org. Stamped from the X-Workspace-Id header (falling back to the - # org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - phone_number = Column(String, nullable=True) # Optional, required only for phone_call - language = Column(String, nullable=False, default=LanguageEnum.ENGLISH.value) - description = Column(String) - provider_prompt = Column(Text, nullable=True) - provider_prompt_synced_at = Column(DateTime(timezone=True), nullable=True) - call_type = Column(String, nullable=False, default=CallTypeEnum.OUTBOUND.value) - call_medium = Column(String, nullable=False, default=CallMediumEnum.PHONE_CALL.value) - telephony_phone_number_id = Column( - UUID(as_uuid=True), - ForeignKey("telephony_phone_numbers.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - - - - # Voice configuration - either voice_bundle_id OR ai_provider_id OR voice_ai_integration_id (mutually exclusive) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True, index=True) - ai_provider_id = Column(UUID(as_uuid=True), ForeignKey("aiproviders.id"), nullable=True, index=True) - - # Voice AI agent integration (Retell, Vapi, etc.) - voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) - voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) - prompt_variables = Column(JSON, nullable=True) - silence_hangup_secs = Column(Integer, nullable=False, server_default="15") - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -class Persona(Base): - """Persona - TTS provider-tied voice identity for testing""" - __tablename__ = "personas" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every persona belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String, nullable=False) - gender = Column(String, nullable=False, default=GenderEnum.NEUTRAL.value) - tts_provider = Column(String(100), nullable=True) - tts_voice_id = Column(String(255), nullable=True) - tts_voice_name = Column(String(255), nullable=True) - is_custom = Column(Boolean, default=False) - description = Column(Text, nullable=True) - tts_config = Column(JSON, nullable=True) - llm_temperature = Column(Float, nullable=True) - llm_max_tokens = Column(Integer, nullable=True) - response_delay_ms = Column(Integer, nullable=True) - max_turns = Column(Integer, nullable=True) - allow_interruptions = Column(Boolean, nullable=True) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -class Scenario(Base): - """Scenario - The conversation scenario/test case""" - __tablename__ = "scenarios" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every scenario belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) - name = Column(String, nullable=False) - description = Column(String) - required_info = Column(JSON) - - created_at = Column(DateTime, server_default=func.now()) - updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) - created_by = Column(String) - - -# Enums moved to enums.py - - -class Integration(Base): - """Integration model for connecting with external voice AI platforms.""" - __tablename__ = "integrations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - platform = Column(String, nullable=False) - - - - name = Column(String, nullable=True) # Optional friendly name - api_key = Column(String, nullable=False) # Encrypted Private API key for the platform - public_key = Column(String, nullable=True) # Optional Public API key (e.g. for Vapi) - is_active = Column(Boolean, default=True, nullable=False) - # Multiple credentials per (org, platform) are allowed. is_default marks - # the row used when a caller does not explicitly select a credential. - # A partial unique index in migration 028 enforces at most one default - # per (org, platform) at the DB level. - is_default = Column(Boolean, default=False, nullable=False) - # inherit | gateway | direct — per-credential LLM routing override - routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated - - -class ManualTranscription(Base): - """Manual transcription model for storing transcriptions from S3 audio files.""" - - __tablename__ = "manual_transcriptions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - name = Column(String(255), nullable=True) # User-friendly name for the transcription - audio_file_key = Column(String(512), nullable=False) # S3 key or file path - transcript = Column(String, nullable=False) # Full transcript text - speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - stt_model = Column(String(100), nullable=True) # STT model used (e.g., "whisper-1", "google-speech-v2") - stt_provider = Column(String, nullable=True) # Provider used - - - - language = Column(String(10), nullable=True) # Detected or specified language - processing_time = Column(Float, nullable=True) # Processing time in seconds - raw_output = Column(JSON, nullable=True) # Full model output for reference - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class ConversationEvaluation(Base): - """Conversation evaluation model for evaluating manual transcriptions against agent objectives.""" - - __tablename__ = "conversation_evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - transcription_id = Column(UUID(as_uuid=True), ForeignKey("manual_transcriptions.id"), nullable=False, index=True) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) - - # Evaluation results - objective_achieved = Column(Boolean, nullable=False) # Binary: was the conversation objective achieved? - objective_achieved_reason = Column(String, nullable=True) # Explanation for the binary result - additional_metrics = Column(JSON, nullable=True) # Additional evaluation metrics (e.g., professionalism, clarity, etc.) - overall_score = Column(Float, nullable=True) # Overall score (0.0 to 1.0) - - # LLM metadata - llm_provider = Column(Enum(ModelProvider, native_enum=False), nullable=True) - - llm_model = Column(String(100), nullable=True) - llm_response = Column(JSON, nullable=True) # Full LLM response for reference - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class AIProvider(Base): - """AI Provider - Stores API keys for different AI platforms.""" - __tablename__ = "aiproviders" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String, nullable=False) - - - - api_key = Column(String, nullable=False) # Encrypted API key - name = Column(String, nullable=True) # Optional friendly name - # Azure OpenAI resource endpoint (e.g. https://my-resource.openai.azure.com). - # Only used when provider is azure; other providers ignore this column. - endpoint_url = Column(String, nullable=True) - is_active = Column(Boolean, default=True, nullable=False) - # Multiple AIProvider rows per (org, provider) are allowed. is_default - # marks the row resolved when no explicit credential id is selected. - # A partial unique index in migration 028 enforces at most one default. - is_default = Column(Boolean, default=False, nullable=False) - # inherit | gateway | direct — per-credential LLM routing override - routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # Bifrost custom model ID used when routing via gateway - gateway_model = Column(String(255), nullable=True) - # inherit | litellm_shim | native_openai — Bifrost API surface override - gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") - # Optional per-credential Bifrost/gateway base URL override - gateway_base_url = Column(String(512), nullable=True) - # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) - gateway_auth_header = Column(String(64), nullable=True) - # Env var name whose value is sent as the gateway auth secret - gateway_auth_secret_env = Column(String(128), nullable=True) - # Encrypted inline gateway auth secret (alternative to env var) - gateway_auth_secret = Column(String, nullable=True) - # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls - gateway_extra_headers = Column(JSON, nullable=True) - # Non-empty list restricts model pickers; null/empty = all catalog models for provider. - enabled_models = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated - - -# Enums moved to enums.py - - -class VoiceBundle(Base): - """VoiceBundle - Composable unit combining STT, LLM, and TTS for voice AI testing, or S2S models.""" - __tablename__ = "voicebundles" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - name = Column(String, nullable=False) - description = Column(String, nullable=True) - - # Bundle type: either STT+LLM+TTS or S2S - # Using String instead of Enum to avoid SQLAlchemy enum conversion issues - # The enum conversion is handled in the Pydantic schemas - bundle_type = Column(String(50), nullable=False, default=VoiceBundleType.STT_LLM_TTS.value) - - # STT Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - stt_provider = Column(String, nullable=True) - # Optional explicit credential row (aiproviders.id or integrations.id). - # When NULL the credential resolver picks the default row for the - # provider. No FK is set because the target table varies by provider. - stt_credential_id = Column(UUID(as_uuid=True), nullable=True) - - stt_model = Column(String, nullable=True) # e.g., "whisper-1", "google-speech-v2" - - # LLM Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - llm_provider = Column(String, nullable=True) - llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - - llm_model = Column(String, nullable=True) # e.g., "gpt-4", "claude-3-opus" - llm_temperature = Column(Float, nullable=True, default=0.7) - llm_max_tokens = Column(Integer, nullable=True) - llm_config = Column(JSON, nullable=True) # Additional LLM configuration (extensible) - - # TTS Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S - tts_provider = Column(String, nullable=True) - tts_credential_id = Column(UUID(as_uuid=True), nullable=True) - - tts_model = Column(String, nullable=True) # e.g., "tts-1", "neural-voice" - tts_voice = Column(String, nullable=True) # Voice selection if applicable - tts_config = Column(JSON, nullable=True) # Additional TTS configuration (extensible) - - # S2S Configuration - required for S2S type, optional for STT_LLM_TTS - s2s_provider = Column(String, nullable=True) - s2s_credential_id = Column(UUID(as_uuid=True), nullable=True) - - - - s2s_model = Column(String, nullable=True) # e.g., "gpt-4o-transcribe", speech-to-speech model - s2s_config = Column(JSON, nullable=True) # Additional S2S configuration (extensible) - - # Additional configuration for extensibility - extra_metadata = Column(JSON, nullable=True) # For future extensions (renamed from 'metadata' to avoid SQLAlchemy conflict) - - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class TestAgentConversation(Base): - """Test Agent Conversation - Records conversations between test AI agent and voice AI agent.""" - __tablename__ = "test_agent_conversations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every playground conversation belongs to a - # workspace within its org. Stamped from the X-Workspace-Id header - # (falling back to the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Configuration - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=False) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) - - # Conversation data - status = Column(String, nullable=False, default=TestAgentConversationStatus.INITIALIZING.value) - - - - live_transcription = Column(JSON, nullable=True) # Array of conversation turns with timestamps - conversation_audio_key = Column(String, nullable=True) # S3 key for recorded conversation audio - full_transcript = Column(String, nullable=True) # Full conversation transcript - - # Metadata - started_at = Column(DateTime(timezone=True), server_default=func.now()) - ended_at = Column(DateTime(timezone=True), nullable=True) - duration_seconds = Column(Float, nullable=True) - - # Additional metadata - conversation_metadata = Column(JSON, nullable=True) # Additional conversation metadata - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorSuite(Base): - """Evaluator suite — one agent + one persona + N scenario combinations.""" - - __tablename__ = "evaluator_suites" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - name = Column(String, nullable=True) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) - metric_ids = Column(JSON, nullable=True) - llm_provider = Column(String, nullable=True) - llm_model = Column(String, nullable=True) - llm_config = Column(JSON, nullable=True) - tags = Column(JSON, nullable=True) - default_runs_per_combination = Column(Integer, nullable=False, default=1) - round_robin_index = Column(Integer, nullable=False, default=0) - is_active = Column(Boolean, nullable=False, default=False) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class Evaluator(Base): - """Evaluator - Configuration for testing agents with specific persona and scenario combinations, or custom prompt evaluators.""" - __tablename__ = "evaluators" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluator_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every evaluator belongs to a workspace within - # its org. Stamped from the X-Workspace-Id header (falling back to - # the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Display name (required for custom evaluators, optional for standard) - name = Column(String, nullable=True) - - # Parent suite (nullable for legacy/custom evaluators) - suite_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_suites.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - - # Standard evaluator configuration (nullable for custom evaluators) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) - - # Custom evaluator prompt (used instead of agent/persona/scenario) - custom_prompt = Column(Text, nullable=True) - - # Custom evaluator metric selection. When set, the worker filters the - # enabled-org metrics down to only these IDs (list of metric UUID strings). - # Standard evaluators leave this NULL and use all enabled agent metrics. - metric_ids = Column(JSON, nullable=True) - - # LLM configuration for evaluation (overrides hardcoded defaults) - llm_provider = Column(String, nullable=True) # e.g. "openai", "anthropic", "google" - llm_model = Column(String, nullable=True) # e.g. "gpt-4.1", "claude-sonnet-4-20250514" - llm_config = Column(JSON, nullable=True) - - # Tags for categorization - tags = Column(JSON, nullable=True) # Array of tag strings - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class Metric(Base): - """Metric - Configuration for evaluation metrics. - - Supports a 2-level hierarchy via ``parent_metric_id``: a "category" - parent metric (e.g. "Call Outcome") owns N child sub-metric labels - (e.g. "happy_completion", "angry_hangup"). ``selection_mode`` is set - only on parents and controls how the LLM scores children together - (``single_choice`` = pick exactly one; ``multi_label`` = independent - yes/no with logical consistency). Children are always boolean. - """ - __tablename__ = "metrics" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: two-shape column. - # - # * ``workspace_id = `` — workspace-scoped metric. Only - # visible inside that workspace (the default behavior; existing - # rows all look like this). - # * ``workspace_id IS NULL`` — org-shared metric. Surfaces in - # every workspace's listing under this org so users don't have - # to recreate the same metric per workspace. - # - # Children always inherit their parent's ``workspace_id`` (including - # NULL) so a category metric's whole subtree shares one scope; the - # add-child / promote-discovered endpoints enforce this. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=True, - index=True, - ) - - # Basic information - name = Column(String, nullable=False) - description = Column(String, nullable=True) - # Free-form illustrative example used to sharpen the LLM judge's - # rubric. Today this is consumed by child sub-labels of a - # categorization parent metric so each label can carry "what does - # this look like in a transcript?" text alongside the rubric in - # ``description``. The column lives on every Metric row for - # forward-compat: a standalone metric could later surface its own - # example without another migration. - example = Column(Text, nullable=True) - - # Configuration - metric_type = Column(String, nullable=False, default=MetricType.RATING.value) - metric_category = Column( - String(30), - nullable=False, - default=MetricCategory.QUALITY.value, - server_default=MetricCategory.QUALITY.value, - ) - trigger = Column(String, nullable=False, default=MetricTrigger.ALWAYS.value) - metric_origin = Column(String(30), nullable=False, default="default") - supported_surfaces = Column(JSON, nullable=False, default=list) # ["agent", "voice_playground", "blind_test"] - enabled_surfaces = Column(JSON, nullable=False, default=list) # subset of supported_surfaces - custom_data_type = Column(String(30), nullable=True) # "boolean" | "enum" | "number_range" - custom_config = Column(JSON, nullable=True) # enum options / number range config - tags = Column(JSON, nullable=True) # ["tone", "latency", ...] - - # Hierarchy: NULL = standalone or parent. When set, this row is a - # child sub-metric of the referenced parent. ON DELETE CASCADE so - # deleting a category removes its children atomically. - parent_metric_id = Column( - UUID(as_uuid=True), - ForeignKey("metrics.id", ondelete="CASCADE"), - nullable=True, - index=True, - ) - # Set only on parent rows (``parent_metric_id IS NULL``). Either - # ``single_choice`` or ``multi_label``. NULL = legacy / non-hierarchical - # metric (no children). - selection_mode = Column(String(20), nullable=True) - - # When true on a parent metric (any selection_mode), the LLM is - # invited during call-import evaluation to emit additional - # candidate sub-labels beyond the user-defined children. The - # candidates surface in a "Discovered labels" panel where the user - # manually promotes them into real child Metric rows. For - # ``single_choice`` parents the discovered entries are - # supplemental — the chosen child is still picked from the - # predefined children so the exactly-one-true invariant holds. - # The validator rejects this flag on standalone / child metrics. - allow_discovery = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - - # When True, this metric is a "transcript-compare judge": the - # call-import evaluator feeds BOTH the production transcript - # (``call_import_rows.transcript``, CSV-supplied) and the diarised - # transcript (``call_import_rows.diarised_transcript``, worker- - # produced by the STT/diarisation pipeline) to the LLM as a - # labeled pair instead of feeding one transcript. The parent - # evaluation's ``CallImportEvaluation.transcript_source`` is - # ignored for these metrics — they always read both columns. - # Rows where either transcript is missing are skipped per-metric - # with ``skipped="comparison_missing_transcript"`` so the rest of - # the row's metrics still produce scores. The Pydantic validator - # rejects ``compare_transcripts`` combined with ``parent_metric_id`` - # or ``selection_mode`` (i.e. it can't simultaneously be part of - # a parent/child hierarchy). The call-import worker also - # auto-promotes a metric to comparison mode when its description - # references the production / diarised transcripts in well-known - # phrases (see ``_metric_text_references_production`` in - # ``app.workers.tasks.evaluate_call_import_row``). - compare_transcripts = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - - parent = relationship( - "Metric", - remote_side=[id], - backref="children", - ) - - # When true, the LLM-judge is asked to also return a short free-form - # rationale alongside the value (stored under ``metric_scores[id].rationale``). - # Adds a second " - LLM Rationale" column in the call-import CSV export. - capture_rationale = Column(Boolean, nullable=False, default=False) - - enabled = Column(Boolean, nullable=False, default=True) - - # Studio draft lifecycle: ``draft`` metrics are visible only in Metrics - # Studio until promoted to ``active``. - lifecycle = Column( - String(20), - nullable=False, - default="active", - server_default="active", - ) - promoted_from_draft_at = Column(DateTime(timezone=True), nullable=True) - studio_notes = Column(Text, nullable=True) - - # Metadata - is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class EvaluatorResult(Base): - """EvaluatorResult - Results from running an evaluator with transcription and metric evaluations.""" - __tablename__ = "evaluator_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - result_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every evaluator result belongs to a workspace - # within its org. Stamped from the active workspace at creation time - # (either the X-Workspace-Id header or the org's Default workspace). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # References - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True, index=True) # Optional - can be None for test calls without persona/scenario - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Nullable for custom evaluators - persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) # Optional - can be None for test calls - scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) # Optional - can be None for test calls - - # Result data - name = Column(String, nullable=True) # Scenario name or test call name (optional) - timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - duration_seconds = Column(Float, nullable=True) # Call duration - status = Column(String(20), nullable=False, default=EvaluatorResultStatus.QUEUED.value) - - # Audio and transcription - audio_s3_key = Column(String, nullable=True) # S3 key for audio file - transcription = Column(String, nullable=True) # Full transcription - speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - - # Metric scores - JSON object with metric_id as key and score as value - # Format: {"metric_id_1": {"value": 85, "type": "rating"}, "metric_id_2": {"value": true, "type": "boolean"}} - metric_scores = Column(JSON, nullable=True) - - # Celery task tracking - celery_task_id = Column(String, nullable=True, index=True) # Celery task ID for tracking - - # Error information - error_message = Column(String, nullable=True) - - # Call event tracking (similar to CallRecording) - call_event = Column(String, nullable=True, index=True) # Latest call event (e.g., call_started, call_ended) - provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) - provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" - call_data = Column(JSON, nullable=True) # Full call details from provider (like CallRecording) - - # Data-plane shard routing (payload rows on shard DBs when sharding enabled) - shard_id = Column(String(64), nullable=True, index=True) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -# Enums moved to enums.py - - -class CallRecordingSource(str, enum.Enum): - """Source of the call recording data.""" - - PLAYGROUND = "playground" - WEBHOOK = "webhook" - - -class CallRecording(Base): - """Call Recording model for tracking voice provider calls.""" - __tablename__ = "call_recordings" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every recording belongs to a workspace within - # its org. For playground-origin rows this is stamped from the active - # workspace at creation time; for webhook-origin rows the worker - # looks up the recording's agent and inherits its workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - call_short_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID - status = Column(Enum(CallRecordingStatus), nullable=False, default=CallRecordingStatus.PENDING, index=True) - call_event = Column(String, nullable=True, index=True) # Latest webhook event (e.g., call_started, call_ended) - source = Column(Enum(CallRecordingSource), nullable=False, default=CallRecordingSource.PLAYGROUND, index=True) - call_data = Column(JSON, nullable=True) # JSON blob for provider response - provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) - provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent - - # Link to EvaluatorResult for metric evaluations - evaluator_result_id = Column( - UUID(as_uuid=True), - ForeignKey("evaluator_results.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - shard_id = Column(String(64), nullable=True, index=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class EvaluatorResultPayload(Base): - """Heavy evaluator result fields stored on data shards when sharding is enabled.""" - - __tablename__ = "evaluator_result_payloads" - - evaluator_result_id = Column(UUID(as_uuid=True), primary_key=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - audio_s3_key = Column(String, nullable=True) - transcription = Column(String, nullable=True) - speaker_segments = Column(JSON, nullable=True) - metric_scores = Column(JSON, nullable=True) - call_data = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class CallRecordingPayload(Base): - """Heavy call recording fields stored on data shards when sharding is enabled.""" - - __tablename__ = "call_recording_payloads" - - call_recording_id = Column(UUID(as_uuid=True), primary_key=True) - workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) - call_data = Column(JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class Alert(Base): - """Alert model for configuring monitoring alerts.""" - __tablename__ = "alerts" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - - # Basic information - name = Column(String(255), nullable=False) - description = Column(String, nullable=True) - - # Metric condition configuration - metric_type = Column(String, nullable=False, default=AlertMetricType.NUMBER_OF_CALLS.value) - aggregation = Column(String, nullable=False, default=AlertAggregation.SUM.value) - operator = Column(String, nullable=False, default=AlertOperator.GREATER_THAN.value) - threshold_value = Column(Float, nullable=False) - time_window_minutes = Column(Integer, nullable=False, default=60) # Time window for aggregation - - # Agent selection (JSON array of agent UUIDs, null means all agents) - agent_ids = Column(JSON, nullable=True) - - # Notification configuration - notify_frequency = Column(String, nullable=False, default=AlertNotifyFrequency.IMMEDIATE.value) - notify_emails = Column(JSON, nullable=True) # Array of email addresses - notify_webhooks = Column(JSON, nullable=True) # Array of webhook URLs (Slack, etc.) - - # Status - status = Column(String, nullable=False, default=AlertStatus.ACTIVE.value) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - # Relationships - alert_history = relationship("AlertHistory", back_populates="alert", cascade="all, delete-orphan") - - -class AlertHistory(Base): - """Alert history model for tracking triggered alerts.""" - __tablename__ = "alert_history" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - alert_id = Column(UUID(as_uuid=True), ForeignKey("alerts.id"), nullable=False, index=True) - - # Trigger information - triggered_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) - triggered_value = Column(Float, nullable=False) # The actual value that triggered the alert - threshold_value = Column(Float, nullable=False) # The threshold at time of trigger - - # Status tracking - status = Column(String, nullable=False, default=AlertHistoryStatus.TRIGGERED.value) - - # Notification tracking - notified_at = Column(DateTime(timezone=True), nullable=True) - notification_details = Column(JSON, nullable=True) # Details of sent notifications - - # Resolution - acknowledged_at = Column(DateTime(timezone=True), nullable=True) - acknowledged_by = Column(String, nullable=True) - resolved_at = Column(DateTime(timezone=True), nullable=True) - resolved_by = Column(String, nullable=True) - resolution_notes = Column(String, nullable=True) - - # Additional context - context_data = Column(JSON, nullable=True) # Additional data about the trigger - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - # Relationships - alert = relationship("Alert", back_populates="alert_history") - - -class CronJob(Base): - """Cron job model for scheduling automated evaluator runs.""" - __tablename__ = "cron_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=True, index=True) - - # Basic information - name = Column(String(255), nullable=False) - job_type = Column(String(64), nullable=False, default="evaluator_run") - is_system = Column(Boolean, nullable=False, default=False) - config = Column(JSON, nullable=False, default=dict) - cron_expression = Column(String(100), nullable=False) # e.g., "0 9 * * 1-5" - timezone = Column(String(100), nullable=False, default="UTC") - - # Run configuration - max_runs = Column(Integer, nullable=False, default=10) - current_runs = Column(Integer, nullable=False, default=0) - - # Evaluators to trigger (JSON array of evaluator UUIDs) - evaluator_ids = Column(JSON, nullable=False) - - # Status - status = Column(String, nullable=False, default=CronJobStatus.ACTIVE.value) - - # Run tracking - next_run_at = Column(DateTime(timezone=True), nullable=True) - last_run_at = Column(DateTime(timezone=True), nullable=True) - - # Metadata - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - -class TTSComparisonStatus(str, enum.Enum): - PENDING = "pending" - GENERATING = "generating" - EVALUATING = "evaluating" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSSampleStatus(str, enum.Enum): - PENDING = "pending" - GENERATING = "generating" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSReportJobStatus(str, enum.Enum): - PENDING = "pending" - PROCESSING = "processing" - COMPLETED = "completed" - FAILED = "failed" - - -class TTSComparison(Base): - """TTS Comparison session for A/B testing voice providers.""" - __tablename__ = "tts_comparisons" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every voice playground comparison belongs to - # a workspace within its org. Children (samples, report jobs, blind - # test shares) inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - simulation_id = Column(String(6), unique=True, index=True, nullable=True) - - name = Column(String(255), nullable=True) - status = Column(String(50), nullable=False, default=TTSComparisonStatus.PENDING.value) - - # 'benchmark' = traditional TTS A/B benchmark (provider-generated audio). - # 'blind_test_only' = standalone blind test built from existing recordings - # / uploads / past TTS samples; no TTS generation happens. - mode = Column(String(32), nullable=False, default="benchmark") - - provider_a = Column(String(100), nullable=True) - model_a = Column(String(100), nullable=True) - voices_a = Column(JSON, nullable=True) - - provider_b = Column(String(100), nullable=True) - model_b = Column(String(100), nullable=True) - voices_b = Column(JSON, nullable=True) - - sample_texts = Column(JSON, nullable=False) - num_runs = Column(Integer, nullable=False, default=1) - - blind_test_results = Column(JSON, nullable=True) - evaluation_summary = Column(JSON, nullable=True) - - eval_stt_provider = Column(String(100), nullable=True) - eval_stt_model = Column(String(100), nullable=True) - - celery_task_id = Column(String, nullable=True, index=True) - error_message = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - samples = relationship("TTSSample", back_populates="comparison", cascade="all, delete-orphan") - - -class TTSSample(Base): - """Individual TTS audio sample within a comparison.""" - __tablename__ = "tts_samples" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - provider = Column(String(100), nullable=True) - model = Column(String(100), nullable=True) - voice_id = Column(String(255), nullable=True) - voice_name = Column(String(255), nullable=True) - side = Column(String(1), nullable=True) # "A" or "B" - sample_index = Column(Integer, nullable=False) - run_index = Column(Integer, nullable=False, default=0) - - # 'tts' (default, audio is synthesized by a provider), 'recording' (audio - # is reused from a CallImportRow recording), or 'upload' (audio was - # uploaded by the user). Non-tts samples are marked completed up-front - # by the API and skipped by the generation worker. - source_type = Column(String(32), nullable=False, default="tts") - # When source_type == 'recording', references CallImportRow.id (no FK - # constraint to keep cascading deletes simple if a call import is later - # removed; the audio_s3_key is what's actually used). - source_ref_id = Column(UUID(as_uuid=True), nullable=True) - - text = Column(String, nullable=False) - audio_s3_key = Column(String(512), nullable=True) - duration_seconds = Column(Float, nullable=True) - latency_ms = Column(Float, nullable=True) - ttfb_ms = Column(Float, nullable=True) - - evaluation_metrics = Column(JSON, nullable=True) - status = Column(String(50), nullable=False, default=TTSSampleStatus.PENDING.value) - error_message = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - comparison = relationship("TTSComparison", back_populates="samples") - - -class TTSReportJob(Base): - """Asynchronous PDF report generation jobs for Voice Playground.""" - __tablename__ = "tts_report_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) - - status = Column(String(50), nullable=False, default=TTSReportJobStatus.PENDING.value) - format = Column(String(20), nullable=False, default="pdf") - filename = Column(String(255), nullable=True) - s3_key = Column(String(512), nullable=True) - error_message = Column(String, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - comparison = relationship("TTSComparison") - - -class TTSBlindTestShareStatus(str, enum.Enum): - OPEN = "open" - CLOSED = "closed" - - -class TTSBlindTestShare(Base): - """A publicly sharable blind test for a TTSComparison. - - The share_token is the capability: anyone holding it can open the public - form and submit a response. Each comparison has at most one share row. - """ - __tablename__ = "tts_blind_test_shares" - __table_args__ = ( - UniqueConstraint("comparison_id", name="uq_blind_test_shares_comparison"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - comparison_id = Column( - UUID(as_uuid=True), - ForeignKey("tts_comparisons.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: mirrors the parent TTSComparison's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - share_token = Column(String(64), unique=True, nullable=False, index=True) - - title = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - - # Internal notes visible only to the share creator (e.g. which voice - # corresponds to which side, source notes for standalone blind tests). - # Never exposed via the public blind test payload. - creator_notes = Column(Text, nullable=True) - - # JSON list: [{ "key": str, "label": str, "type": "rating"|"comment", "scale": int? }] - custom_metrics = Column(JSON, nullable=False) - - status = Column(String(20), nullable=False, default=TTSBlindTestShareStatus.OPEN.value) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - closed_at = Column(DateTime(timezone=True), nullable=True) - created_by = Column(String, nullable=True) - - comparison = relationship("TTSComparison") - responses = relationship( - "TTSBlindTestResponse", - back_populates="share", - cascade="all, delete-orphan", - ) - - -class TTSBlindTestResponse(Base): - """A single rater's submission against a TTSBlindTestShare.""" - __tablename__ = "tts_blind_test_responses" - __table_args__ = ( - UniqueConstraint("share_id", "rater_email", name="uq_blind_test_response_share_email"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - share_id = Column( - UUID(as_uuid=True), - ForeignKey("tts_blind_test_shares.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent TTSBlindTestShare's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - rater_name = Column(String(255), nullable=False) - rater_email = Column(String(320), nullable=False, index=True) - - # JSON list keyed by sample_index. Server stores in TRUE A/B orientation - # (already de-flipped from whatever the rater's UI showed): - # [{ - # "sample_index": int, - # "preferred": "A" | "B", - # "ratings_a": { metric_key: number }, - # "ratings_b": { metric_key: number }, - # "comment": str? - # }] - responses = Column(JSON, nullable=False) - - ip = Column(String(64), nullable=True) - user_agent = Column(String(512), nullable=True) - - submitted_at = Column(DateTime(timezone=True), server_default=func.now()) - - share = relationship("TTSBlindTestShare", back_populates="responses") - - -class PromptPartial(Base): - """Prompt Partial - Reusable prompt templates with version history.""" - __tablename__ = "prompt_partials" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every prompt partial belongs to a workspace - # within its org. Versions inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(String, nullable=True) - content = Column(Text, nullable=False) - tags = Column(JSON, nullable=True) - current_version = Column(Integer, nullable=False, default=1) - # Cached LLM-generated flowchart for imported production agent prompts. - # Shape: AgentFlowGraph JSON (nodes[], edges[]). - agent_flowchart = Column(JSON, nullable=True) - agent_flowchart_status = Column(String(20), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - versions = relationship("PromptPartialVersion", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPartialVersion.version.desc()") - - -class PromptPartialVersion(Base): - """Version history for a prompt partial.""" - __tablename__ = "prompt_partial_versions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) - # Workspace isolation: mirrors the parent PromptPartial's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - version = Column(Integer, nullable=False) - content = Column(Text, nullable=False) - change_summary = Column(String, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - created_by = Column(String, nullable=True) - - prompt_partial = relationship("PromptPartial", back_populates="versions") - - __table_args__ = ( - UniqueConstraint('prompt_partial_id', 'version', name='uq_prompt_partial_version'), - ) - - -class CustomTTSVoice(Base): - """Organization-scoped custom TTS voice metadata.""" - __tablename__ = "custom_tts_voices" - __table_args__ = ( - UniqueConstraint("organization_id", "provider", "voice_id", name="uq_custom_tts_voice_org_provider_voice_id"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String(100), nullable=False, index=True) - voice_id = Column(String(255), nullable=False) - name = Column(String(255), nullable=False) - gender = Column(String(50), nullable=True) - accent = Column(String(100), nullable=True) - description = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - -class PromptOptimizationRun(Base): - """A single GEPA prompt optimization run for an agent.""" - __tablename__ = "prompt_optimization_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every optimization run belongs to a workspace - # within its org. Candidates inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) - evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True) - voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) - - seed_prompt = Column(Text, nullable=False) - best_prompt = Column(Text, nullable=True) - best_score = Column(Float, nullable=True) - - status = Column(String(20), nullable=False, default=PromptOptimizationStatus.PENDING.value) - config = Column(JSON, nullable=True) - reflection_trace = Column(JSON, nullable=True) - metric_history = Column(JSON, nullable=True) - - num_iterations = Column(Integer, nullable=True) - num_metric_calls = Column(Integer, nullable=True) - - celery_task_id = Column(String, nullable=True, index=True) - error_message = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - candidates = relationship("PromptOptimizationCandidate", back_populates="optimization_run", cascade="all, delete-orphan") - - -class PromptOptimizationCandidate(Base): - """A candidate prompt generated during an optimization run.""" - __tablename__ = "prompt_optimization_candidates" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - optimization_run_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_runs.id", ondelete="CASCADE"), nullable=False, index=True) - # Workspace isolation: mirrors the parent PromptOptimizationRun's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - prompt_text = Column(Text, nullable=False) - score = Column(Float, nullable=True) - metric_breakdown = Column(JSON, nullable=True) - reflection_summary = Column(Text, nullable=True) - - parent_candidate_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_candidates.id"), nullable=True) - - is_accepted = Column(Boolean, nullable=False, default=False) - pushed_to_provider_at = Column(DateTime(timezone=True), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - optimization_run = relationship("PromptOptimizationRun", back_populates="candidates") - - -class TelephonyIntegration(Base): - """Per-organization telephony provider credentials and configuration. - - Multiple rows per (organization_id, provider) are allowed so that an - organization can keep several Plivo / Exotel accounts side-by-side. - A partial unique index in migration 028 enforces at most one row with - is_default = TRUE per (org, provider); resolution falls back to that - default row when the caller does not pin a specific credential. - """ - - __tablename__ = "telephony_integrations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider = Column(String(50), nullable=False, default="plivo") - name = Column(String(255), nullable=True) # Optional friendly name to disambiguate multiple credentials - - auth_id = Column(String(255), nullable=False) - auth_token = Column(String(512), nullable=False) - - verify_app_uuid = Column(String(255), nullable=True) - voice_app_id = Column(String(255), nullable=True) - sip_domain = Column(String(255), nullable=True) - masking_config = Column(JSON, nullable=True) - - is_active = Column(Boolean, default=True, nullable=False) - is_default = Column(Boolean, default=False, nullable=False) - last_tested_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyPhoneNumber(Base): - """Inventory of telephony phone numbers owned by an organization.""" - - __tablename__ = "telephony_phone_numbers" - __table_args__ = ( - UniqueConstraint("organization_id", "phone_number", name="uq_telephony_number_org_phone"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - telephony_integration_id = Column( - UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=True, index=True - ) - - phone_number = Column(String(20), nullable=False, index=True) - country_iso2 = Column(String(2), nullable=True) - region = Column(String(100), nullable=True) - number_type = Column(String(20), nullable=True) - capabilities = Column(JSON, nullable=True) - provider_app_id = Column(String(255), nullable=True) - - is_masking_pool = Column(Boolean, default=False, nullable=False) - inbound_enabled = Column(Boolean, default=True, nullable=False) - outbound_enabled = Column(Boolean, default=True, nullable=False) - source = Column(String(20), nullable=False, default="imported") - agent_id = Column( - UUID(as_uuid=True), - ForeignKey( - "agents.id", - ondelete="SET NULL", - use_alter=True, - name="fk_telephony_phone_numbers_agent_id", - ), - nullable=True, - index=True, - ) - is_active = Column(Boolean, default=True, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyDialTarget(Base): - """Org-scoped saved destination numbers for outbound test calls.""" - - __tablename__ = "telephony_dial_targets" - __table_args__ = ( - UniqueConstraint("organization_id", "phone_number", name="uq_telephony_dial_target_org_phone"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - phone_number = Column(String(20), nullable=False, index=True) - label = Column(String(255), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyVerifySession(Base): - """Tracks voice OTP verification sessions via telephony provider.""" - - __tablename__ = "telephony_verify_sessions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - provider_session_uuid = Column(String(255), nullable=False, unique=True, index=True) - recipient_number = Column(String(20), nullable=False) - channel = Column(String(10), nullable=False, default="voice") - status = Column(String(20), nullable=False, default="pending") - initiated_by = Column(String(255), nullable=True) - verify_app_uuid = Column(String(255), nullable=True) - verified_at = Column(DateTime(timezone=True), nullable=True) - expires_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class TelephonyMaskedSession(Base): - """Number-masking session between two parties through a middle number.""" - - __tablename__ = "telephony_masked_sessions" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - telephony_integration_id = Column(UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=False) - masked_number_id = Column( - UUID(as_uuid=True), ForeignKey("telephony_phone_numbers.id"), nullable=False, index=True - ) - masked_number = Column(String(20), nullable=False) - party_a_number = Column(String(20), nullable=False) - party_b_number = Column(String(20), nullable=False) - status = Column(String(20), nullable=False, default="active") - expires_at = Column(DateTime(timezone=True), nullable=True) - ended_at = Column(DateTime(timezone=True), nullable=True) - session_metadata = Column("metadata", JSON, nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - -class CallImportSchema(Base): - """Reusable Input Parameter schema for the call-uploads flow. - - A schema is workspace-scoped: users define a named bundle of typed - Input Parameters once (e.g. "Standard Voice QA" with conversation_id + - recording_url + transcript + agent_name) and then map those parameters - to CSV/Excel headers each time they upload a new batch. - - Every schema MUST contain exactly one parameter with - ``type='conversation_id'`` and ``is_required=True`` - that's the - mandatory identity field every imported row needs. A schema may - optionally include at most one ``recording_url`` parameter. The - invariant is enforced in app code on create/update (no DB-level - CHECK because the parent + children are written across two tables in - one transaction). - """ - - __tablename__ = "call_import_schemas" - __table_args__ = ( - # Case-insensitive uniqueness is enforced via the matching partial - # index on ``LOWER(name)`` in the migration; this constraint here - # would be case-sensitive and is intentionally omitted to avoid - # confusing the user. - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - created_by_user_id = Column( - UUID(as_uuid=True), - ForeignKey("users.id", ondelete="SET NULL"), - nullable=True, - ) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - parameters = relationship( - "CallImportSchemaParameter", - back_populates="schema", - cascade="all, delete-orphan", - order_by="CallImportSchemaParameter.ordering", - ) - - -class CallImportSchemaParameter(Base): - """A single typed parameter inside a :class:`CallImportSchema`. - - ``type`` is one of the strings tracked by - :data:`app.models.enums.CallImportParameterType`. ``conversation_id`` - is reserved for the mandatory identity parameter every schema must - contain. - """ - - __tablename__ = "call_import_schema_parameters" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - schema_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_schemas.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - name = Column(String(255), nullable=False) - type = Column(String(32), nullable=False) - description = Column(Text, nullable=True) - is_required = Column(Boolean, nullable=False, default=False) - # Stable ordering so the UI renders parameters in the order the - # schema author defined them (matters when conversation_id is pinned - # first and the user re-orders the rest). - ordering = Column(Integer, nullable=False, default=0) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - schema = relationship("CallImportSchema", back_populates="parameters") - - -class CallImport(Base): - """Batch record for a CSV-driven call import job.""" - - __tablename__ = "call_imports" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - # Workspace isolation: every imported batch belongs to a workspace - # within its org. The /upload endpoint stamps it from the active - # workspace header (or the org's Default if absent). - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) - last_updated_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - - # Telephony provider key (e.g. ``'exotel'``, ``'plivo'``). In the - # legacy one-shot ``POST /upload`` endpoint this is supplied with the - # file; in the three-stage flow (UPLOAD -> MAP -> IMPORT) the value - # isn't known until the IMPORT stage, so the column is nullable for - # ``uploaded`` / ``mapped`` batches. - provider = Column(String(50), nullable=True, default="exotel") - # Pin a specific telephony credential for this batch so the worker - # downloads recordings using *that* row instead of the org default. - # NULL preserves legacy behavior (resolve by provider + default). - telephony_integration_id = Column( - UUID(as_uuid=True), - ForeignKey("telephony_integrations.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - original_filename = Column(String(512), nullable=True) - # When the source file was a multi-sheet Excel workbook, this records - # the worksheet the rows came from (one batch per sheet). NULL for CSV - # uploads since CSV has no sheet concept. - sheet_name = Column(String(255), nullable=True) - - # --- Source-file staging (UPLOAD stage) --------------------------- - # The raw CSV / Excel file is stored in S3 between stages so the - # user can come back later to MAP and IMPORT without re-uploading. - # ``source_s3_key`` is NULL on legacy batches that were imported via - # the one-shot endpoint (those batches stay read-only post-import). - source_s3_key = Column(Text, nullable=True) - source_format = Column(String(16), nullable=True) - source_size_bytes = Column(BigInteger, nullable=True) - source_content_type = Column(String(255), nullable=True) - - # Snapshot of the file's sheets + headers captured at UPLOAD time - # so the MAP UI doesn't need to re-fetch the source bytes from S3. - # Shape: ``[{"name": str, "headers": [str, ...], "row_count": int}, ...]``. - available_sheets = Column(JSON, nullable=True) - - # User's explicit "drop these columns" decision captured at MAP - # time. Was validation-only and ephemeral in the legacy flow; now - # persisted so the IMPORT stage can re-parse the file with the same - # mapping/skip intent. - skipped_columns = Column(JSON, nullable=False, default=list) - # Rows skipped at parse time (missing/invalid conversation_id or URL). - # Shape: ``[{"source_row": int, "reason": str, "message": str}, ...]``. - source_row_skips = Column(JSON, nullable=False, default=list) - - # Free-text high-level segregation label. Powers the "Dataset" filter - # at the top of the imports page; multiple imports can share a value. - dataset = Column(String(255), nullable=True, index=True) - - # Reusable Input Parameter schema this batch was uploaded against. - # NULL on legacy batches uploaded before the schema-driven flow - # shipped; those still render via ``column_mapping`` + ``extra_columns`` - # + ``custom_column_mapping`` below. - schema_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_schemas.id", ondelete="RESTRICT"), - nullable=True, - index=True, - ) - # New schema-driven mapping: ``{schema_parameter_name: csv_header}``. - # Populated for new uploads; empty dict on legacy batches. - parameter_mapping = Column(JSON, nullable=False, default=dict) - - # Legacy free-form mapping (pre-schema-flow). Kept on the model so - # batches that were uploaded before the schema feature shipped still - # render correctly on the detail page; new uploads stop writing here. - # Keys: external_call_id (required), transcript, recording_url. - # (DB column ``external_call_id`` is now ``conversation_id``; this - # JSON key stays as-is for historical batches.) - # Values: original CSV header strings (preserve user casing for export). - column_mapping = Column(JSON, nullable=False, default=dict) - # Ordered list of additional CSV header strings the uploader wants - # preserved verbatim into the evaluation export CSV. - extra_columns = Column(JSON, nullable=False, default=list) - # User-defined ``{custom_field_name: csv_header}`` mappings on top of - # the three system fields above. Cells from the mapped CSV columns are - # preserved per row (keyed by the CSV header in ``raw_columns``) and - # surface in the evaluation export under the uploader-chosen name. - custom_column_mapping = Column(JSON, nullable=False, default=dict) - - total_rows = Column(Integer, nullable=False, default=0) - completed_rows = Column(Integer, nullable=False, default=0) - failed_rows = Column(Integer, nullable=False, default=0) - - status = Column( - Enum(CallImportStatus, values_callable=get_enum_values), - nullable=False, - default=CallImportStatus.PENDING, - index=True, - ) - error_message = Column(Text, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - rows = relationship( - "CallImportRow", - back_populates="call_import", - cascade="all, delete-orphan", - order_by="CallImportRow.row_index", - ) - tags = relationship( - "CallImportTag", - secondary="call_import_tag_assignments", - backref="call_imports", - lazy="selectin", - ) - evaluations = relationship( - "CallImportEvaluation", - back_populates="call_import", - cascade="all, delete-orphan", - ) - - -class CallImportShardSlice(Base): - """Registry row: which shard stores a slice of rows for an import.""" - - __tablename__ = "call_import_shard_slices" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - slice_id = Column(Integer, primary_key=True) - shard_id = Column(String(64), nullable=False, index=True) - row_index_min = Column(Integer, nullable=False) - row_index_max = Column(Integer, nullable=False) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportRow(Base): - """A single row within a CallImport batch (one CSV line / one external call).""" - - __tablename__ = "call_import_rows" - __table_args__ = ( - UniqueConstraint("call_import_id", "row_index", name="uq_call_import_row_index"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - row_index = Column(Integer, nullable=False) - # Was historically named ``external_call_id``; renamed to - # ``conversation_id`` so the new schema-driven upload flow can refer - # to it by a single canonical name across the schema definition, - # exports, and downstream evaluation tables. - conversation_id = Column(String(255), nullable=False, index=True) - # Supplied via CSV for Exotel credentialed imports (required per row). - # Nullable in the schema for legacy rows imported before recording_url - # was mandatory on every Exotel upload. - recording_url = Column(Text, nullable=True) - # Date-only call recording date supplied by the import schema. Used - # for historical report comparisons without timezone/time ambiguity. - recording_date = Column(Date, nullable=True, index=True) - # The "production" transcript: the value supplied via the CSV - # upload mapping. Never overwritten by the diarisation worker — - # the worker writes its output into ``diarised_transcript`` so - # the user keeps both versions side by side. - transcript = Column(Text, nullable=True) - # Snapshot of the original CSV row keyed by the user's headers so the - # evaluation export can reproduce every column the uploader supplied - # (mapped + extra). NULL on legacy rows imported before this column. - raw_columns = Column(JSON, nullable=True) - - # Where the value in ``transcript`` came from. ``csv`` = supplied via - # the upload mapping, ``edited`` = manually changed in the UI. NULL - # on rows that have never had a production transcript. - # (Worker-produced transcripts now live in ``diarised_transcript`` - # and are tracked via ``diarised_transcript_*`` metadata below.) - transcript_source = Column(String(20), nullable=True) - # Provider/model recorded by the (legacy) post-hoc transcription - # worker. New worker runs leave these NULL and write into the - # ``diarised_transcript_*`` columns instead; kept on the model for - # backwards compatibility with pre-split rows that still carry the - # original transcription metadata here. - transcript_provider = Column(String(50), nullable=True) - transcript_model = Column(String(100), nullable=True) - # Lifecycle status for the legacy transcription workflow itself, - # independent of the row's recording-fetch ``status``. ``idle`` = - # no transcribe task has touched this column. New diarisation runs - # update ``diarised_transcript_status`` instead. - transcript_status = Column( - String(20), - nullable=False, - default="idle", - ) - transcript_error = Column(Text, nullable=True) - transcribed_at = Column(DateTime(timezone=True), nullable=True) - - # The "diarised" transcript: produced by the post-hoc - # transcription/diarisation worker. Stored separately so a manual - # diarisation run never clobbers the production transcript above. - # Evaluations can be configured to score against either column - # (see ``CallImportEvaluation.transcript_source``). - diarised_transcript = Column(Text, nullable=True) - # Provider/model the diarisation worker used. Surfaced in the UI - # as "Diarised via deepgram/nova-2" next to the diarised - # transcript section. - diarised_transcript_provider = Column(String(50), nullable=True) - diarised_transcript_model = Column(String(100), nullable=True) - # Lifecycle status for the diarisation workflow. - # ``idle`` = no diarisation task has run; ``pending``/``running`` = - # a Celery task is queued or in flight; ``completed``/``failed`` = - # terminal. Independent of ``transcript_status`` so the two - # transcripts can be in different lifecycle states. - diarised_transcript_status = Column( - String(20), - nullable=False, - default="idle", - server_default="idle", - ) - diarised_transcript_error = Column(Text, nullable=True) - diarised_at = Column(DateTime(timezone=True), nullable=True) - - # Structured speaker turns produced by the diarisation worker — - # ``[{ "speaker": "agent"|"user"|"speaker_3", "text": "...", - # "start": float, "end": float, "raw_speaker": "Speaker 1" }, ...]`` - # The plain-text ``diarised_transcript`` above is a rendered view - # of this list (``: `` per line). When the worker - # cannot recover structured turns (no pyannote token / single- - # speaker recording / provider that doesn't surface segments) this - # column stays NULL and the plain-text path is still populated. - diarised_segments = Column(JSON, nullable=True) - # When True the ``agent`` <-> ``user`` mapping inside - # ``diarised_segments`` is inverted at render / export time. The - # worker writes the canonical mapping using the "first speaker is - # the agent" heuristic; reviewers can flip the toggle from the row - # detail panel without re-running diarisation. - diarised_speaker_swap = Column( - Boolean, - nullable=False, - default=False, - server_default="false", - ) - # LLM that turned the STT plain-text output into structured - # ``diarised_segments``. The legacy diarisation worker used - # pyannote and left these NULL; the current path always runs an - # LLM with the operator-supplied (or default) ``diarised_prompt`` - # below, and records exactly which model + prompt produced each - # row so reviewers can reproduce a specific run. - diarised_llm_provider = Column(String(50), nullable=True) - diarised_llm_model = Column(String(100), nullable=True) - diarised_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - diarised_prompt = Column(Text, nullable=True) - # Which diarisation pipeline produced this row's turns. - # * ``"stt_llm"`` (default) — two-stage: STT then LLM diariser. - # ``diarised_transcript_provider``/``_model`` describe the STT - # side; ``diarised_llm_provider``/``_model`` the LLM side. - # * ``"llm_only"`` — single-stage: audio fed straight to a - # multimodal LLM. ``diarised_transcript_provider`` is stamped - # with the sentinel ``"llm_only"``; the real model is on - # ``diarised_llm_*``. - # Persisting it on the row (not just the run) lets the row detail - # panel render the right "Diarised via …" label even for ad-hoc - # standalone transcribes (no parent evaluation). - transcribe_mode = Column( - String(20), - nullable=False, - default="stt_llm", - server_default="stt_llm", - ) - - status = Column( - Enum(CallImportRowStatus, values_callable=get_enum_values), - nullable=False, - default=CallImportRowStatus.PENDING, - index=True, - ) - - recording_s3_key = Column(String(1024), nullable=True) - recording_content_type = Column(String(128), nullable=True) - recording_size_bytes = Column(Integer, nullable=True) - - error_message = Column(Text, nullable=True) - attempts = Column(Integer, nullable=False, default=0) - celery_task_id = Column(String(255), nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - call_import = relationship("CallImport", back_populates="rows") - - -@event.listens_for(CallImportRow, "before_insert") -def _call_import_row_fill_workspace_id(_mapper, connection, target): - """Denormalize workspace_id from the parent import when omitted.""" - if target.workspace_id is not None or target.call_import_id is None: - return - workspace_id = connection.execute( - select(CallImport.workspace_id).where( - CallImport.id == target.call_import_id - ) - ).scalar_one_or_none() - if workspace_id is not None: - target.workspace_id = workspace_id - - -class CallImportTag(Base): - """User-defined tag that can be attached to one or more call imports. - - Tags coexist with the free-text ``CallImport.dataset`` column: dataset - is the primary high-level segregation, tags are an optional secondary - classification (an import can have many tags). - """ - - __tablename__ = "call_import_tags" - __table_args__ = ( - UniqueConstraint("organization_id", "name", name="uq_call_import_tag_org_name"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - name = Column(String(255), nullable=False) - color = Column(String(32), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class CallImportTagAssignment(Base): - """Many-to-many join table between CallImport and CallImportTag.""" - - __tablename__ = "call_import_tag_assignments" - - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - primary_key=True, - ) - tag_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_tags.id", ondelete="CASCADE"), - primary_key=True, - index=True, - ) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -class CallImportEvaluation(Base): - """Parent record for an evaluation run over a CallImport batch. - - A user picks a subset of org ``Metric`` rows and triggers an evaluation; - we fan out one ``CallImportEvaluationRow`` per source row and roll up - counters as workers finish. Status mirrors ``CallImportStatus`` plus a - ``RUNNING`` value so the UI can distinguish "queued" from "in flight". - """ - - __tablename__ = "call_import_evaluations" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent CallImport's workspace. - # Denormalized for fast filter-by-workspace listings without a join. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id"), nullable=True - ) - last_updated_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True - ) - - # Optional user-supplied label for this run. Lets the UI surface - # something more meaningful than the UUID prefix (e.g. "March QA pass"). - name = Column(String(255), nullable=True) - - # JSON list of Metric UUID strings selected for this run. Stored as text - # in JSON so we don't have to deal with PG arrays of UUIDs / cascade - # delete policies when metrics are removed; the loader filters for - # still-existing org metrics at run time. - selected_metric_ids = Column(JSON, nullable=False, default=list) - # Hierarchy grouping snapshot: ``{parent_id_str: [child_id_str, ...]}``. - # Captures which children belong to which parent for THIS run so the UI - # / aggregator can reconstruct the tree even when the user selected - # only a subset of children, or after metrics are deleted / renamed. - # NULL on legacy rows means "no hierarchy" → fall back to flat - # ``selected_metric_ids`` semantics. - selected_metric_groups = Column(JSON, nullable=True) - # User-driven merges of LLM-discovered candidate sub-labels for - # ``allow_discovery`` parents. Shape: - # ``{"": {"": "", ...}}``. - # Populated via ``POST .../discovered-labels/merge``; consulted by - # the discovered-labels aggregator, the flow graph builder, and the - # worker so that rows finishing AFTER a merge cannot reintroduce - # the merged-away slug. Empty dict on fresh rows. - discovered_label_aliases = Column( - JSON, nullable=False, default=dict, server_default="{}" - ) - - # Per-run opt-in for top-level metric discovery. When True, the LLM - # is asked to propose brand-new top-level metrics (boolean / rating / - # category) observed in the transcripts in addition to scoring the - # ``selected_metric_ids`` for the row. Candidates surface in a - # "Discovered metrics" panel on the evaluation's Flow tab and can - # be promoted into real standalone ``Metric`` rows via - # ``POST /metrics/from-discovered``. Defaults to False so existing - # evaluation creation payloads keep their previous behaviour. - discover_new_metrics = Column( - Boolean, nullable=False, default=False, server_default="false" - ) - # Flat slug-to-slug redirect map for user merges + tombstones of - # discovered top-level metric candidates. Mirrors - # ``discovered_label_aliases`` but is NOT nested per parent — - # top-level metric discovery is not scoped to any parent. Shape:: - # - # {"": "", ...} - # - # An empty-string value tombstones the slug so workers finishing - # later can't re-introduce it. - discovered_metric_aliases = Column( - JSON, nullable=False, default=dict, server_default="{}" - ) - - # Run-level LLM config picked from the Run Evaluation modal. NULL on - # legacy rows means "use the historical OpenAI/gpt-4o default" — the - # worker checks for this and falls back accordingly. ``llm_credential_id`` - # pins a specific AIProvider row when the org has multiple credentials - # for the same provider. - llm_provider = Column(String(50), nullable=True) - llm_model = Column(String(100), nullable=True) - llm_credential_id = Column( - UUID(as_uuid=True), - ForeignKey("aiproviders.id", ondelete="SET NULL"), - nullable=True, - ) - llm_config = Column(JSON, nullable=True) - # Optional per-metric LLM override: - # ``{"": {"provider": "...", "model": "...", "credential_id": "..."}}``. - # Each entry overrides the run-level default for that metric only; - # missing keys = use run-level default. Stored as JSON so the UI can - # round-trip arbitrary {provider, model} pairs without migrations. - metric_llm_overrides = Column(JSON, nullable=True) - - # When ``auto_transcribe`` was set on the create payload, record the - # STT provider/model used so the UI can show "Auto-transcribed via - # deepgram/nova-2" on the evaluation header. ``stt_credential_id`` is - # untyped (no FK) because STT keys may live in either ``aiproviders`` - # (OpenAI) or ``integrations`` (Deepgram, ElevenLabs) — the - # transcription service handles the lookup. - stt_provider = Column(String(50), nullable=True) - stt_model = Column(String(100), nullable=True) - stt_credential_id = Column(UUID(as_uuid=True), nullable=True) - - # Run-level LLM diariser config. Used when the create-run / - # retry-run paths chain a ``transcribe_call_import_row_task`` - # because the row is missing a diarised transcript. Persisted on - # the run so a retry uses the same diariser the original create - # call picked (unless the retry payload explicitly overrides). - diarisation_llm_provider = Column(String(50), nullable=True) - diarisation_llm_model = Column(String(100), nullable=True) - diarisation_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) - diarisation_prompt = Column(Text, nullable=True) - # Mode the run was *created* with for its auto-transcribe step. - # Retry chains read this to decide whether to enqueue an STT+LLM - # transcribe or a single-stage multimodal LLM transcribe — without - # it we'd have to infer the mode from "stt_provider is NULL", which - # would silently break legacy rows that simply never configured - # auto-transcribe. See migration 041 for the column DDL. - transcribe_mode = Column( - String(20), - nullable=False, - default="stt_llm", - server_default="stt_llm", - ) - - # Which of the two transcripts on each ``CallImportRow`` this run - # scored against. ``'production'`` reads ``CallImportRow.transcript`` - # (the CSV-supplied value); ``'diarised'`` reads - # ``CallImportRow.diarised_transcript`` (the worker output). When - # the user ticks both checkboxes in the Run Evaluation modal we - # create two ``CallImportEvaluation`` rows — one per source — so - # the two scorings can be compared side-by-side. Defaults to - # ``'production'`` so legacy runs (which always read the single - # historical ``transcript`` column) keep their semantics. - transcript_source = Column( - String(20), - nullable=False, - default="production", - server_default="production", - ) - - # Cached LLM-generated TLDR rendered above the Visualizations charts. - # Populated lazily by ``POST /evaluations/{eval_id}/insights`` so we - # never auto-burn LLM tokens on page load. Shape:: - # {"narrative": str, "patterns": [str, ...], - # "generated_at": iso8601, "generated_at_completed_rows": int, - # "provider": str, "model": str} - # NULL on rows that have never been summarised. - tldr_summary = Column(JSON, nullable=True) - - # Cached LLM-generated user insights for External Audit PDF section 03. - # Populated by a background Celery job triggered alongside TLDR generation. - # Shape: EvaluationUserInsightsState JSON (status, insights[], progress, …). - user_insights = Column(JSON, nullable=True) - - # Cached per-metric failure clustering for internal diagnostics PDF/UI. - # Shape: EvaluationMetricClustersState JSON (status, groups[], …). - metric_clusters = Column(JSON, nullable=True) - - # Cached LLM-generated prompt improvement suggestions keyed to an - # imported agent (PromptPartial tagged __imported_agent__). - # Shape: EvaluationPromptImprovementsState JSON. - prompt_improvements = Column(JSON, nullable=True) - - # Cached LLM explanations for week-over-week metric deltas keyed by - # baseline evaluation id + completed row counts. - period_delta_explanations = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - - total_rows = Column(Integer, nullable=False, default=0) - completed_rows = Column(Integer, nullable=False, default=0) - failed_rows = Column(Integer, nullable=False, default=0) - # Flexprice pass-level delta billing watermark: rows already emitted - # on ``call_import.evaluation_completed`` for this evaluation run. - billed_completed_rows = Column( - Integer, nullable=False, default=0, server_default="0" - ) - error_message = Column(Text, nullable=True) - celery_group_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - call_import = relationship("CallImport", back_populates="evaluations") - row_results = relationship( - "CallImportEvaluationRow", - back_populates="evaluation", - cascade="all, delete-orphan", - ) - - -class CallImportEvaluationRow(Base): - """Per-source-row scoring output for a CallImportEvaluation parent.""" - - __tablename__ = "call_import_evaluation_rows" - __table_args__ = ( - UniqueConstraint( - "evaluation_id", "call_import_row_id", name="uq_call_import_evaluation_row" - ), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_row_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_rows.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - status = Column(String(20), nullable=False, default="pending", index=True) - # Same shape as EvaluatorResult.metric_scores: {metric_id_str: {value, type, metric_name, ...}} - metric_scores = Column(JSON, nullable=False, default=dict) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - evaluation = relationship("CallImportEvaluation", back_populates="row_results") - source_row = relationship("CallImportRow") - - -@event.listens_for(CallImportEvaluationRow, "before_insert") -def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): - """Denormalize workspace_id from the parent evaluation when omitted.""" - if target.workspace_id is not None or target.evaluation_id is None: - return - workspace_id = connection.execute( - select(CallImportEvaluation.workspace_id).where( - CallImportEvaluation.id == target.evaluation_id - ) - ).scalar_one_or_none() - if workspace_id is not None: - target.workspace_id = workspace_id - - -class MetricStudioRun(Base): - """Ad-hoc metric experiment run in Metrics Studio.""" - - __tablename__ = "metric_studio_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - created_by_user_id = Column( - UUID(as_uuid=True), ForeignKey("users.id"), nullable=True - ) - - name = Column(String(255), nullable=True) - selected_metric_ids = Column(JSON, nullable=False, default=list) - selected_metric_groups = Column(JSON, nullable=True) - transcript_source = Column( - String(20), - nullable=False, - default="diarised", - server_default="diarised", - ) - - llm_provider = Column(String(50), nullable=True) - llm_model = Column(String(100), nullable=True) - llm_credential_id = Column( - UUID(as_uuid=True), - ForeignKey("aiproviders.id", ondelete="SET NULL"), - nullable=True, - ) - llm_config = Column(JSON, nullable=True) - metric_llm_overrides = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - total_items = Column(Integer, nullable=False, default=0) - completed_items = Column(Integer, nullable=False, default=0) - failed_items = Column(Integer, nullable=False, default=0) - error_message = Column(Text, nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - results = relationship( - "MetricStudioRunResult", - back_populates="run", - cascade="all, delete-orphan", - ) - - -class MetricStudioRunResult(Base): - """Per-source scoring output for a MetricStudioRun.""" - - __tablename__ = "metric_studio_run_results" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - run_id = Column( - UUID(as_uuid=True), - ForeignKey("metric_studio_runs.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - source_kind = Column(String(40), nullable=False) - source_ref = Column(String(255), nullable=False) - display_label = Column(String(512), nullable=True) - source_metadata = Column(JSON, nullable=True) - - status = Column(String(20), nullable=False, default="pending", index=True) - metric_scores = Column(JSON, nullable=False, default=dict) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String(255), nullable=True) - - started_at = Column(DateTime(timezone=True), nullable=True) - finished_at = Column(DateTime(timezone=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - run = relationship("MetricStudioRun", back_populates="results") - - -class CallImportEvaluationReportSnapshot(Base): - """Persisted PDF-report aggregate used for period-over-period deltas.""" - - __tablename__ = "call_import_evaluation_report_snapshots" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - period_label = Column(String(64), nullable=True, index=True) - period_start = Column(Date, nullable=True, index=True) - period_end = Column(Date, nullable=True, index=True) - report_config = Column(JSON, nullable=False, default=dict, server_default="{}") - selected_metric_ids = Column(JSON, nullable=False, default=list, server_default="[]") - metric_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") - insight_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") - narrative = Column(JSON, nullable=True) - total_calls = Column(Integer, nullable=False, default=0) - selected_metric_count = Column(Integer, nullable=False, default=0) - total_metric_count = Column(Integer, nullable=False, default=0) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) - - -class CallImportEvaluationPdfReport(Base): - """Stored PDF artifact for a call import evaluation report generation.""" - - __tablename__ = "call_import_evaluation_pdf_reports" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - evaluation_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - call_import_id = Column( - UUID(as_uuid=True), - ForeignKey("call_imports.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - snapshot_id = Column( - UUID(as_uuid=True), - ForeignKey("call_import_evaluation_report_snapshots.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - vendor_name = Column(String(120), nullable=False) - report_type = Column(String(20), nullable=False, default="external") - filename = Column(String(255), nullable=True) - s3_key = Column(String(512), nullable=True) - report_config = Column(JSON, nullable=False, default=dict, server_default="{}") - cache_fingerprint = Column(String(64), nullable=True) - created_by = Column(String, nullable=True) - created_by_user_id = Column(UUID(as_uuid=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - - -# --------------------------------------------------------------------------- -# Judge Alignment (AlignEval-style hybrid integration) -# -# Three tables back the "Judge Alignment" surface: -# - JudgeDataset: a labeled dataset materialised from one of three sources -# (voice transcripts, existing Metric/Evaluator outputs, -# or a generic CSV upload). Holds the dataset's source -# config + which fields play the role of input/output. -# - JudgeSample: one row in a dataset (input/output pair plus an -# optional binary pass/fail human label). -# - JudgeRun: a single run of an LLM-judge (existing Evaluator) over -# a subset of samples, with computed alignment metrics -# (precision/recall/F1/Cohen's kappa) and per-sample -# predictions. Optionally links to a GEPA optimization -# run when the user kicks off prompt tuning from a -# dataset. -# --------------------------------------------------------------------------- - - -class JudgeDataset(Base): - """Container for binary-labeled samples used to calibrate an LLM-judge.""" - - __tablename__ = "judge_datasets" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - # Workspace isolation: every judge dataset belongs to a workspace - # within its org. Samples and runs inherit this workspace_id. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - name = Column(String(255), nullable=False) - description = Column(Text, nullable=True) - - # One of: "transcript", "metric_output", "csv" - source_type = Column(String(32), nullable=False, index=True) - # Source-specific config. Examples: - # transcript: {"transcription_ids": [...]} or {"agent_id": "..."} - # metric_output: {"metric_id": "...", "evaluator_id": "..."} - # csv: {"s3_key": "...", "filename": "..."} - source_config = Column(JSON, nullable=False, default=dict) - - # Field roles - which textual content is "input" vs "output" for the judge. - # For voice transcripts both default to the transcript text but can be - # tightened (e.g. agent-only turns vs full conversation). - input_field = Column(String(64), nullable=False, default="input") - output_field = Column(String(64), nullable=False, default="output") - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - samples = relationship( - "JudgeSample", - back_populates="dataset", - cascade="all, delete-orphan", - order_by="JudgeSample.created_at", - ) - runs = relationship( - "JudgeRun", - back_populates="dataset", - cascade="all, delete-orphan", - order_by="JudgeRun.created_at.desc()", - ) - - -class JudgeSample(Base): - """One labelable input/output pair within a JudgeDataset.""" - - __tablename__ = "judge_samples" - __table_args__ = ( - UniqueConstraint("dataset_id", "external_id", name="uq_judge_samples_dataset_external"), - ) - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - dataset_id = Column( - UUID(as_uuid=True), - ForeignKey("judge_datasets.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - # Workspace isolation: mirrors the parent JudgeDataset's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Stable identifier within the source (e.g. transcription UUID, CSV row id). - # Used to dedupe re-imports and link back to the originating record. - external_id = Column(String(128), nullable=True, index=True) - - input_text = Column(Text, nullable=False) - output_text = Column(Text, nullable=False) - - # Binary human label: "pass" | "fail" | null (unlabeled). - # Stored as string (rather than enum) so it stays trivially extendable. - label = Column(String(16), nullable=True, index=True) - labeled_by = Column(String(255), nullable=True) - labeled_at = Column(DateTime(timezone=True), nullable=True) - - # Source-specific context (e.g. agent_id, original metric value, csv row). - extra = Column(JSON, nullable=True) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - - dataset = relationship("JudgeDataset", back_populates="samples") - - -class JudgeRun(Base): - """One execution of an LLM-judge against a JudgeDataset, with alignment metrics.""" - - __tablename__ = "judge_runs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - dataset_id = Column( - UUID(as_uuid=True), - ForeignKey("judge_datasets.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - organization_id = Column( - UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True - ) - # Workspace isolation: mirrors the parent JudgeDataset's workspace. - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="RESTRICT"), - nullable=False, - index=True, - ) - - # Reuses the existing Evaluator row (its custom_prompt + llm_provider + llm_model - # define the judge under test). Nullable so a run may target an inline prompt - # in the future without inflating the Evaluator table. - evaluator_id = Column( - UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True, index=True - ) - - # Which subset was scored: "all" | "dev" | "test" - split = Column(String(16), nullable=False, default="all") - - # Snapshot of the model used (so a later Evaluator edit doesn't rewrite history). - llm_provider = Column(String(64), nullable=True) - llm_model = Column(String(128), nullable=True) - - # Computed alignment metrics: - # {"precision": float, "recall": float, "f1": float, "kappa": float, - # "tp": int, "fp": int, "tn": int, "fn": int, "n": int} - metrics = Column(JSON, nullable=True) - - # Per-sample predictions, keyed by sample_id (UUID string): - # {sample_id: {"prediction": "pass"|"fail", "explanation": str, "raw": str}} - predictions = Column(JSON, nullable=True) - - # Run lifecycle. - status = Column(String(20), nullable=False, default="pending", index=True) - error_message = Column(Text, nullable=True) - celery_task_id = Column(String, nullable=True, index=True) - - # Optional link to a GEPA optimization run kicked off from this dataset. - gepa_optimization_id = Column( - UUID(as_uuid=True), - ForeignKey("prompt_optimization_runs.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - created_by = Column(String, nullable=True) - - dataset = relationship("JudgeDataset", back_populates="runs") - - -class UsageCostRecomputeJob(Base): - """Async job tracking for retroactive usage cost recompute.""" - - __tablename__ = "usage_cost_recompute_jobs" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - status = Column(String(32), nullable=False, default="pending", server_default="pending") - model = Column(String(255), nullable=True) - usage_kind = Column(String(16), nullable=True) - start_date = Column(Date, nullable=True) - end_date = Column(Date, nullable=True) - updated_rows = Column(BigInteger, nullable=False, default=0, server_default="0") - error_message = Column(String, nullable=True) - celery_task_id = Column(String(255), nullable=True, index=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) - completed_at = Column(DateTime(timezone=True), nullable=True) - - -class LLMUsageDaily(Base): - """Daily LLM/STT usage rollups for org-scoped Usage reporting.""" - - __tablename__ = "llm_usage_daily" - - id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - organization_id = Column( - UUID(as_uuid=True), - ForeignKey("organizations.id", ondelete="CASCADE"), - nullable=False, - index=True, - ) - workspace_id = Column( - UUID(as_uuid=True), - ForeignKey("workspaces.id", ondelete="SET NULL"), - nullable=True, - index=True, - ) - product_section = Column(String(64), nullable=False, index=True) - model = Column(String(255), nullable=False, index=True) - context = Column(JSONB, nullable=False, server_default="{}", default=dict) - usage_date = Column(Date, nullable=False, index=True) - usage_kind = Column(String(16), nullable=False, default="llm", server_default="llm") - prompt_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") - completion_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") - cache_read_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") - cache_creation_tokens = Column( - BigInteger, nullable=False, default=0, server_default="0" - ) - reasoning_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") - audio_seconds = Column(BigInteger, nullable=False, default=0, server_default="0") - tts_characters = Column(BigInteger, nullable=False, default=0, server_default="0") - call_count = Column(BigInteger, nullable=False, default=0, server_default="0") - input_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") - output_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") - cache_read_cost_micro_usd = Column( - BigInteger, nullable=False, default=0, server_default="0" - ) - cache_creation_cost_micro_usd = Column( - BigInteger, nullable=False, default=0, server_default="0" - ) - reasoning_cost_micro_usd = Column( - BigInteger, nullable=False, default=0, server_default="0" - ) - audio_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") - tts_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") - total_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") - pricing_rate_source = Column(String(16), nullable=True) - pricing_rate_id = Column(UUID(as_uuid=True), nullable=True) - created_at = Column(DateTime(timezone=True), server_default=func.now()) - updated_at = Column( - DateTime(timezone=True), server_default=func.now(), onupdate=func.now() - ) +"""SQLAlchemy database models.""" + +from sqlalchemy import ( + BigInteger, + Boolean, + Column, + Date, + DateTime, + DDL, + Enum, + event, + Float, + ForeignKey, + Integer, + JSON, + String, + Text, + UniqueConstraint, + select, + text, +) +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import relationship +from sqlalchemy.sql import func +import uuid +import enum +from app.models.enums import ( + EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, + LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, + IntegrationPlatform, ModelProvider, VoiceBundleType, TestAgentConversationStatus, + MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, + AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, + PromptOptimizationStatus, CallImportStatus, CallImportRowStatus, +) + +def get_enum_values(enum_class): + """Helper to get values from enum class for SQLAlchemy.""" + return [e.value for e in enum_class] + +from app.database import Base + + +# Enums moved to enums.py + + +class Organization(Base): + """Organization model for multi-tenancy.""" + + __tablename__ = "organizations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + name = Column(String(255), nullable=False) + voice_playground_threshold_overrides = Column(JSON, nullable=True) + # AlignEval-style judge alignment thresholds. + # Shape: {"min_labels_to_evaluate": int, "min_labels_to_optimize": int} + # Falls back to system defaults (20 / 50) when null. + judge_alignment_settings = Column(JSON, nullable=True) + # Per-org LLM gateway overrides (enabled, gateway_type, base_url, keys). + llm_gateway_settings = Column(JSON, nullable=True) + is_active = Column(Boolean, default=True, nullable=False, server_default=text("true"), index=True) + disabled_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + api_keys = relationship("APIKey", back_populates="organization") + members = relationship("OrganizationMember", back_populates="organization", cascade="all, delete-orphan") + invitations = relationship("Invitation", back_populates="organization", cascade="all, delete-orphan") + workspaces = relationship( + "Workspace", + back_populates="organization", + cascade="all, delete-orphan", + ) + workspace_roles = relationship( + "WorkspaceRole", + back_populates="organization", + cascade="all, delete-orphan", + ) + + +class Workspace(Base): + """Workspace - in-org isolation boundary for call imports and metrics. + + Every organization has at least one workspace (``is_default = True``, + seeded by migration 033). Users pick an "active workspace" in the UI; + list endpoints filter by it so users only see calls/metrics from the + project they're currently working in. Access is governed by + ``workspace_members`` and org-scoped ``workspace_roles`` (capability + bundles); org admins implicitly access all workspaces. + """ + + __tablename__ = "workspaces" + __table_args__ = ( + UniqueConstraint("organization_id", "slug", name="uq_workspaces_org_slug"), + ) + + # ``server_default`` is required so that raw-SQL INSERTs (e.g. the + # per-org Default seed in migration 033) can omit ``id`` and let the + # database fill it in. Without it, ``create_all`` produces a column + # with NOT NULL but no DEFAULT, and the migration crashes with + # ``null value in column "id"``. + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + slug = Column(String(255), nullable=False) + # At most one default per org. Enforced on Postgres by the partial + # unique index attached via the after_create event below; on + # SQLite (test runs) we rely on the route-level _check_slug_unique + # check + the Default-workspace conftest fixture instead, because + # SQLite doesn't support partial indexes the same way. + is_default = Column(Boolean, nullable=False, default=False, server_default="false") + is_active = Column(Boolean, nullable=False, default=True, server_default="true") + # Reusable PDF/report branding metadata scoped to this workspace. Images + # live in S3. Shape: {"heading": str|null, "images": [{id, s3_key, + # content_type, filename, size_bytes, updated_at}, ...]}. + report_branding = Column(JSON, nullable=True) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + organization = relationship("Organization", back_populates="workspaces") + members = relationship( + "WorkspaceMember", + back_populates="workspace", + cascade="all, delete-orphan", + ) + + +class WorkspaceRole(Base): + """Org-scoped workspace role (system or custom) as a capability bundle.""" + + __tablename__ = "workspace_roles" + __table_args__ = ( + UniqueConstraint("organization_id", "name", name="uq_workspace_roles_org_name"), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + capabilities = Column(JSON, nullable=False, default=list) + is_system = Column(Boolean, nullable=False, default=False, server_default="false") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + organization = relationship("Organization", back_populates="workspace_roles") + members = relationship("WorkspaceMember", back_populates="role") + + +class WorkspaceMember(Base): + """User membership in a workspace with an assigned workspace role.""" + + __tablename__ = "workspace_members" + __table_args__ = ( + UniqueConstraint("workspace_id", "user_id", name="uq_workspace_members_ws_user"), + ) + + id = Column( + UUID(as_uuid=True), + primary_key=True, + default=uuid.uuid4, + server_default=text("gen_random_uuid()"), + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + role_id = Column( + UUID(as_uuid=True), + ForeignKey("workspace_roles.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + added_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + workspace = relationship("Workspace", back_populates="members") + user = relationship("User", foreign_keys=[user_id]) + role = relationship("WorkspaceRole", back_populates="members") + added_by = relationship("User", foreign_keys=[added_by_user_id]) + + +# Partial unique index: "at most one default workspace per org". This +# is attached as an after_create event (rather than declared in +# ``__table_args__``) because SQLAlchemy's ``Index(..., +# postgresql_where=...)`` silently degrades to a *full* unique index on +# SQLite - which then forbids any second workspace per org and breaks +# the test suite. ``execute_if(dialect="postgresql")`` makes this DDL +# a no-op on SQLite while still emitting it on Postgres (prod, CI). +event.listen( + Workspace.__table__, + "after_create", + DDL( + "CREATE UNIQUE INDEX IF NOT EXISTS uq_workspaces_org_default " + "ON workspaces (organization_id) WHERE is_default" + ).execute_if(dialect="postgresql"), +) + + +class User(Base): + """User model for authentication and profile management.""" + + __tablename__ = "users" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + email = Column(String(255), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + first_name = Column(String(255), nullable=True) + last_name = Column(String(255), nullable=True) + password_hash = Column(String(255), nullable=True) # Nullable for users created via invitation + external_id = Column(String(255), unique=True, nullable=True, index=True) + auth_provider = Column(String(50), nullable=True) + mfa_enabled = Column(Boolean, default=False, nullable=False) + last_login_at = Column(DateTime(timezone=True), nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + organization_memberships = relationship("OrganizationMember", back_populates="user", cascade="all, delete-orphan") + api_keys = relationship("APIKey", back_populates="user") + invitations = relationship("Invitation", back_populates="invited_user", foreign_keys="Invitation.invited_user_id") + refresh_tokens = relationship("RefreshToken", back_populates="user", cascade="all, delete-orphan") + + +class PlatformAdmin(Base): + """Platform-level administrator (separate from org-scoped users).""" + + __tablename__ = "platform_admins" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + email = Column(String(255), unique=True, nullable=False, index=True) + password_hash = Column(String(255), nullable=False) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + last_login_at = Column(DateTime(timezone=True), nullable=True) + + signup_reference_codes = relationship( + "SignupReferenceCode", + back_populates="created_by_admin", + foreign_keys="SignupReferenceCode.created_by", + ) + + +class SignupReferenceCode(Base): + """Single- or multi-use reference code required for gated self-service signup.""" + + __tablename__ = "signup_reference_codes" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + code_hash = Column(String(64), unique=True, nullable=False) + label = Column(String(255), nullable=True) + max_uses = Column(Integer, nullable=True) + use_count = Column(Integer, default=0, nullable=False) + expires_at = Column(DateTime(timezone=True), nullable=True) + is_active = Column(Boolean, default=True, nullable=False, index=True) + created_by = Column(UUID(as_uuid=True), ForeignKey("platform_admins.id"), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + created_by_admin = relationship( + "PlatformAdmin", + back_populates="signup_reference_codes", + foreign_keys=[created_by], + ) + + +class RefreshToken(Base): + """Opaque refresh token for extending local-password sessions.""" + + __tablename__ = "refresh_tokens" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False, index=True) + token_hash = Column(String(64), unique=True, nullable=False, index=True) + expires_at = Column(DateTime(timezone=True), nullable=False) + revoked_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + user = relationship("User", back_populates="refresh_tokens") + + +class OrganizationMember(Base): + """Organization membership with role.""" + + __tablename__ = "organization_members" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False, index=True) + role = Column(String, nullable=False, default=RoleEnum.READER.value) + + # User preferences for this organization + default_agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) + + joined_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Unique constraint: one membership per user per organization + __table_args__ = ( + UniqueConstraint('organization_id', 'user_id', name='uq_org_user'), + ) + + # Relationships + organization = relationship("Organization", back_populates="members") + user = relationship("User", back_populates="organization_memberships") + default_agent = relationship("Agent", foreign_keys=[default_agent_id]) + + +class Invitation(Base): + """Invitation model for inviting users to organizations.""" + + __tablename__ = "invitations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + invited_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Null if user doesn't exist yet + invited_by_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) + email = Column(String(255), nullable=False) # Email of invited user + role = Column(String, nullable=False, default=RoleEnum.READER.value) + status = Column(String, nullable=False, default=InvitationStatus.PENDING.value) + + + + token = Column(String(255), unique=True, nullable=False, index=True) # Invitation token + expires_at = Column(DateTime(timezone=True), nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + accepted_at = Column(DateTime(timezone=True), nullable=True) + + # Relationships + organization = relationship("Organization", back_populates="invitations") + invited_user = relationship("User", foreign_keys=[invited_user_id], back_populates="invitations") + invited_by = relationship("User", foreign_keys=[invited_by_id]) + + +class APIKey(Base): + """API Key model for authentication.""" + + __tablename__ = "api_keys" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + key = Column(String(255), unique=True, nullable=False, index=True) + name = Column(String(255), nullable=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True, index=True) # Optional: link to user + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + last_used = Column(DateTime(timezone=True), nullable=True) + + # Relationships + organization = relationship("Organization", back_populates="api_keys") + user = relationship("User", back_populates="api_keys") + + +class AudioFile(Base): + """Audio file model.""" + + __tablename__ = "audio_files" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + filename = Column(String(255), nullable=False) + file_path = Column(String(512), nullable=False) + file_size = Column(Integer, nullable=False) # Size in bytes + duration = Column(Float, nullable=True) # Duration in seconds + sample_rate = Column(Integer, nullable=True) + channels = Column(Integer, nullable=True) + format = Column(String(10), nullable=False) # wav, mp3, flac, etc. + uploaded_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + evaluations = relationship("Evaluation", back_populates="audio_file") + + +class Evaluation(Base): + """Evaluation job model.""" + + __tablename__ = "evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every legacy audio evaluation belongs to a + # workspace within its org. Stamped from the X-Workspace-Id header + # (falling back to the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + audio_id = Column(UUID(as_uuid=True), ForeignKey("audio_files.id"), nullable=False) + reference_text = Column(String, nullable=True) # For WER calculation + evaluation_type = Column(String, nullable=False) + model_name = Column(String(100), nullable=True) + status = Column(String, default=EvaluationStatus.PENDING.value, nullable=False) + + + + metrics_requested = Column(JSON, nullable=True) # List of requested metrics + created_at = Column(DateTime(timezone=True), server_default=func.now()) + started_at = Column(DateTime(timezone=True), nullable=True) + completed_at = Column(DateTime(timezone=True), nullable=True) + error_message = Column(String, nullable=True) + + # Relationships + audio_file = relationship("AudioFile", back_populates="evaluations") + result = relationship("EvaluationResult", back_populates="evaluation", uselist=False) + + +class EvaluationResult(Base): + """Evaluation result model.""" + + __tablename__ = "evaluation_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column(UUID(as_uuid=True), ForeignKey("evaluations.id"), nullable=False, unique=True) + # Workspace isolation: mirrors the parent Evaluation's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + transcript = Column(String, nullable=True) + metrics = Column(JSON, nullable=False) # {"wer": 0.05, "latency_ms": 1250, ...} + raw_output = Column(JSON, nullable=True) # Full model output + processing_time = Column(Float, nullable=True) # Processing time in seconds + model_used = Column(String(100), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + # Relationships + evaluation = relationship("Evaluation", back_populates="result") + + +# ============================================ +# VAIOPS MODELS - Voice AI Ops +# ============================================ + +# Enums moved to enums.py + + +class Agent(Base): + """Test Agent - The voice AI agent being evaluated""" + __tablename__ = "agents" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + agent_id = Column(String(6), unique=True, nullable=True, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every agent belongs to a workspace within its + # org. Stamped from the X-Workspace-Id header (falling back to the + # org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String, nullable=False) + phone_number = Column(String, nullable=True) # Optional, required only for phone_call + language = Column(String, nullable=False, default=LanguageEnum.ENGLISH.value) + description = Column(String) + provider_prompt = Column(Text, nullable=True) + provider_prompt_synced_at = Column(DateTime(timezone=True), nullable=True) + call_type = Column(String, nullable=False, default=CallTypeEnum.OUTBOUND.value) + call_medium = Column(String, nullable=False, default=CallMediumEnum.PHONE_CALL.value) + telephony_phone_number_id = Column( + UUID(as_uuid=True), + ForeignKey("telephony_phone_numbers.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + + + + # Voice configuration - either voice_bundle_id OR ai_provider_id OR voice_ai_integration_id (mutually exclusive) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True, index=True) + ai_provider_id = Column(UUID(as_uuid=True), ForeignKey("aiproviders.id"), nullable=True, index=True) + + # Voice AI agent integration (Retell, Vapi, etc.) + voice_ai_integration_id = Column(UUID(as_uuid=True), ForeignKey("integrations.id"), nullable=True, index=True) + voice_ai_agent_id = Column(String, nullable=True) # Agent ID from the external provider (Retell/Vapi) + prompt_variables = Column(JSON, nullable=True) + silence_hangup_secs = Column(Integer, nullable=False, server_default="15") + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class Persona(Base): + """Persona - TTS provider-tied voice identity for testing""" + __tablename__ = "personas" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every persona belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String, nullable=False) + gender = Column(String, nullable=False, default=GenderEnum.NEUTRAL.value) + tts_provider = Column(String(100), nullable=True) + tts_voice_id = Column(String(255), nullable=True) + tts_voice_name = Column(String(255), nullable=True) + is_custom = Column(Boolean, default=False) + description = Column(Text, nullable=True) + tts_config = Column(JSON, nullable=True) + llm_temperature = Column(Float, nullable=True) + llm_max_tokens = Column(Integer, nullable=True) + response_delay_ms = Column(Integer, nullable=True) + max_turns = Column(Integer, nullable=True) + allow_interruptions = Column(Boolean, nullable=True) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +class Scenario(Base): + """Scenario - The conversation scenario/test case""" + __tablename__ = "scenarios" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every scenario belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) + name = Column(String, nullable=False) + description = Column(String) + required_info = Column(JSON) + + created_at = Column(DateTime, server_default=func.now()) + updated_at = Column(DateTime, server_default=func.now(), onupdate=func.now()) + created_by = Column(String) + + +# Enums moved to enums.py + + +class Integration(Base): + """Integration model for connecting with external voice AI platforms.""" + __tablename__ = "integrations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + platform = Column(String, nullable=False) + + + + name = Column(String, nullable=True) # Optional friendly name + api_key = Column(String, nullable=False) # Encrypted Private API key for the platform + public_key = Column(String, nullable=True) # Optional Public API key (e.g. for Vapi) + is_active = Column(Boolean, default=True, nullable=False) + # Multiple credentials per (org, platform) are allowed. is_default marks + # the row used when a caller does not explicitly select a credential. + # A partial unique index in migration 028 enforces at most one default + # per (org, platform) at the DB level. + is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated + + +class ManualTranscription(Base): + """Manual transcription model for storing transcriptions from S3 audio files.""" + + __tablename__ = "manual_transcriptions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + name = Column(String(255), nullable=True) # User-friendly name for the transcription + audio_file_key = Column(String(512), nullable=False) # S3 key or file path + transcript = Column(String, nullable=False) # Full transcript text + speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + stt_model = Column(String(100), nullable=True) # STT model used (e.g., "whisper-1", "google-speech-v2") + stt_provider = Column(String, nullable=True) # Provider used + + + + language = Column(String(10), nullable=True) # Detected or specified language + processing_time = Column(Float, nullable=True) # Processing time in seconds + raw_output = Column(JSON, nullable=True) # Full model output for reference + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class ConversationEvaluation(Base): + """Conversation evaluation model for evaluating manual transcriptions against agent objectives.""" + + __tablename__ = "conversation_evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + transcription_id = Column(UUID(as_uuid=True), ForeignKey("manual_transcriptions.id"), nullable=False, index=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) + + # Evaluation results + objective_achieved = Column(Boolean, nullable=False) # Binary: was the conversation objective achieved? + objective_achieved_reason = Column(String, nullable=True) # Explanation for the binary result + additional_metrics = Column(JSON, nullable=True) # Additional evaluation metrics (e.g., professionalism, clarity, etc.) + overall_score = Column(Float, nullable=True) # Overall score (0.0 to 1.0) + + # LLM metadata + llm_provider = Column(Enum(ModelProvider, native_enum=False), nullable=True) + + llm_model = Column(String(100), nullable=True) + llm_response = Column(JSON, nullable=True) # Full LLM response for reference + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class AIProvider(Base): + """AI Provider - Stores API keys for different AI platforms.""" + __tablename__ = "aiproviders" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String, nullable=False) + + + + api_key = Column(String, nullable=False) # Encrypted API key + name = Column(String, nullable=True) # Optional friendly name + # Azure OpenAI resource endpoint (e.g. https://my-resource.openai.azure.com). + # Only used when provider is azure; other providers ignore this column. + endpoint_url = Column(String, nullable=True) + is_active = Column(Boolean, default=True, nullable=False) + # Multiple AIProvider rows per (org, provider) are allowed. is_default + # marks the row resolved when no explicit credential id is selected. + # A partial unique index in migration 028 enforces at most one default. + is_default = Column(Boolean, default=False, nullable=False) + # inherit | gateway | direct — per-credential LLM routing override + routing_mode = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Bifrost custom model ID used when routing via gateway + gateway_model = Column(String(255), nullable=True) + # inherit | litellm_shim | native_openai — Bifrost API surface override + gateway_interface = Column(String(20), nullable=False, default="inherit", server_default="inherit") + # Optional per-credential Bifrost/gateway base URL override + gateway_base_url = Column(String(512), nullable=True) + # Optional auth header for Bifrost (e.g. x-bf-vk, Authorization, x-api-key) + gateway_auth_header = Column(String(64), nullable=True) + # Env var name whose value is sent as the gateway auth secret + gateway_auth_secret_env = Column(String(128), nullable=True) + # Encrypted inline gateway auth secret (alternative to env var) + gateway_auth_secret = Column(String, nullable=True) + # Arbitrary HTTP headers sent with gateway-routed LiteLLM calls + gateway_extra_headers = Column(JSON, nullable=True) + # Non-empty list restricts model pickers; null/empty = all catalog models for provider. + enabled_models = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + last_tested_at = Column(DateTime(timezone=True), nullable=True) # When API key was last validated + + +# Enums moved to enums.py + + +class VoiceBundle(Base): + """VoiceBundle - Composable unit combining STT, LLM, and TTS for voice AI testing, or S2S models.""" + __tablename__ = "voicebundles" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + name = Column(String, nullable=False) + description = Column(String, nullable=True) + + # Bundle type: either STT+LLM+TTS or S2S + # Using String instead of Enum to avoid SQLAlchemy enum conversion issues + # The enum conversion is handled in the Pydantic schemas + bundle_type = Column(String(50), nullable=False, default=VoiceBundleType.STT_LLM_TTS.value) + + # STT Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + stt_provider = Column(String, nullable=True) + # Optional explicit credential row (aiproviders.id or integrations.id). + # When NULL the credential resolver picks the default row for the + # provider. No FK is set because the target table varies by provider. + stt_credential_id = Column(UUID(as_uuid=True), nullable=True) + + stt_model = Column(String, nullable=True) # e.g., "whisper-1", "google-speech-v2" + + # LLM Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + llm_provider = Column(String, nullable=True) + llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + + llm_model = Column(String, nullable=True) # e.g., "gpt-4", "claude-3-opus" + llm_temperature = Column(Float, nullable=True, default=0.7) + llm_max_tokens = Column(Integer, nullable=True) + llm_config = Column(JSON, nullable=True) # Additional LLM configuration (extensible) + + # TTS Configuration (references AIProvider via provider name) - required for STT_LLM_TTS, optional for S2S + tts_provider = Column(String, nullable=True) + tts_credential_id = Column(UUID(as_uuid=True), nullable=True) + + tts_model = Column(String, nullable=True) # e.g., "tts-1", "neural-voice" + tts_voice = Column(String, nullable=True) # Voice selection if applicable + tts_config = Column(JSON, nullable=True) # Additional TTS configuration (extensible) + + # S2S Configuration - required for S2S type, optional for STT_LLM_TTS + s2s_provider = Column(String, nullable=True) + s2s_credential_id = Column(UUID(as_uuid=True), nullable=True) + + + + s2s_model = Column(String, nullable=True) # e.g., "gpt-4o-transcribe", speech-to-speech model + s2s_config = Column(JSON, nullable=True) # Additional S2S configuration (extensible) + + # Additional configuration for extensibility + extra_metadata = Column(JSON, nullable=True) # For future extensions (renamed from 'metadata' to avoid SQLAlchemy conflict) + + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class TestAgentConversation(Base): + """Test Agent Conversation - Records conversations between test AI agent and voice AI agent.""" + __tablename__ = "test_agent_conversations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every playground conversation belongs to a + # workspace within its org. Stamped from the X-Workspace-Id header + # (falling back to the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Configuration + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=False) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) + + # Conversation data + status = Column(String, nullable=False, default=TestAgentConversationStatus.INITIALIZING.value) + + + + live_transcription = Column(JSON, nullable=True) # Array of conversation turns with timestamps + conversation_audio_key = Column(String, nullable=True) # S3 key for recorded conversation audio + full_transcript = Column(String, nullable=True) # Full conversation transcript + + # Metadata + started_at = Column(DateTime(timezone=True), server_default=func.now()) + ended_at = Column(DateTime(timezone=True), nullable=True) + duration_seconds = Column(Float, nullable=True) + + # Additional metadata + conversation_metadata = Column(JSON, nullable=True) # Additional conversation metadata + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorSuite(Base): + """Evaluator suite — one agent + one persona + N scenario combinations.""" + + __tablename__ = "evaluator_suites" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + name = Column(String, nullable=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=False) + metric_ids = Column(JSON, nullable=True) + llm_provider = Column(String, nullable=True) + llm_model = Column(String, nullable=True) + llm_config = Column(JSON, nullable=True) + tags = Column(JSON, nullable=True) + default_runs_per_combination = Column(Integer, nullable=False, default=1) + round_robin_index = Column(Integer, nullable=False, default=0) + is_active = Column(Boolean, nullable=False, default=False) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class Evaluator(Base): + """Evaluator - Configuration for testing agents with specific persona and scenario combinations, or custom prompt evaluators.""" + __tablename__ = "evaluators" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluator_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every evaluator belongs to a workspace within + # its org. Stamped from the X-Workspace-Id header (falling back to + # the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Display name (required for custom evaluators, optional for standard) + name = Column(String, nullable=True) + + # Parent suite (nullable for legacy/custom evaluators) + suite_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_suites.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + + # Standard evaluator configuration (nullable for custom evaluators) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) + + # Custom evaluator prompt (used instead of agent/persona/scenario) + custom_prompt = Column(Text, nullable=True) + + # Custom evaluator metric selection. When set, the worker filters the + # enabled-org metrics down to only these IDs (list of metric UUID strings). + # Standard evaluators leave this NULL and use all enabled agent metrics. + metric_ids = Column(JSON, nullable=True) + + # LLM configuration for evaluation (overrides hardcoded defaults) + llm_provider = Column(String, nullable=True) # e.g. "openai", "anthropic", "google" + llm_model = Column(String, nullable=True) # e.g. "gpt-4.1", "claude-sonnet-4-20250514" + llm_config = Column(JSON, nullable=True) + + # Tags for categorization + tags = Column(JSON, nullable=True) # Array of tag strings + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class Metric(Base): + """Metric - Configuration for evaluation metrics. + + Supports a 2-level hierarchy via ``parent_metric_id``: a "category" + parent metric (e.g. "Call Outcome") owns N child sub-metric labels + (e.g. "happy_completion", "angry_hangup"). ``selection_mode`` is set + only on parents and controls how the LLM scores children together + (``single_choice`` = pick exactly one; ``multi_label`` = independent + yes/no with logical consistency). Children are always boolean. + """ + __tablename__ = "metrics" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: two-shape column. + # + # * ``workspace_id = `` — workspace-scoped metric. Only + # visible inside that workspace (the default behavior; existing + # rows all look like this). + # * ``workspace_id IS NULL`` — org-shared metric. Surfaces in + # every workspace's listing under this org so users don't have + # to recreate the same metric per workspace. + # + # Children always inherit their parent's ``workspace_id`` (including + # NULL) so a category metric's whole subtree shares one scope; the + # add-child / promote-discovered endpoints enforce this. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + + # Basic information + name = Column(String, nullable=False) + description = Column(String, nullable=True) + # Free-form illustrative example used to sharpen the LLM judge's + # rubric. Today this is consumed by child sub-labels of a + # categorization parent metric so each label can carry "what does + # this look like in a transcript?" text alongside the rubric in + # ``description``. The column lives on every Metric row for + # forward-compat: a standalone metric could later surface its own + # example without another migration. + example = Column(Text, nullable=True) + + # Configuration + metric_type = Column(String, nullable=False, default=MetricType.RATING.value) + metric_category = Column( + String(30), + nullable=False, + default=MetricCategory.QUALITY.value, + server_default=MetricCategory.QUALITY.value, + ) + trigger = Column(String, nullable=False, default=MetricTrigger.ALWAYS.value) + metric_origin = Column(String(30), nullable=False, default="default") + supported_surfaces = Column(JSON, nullable=False, default=list) # ["agent", "voice_playground", "blind_test"] + enabled_surfaces = Column(JSON, nullable=False, default=list) # subset of supported_surfaces + custom_data_type = Column(String(30), nullable=True) # "boolean" | "enum" | "number_range" + custom_config = Column(JSON, nullable=True) # enum options / number range config + tags = Column(JSON, nullable=True) # ["tone", "latency", ...] + + # Hierarchy: NULL = standalone or parent. When set, this row is a + # child sub-metric of the referenced parent. ON DELETE CASCADE so + # deleting a category removes its children atomically. + parent_metric_id = Column( + UUID(as_uuid=True), + ForeignKey("metrics.id", ondelete="CASCADE"), + nullable=True, + index=True, + ) + # Set only on parent rows (``parent_metric_id IS NULL``). Either + # ``single_choice`` or ``multi_label``. NULL = legacy / non-hierarchical + # metric (no children). + selection_mode = Column(String(20), nullable=True) + + # When true on a parent metric (any selection_mode), the LLM is + # invited during call-import evaluation to emit additional + # candidate sub-labels beyond the user-defined children. The + # candidates surface in a "Discovered labels" panel where the user + # manually promotes them into real child Metric rows. For + # ``single_choice`` parents the discovered entries are + # supplemental — the chosen child is still picked from the + # predefined children so the exactly-one-true invariant holds. + # The validator rejects this flag on standalone / child metrics. + allow_discovery = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + + # When True, this metric is a "transcript-compare judge": the + # call-import evaluator feeds BOTH the production transcript + # (``call_import_rows.transcript``, CSV-supplied) and the diarised + # transcript (``call_import_rows.diarised_transcript``, worker- + # produced by the STT/diarisation pipeline) to the LLM as a + # labeled pair instead of feeding one transcript. The parent + # evaluation's ``CallImportEvaluation.transcript_source`` is + # ignored for these metrics — they always read both columns. + # Rows where either transcript is missing are skipped per-metric + # with ``skipped="comparison_missing_transcript"`` so the rest of + # the row's metrics still produce scores. The Pydantic validator + # rejects ``compare_transcripts`` combined with ``parent_metric_id`` + # or ``selection_mode`` (i.e. it can't simultaneously be part of + # a parent/child hierarchy). The call-import worker also + # auto-promotes a metric to comparison mode when its description + # references the production / diarised transcripts in well-known + # phrases (see ``_metric_text_references_production`` in + # ``app.workers.tasks.evaluate_call_import_row``). + compare_transcripts = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + + parent = relationship( + "Metric", + remote_side=[id], + backref="children", + ) + + # When true, the LLM-judge is asked to also return a short free-form + # rationale alongside the value (stored under ``metric_scores[id].rationale``). + # Adds a second " - LLM Rationale" column in the call-import CSV export. + capture_rationale = Column(Boolean, nullable=False, default=False) + + enabled = Column(Boolean, nullable=False, default=True) + + # Studio draft lifecycle: ``draft`` metrics are visible only in Metrics + # Studio until promoted to ``active``. + lifecycle = Column( + String(20), + nullable=False, + default="active", + server_default="active", + ) + promoted_from_draft_at = Column(DateTime(timezone=True), nullable=True) + studio_notes = Column(Text, nullable=True) + + # Metadata + is_default = Column(Boolean, nullable=False, default=False) # Pre-defined metrics + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class EvaluatorResult(Base): + """EvaluatorResult - Results from running an evaluator with transcription and metric evaluations.""" + __tablename__ = "evaluator_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + result_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every evaluator result belongs to a workspace + # within its org. Stamped from the active workspace at creation time + # (either the X-Workspace-Id header or the org's Default workspace). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # References + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True, index=True) # Optional - can be None for test calls without persona/scenario + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Nullable for custom evaluators + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id"), nullable=True) # Optional - can be None for test calls + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id"), nullable=True) # Optional - can be None for test calls + + # Result data + name = Column(String, nullable=True) # Scenario name or test call name (optional) + timestamp = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + duration_seconds = Column(Float, nullable=True) # Call duration + status = Column(String(20), nullable=False, default=EvaluatorResultStatus.QUEUED.value) + + # Audio and transcription + audio_s3_key = Column(String, nullable=True) # S3 key for audio file + transcription = Column(String, nullable=True) # Full transcription + speaker_segments = Column(JSON, nullable=True) # List of segments with speaker labels: [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + + # Metric scores - JSON object with metric_id as key and score as value + # Format: {"metric_id_1": {"value": 85, "type": "rating"}, "metric_id_2": {"value": true, "type": "boolean"}} + metric_scores = Column(JSON, nullable=True) + + # Celery task tracking + celery_task_id = Column(String, nullable=True, index=True) # Celery task ID for tracking + + # Error information + error_message = Column(String, nullable=True) + + # Call event tracking (similar to CallRecording) + call_event = Column(String, nullable=True, index=True) # Latest call event (e.g., call_started, call_ended) + provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) + provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + call_data = Column(JSON, nullable=True) # Full call details from provider (like CallRecording) + + # Data-plane shard routing (payload rows on shard DBs when sharding enabled) + shard_id = Column(String(64), nullable=True, index=True) + synthetic_call_trace_id = Column( + UUID(as_uuid=True), + ForeignKey("synthetic_call_traces.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +# Enums moved to enums.py + + +class CallRecordingSource(str, enum.Enum): + """Source of the call recording data.""" + + PLAYGROUND = "playground" + WEBHOOK = "webhook" + + +class CallRecording(Base): + """Call Recording model for tracking voice provider calls.""" + __tablename__ = "call_recordings" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every recording belongs to a workspace within + # its org. For playground-origin rows this is stamped from the active + # workspace at creation time; for webhook-origin rows the worker + # looks up the recording's agent and inherits its workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + call_short_id = Column(String(6), unique=True, nullable=False, index=True) # 6-digit ID + status = Column(Enum(CallRecordingStatus), nullable=False, default=CallRecordingStatus.PENDING, index=True) + call_event = Column(String, nullable=True, index=True) # Latest webhook event (e.g., call_started, call_ended) + source = Column(Enum(CallRecordingSource), nullable=False, default=CallRecordingSource.PLAYGROUND, index=True) + call_data = Column(JSON, nullable=True) # JSON blob for provider response + provider_call_id = Column(String, nullable=True, index=True) # Provider's call_id (e.g., Retell call_id) + provider_platform = Column(String, nullable=True) # e.g., "retell", "vapi" + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=True) # Reference to our agent + + # Link to EvaluatorResult for metric evaluations + evaluator_result_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_results.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + shard_id = Column(String(64), nullable=True, index=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class EvaluatorResultPayload(Base): + """Heavy evaluator result fields stored on data shards when sharding is enabled.""" + + __tablename__ = "evaluator_result_payloads" + + evaluator_result_id = Column(UUID(as_uuid=True), primary_key=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + audio_s3_key = Column(String, nullable=True) + transcription = Column(String, nullable=True) + speaker_segments = Column(JSON, nullable=True) + metric_scores = Column(JSON, nullable=True) + call_data = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class CallRecordingPayload(Base): + """Heavy call recording fields stored on data shards when sharding is enabled.""" + + __tablename__ = "call_recording_payloads" + + call_recording_id = Column(UUID(as_uuid=True), primary_key=True) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + call_data = Column(JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class SyntheticCallTrace(Base): + """Catalog row for synthetic test call timing and OTLP traces.""" + + __tablename__ = "synthetic_call_traces" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + evaluator_result_id = Column( + UUID(as_uuid=True), + ForeignKey("evaluator_results.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True) + persona_id = Column(UUID(as_uuid=True), ForeignKey("personas.id", ondelete="SET NULL"), nullable=True) + scenario_id = Column(UUID(as_uuid=True), ForeignKey("scenarios.id", ondelete="SET NULL"), nullable=True) + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True) + call_recording_id = Column( + UUID(as_uuid=True), + ForeignKey("call_recordings.id", ondelete="SET NULL"), + nullable=True, + ) + call_short_id = Column(String(6), nullable=True, index=True) + environment = Column(String(32), nullable=False, default="pre_prod") + provider_platform = Column(String(64), nullable=True) + transport = Column(String(32), nullable=False, default="phone") + tier = Column(String(32), nullable=False, default="black_box") + status = Column(String(32), nullable=False, default="open") + started_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + ended_at = Column(DateTime(timezone=True), nullable=True) + turn_count = Column(Integer, nullable=False, default=0) + response_latency_p50_ms = Column(Float, nullable=True) + response_latency_p90_ms = Column(Float, nullable=True) + response_latency_p95_ms = Column(Float, nullable=True) + component_aggregates = Column(JSON, nullable=True) + failure_flags = Column(JSON, nullable=True) + trace_version = Column(Integer, nullable=False, default=1) + shard_id = Column(String(64), nullable=True, index=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class SyntheticTracePayload(Base): + """Per-turn timing payload for a synthetic call trace.""" + + __tablename__ = "synthetic_trace_payloads" + + synthetic_call_trace_id = Column( + UUID(as_uuid=True), + ForeignKey("synthetic_call_traces.id", ondelete="CASCADE"), + primary_key=True, + ) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + turns = Column(JSON, nullable=False, default=list) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class SyntheticTraceOtelPayload(Base): + """Full OTLP span tree for a synthetic call trace.""" + + __tablename__ = "synthetic_trace_otel_payloads" + + synthetic_call_trace_id = Column( + UUID(as_uuid=True), + ForeignKey("synthetic_call_traces.id", ondelete="CASCADE"), + primary_key=True, + ) + workspace_id = Column(UUID(as_uuid=True), nullable=False, index=True) + spans = Column(JSON, nullable=False, default=list) + trace_ids = Column(JSON, nullable=False, default=list) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class Alert(Base): + """Alert model for configuring monitoring alerts.""" + __tablename__ = "alerts" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + + # Basic information + name = Column(String(255), nullable=False) + description = Column(String, nullable=True) + + # Metric condition configuration + metric_type = Column(String, nullable=False, default=AlertMetricType.NUMBER_OF_CALLS.value) + aggregation = Column(String, nullable=False, default=AlertAggregation.SUM.value) + operator = Column(String, nullable=False, default=AlertOperator.GREATER_THAN.value) + threshold_value = Column(Float, nullable=False) + time_window_minutes = Column(Integer, nullable=False, default=60) # Time window for aggregation + + # Agent selection (JSON array of agent UUIDs, null means all agents) + agent_ids = Column(JSON, nullable=True) + + # Notification configuration + notify_frequency = Column(String, nullable=False, default=AlertNotifyFrequency.IMMEDIATE.value) + notify_emails = Column(JSON, nullable=True) # Array of email addresses + notify_webhooks = Column(JSON, nullable=True) # Array of webhook URLs (Slack, etc.) + + # Status + status = Column(String, nullable=False, default=AlertStatus.ACTIVE.value) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + # Relationships + alert_history = relationship("AlertHistory", back_populates="alert", cascade="all, delete-orphan") + + +class AlertHistory(Base): + """Alert history model for tracking triggered alerts.""" + __tablename__ = "alert_history" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + alert_id = Column(UUID(as_uuid=True), ForeignKey("alerts.id"), nullable=False, index=True) + + # Trigger information + triggered_at = Column(DateTime(timezone=True), server_default=func.now(), nullable=False) + triggered_value = Column(Float, nullable=False) # The actual value that triggered the alert + threshold_value = Column(Float, nullable=False) # The threshold at time of trigger + + # Status tracking + status = Column(String, nullable=False, default=AlertHistoryStatus.TRIGGERED.value) + + # Notification tracking + notified_at = Column(DateTime(timezone=True), nullable=True) + notification_details = Column(JSON, nullable=True) # Details of sent notifications + + # Resolution + acknowledged_at = Column(DateTime(timezone=True), nullable=True) + acknowledged_by = Column(String, nullable=True) + resolved_at = Column(DateTime(timezone=True), nullable=True) + resolved_by = Column(String, nullable=True) + resolution_notes = Column(String, nullable=True) + + # Additional context + context_data = Column(JSON, nullable=True) # Additional data about the trigger + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + # Relationships + alert = relationship("Alert", back_populates="alert_history") + + +class CronJob(Base): + """Cron job model for scheduling automated evaluator runs.""" + __tablename__ = "cron_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=True, index=True) + + # Basic information + name = Column(String(255), nullable=False) + job_type = Column(String(64), nullable=False, default="evaluator_run") + is_system = Column(Boolean, nullable=False, default=False) + config = Column(JSON, nullable=False, default=dict) + cron_expression = Column(String(100), nullable=False) # e.g., "0 9 * * 1-5" + timezone = Column(String(100), nullable=False, default="UTC") + + # Run configuration + max_runs = Column(Integer, nullable=False, default=10) + current_runs = Column(Integer, nullable=False, default=0) + + # Evaluators to trigger (JSON array of evaluator UUIDs) + evaluator_ids = Column(JSON, nullable=False) + + # Status + status = Column(String, nullable=False, default=CronJobStatus.ACTIVE.value) + + # Run tracking + next_run_at = Column(DateTime(timezone=True), nullable=True) + last_run_at = Column(DateTime(timezone=True), nullable=True) + + # Metadata + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + +class TTSComparisonStatus(str, enum.Enum): + PENDING = "pending" + GENERATING = "generating" + EVALUATING = "evaluating" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSSampleStatus(str, enum.Enum): + PENDING = "pending" + GENERATING = "generating" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSReportJobStatus(str, enum.Enum): + PENDING = "pending" + PROCESSING = "processing" + COMPLETED = "completed" + FAILED = "failed" + + +class TTSComparison(Base): + """TTS Comparison session for A/B testing voice providers.""" + __tablename__ = "tts_comparisons" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every voice playground comparison belongs to + # a workspace within its org. Children (samples, report jobs, blind + # test shares) inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + simulation_id = Column(String(6), unique=True, index=True, nullable=True) + + name = Column(String(255), nullable=True) + status = Column(String(50), nullable=False, default=TTSComparisonStatus.PENDING.value) + + # 'benchmark' = traditional TTS A/B benchmark (provider-generated audio). + # 'blind_test_only' = standalone blind test built from existing recordings + # / uploads / past TTS samples; no TTS generation happens. + mode = Column(String(32), nullable=False, default="benchmark") + + provider_a = Column(String(100), nullable=True) + model_a = Column(String(100), nullable=True) + voices_a = Column(JSON, nullable=True) + + provider_b = Column(String(100), nullable=True) + model_b = Column(String(100), nullable=True) + voices_b = Column(JSON, nullable=True) + + sample_texts = Column(JSON, nullable=False) + num_runs = Column(Integer, nullable=False, default=1) + + blind_test_results = Column(JSON, nullable=True) + evaluation_summary = Column(JSON, nullable=True) + + eval_stt_provider = Column(String(100), nullable=True) + eval_stt_model = Column(String(100), nullable=True) + + celery_task_id = Column(String, nullable=True, index=True) + error_message = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + samples = relationship("TTSSample", back_populates="comparison", cascade="all, delete-orphan") + + +class TTSSample(Base): + """Individual TTS audio sample within a comparison.""" + __tablename__ = "tts_samples" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + provider = Column(String(100), nullable=True) + model = Column(String(100), nullable=True) + voice_id = Column(String(255), nullable=True) + voice_name = Column(String(255), nullable=True) + side = Column(String(1), nullable=True) # "A" or "B" + sample_index = Column(Integer, nullable=False) + run_index = Column(Integer, nullable=False, default=0) + + # 'tts' (default, audio is synthesized by a provider), 'recording' (audio + # is reused from a CallImportRow recording), or 'upload' (audio was + # uploaded by the user). Non-tts samples are marked completed up-front + # by the API and skipped by the generation worker. + source_type = Column(String(32), nullable=False, default="tts") + # When source_type == 'recording', references CallImportRow.id (no FK + # constraint to keep cascading deletes simple if a call import is later + # removed; the audio_s3_key is what's actually used). + source_ref_id = Column(UUID(as_uuid=True), nullable=True) + + text = Column(String, nullable=False) + audio_s3_key = Column(String(512), nullable=True) + duration_seconds = Column(Float, nullable=True) + latency_ms = Column(Float, nullable=True) + ttfb_ms = Column(Float, nullable=True) + + evaluation_metrics = Column(JSON, nullable=True) + status = Column(String(50), nullable=False, default=TTSSampleStatus.PENDING.value) + error_message = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + comparison = relationship("TTSComparison", back_populates="samples") + + +class TTSReportJob(Base): + """Asynchronous PDF report generation jobs for Voice Playground.""" + __tablename__ = "tts_report_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + comparison_id = Column(UUID(as_uuid=True), ForeignKey("tts_comparisons.id", ondelete="CASCADE"), nullable=False, index=True) + + status = Column(String(50), nullable=False, default=TTSReportJobStatus.PENDING.value) + format = Column(String(20), nullable=False, default="pdf") + filename = Column(String(255), nullable=True) + s3_key = Column(String(512), nullable=True) + error_message = Column(String, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + comparison = relationship("TTSComparison") + + +class TTSBlindTestShareStatus(str, enum.Enum): + OPEN = "open" + CLOSED = "closed" + + +class TTSBlindTestShare(Base): + """A publicly sharable blind test for a TTSComparison. + + The share_token is the capability: anyone holding it can open the public + form and submit a response. Each comparison has at most one share row. + """ + __tablename__ = "tts_blind_test_shares" + __table_args__ = ( + UniqueConstraint("comparison_id", name="uq_blind_test_shares_comparison"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + comparison_id = Column( + UUID(as_uuid=True), + ForeignKey("tts_comparisons.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: mirrors the parent TTSComparison's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + share_token = Column(String(64), unique=True, nullable=False, index=True) + + title = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # Internal notes visible only to the share creator (e.g. which voice + # corresponds to which side, source notes for standalone blind tests). + # Never exposed via the public blind test payload. + creator_notes = Column(Text, nullable=True) + + # JSON list: [{ "key": str, "label": str, "type": "rating"|"comment", "scale": int? }] + custom_metrics = Column(JSON, nullable=False) + + status = Column(String(20), nullable=False, default=TTSBlindTestShareStatus.OPEN.value) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + closed_at = Column(DateTime(timezone=True), nullable=True) + created_by = Column(String, nullable=True) + + comparison = relationship("TTSComparison") + responses = relationship( + "TTSBlindTestResponse", + back_populates="share", + cascade="all, delete-orphan", + ) + + +class TTSBlindTestResponse(Base): + """A single rater's submission against a TTSBlindTestShare.""" + __tablename__ = "tts_blind_test_responses" + __table_args__ = ( + UniqueConstraint("share_id", "rater_email", name="uq_blind_test_response_share_email"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + share_id = Column( + UUID(as_uuid=True), + ForeignKey("tts_blind_test_shares.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent TTSBlindTestShare's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + rater_name = Column(String(255), nullable=False) + rater_email = Column(String(320), nullable=False, index=True) + + # JSON list keyed by sample_index. Server stores in TRUE A/B orientation + # (already de-flipped from whatever the rater's UI showed): + # [{ + # "sample_index": int, + # "preferred": "A" | "B", + # "ratings_a": { metric_key: number }, + # "ratings_b": { metric_key: number }, + # "comment": str? + # }] + responses = Column(JSON, nullable=False) + + ip = Column(String(64), nullable=True) + user_agent = Column(String(512), nullable=True) + + submitted_at = Column(DateTime(timezone=True), server_default=func.now()) + + share = relationship("TTSBlindTestShare", back_populates="responses") + + +class PromptPartial(Base): + """Prompt Partial - Reusable prompt templates with version history.""" + __tablename__ = "prompt_partials" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every prompt partial belongs to a workspace + # within its org. Versions inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(String, nullable=True) + content = Column(Text, nullable=False) + tags = Column(JSON, nullable=True) + current_version = Column(Integer, nullable=False, default=1) + # Cached LLM-generated flowchart for imported production agent prompts. + # Shape: AgentFlowGraph JSON (nodes[], edges[]). + agent_flowchart = Column(JSON, nullable=True) + agent_flowchart_status = Column(String(20), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + versions = relationship("PromptPartialVersion", back_populates="prompt_partial", cascade="all, delete-orphan", order_by="PromptPartialVersion.version.desc()") + + +class PromptPartialVersion(Base): + """Version history for a prompt partial.""" + __tablename__ = "prompt_partial_versions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + prompt_partial_id = Column(UUID(as_uuid=True), ForeignKey("prompt_partials.id", ondelete="CASCADE"), nullable=False, index=True) + # Workspace isolation: mirrors the parent PromptPartial's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + version = Column(Integer, nullable=False) + content = Column(Text, nullable=False) + change_summary = Column(String, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + created_by = Column(String, nullable=True) + + prompt_partial = relationship("PromptPartial", back_populates="versions") + + __table_args__ = ( + UniqueConstraint('prompt_partial_id', 'version', name='uq_prompt_partial_version'), + ) + + +class CustomTTSVoice(Base): + """Organization-scoped custom TTS voice metadata.""" + __tablename__ = "custom_tts_voices" + __table_args__ = ( + UniqueConstraint("organization_id", "provider", "voice_id", name="uq_custom_tts_voice_org_provider_voice_id"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String(100), nullable=False, index=True) + voice_id = Column(String(255), nullable=False) + name = Column(String(255), nullable=False) + gender = Column(String(50), nullable=True) + accent = Column(String(100), nullable=True) + description = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + +class PromptOptimizationRun(Base): + """A single GEPA prompt optimization run for an agent.""" + __tablename__ = "prompt_optimization_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every optimization run belongs to a workspace + # within its org. Candidates inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id"), nullable=False, index=True) + evaluator_id = Column(UUID(as_uuid=True), ForeignKey("evaluators.id"), nullable=True) + voice_bundle_id = Column(UUID(as_uuid=True), ForeignKey("voicebundles.id"), nullable=True) + + seed_prompt = Column(Text, nullable=False) + best_prompt = Column(Text, nullable=True) + best_score = Column(Float, nullable=True) + + status = Column(String(20), nullable=False, default=PromptOptimizationStatus.PENDING.value) + config = Column(JSON, nullable=True) + reflection_trace = Column(JSON, nullable=True) + metric_history = Column(JSON, nullable=True) + + num_iterations = Column(Integer, nullable=True) + num_metric_calls = Column(Integer, nullable=True) + + celery_task_id = Column(String, nullable=True, index=True) + error_message = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + candidates = relationship("PromptOptimizationCandidate", back_populates="optimization_run", cascade="all, delete-orphan") + + +class PromptOptimizationCandidate(Base): + """A candidate prompt generated during an optimization run.""" + __tablename__ = "prompt_optimization_candidates" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + optimization_run_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_runs.id", ondelete="CASCADE"), nullable=False, index=True) + # Workspace isolation: mirrors the parent PromptOptimizationRun's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + prompt_text = Column(Text, nullable=False) + score = Column(Float, nullable=True) + metric_breakdown = Column(JSON, nullable=True) + reflection_summary = Column(Text, nullable=True) + + parent_candidate_id = Column(UUID(as_uuid=True), ForeignKey("prompt_optimization_candidates.id"), nullable=True) + + is_accepted = Column(Boolean, nullable=False, default=False) + pushed_to_provider_at = Column(DateTime(timezone=True), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + optimization_run = relationship("PromptOptimizationRun", back_populates="candidates") + + +class TelephonyIntegration(Base): + """Per-organization telephony provider credentials and configuration. + + Multiple rows per (organization_id, provider) are allowed so that an + organization can keep several Plivo / Exotel accounts side-by-side. + A partial unique index in migration 028 enforces at most one row with + is_default = TRUE per (org, provider); resolution falls back to that + default row when the caller does not pin a specific credential. + """ + + __tablename__ = "telephony_integrations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider = Column(String(50), nullable=False, default="plivo") + name = Column(String(255), nullable=True) # Optional friendly name to disambiguate multiple credentials + + auth_id = Column(String(255), nullable=False) + auth_token = Column(String(512), nullable=False) + + verify_app_uuid = Column(String(255), nullable=True) + voice_app_id = Column(String(255), nullable=True) + sip_domain = Column(String(255), nullable=True) + masking_config = Column(JSON, nullable=True) + + is_active = Column(Boolean, default=True, nullable=False) + is_default = Column(Boolean, default=False, nullable=False) + last_tested_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyPhoneNumber(Base): + """Inventory of telephony phone numbers owned by an organization.""" + + __tablename__ = "telephony_phone_numbers" + __table_args__ = ( + UniqueConstraint("organization_id", "phone_number", name="uq_telephony_number_org_phone"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + telephony_integration_id = Column( + UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=True, index=True + ) + + phone_number = Column(String(20), nullable=False, index=True) + country_iso2 = Column(String(2), nullable=True) + region = Column(String(100), nullable=True) + number_type = Column(String(20), nullable=True) + capabilities = Column(JSON, nullable=True) + provider_app_id = Column(String(255), nullable=True) + + is_masking_pool = Column(Boolean, default=False, nullable=False) + inbound_enabled = Column(Boolean, default=True, nullable=False) + outbound_enabled = Column(Boolean, default=True, nullable=False) + source = Column(String(20), nullable=False, default="imported") + agent_id = Column( + UUID(as_uuid=True), + ForeignKey( + "agents.id", + ondelete="SET NULL", + use_alter=True, + name="fk_telephony_phone_numbers_agent_id", + ), + nullable=True, + index=True, + ) + is_active = Column(Boolean, default=True, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyDialTarget(Base): + """Org-scoped saved destination numbers for outbound test calls.""" + + __tablename__ = "telephony_dial_targets" + __table_args__ = ( + UniqueConstraint("organization_id", "phone_number", name="uq_telephony_dial_target_org_phone"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + phone_number = Column(String(20), nullable=False, index=True) + label = Column(String(255), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyVerifySession(Base): + """Tracks voice OTP verification sessions via telephony provider.""" + + __tablename__ = "telephony_verify_sessions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + provider_session_uuid = Column(String(255), nullable=False, unique=True, index=True) + recipient_number = Column(String(20), nullable=False) + channel = Column(String(10), nullable=False, default="voice") + status = Column(String(20), nullable=False, default="pending") + initiated_by = Column(String(255), nullable=True) + verify_app_uuid = Column(String(255), nullable=True) + verified_at = Column(DateTime(timezone=True), nullable=True) + expires_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class TelephonyMaskedSession(Base): + """Number-masking session between two parties through a middle number.""" + + __tablename__ = "telephony_masked_sessions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + telephony_integration_id = Column(UUID(as_uuid=True), ForeignKey("telephony_integrations.id"), nullable=False) + masked_number_id = Column( + UUID(as_uuid=True), ForeignKey("telephony_phone_numbers.id"), nullable=False, index=True + ) + masked_number = Column(String(20), nullable=False) + party_a_number = Column(String(20), nullable=False) + party_b_number = Column(String(20), nullable=False) + status = Column(String(20), nullable=False, default="active") + expires_at = Column(DateTime(timezone=True), nullable=True) + ended_at = Column(DateTime(timezone=True), nullable=True) + session_metadata = Column("metadata", JSON, nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +class CallImportSchema(Base): + """Reusable Input Parameter schema for the call-uploads flow. + + A schema is workspace-scoped: users define a named bundle of typed + Input Parameters once (e.g. "Standard Voice QA" with conversation_id + + recording_url + transcript + agent_name) and then map those parameters + to CSV/Excel headers each time they upload a new batch. + + Every schema MUST contain exactly one parameter with + ``type='conversation_id'`` and ``is_required=True`` - that's the + mandatory identity field every imported row needs. A schema may + optionally include at most one ``recording_url`` parameter. The + invariant is enforced in app code on create/update (no DB-level + CHECK because the parent + children are written across two tables in + one transaction). + """ + + __tablename__ = "call_import_schemas" + __table_args__ = ( + # Case-insensitive uniqueness is enforced via the matching partial + # index on ``LOWER(name)`` in the migration; this constraint here + # would be case-sensitive and is intentionally omitted to avoid + # confusing the user. + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + created_by_user_id = Column( + UUID(as_uuid=True), + ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + parameters = relationship( + "CallImportSchemaParameter", + back_populates="schema", + cascade="all, delete-orphan", + order_by="CallImportSchemaParameter.ordering", + ) + + +class CallImportSchemaParameter(Base): + """A single typed parameter inside a :class:`CallImportSchema`. + + ``type`` is one of the strings tracked by + :data:`app.models.enums.CallImportParameterType`. ``conversation_id`` + is reserved for the mandatory identity parameter every schema must + contain. + """ + + __tablename__ = "call_import_schema_parameters" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + schema_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_schemas.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + name = Column(String(255), nullable=False) + type = Column(String(32), nullable=False) + description = Column(Text, nullable=True) + is_required = Column(Boolean, nullable=False, default=False) + # Stable ordering so the UI renders parameters in the order the + # schema author defined them (matters when conversation_id is pinned + # first and the user re-orders the rest). + ordering = Column(Integer, nullable=False, default=0) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + schema = relationship("CallImportSchema", back_populates="parameters") + + +class CallImport(Base): + """Batch record for a CSV-driven call import job.""" + + __tablename__ = "call_imports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + # Workspace isolation: every imported batch belongs to a workspace + # within its org. The /upload endpoint stamps it from the active + # workspace header (or the org's Default if absent). + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=True) + last_updated_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + # Telephony provider key (e.g. ``'exotel'``, ``'plivo'``). In the + # legacy one-shot ``POST /upload`` endpoint this is supplied with the + # file; in the three-stage flow (UPLOAD -> MAP -> IMPORT) the value + # isn't known until the IMPORT stage, so the column is nullable for + # ``uploaded`` / ``mapped`` batches. + provider = Column(String(50), nullable=True, default="exotel") + # Pin a specific telephony credential for this batch so the worker + # downloads recordings using *that* row instead of the org default. + # NULL preserves legacy behavior (resolve by provider + default). + telephony_integration_id = Column( + UUID(as_uuid=True), + ForeignKey("telephony_integrations.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + original_filename = Column(String(512), nullable=True) + # When the source file was a multi-sheet Excel workbook, this records + # the worksheet the rows came from (one batch per sheet). NULL for CSV + # uploads since CSV has no sheet concept. + sheet_name = Column(String(255), nullable=True) + + # --- Source-file staging (UPLOAD stage) --------------------------- + # The raw CSV / Excel file is stored in S3 between stages so the + # user can come back later to MAP and IMPORT without re-uploading. + # ``source_s3_key`` is NULL on legacy batches that were imported via + # the one-shot endpoint (those batches stay read-only post-import). + source_s3_key = Column(Text, nullable=True) + source_format = Column(String(16), nullable=True) + source_size_bytes = Column(BigInteger, nullable=True) + source_content_type = Column(String(255), nullable=True) + + # Snapshot of the file's sheets + headers captured at UPLOAD time + # so the MAP UI doesn't need to re-fetch the source bytes from S3. + # Shape: ``[{"name": str, "headers": [str, ...], "row_count": int}, ...]``. + available_sheets = Column(JSON, nullable=True) + + # User's explicit "drop these columns" decision captured at MAP + # time. Was validation-only and ephemeral in the legacy flow; now + # persisted so the IMPORT stage can re-parse the file with the same + # mapping/skip intent. + skipped_columns = Column(JSON, nullable=False, default=list) + # Rows skipped at parse time (missing/invalid conversation_id or URL). + # Shape: ``[{"source_row": int, "reason": str, "message": str}, ...]``. + source_row_skips = Column(JSON, nullable=False, default=list) + + # Free-text high-level segregation label. Powers the "Dataset" filter + # at the top of the imports page; multiple imports can share a value. + dataset = Column(String(255), nullable=True, index=True) + + # Reusable Input Parameter schema this batch was uploaded against. + # NULL on legacy batches uploaded before the schema-driven flow + # shipped; those still render via ``column_mapping`` + ``extra_columns`` + # + ``custom_column_mapping`` below. + schema_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_schemas.id", ondelete="RESTRICT"), + nullable=True, + index=True, + ) + # New schema-driven mapping: ``{schema_parameter_name: csv_header}``. + # Populated for new uploads; empty dict on legacy batches. + parameter_mapping = Column(JSON, nullable=False, default=dict) + + # Legacy free-form mapping (pre-schema-flow). Kept on the model so + # batches that were uploaded before the schema feature shipped still + # render correctly on the detail page; new uploads stop writing here. + # Keys: external_call_id (required), transcript, recording_url. + # (DB column ``external_call_id`` is now ``conversation_id``; this + # JSON key stays as-is for historical batches.) + # Values: original CSV header strings (preserve user casing for export). + column_mapping = Column(JSON, nullable=False, default=dict) + # Ordered list of additional CSV header strings the uploader wants + # preserved verbatim into the evaluation export CSV. + extra_columns = Column(JSON, nullable=False, default=list) + # User-defined ``{custom_field_name: csv_header}`` mappings on top of + # the three system fields above. Cells from the mapped CSV columns are + # preserved per row (keyed by the CSV header in ``raw_columns``) and + # surface in the evaluation export under the uploader-chosen name. + custom_column_mapping = Column(JSON, nullable=False, default=dict) + + total_rows = Column(Integer, nullable=False, default=0) + completed_rows = Column(Integer, nullable=False, default=0) + failed_rows = Column(Integer, nullable=False, default=0) + + status = Column( + Enum(CallImportStatus, values_callable=get_enum_values), + nullable=False, + default=CallImportStatus.PENDING, + index=True, + ) + error_message = Column(Text, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + rows = relationship( + "CallImportRow", + back_populates="call_import", + cascade="all, delete-orphan", + order_by="CallImportRow.row_index", + ) + tags = relationship( + "CallImportTag", + secondary="call_import_tag_assignments", + backref="call_imports", + lazy="selectin", + ) + evaluations = relationship( + "CallImportEvaluation", + back_populates="call_import", + cascade="all, delete-orphan", + ) + + +class CallImportShardSlice(Base): + """Registry row: which shard stores a slice of rows for an import.""" + + __tablename__ = "call_import_shard_slices" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + slice_id = Column(Integer, primary_key=True) + shard_id = Column(String(64), nullable=False, index=True) + row_index_min = Column(Integer, nullable=False) + row_index_max = Column(Integer, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportRow(Base): + """A single row within a CallImport batch (one CSV line / one external call).""" + + __tablename__ = "call_import_rows" + __table_args__ = ( + UniqueConstraint("call_import_id", "row_index", name="uq_call_import_row_index"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + row_index = Column(Integer, nullable=False) + # Was historically named ``external_call_id``; renamed to + # ``conversation_id`` so the new schema-driven upload flow can refer + # to it by a single canonical name across the schema definition, + # exports, and downstream evaluation tables. + conversation_id = Column(String(255), nullable=False, index=True) + # Supplied via CSV for Exotel credentialed imports (required per row). + # Nullable in the schema for legacy rows imported before recording_url + # was mandatory on every Exotel upload. + recording_url = Column(Text, nullable=True) + # Date-only call recording date supplied by the import schema. Used + # for historical report comparisons without timezone/time ambiguity. + recording_date = Column(Date, nullable=True, index=True) + # The "production" transcript: the value supplied via the CSV + # upload mapping. Never overwritten by the diarisation worker — + # the worker writes its output into ``diarised_transcript`` so + # the user keeps both versions side by side. + transcript = Column(Text, nullable=True) + # Snapshot of the original CSV row keyed by the user's headers so the + # evaluation export can reproduce every column the uploader supplied + # (mapped + extra). NULL on legacy rows imported before this column. + raw_columns = Column(JSON, nullable=True) + + # Where the value in ``transcript`` came from. ``csv`` = supplied via + # the upload mapping, ``edited`` = manually changed in the UI. NULL + # on rows that have never had a production transcript. + # (Worker-produced transcripts now live in ``diarised_transcript`` + # and are tracked via ``diarised_transcript_*`` metadata below.) + transcript_source = Column(String(20), nullable=True) + # Provider/model recorded by the (legacy) post-hoc transcription + # worker. New worker runs leave these NULL and write into the + # ``diarised_transcript_*`` columns instead; kept on the model for + # backwards compatibility with pre-split rows that still carry the + # original transcription metadata here. + transcript_provider = Column(String(50), nullable=True) + transcript_model = Column(String(100), nullable=True) + # Lifecycle status for the legacy transcription workflow itself, + # independent of the row's recording-fetch ``status``. ``idle`` = + # no transcribe task has touched this column. New diarisation runs + # update ``diarised_transcript_status`` instead. + transcript_status = Column( + String(20), + nullable=False, + default="idle", + ) + transcript_error = Column(Text, nullable=True) + transcribed_at = Column(DateTime(timezone=True), nullable=True) + + # The "diarised" transcript: produced by the post-hoc + # transcription/diarisation worker. Stored separately so a manual + # diarisation run never clobbers the production transcript above. + # Evaluations can be configured to score against either column + # (see ``CallImportEvaluation.transcript_source``). + diarised_transcript = Column(Text, nullable=True) + # Provider/model the diarisation worker used. Surfaced in the UI + # as "Diarised via deepgram/nova-2" next to the diarised + # transcript section. + diarised_transcript_provider = Column(String(50), nullable=True) + diarised_transcript_model = Column(String(100), nullable=True) + # Lifecycle status for the diarisation workflow. + # ``idle`` = no diarisation task has run; ``pending``/``running`` = + # a Celery task is queued or in flight; ``completed``/``failed`` = + # terminal. Independent of ``transcript_status`` so the two + # transcripts can be in different lifecycle states. + diarised_transcript_status = Column( + String(20), + nullable=False, + default="idle", + server_default="idle", + ) + diarised_transcript_error = Column(Text, nullable=True) + diarised_at = Column(DateTime(timezone=True), nullable=True) + + # Structured speaker turns produced by the diarisation worker — + # ``[{ "speaker": "agent"|"user"|"speaker_3", "text": "...", + # "start": float, "end": float, "raw_speaker": "Speaker 1" }, ...]`` + # The plain-text ``diarised_transcript`` above is a rendered view + # of this list (``: `` per line). When the worker + # cannot recover structured turns (no pyannote token / single- + # speaker recording / provider that doesn't surface segments) this + # column stays NULL and the plain-text path is still populated. + diarised_segments = Column(JSON, nullable=True) + # When True the ``agent`` <-> ``user`` mapping inside + # ``diarised_segments`` is inverted at render / export time. The + # worker writes the canonical mapping using the "first speaker is + # the agent" heuristic; reviewers can flip the toggle from the row + # detail panel without re-running diarisation. + diarised_speaker_swap = Column( + Boolean, + nullable=False, + default=False, + server_default="false", + ) + # LLM that turned the STT plain-text output into structured + # ``diarised_segments``. The legacy diarisation worker used + # pyannote and left these NULL; the current path always runs an + # LLM with the operator-supplied (or default) ``diarised_prompt`` + # below, and records exactly which model + prompt produced each + # row so reviewers can reproduce a specific run. + diarised_llm_provider = Column(String(50), nullable=True) + diarised_llm_model = Column(String(100), nullable=True) + diarised_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + diarised_prompt = Column(Text, nullable=True) + # Which diarisation pipeline produced this row's turns. + # * ``"stt_llm"`` (default) — two-stage: STT then LLM diariser. + # ``diarised_transcript_provider``/``_model`` describe the STT + # side; ``diarised_llm_provider``/``_model`` the LLM side. + # * ``"llm_only"`` — single-stage: audio fed straight to a + # multimodal LLM. ``diarised_transcript_provider`` is stamped + # with the sentinel ``"llm_only"``; the real model is on + # ``diarised_llm_*``. + # Persisting it on the row (not just the run) lets the row detail + # panel render the right "Diarised via …" label even for ad-hoc + # standalone transcribes (no parent evaluation). + transcribe_mode = Column( + String(20), + nullable=False, + default="stt_llm", + server_default="stt_llm", + ) + + status = Column( + Enum(CallImportRowStatus, values_callable=get_enum_values), + nullable=False, + default=CallImportRowStatus.PENDING, + index=True, + ) + + recording_s3_key = Column(String(1024), nullable=True) + recording_content_type = Column(String(128), nullable=True) + recording_size_bytes = Column(Integer, nullable=True) + + error_message = Column(Text, nullable=True) + attempts = Column(Integer, nullable=False, default=0) + celery_task_id = Column(String(255), nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + call_import = relationship("CallImport", back_populates="rows") + + +@event.listens_for(CallImportRow, "before_insert") +def _call_import_row_fill_workspace_id(_mapper, connection, target): + """Denormalize workspace_id from the parent import when omitted.""" + if target.workspace_id is not None or target.call_import_id is None: + return + workspace_id = connection.execute( + select(CallImport.workspace_id).where( + CallImport.id == target.call_import_id + ) + ).scalar_one_or_none() + if workspace_id is not None: + target.workspace_id = workspace_id + + +class CallImportTag(Base): + """User-defined tag that can be attached to one or more call imports. + + Tags coexist with the free-text ``CallImport.dataset`` column: dataset + is the primary high-level segregation, tags are an optional secondary + classification (an import can have many tags). + """ + + __tablename__ = "call_import_tags" + __table_args__ = ( + UniqueConstraint("organization_id", "name", name="uq_call_import_tag_org_name"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + name = Column(String(255), nullable=False) + color = Column(String(32), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class CallImportTagAssignment(Base): + """Many-to-many join table between CallImport and CallImportTag.""" + + __tablename__ = "call_import_tag_assignments" + + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + primary_key=True, + ) + tag_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_tags.id", ondelete="CASCADE"), + primary_key=True, + index=True, + ) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +class CallImportEvaluation(Base): + """Parent record for an evaluation run over a CallImport batch. + + A user picks a subset of org ``Metric`` rows and triggers an evaluation; + we fan out one ``CallImportEvaluationRow`` per source row and roll up + counters as workers finish. Status mirrors ``CallImportStatus`` plus a + ``RUNNING`` value so the UI can distinguish "queued" from "in flight". + """ + + __tablename__ = "call_import_evaluations" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent CallImport's workspace. + # Denormalized for fast filter-by-workspace listings without a join. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id"), nullable=True + ) + last_updated_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL"), nullable=True + ) + + # Optional user-supplied label for this run. Lets the UI surface + # something more meaningful than the UUID prefix (e.g. "March QA pass"). + name = Column(String(255), nullable=True) + + # JSON list of Metric UUID strings selected for this run. Stored as text + # in JSON so we don't have to deal with PG arrays of UUIDs / cascade + # delete policies when metrics are removed; the loader filters for + # still-existing org metrics at run time. + selected_metric_ids = Column(JSON, nullable=False, default=list) + # Hierarchy grouping snapshot: ``{parent_id_str: [child_id_str, ...]}``. + # Captures which children belong to which parent for THIS run so the UI + # / aggregator can reconstruct the tree even when the user selected + # only a subset of children, or after metrics are deleted / renamed. + # NULL on legacy rows means "no hierarchy" → fall back to flat + # ``selected_metric_ids`` semantics. + selected_metric_groups = Column(JSON, nullable=True) + # User-driven merges of LLM-discovered candidate sub-labels for + # ``allow_discovery`` parents. Shape: + # ``{"": {"": "", ...}}``. + # Populated via ``POST .../discovered-labels/merge``; consulted by + # the discovered-labels aggregator, the flow graph builder, and the + # worker so that rows finishing AFTER a merge cannot reintroduce + # the merged-away slug. Empty dict on fresh rows. + discovered_label_aliases = Column( + JSON, nullable=False, default=dict, server_default="{}" + ) + + # Per-run opt-in for top-level metric discovery. When True, the LLM + # is asked to propose brand-new top-level metrics (boolean / rating / + # category) observed in the transcripts in addition to scoring the + # ``selected_metric_ids`` for the row. Candidates surface in a + # "Discovered metrics" panel on the evaluation's Flow tab and can + # be promoted into real standalone ``Metric`` rows via + # ``POST /metrics/from-discovered``. Defaults to False so existing + # evaluation creation payloads keep their previous behaviour. + discover_new_metrics = Column( + Boolean, nullable=False, default=False, server_default="false" + ) + # Flat slug-to-slug redirect map for user merges + tombstones of + # discovered top-level metric candidates. Mirrors + # ``discovered_label_aliases`` but is NOT nested per parent — + # top-level metric discovery is not scoped to any parent. Shape:: + # + # {"": "", ...} + # + # An empty-string value tombstones the slug so workers finishing + # later can't re-introduce it. + discovered_metric_aliases = Column( + JSON, nullable=False, default=dict, server_default="{}" + ) + + # Run-level LLM config picked from the Run Evaluation modal. NULL on + # legacy rows means "use the historical OpenAI/gpt-4o default" — the + # worker checks for this and falls back accordingly. ``llm_credential_id`` + # pins a specific AIProvider row when the org has multiple credentials + # for the same provider. + llm_provider = Column(String(50), nullable=True) + llm_model = Column(String(100), nullable=True) + llm_credential_id = Column( + UUID(as_uuid=True), + ForeignKey("aiproviders.id", ondelete="SET NULL"), + nullable=True, + ) + llm_config = Column(JSON, nullable=True) + # Optional per-metric LLM override: + # ``{"": {"provider": "...", "model": "...", "credential_id": "..."}}``. + # Each entry overrides the run-level default for that metric only; + # missing keys = use run-level default. Stored as JSON so the UI can + # round-trip arbitrary {provider, model} pairs without migrations. + metric_llm_overrides = Column(JSON, nullable=True) + + # When ``auto_transcribe`` was set on the create payload, record the + # STT provider/model used so the UI can show "Auto-transcribed via + # deepgram/nova-2" on the evaluation header. ``stt_credential_id`` is + # untyped (no FK) because STT keys may live in either ``aiproviders`` + # (OpenAI) or ``integrations`` (Deepgram, ElevenLabs) — the + # transcription service handles the lookup. + stt_provider = Column(String(50), nullable=True) + stt_model = Column(String(100), nullable=True) + stt_credential_id = Column(UUID(as_uuid=True), nullable=True) + + # Run-level LLM diariser config. Used when the create-run / + # retry-run paths chain a ``transcribe_call_import_row_task`` + # because the row is missing a diarised transcript. Persisted on + # the run so a retry uses the same diariser the original create + # call picked (unless the retry payload explicitly overrides). + diarisation_llm_provider = Column(String(50), nullable=True) + diarisation_llm_model = Column(String(100), nullable=True) + diarisation_llm_credential_id = Column(UUID(as_uuid=True), nullable=True) + diarisation_prompt = Column(Text, nullable=True) + # Mode the run was *created* with for its auto-transcribe step. + # Retry chains read this to decide whether to enqueue an STT+LLM + # transcribe or a single-stage multimodal LLM transcribe — without + # it we'd have to infer the mode from "stt_provider is NULL", which + # would silently break legacy rows that simply never configured + # auto-transcribe. See migration 041 for the column DDL. + transcribe_mode = Column( + String(20), + nullable=False, + default="stt_llm", + server_default="stt_llm", + ) + + # Which of the two transcripts on each ``CallImportRow`` this run + # scored against. ``'production'`` reads ``CallImportRow.transcript`` + # (the CSV-supplied value); ``'diarised'`` reads + # ``CallImportRow.diarised_transcript`` (the worker output). When + # the user ticks both checkboxes in the Run Evaluation modal we + # create two ``CallImportEvaluation`` rows — one per source — so + # the two scorings can be compared side-by-side. Defaults to + # ``'production'`` so legacy runs (which always read the single + # historical ``transcript`` column) keep their semantics. + transcript_source = Column( + String(20), + nullable=False, + default="production", + server_default="production", + ) + + # Cached LLM-generated TLDR rendered above the Visualizations charts. + # Populated lazily by ``POST /evaluations/{eval_id}/insights`` so we + # never auto-burn LLM tokens on page load. Shape:: + # {"narrative": str, "patterns": [str, ...], + # "generated_at": iso8601, "generated_at_completed_rows": int, + # "provider": str, "model": str} + # NULL on rows that have never been summarised. + tldr_summary = Column(JSON, nullable=True) + + # Cached LLM-generated user insights for External Audit PDF section 03. + # Populated by a background Celery job triggered alongside TLDR generation. + # Shape: EvaluationUserInsightsState JSON (status, insights[], progress, …). + user_insights = Column(JSON, nullable=True) + + # Cached per-metric failure clustering for internal diagnostics PDF/UI. + # Shape: EvaluationMetricClustersState JSON (status, groups[], …). + metric_clusters = Column(JSON, nullable=True) + + # Cached LLM-generated prompt improvement suggestions keyed to an + # imported agent (PromptPartial tagged __imported_agent__). + # Shape: EvaluationPromptImprovementsState JSON. + prompt_improvements = Column(JSON, nullable=True) + + # Cached LLM explanations for week-over-week metric deltas keyed by + # baseline evaluation id + completed row counts. + period_delta_explanations = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + + total_rows = Column(Integer, nullable=False, default=0) + completed_rows = Column(Integer, nullable=False, default=0) + failed_rows = Column(Integer, nullable=False, default=0) + # Flexprice pass-level delta billing watermark: rows already emitted + # on ``call_import.evaluation_completed`` for this evaluation run. + billed_completed_rows = Column( + Integer, nullable=False, default=0, server_default="0" + ) + error_message = Column(Text, nullable=True) + celery_group_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + call_import = relationship("CallImport", back_populates="evaluations") + row_results = relationship( + "CallImportEvaluationRow", + back_populates="evaluation", + cascade="all, delete-orphan", + ) + + +class CallImportEvaluationRow(Base): + """Per-source-row scoring output for a CallImportEvaluation parent.""" + + __tablename__ = "call_import_evaluation_rows" + __table_args__ = ( + UniqueConstraint( + "evaluation_id", "call_import_row_id", name="uq_call_import_evaluation_row" + ), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_row_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_rows.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + status = Column(String(20), nullable=False, default="pending", index=True) + # Same shape as EvaluatorResult.metric_scores: {metric_id_str: {value, type, metric_name, ...}} + metric_scores = Column(JSON, nullable=False, default=dict) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + evaluation = relationship("CallImportEvaluation", back_populates="row_results") + source_row = relationship("CallImportRow") + + +@event.listens_for(CallImportEvaluationRow, "before_insert") +def _call_import_evaluation_row_fill_workspace_id(_mapper, connection, target): + """Denormalize workspace_id from the parent evaluation when omitted.""" + if target.workspace_id is not None or target.evaluation_id is None: + return + workspace_id = connection.execute( + select(CallImportEvaluation.workspace_id).where( + CallImportEvaluation.id == target.evaluation_id + ) + ).scalar_one_or_none() + if workspace_id is not None: + target.workspace_id = workspace_id + + +class MetricStudioRun(Base): + """Ad-hoc metric experiment run in Metrics Studio.""" + + __tablename__ = "metric_studio_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + created_by_user_id = Column( + UUID(as_uuid=True), ForeignKey("users.id"), nullable=True + ) + + name = Column(String(255), nullable=True) + selected_metric_ids = Column(JSON, nullable=False, default=list) + selected_metric_groups = Column(JSON, nullable=True) + transcript_source = Column( + String(20), + nullable=False, + default="diarised", + server_default="diarised", + ) + + llm_provider = Column(String(50), nullable=True) + llm_model = Column(String(100), nullable=True) + llm_credential_id = Column( + UUID(as_uuid=True), + ForeignKey("aiproviders.id", ondelete="SET NULL"), + nullable=True, + ) + llm_config = Column(JSON, nullable=True) + metric_llm_overrides = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + total_items = Column(Integer, nullable=False, default=0) + completed_items = Column(Integer, nullable=False, default=0) + failed_items = Column(Integer, nullable=False, default=0) + error_message = Column(Text, nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + results = relationship( + "MetricStudioRunResult", + back_populates="run", + cascade="all, delete-orphan", + ) + + +class MetricStudioRunResult(Base): + """Per-source scoring output for a MetricStudioRun.""" + + __tablename__ = "metric_studio_run_results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + run_id = Column( + UUID(as_uuid=True), + ForeignKey("metric_studio_runs.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + source_kind = Column(String(40), nullable=False) + source_ref = Column(String(255), nullable=False) + display_label = Column(String(512), nullable=True) + source_metadata = Column(JSON, nullable=True) + + status = Column(String(20), nullable=False, default="pending", index=True) + metric_scores = Column(JSON, nullable=False, default=dict) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String(255), nullable=True) + + started_at = Column(DateTime(timezone=True), nullable=True) + finished_at = Column(DateTime(timezone=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + run = relationship("MetricStudioRun", back_populates="results") + + +class CallImportEvaluationReportSnapshot(Base): + """Persisted PDF-report aggregate used for period-over-period deltas.""" + + __tablename__ = "call_import_evaluation_report_snapshots" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + period_label = Column(String(64), nullable=True, index=True) + period_start = Column(Date, nullable=True, index=True) + period_end = Column(Date, nullable=True, index=True) + report_config = Column(JSON, nullable=False, default=dict, server_default="{}") + selected_metric_ids = Column(JSON, nullable=False, default=list, server_default="[]") + metric_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") + insight_aggregates = Column(JSON, nullable=False, default=list, server_default="[]") + narrative = Column(JSON, nullable=True) + total_calls = Column(Integer, nullable=False, default=0) + selected_metric_count = Column(Integer, nullable=False, default=0) + total_metric_count = Column(Integer, nullable=False, default=0) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + +class CallImportEvaluationPdfReport(Base): + """Stored PDF artifact for a call import evaluation report generation.""" + + __tablename__ = "call_import_evaluation_pdf_reports" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + evaluation_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + call_import_id = Column( + UUID(as_uuid=True), + ForeignKey("call_imports.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + snapshot_id = Column( + UUID(as_uuid=True), + ForeignKey("call_import_evaluation_report_snapshots.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + vendor_name = Column(String(120), nullable=False) + report_type = Column(String(20), nullable=False, default="external") + filename = Column(String(255), nullable=True) + s3_key = Column(String(512), nullable=True) + report_config = Column(JSON, nullable=False, default=dict, server_default="{}") + cache_fingerprint = Column(String(64), nullable=True) + created_by = Column(String, nullable=True) + created_by_user_id = Column(UUID(as_uuid=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + + +# --------------------------------------------------------------------------- +# Judge Alignment (AlignEval-style hybrid integration) +# +# Three tables back the "Judge Alignment" surface: +# - JudgeDataset: a labeled dataset materialised from one of three sources +# (voice transcripts, existing Metric/Evaluator outputs, +# or a generic CSV upload). Holds the dataset's source +# config + which fields play the role of input/output. +# - JudgeSample: one row in a dataset (input/output pair plus an +# optional binary pass/fail human label). +# - JudgeRun: a single run of an LLM-judge (existing Evaluator) over +# a subset of samples, with computed alignment metrics +# (precision/recall/F1/Cohen's kappa) and per-sample +# predictions. Optionally links to a GEPA optimization +# run when the user kicks off prompt tuning from a +# dataset. +# --------------------------------------------------------------------------- + + +class JudgeDataset(Base): + """Container for binary-labeled samples used to calibrate an LLM-judge.""" + + __tablename__ = "judge_datasets" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + # Workspace isolation: every judge dataset belongs to a workspace + # within its org. Samples and runs inherit this workspace_id. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + name = Column(String(255), nullable=False) + description = Column(Text, nullable=True) + + # One of: "transcript", "metric_output", "csv" + source_type = Column(String(32), nullable=False, index=True) + # Source-specific config. Examples: + # transcript: {"transcription_ids": [...]} or {"agent_id": "..."} + # metric_output: {"metric_id": "...", "evaluator_id": "..."} + # csv: {"s3_key": "...", "filename": "..."} + source_config = Column(JSON, nullable=False, default=dict) + + # Field roles - which textual content is "input" vs "output" for the judge. + # For voice transcripts both default to the transcript text but can be + # tightened (e.g. agent-only turns vs full conversation). + input_field = Column(String(64), nullable=False, default="input") + output_field = Column(String(64), nullable=False, default="output") + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + samples = relationship( + "JudgeSample", + back_populates="dataset", + cascade="all, delete-orphan", + order_by="JudgeSample.created_at", + ) + runs = relationship( + "JudgeRun", + back_populates="dataset", + cascade="all, delete-orphan", + order_by="JudgeRun.created_at.desc()", + ) + + +class JudgeSample(Base): + """One labelable input/output pair within a JudgeDataset.""" + + __tablename__ = "judge_samples" + __table_args__ = ( + UniqueConstraint("dataset_id", "external_id", name="uq_judge_samples_dataset_external"), + ) + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey("judge_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + # Workspace isolation: mirrors the parent JudgeDataset's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Stable identifier within the source (e.g. transcription UUID, CSV row id). + # Used to dedupe re-imports and link back to the originating record. + external_id = Column(String(128), nullable=True, index=True) + + input_text = Column(Text, nullable=False) + output_text = Column(Text, nullable=False) + + # Binary human label: "pass" | "fail" | null (unlabeled). + # Stored as string (rather than enum) so it stays trivially extendable. + label = Column(String(16), nullable=True, index=True) + labeled_by = Column(String(255), nullable=True) + labeled_at = Column(DateTime(timezone=True), nullable=True) + + # Source-specific context (e.g. agent_id, original metric value, csv row). + extra = Column(JSON, nullable=True) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + dataset = relationship("JudgeDataset", back_populates="samples") + + +class JudgeRun(Base): + """One execution of an LLM-judge against a JudgeDataset, with alignment metrics.""" + + __tablename__ = "judge_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id = Column( + UUID(as_uuid=True), + ForeignKey("judge_datasets.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + organization_id = Column( + UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True + ) + # Workspace isolation: mirrors the parent JudgeDataset's workspace. + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="RESTRICT"), + nullable=False, + index=True, + ) + + # Reuses the existing Evaluator row (its custom_prompt + llm_provider + llm_model + # define the judge under test). Nullable so a run may target an inline prompt + # in the future without inflating the Evaluator table. + evaluator_id = Column( + UUID(as_uuid=True), ForeignKey("evaluators.id", ondelete="SET NULL"), nullable=True, index=True + ) + + # Which subset was scored: "all" | "dev" | "test" + split = Column(String(16), nullable=False, default="all") + + # Snapshot of the model used (so a later Evaluator edit doesn't rewrite history). + llm_provider = Column(String(64), nullable=True) + llm_model = Column(String(128), nullable=True) + + # Computed alignment metrics: + # {"precision": float, "recall": float, "f1": float, "kappa": float, + # "tp": int, "fp": int, "tn": int, "fn": int, "n": int} + metrics = Column(JSON, nullable=True) + + # Per-sample predictions, keyed by sample_id (UUID string): + # {sample_id: {"prediction": "pass"|"fail", "explanation": str, "raw": str}} + predictions = Column(JSON, nullable=True) + + # Run lifecycle. + status = Column(String(20), nullable=False, default="pending", index=True) + error_message = Column(Text, nullable=True) + celery_task_id = Column(String, nullable=True, index=True) + + # Optional link to a GEPA optimization run kicked off from this dataset. + gepa_optimization_id = Column( + UUID(as_uuid=True), + ForeignKey("prompt_optimization_runs.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + created_by = Column(String, nullable=True) + + dataset = relationship("JudgeDataset", back_populates="runs") + + +class UsageCostRecomputeJob(Base): + """Async job tracking for retroactive usage cost recompute.""" + + __tablename__ = "usage_cost_recompute_jobs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + status = Column(String(32), nullable=False, default="pending", server_default="pending") + model = Column(String(255), nullable=True) + usage_kind = Column(String(16), nullable=True) + start_date = Column(Date, nullable=True) + end_date = Column(Date, nullable=True) + updated_rows = Column(BigInteger, nullable=False, default=0, server_default="0") + error_message = Column(String, nullable=True) + celery_task_id = Column(String(255), nullable=True, index=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + completed_at = Column(DateTime(timezone=True), nullable=True) + + +class LLMUsageDaily(Base): + """Daily LLM/STT usage rollups for org-scoped Usage reporting.""" + + __tablename__ = "llm_usage_daily" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column( + UUID(as_uuid=True), + ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + index=True, + ) + workspace_id = Column( + UUID(as_uuid=True), + ForeignKey("workspaces.id", ondelete="SET NULL"), + nullable=True, + index=True, + ) + product_section = Column(String(64), nullable=False, index=True) + model = Column(String(255), nullable=False, index=True) + context = Column(JSONB, nullable=False, server_default="{}", default=dict) + usage_date = Column(Date, nullable=False, index=True) + usage_kind = Column(String(16), nullable=False, default="llm", server_default="llm") + prompt_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + completion_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + cache_read_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + cache_creation_tokens = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + reasoning_tokens = Column(BigInteger, nullable=False, default=0, server_default="0") + audio_seconds = Column(BigInteger, nullable=False, default=0, server_default="0") + tts_characters = Column(BigInteger, nullable=False, default=0, server_default="0") + call_count = Column(BigInteger, nullable=False, default=0, server_default="0") + input_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + output_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + cache_read_cost_micro_usd = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + cache_creation_cost_micro_usd = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + reasoning_cost_micro_usd = Column( + BigInteger, nullable=False, default=0, server_default="0" + ) + audio_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + tts_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + total_cost_micro_usd = Column(BigInteger, nullable=False, default=0, server_default="0") + pricing_rate_source = Column(String(16), nullable=True) + pricing_rate_id = Column(UUID(as_uuid=True), nullable=True) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) diff --git a/app/models/schemas.py b/app/models/schemas.py index 40e988e9..32d66a03 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -1,5450 +1,5465 @@ -"""Pydantic schemas for request/response validation.""" - -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, validator -import re -from typing import Optional, List, Dict, Any, Literal -from datetime import date, datetime -from uuid import UUID -from app.models.enums import ( - EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, - LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, - IntegrationPlatform, ModelProvider, CredentialRoutingMode, GatewayInterfaceMode, VoiceBundleType, TestAgentConversationStatus, - MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, - AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, - CallImportStatus, CallImportRowStatus, CallImportParameterType, -) - - - -# Audio File Schemas -class AudioFileBase(BaseModel): - """Base audio file schema.""" - - filename: str - format: str - - -class AudioFileCreate(AudioFileBase): - """Schema for audio file creation.""" - - file_size: int - duration: Optional[float] = None - sample_rate: Optional[int] = None - channels: Optional[int] = None - - -class AudioFileResponse(AudioFileBase): - """Schema for audio file response.""" - - id: UUID - file_size: int - duration: Optional[float] = None - sample_rate: Optional[int] = None - channels: Optional[int] = None - uploaded_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -# Evaluation Schemas -class EvaluationCreate(BaseModel): - """Schema for creating an evaluation.""" - - audio_id: UUID - reference_text: Optional[str] = None - evaluation_type: EvaluationType - model_name: Optional[str] = Field(None, description="Model to use for evaluation") - metrics: Optional[List[str]] = Field( - default=["wer", "latency"], description="Metrics to calculate" - ) - - @field_validator("metrics") - @classmethod - def validate_metrics(cls, v): - """Validate metrics list.""" - allowed_metrics = ["wer", "cer", "latency", "quality_score", "rtf"] - if v: - invalid = [m for m in v if m not in allowed_metrics] - if invalid: - raise ValueError(f"Invalid metrics: {invalid}") - return v - - -class EvaluationResponse(BaseModel): - """Schema for evaluation response.""" - - id: UUID - audio_id: UUID - reference_text: Optional[str] = None - evaluation_type: EvaluationType - model_name: Optional[str] = None - status: EvaluationStatus - metrics_requested: Optional[List[str]] = None - created_at: datetime - started_at: Optional[datetime] = None - completed_at: Optional[datetime] = None - error_message: Optional[str] = None - - model_config = ConfigDict(from_attributes=True) - - -class EvaluationStatusResponse(BaseModel): - """Schema for evaluation status response.""" - - id: UUID - status: EvaluationStatus - created_at: datetime - completed_at: Optional[datetime] = None - error_message: Optional[str] = None - - -# Evaluation Result Schemas -class EvaluationResultResponse(BaseModel): - """Schema for evaluation result response.""" - - evaluation_id: UUID - status: EvaluationStatus - transcript: Optional[str] = None - metrics: Dict[str, Any] - processing_time: Optional[float] = None - model_used: Optional[str] = None - created_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class MetricsResponse(BaseModel): - """Schema for metrics breakdown.""" - - evaluation_id: UUID - metrics: Dict[str, Any] - processing_time: Optional[float] = None - - -# Comparison Schema -class ComparisonRequest(BaseModel): - """Schema for comparing multiple evaluations.""" - - evaluation_ids: List[UUID] = Field(..., min_length=2, description="At least 2 evaluation IDs to compare") - - -class ComparisonResponse(BaseModel): - """Schema for comparison results.""" - - evaluations: List[EvaluationResultResponse] - comparison_metrics: Dict[str, Any] - - -# API Key Schemas -class APIKeyCreate(BaseModel): - """Schema for creating API key.""" - - name: Optional[str] = None - - -class APIKeyResponse(BaseModel): - """Schema for API key response.""" - - id: UUID - key: str - name: Optional[str] = None - is_active: bool - created_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -# Generic Response Schemas -class MessageResponse(BaseModel): - """Generic message response.""" - - message: str - - -class ErrorResponse(BaseModel): - """Error response schema.""" - - detail: str - -# ============================================ -# VAIOPS SCHEMAS - Voice AI Ops -# ============================================ - -# Enums moved to enums.py - - -# Agent Schemas -class AgentCreate(BaseModel): - """Schema for creating a new agent""" - name: str = Field(..., min_length=1, max_length=255) - phone_number: Optional[str] = None - language: LanguageEnum = LanguageEnum.ENGLISH - description: str = Field(..., min_length=1) - call_type: CallTypeEnum = CallTypeEnum.OUTBOUND - call_medium: CallMediumEnum = CallMediumEnum.PHONE_CALL - telephony_phone_number_id: Optional[UUID] = None - voice_bundle_id: UUID = Field(..., description="Required voice bundle for test agent execution") - ai_provider_id: Optional[UUID] = None - voice_ai_integration_id: Optional[UUID] = None - voice_ai_agent_id: Optional[str] = None - provider_prompt: Optional[str] = None - silence_hangup_secs: int = Field( - default=15, - ge=0, - le=600, - description="End live calls after this many seconds of silence (0 disables)", - ) - - @field_validator('description') - @classmethod - def description_min_words(cls, v: str) -> str: - if len(v.split()) < 10: - raise ValueError('Description must be at least 10 words.') - return v - - @field_validator('phone_number') - @classmethod - def phone_number_format(cls, v: Optional[str]) -> Optional[str]: - if v is not None and v != '': - import re - if not re.fullmatch(r'[\d+]+', v): - raise ValueError('Phone number must contain only digits and the + character.') - return v - - @model_validator(mode='after') - def validate_phone_number(self): - """Ensure phone_number is provided when call_medium is phone_call""" - if self.call_medium == CallMediumEnum.PHONE_CALL and not self.phone_number: - raise ValueError('phone_number is required when call_medium is phone_call') - return self - - model_config = ConfigDict(json_schema_extra={ - "example": { - "name": "Customer Support Bot", - "phone_number": "+1234567890", - "language": "en", - "description": "A customer support bot that handles inquiries about orders, returns, and general questions", - "call_type": "outbound", - "voice_bundle_id": "123e4567-e89b-12d3-a456-426614174000", - "voice_ai_integration_id": "123e4567-e89b-12d3-a456-426614174001", - "voice_ai_agent_id": "agent_abc123" - } - }) - - -class AgentUpdate(BaseModel): - """Schema for updating an agent""" - name: Optional[str] = None - phone_number: Optional[str] = None - language: Optional[LanguageEnum] = None - description: Optional[str] = None - call_type: Optional[CallTypeEnum] = None - call_medium: Optional[CallMediumEnum] = None - telephony_phone_number_id: Optional[UUID] = None - voice_bundle_id: Optional[UUID] = None - voice_ai_integration_id: Optional[UUID] = None - voice_ai_agent_id: Optional[str] = None - provider_prompt: Optional[str] = None - prompt_variables: Optional[Dict[str, str]] = None - silence_hangup_secs: Optional[int] = Field(default=None, ge=0, le=600) - - @model_validator(mode='after') - def validate_voice_config(self): - """Validate voice configuration - both voice_bundle_id and voice_ai_integration_id can be provided independently""" - voice_bundle = self.voice_bundle_id - voice_ai_integration = self.voice_ai_integration_id - - # If voice_ai_integration_id is provided, voice_ai_agent_id must also be provided - if voice_ai_integration and not self.voice_ai_agent_id: - raise ValueError('voice_ai_agent_id is required when voice_ai_integration_id is provided.') - - return self - - @model_validator(mode='after') - def validate_phone_number(self): - """Ensure phone_number is provided when call_medium is phone_call""" - # Only validate if call_medium is being set to phone_call - if self.call_medium == CallMediumEnum.PHONE_CALL and not self.phone_number: - # If phone_number is not being updated, we need to check existing value - # This will be handled in the route - pass - return self - - - -class PreviewIntegrationAgentPromptRequest(BaseModel): - """Fetch a provider agent prompt before an EfficientAI agent exists.""" - voice_ai_agent_id: str = Field(..., min_length=1) - - - - -class PreviewIntegrationAgentPromptResponse(BaseModel): - provider_prompt: str - - - - -class AgentPhoneAssignmentConflict(BaseModel): - """Another agent already owns this phone number.""" - agent_id: UUID - agent_name: str - phone_number: str - - - - -class AgentPhoneAssignmentCheckResponse(BaseModel): - """Result of checking whether a phone number is free to assign.""" - available: bool - phone_number: Optional[str] = None - conflict: Optional[AgentPhoneAssignmentConflict] = None - - - - -class TestPromptSectionResponse(BaseModel): - """One canonical section of a generated test agent prompt.""" - key: str - title: str - content: str - - - - -class GeneratedScenarioDraftResponse(BaseModel): - """LLM-generated scenario draft before persistence.""" - name: str - description: str - goal: Optional[str] = None - - - - -class GenerateTestPromptRequest(BaseModel): - """Stage 1: generate foundational test agent prompt from production prompt.""" - production_prompt: str = Field(..., min_length=1) - agent_name: str = Field(..., min_length=1, max_length=255) - language: Optional[str] = None - call_type: Optional[str] = None - provider: Optional[str] = None - model: Optional[str] = None - credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - additional_context: Optional[str] = None - - - - -class GenerateTestPromptResponse(BaseModel): - sections: List[TestPromptSectionResponse] - test_agent_prompt: str - provider: str - model: str - - - - -class GenerateScenariosFromPromptRequest(BaseModel): - """Stage 2: generate scenario drafts from test agent prompt.""" - test_agent_prompt: str = Field(..., min_length=1) - agent_name: str = Field(..., min_length=1, max_length=255) - scenario_count: int = Field(default=5, ge=1, le=10) - language: Optional[str] = None - call_type: Optional[str] = None - provider: Optional[str] = None - model: Optional[str] = None - credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - additional_context: Optional[str] = None - - - - -class GenerateScenariosFromPromptResponse(BaseModel): - scenarios: List[GeneratedScenarioDraftResponse] - provider: str - model: str - - - - -class GenerateTestSetupRequest(BaseModel): - """Convenience: run stage 1 then stage 2 sequentially.""" - production_prompt: str = Field(..., min_length=1) - agent_name: str = Field(..., min_length=1, max_length=255) - scenario_count: int = Field(default=5, ge=1, le=10) - language: Optional[str] = None - call_type: Optional[str] = None - provider: Optional[str] = None - model: Optional[str] = None - credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - additional_context: Optional[str] = None - - - - -class GenerateTestSetupResponse(BaseModel): - sections: List[TestPromptSectionResponse] - test_agent_prompt: str - scenarios: List[GeneratedScenarioDraftResponse] - provider: str - model: str - - - -class AgentResponse(BaseModel): - """Schema for agent response""" - id: UUID - agent_id: Optional[str] = None - name: str - phone_number: Optional[str] = None - language: LanguageEnum - description: Optional[str] - call_type: CallTypeEnum - call_medium: CallMediumEnum - telephony_phone_number_id: Optional[UUID] = None - voice_bundle_id: Optional[UUID] - ai_provider_id: Optional[UUID] - voice_ai_integration_id: Optional[UUID] - voice_ai_agent_id: Optional[str] - provider_prompt: Optional[str] = None - prompt_variables: Optional[Dict[str, str]] = None - silence_hangup_secs: int = Field( - default=15, - ge=0, - le=600, - description="End live calls after this many seconds of silence (0 disables)", - ) - provider_prompt_synced_at: Optional[datetime] = None - created_at: datetime - updated_at: datetime - - @field_validator('language', mode='before') - @classmethod - def convert_language(cls, v): - """Convert string to LanguageEnum (handles uppercase DB values like ENGLISH -> en).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - # Map old uppercase names to new values - language_map = {'english': 'en', 'spanish': 'es', 'french': 'fr', 'german': 'de', - 'chinese': 'zh', 'japanese': 'ja', 'hindi': 'hi', 'arabic': 'ar'} - if v_lower in language_map: - return LanguageEnum(language_map[v_lower]) - try: - return LanguageEnum(v_lower) - except ValueError: - for enum_member in LanguageEnum: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid LanguageEnum value: {v}") - return v - - @field_validator('call_type', mode='before') - @classmethod - def convert_call_type(cls, v): - """Convert string to CallTypeEnum (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return CallTypeEnum(v_lower) - except ValueError: - for enum_member in CallTypeEnum: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid CallTypeEnum value: {v}") - return v - - @field_validator('call_medium', mode='before') - @classmethod - def convert_call_medium(cls, v): - """Convert string to CallMediumEnum (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return CallMediumEnum(v_lower) - except ValueError: - for enum_member in CallMediumEnum: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid CallMediumEnum value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -# Persona Schemas -class PersonaCreate(BaseModel): - """Schema for creating a new persona (TTS provider-tied voice identity)""" - name: str = Field(..., min_length=1, max_length=255) - gender: GenderEnum = GenderEnum.NEUTRAL - tts_provider: Optional[str] = None - tts_voice_id: Optional[str] = None - tts_voice_name: Optional[str] = None - is_custom: bool = False - description: Optional[str] = None - tts_config: Optional[Dict[str, Any]] = None - llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - llm_max_tokens: Optional[int] = Field(None, gt=0, le=8192) - response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) - max_turns: Optional[int] = Field(None, ge=1, le=100) - allow_interruptions: Optional[bool] = None - - @model_validator(mode="after") - def validate_tts_config(self): - from app.services.personas.persona_tts_config import validate_persona_tts_config - - validate_persona_tts_config(self.tts_provider, self.tts_config) - return self - - -class PersonaUpdate(BaseModel): - """Schema for updating a persona""" - name: Optional[str] = None - gender: Optional[GenderEnum] = None - tts_provider: Optional[str] = None - tts_voice_id: Optional[str] = None - tts_voice_name: Optional[str] = None - is_custom: Optional[bool] = None - description: Optional[str] = None - tts_config: Optional[Dict[str, Any]] = None - llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - llm_max_tokens: Optional[int] = Field(None, gt=0, le=8192) - response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) - max_turns: Optional[int] = Field(None, ge=1, le=100) - allow_interruptions: Optional[bool] = None - - @model_validator(mode="after") - def validate_tts_config(self): - from app.services.personas.persona_tts_config import validate_persona_tts_config - - if self.tts_config is not None: - validate_persona_tts_config(self.tts_provider, self.tts_config) - return self - - -class PersonaResponse(BaseModel): - """Schema for persona response""" - id: UUID - name: str - gender: str - tts_provider: Optional[str] = None - tts_voice_id: Optional[str] = None - tts_voice_name: Optional[str] = None - is_custom: bool = False - description: Optional[str] = None - tts_config: Optional[Dict[str, Any]] = None - llm_temperature: Optional[float] = None - llm_max_tokens: Optional[int] = None - response_delay_ms: Optional[int] = None - max_turns: Optional[int] = None - allow_interruptions: Optional[bool] = None - created_at: datetime - updated_at: datetime - - @field_validator('gender', mode='before') - @classmethod - def convert_gender(cls, v): - if v is None: - return "neutral" - if isinstance(v, str): - return v.lower() - if hasattr(v, 'value'): - return v.value - return v - - model_config = ConfigDict(from_attributes=True) - - -class PersonaCloneRequest(BaseModel): - """Schema for cloning a persona""" - name: Optional[str] = None - - -# Scenario Schemas - -class AgentPromptSourcesResponse(BaseModel): - """Prompt texts from an agent that can seed a persona description.""" - agent_id: UUID - agent_name: str - test_agent_prompt: str - agent_prompt: str - - - - -class GeneratePersonaPromptRequest(BaseModel): - """Generate a persona caller prompt from an agent prompt via LLM.""" - agent_id: UUID - source: str = Field(default="auto", pattern="^(test_agent|agent|auto)$") - persona_name: Optional[str] = Field(None, max_length=255) - persona_gender: Optional[str] = None - additional_context: Optional[str] = None - provider: Optional[str] = None - model: Optional[str] = None - credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - - - - -class GeneratePersonaPromptResponse(BaseModel): - persona_prompt: str - source_used: str - provider: str - model: str - - -# Scenario Schemas - -class ScenarioCreate(BaseModel): - """Schema for creating a new scenario""" - name: str = Field(..., min_length=1, max_length=255) - agent_id: Optional[UUID] = None - description: Optional[str] = None - required_info: Dict[str, str] = Field(default_factory=dict) - - -class ScenarioUpdate(BaseModel): - """Schema for updating a scenario""" - name: Optional[str] = None - agent_id: Optional[UUID] = None - description: Optional[str] = None - required_info: Optional[Dict[str, str]] = None - - -class ScenarioResponse(BaseModel): - """Schema for scenario response""" - id: UUID - name: str - agent_id: Optional[UUID] - description: Optional[str] - required_info: Dict[str, str] - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# IAM & USER SCHEMAS -# ============================================ - -# User Schemas -class UserCreate(BaseModel): - """Schema for creating a user.""" - email: str = Field(..., description="User email address") - name: Optional[str] = None - password: Optional[str] = None # Optional for invitation-based signup - - -class UserUpdate(BaseModel): - """Schema for updating user profile.""" - name: Optional[str] = None - first_name: Optional[str] = None - last_name: Optional[str] = None - email: Optional[str] = None - - -class UserResponse(BaseModel): - """Schema for user response.""" - id: UUID - email: str - name: Optional[str] - first_name: Optional[str] - last_name: Optional[str] - is_active: bool - created_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class OrganizationMemberResponse(BaseModel): - """Schema for organization member response.""" - id: UUID - user_id: UUID - organization_id: UUID - role: RoleEnum - joined_at: datetime - user: UserResponse # Include user details - - model_config = ConfigDict(from_attributes=True) - - -# Invitation Schemas -class InvitationCreate(BaseModel): - """Schema for creating an invitation.""" - email: str = Field(..., description="Email address of the user to invite") - role: RoleEnum = RoleEnum.READER - - -class InvitationResponse(BaseModel): - """Schema for invitation response.""" - id: UUID - organization_id: UUID - email: str - role: RoleEnum - status: InvitationStatus - expires_at: datetime - created_at: datetime - organization_name: Optional[str] = None # Include organization name - invite_path: Optional[str] = None - invite_url: Optional[str] = None - - @field_validator('role', mode='before') - @classmethod - def convert_role(cls, v): - """Convert string to RoleEnum (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return RoleEnum(v_lower) - except ValueError: - for enum_member in RoleEnum: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid RoleEnum value: {v}") - return v - - @field_validator('status', mode='before') - @classmethod - def convert_status(cls, v): - """Convert string to InvitationStatus (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return InvitationStatus(v_lower) - except ValueError: - for enum_member in InvitationStatus: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid InvitationStatus value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -class InvitationUpdate(BaseModel): - """Schema for updating invitation (accept/decline).""" - token: str - - -class RoleUpdate(BaseModel): - """Schema for updating user role in organization.""" - role: RoleEnum - - -# Profile Schemas -class ProfileResponse(BaseModel): - """Schema for user profile response.""" - id: UUID - email: str - name: Optional[str] - first_name: Optional[str] - last_name: Optional[str] - created_at: datetime - organizations: List[dict] = Field(default_factory=list) # List of org memberships - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# INTEGRATION SCHEMAS -# ============================================ - -class IntegrationCreate(BaseModel): - """Schema for creating an integration.""" - platform: IntegrationPlatform - api_key: str = Field(..., description="Private API key for the platform") - public_key: Optional[str] = Field(None, description="Optional public API key (e.g. for Vapi)") - name: Optional[str] = Field(None, description="Optional friendly name for the integration") - routing_mode: CredentialRoutingMode = Field( - CredentialRoutingMode.INHERIT, - description="LLM routing preference: inherit org default, force gateway, or direct API key.", - ) - is_default: Optional[bool] = Field( - None, - description=( - "Mark this credential as the default for the (org, platform). " - "If omitted and no default exists yet, this row becomes the default." - ), - ) - - -class IntegrationUpdate(BaseModel): - """Schema for updating an integration.""" - name: Optional[str] = None - api_key: Optional[str] = None - public_key: Optional[str] = None - is_active: Optional[bool] = None - routing_mode: Optional[CredentialRoutingMode] = None - - -class IntegrationResponse(BaseModel): - """Schema for integration response.""" - id: UUID - organization_id: UUID - platform: IntegrationPlatform - name: Optional[str] - public_key: Optional[str] = None - is_active: bool - is_default: bool = False - routing_mode: CredentialRoutingMode = CredentialRoutingMode.INHERIT - effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" - created_at: datetime - updated_at: datetime - last_tested_at: Optional[datetime] = None - # Note: api_key is NOT included in response for security - - @field_validator('platform', mode='before') - @classmethod - def convert_platform(cls, v): - """Convert string to IntegrationPlatform enum if needed (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - # Try lowercase first (enum value) - v_lower = v.lower() - try: - return IntegrationPlatform(v_lower) - except ValueError: - # Try to find by enum name (uppercase) - for enum_member in IntegrationPlatform: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid IntegrationPlatform value: {v}") - return v - - @field_validator('routing_mode', mode='before') - @classmethod - def convert_routing_mode(cls, v): - if v is None: - return CredentialRoutingMode.INHERIT - if isinstance(v, str): - try: - return CredentialRoutingMode(v.lower()) - except ValueError: - return CredentialRoutingMode.INHERIT - return v - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# DATA SOURCES SCHEMAS -# ============================================ - -class S3ConnectionTest(BaseModel): - """Schema for testing S3 connection.""" - bucket_name: str - region: str = "us-east-1" - access_key_id: str - secret_access_key: str - endpoint_url: Optional[str] = None - - -class S3ConnectionTestResponse(BaseModel): - """Schema for S3 connection test response.""" - success: bool - message: str - bucket_name: Optional[str] = None - - -class S3FileInfo(BaseModel): - """Schema for S3 file information.""" - key: str - filename: str - size: int - last_modified: str - - -class S3ListFilesResponse(BaseModel): - """Schema for listing S3 files response.""" - files: List[S3FileInfo] - total: int - prefix: Optional[str] = None - - -class S3FolderInfo(BaseModel): - """Schema for S3 folder information.""" - name: str - path: str - - -class S3BrowseResponse(BaseModel): - """Schema for browsing S3 folders within an organization.""" - folders: List[S3FolderInfo] - files: List[S3FileInfo] - current_path: str - organization_id: str - - -class S3UploadResponse(BaseModel): - """Schema for S3 upload response.""" - key: str - bucket: str - file_id: UUID - message: str - - -# AIProvider Schemas -_MAX_GATEWAY_EXTRA_HEADERS = 20 - - -def _validate_gateway_extra_headers( - value: Optional[Dict[str, Any]], -) -> Optional[Dict[str, str]]: - if value is None: - return None - if not isinstance(value, dict): - raise ValueError("gateway_extra_headers must be a JSON object of string keys and values.") - if len(value) > _MAX_GATEWAY_EXTRA_HEADERS: - raise ValueError( - f"gateway_extra_headers supports at most {_MAX_GATEWAY_EXTRA_HEADERS} headers." - ) - normalized: Dict[str, str] = {} - for raw_key, raw_val in value.items(): - key = str(raw_key).strip() - if not key: - raise ValueError("gateway_extra_headers keys must be non-empty strings.") - if len(key) > 64 or any(ch.isspace() for ch in key): - raise ValueError(f"Invalid gateway header name: {key!r}") - if raw_val is None: - raise ValueError(f"gateway_extra_headers[{key!r}] must be a string value.") - val = str(raw_val).strip() - if not val: - raise ValueError(f"gateway_extra_headers[{key!r}] must be a non-empty string.") - if len(val) > 1024 or "\n" in val or "\r" in val: - raise ValueError(f"gateway_extra_headers[{key!r}] value is invalid.") - normalized[key] = val - return normalized or None - - -class AIProviderCreate(BaseModel): - """Schema for creating an AI Provider.""" - provider: ModelProvider - api_key: Optional[str] = Field( - None, - description=( - "Provider API key. Optional when routing via gateway with " - "gateway-managed credentials (passthrough_provider_keys: false)." - ), - ) - name: Optional[str] = None - routing_mode: CredentialRoutingMode = Field( - CredentialRoutingMode.INHERIT, - description="LLM routing preference: inherit org default, force gateway, or direct API key.", - ) - gateway_model: Optional[str] = Field( - None, - min_length=1, - max_length=255, - description="Bifrost custom model ID sent when routing via gateway.", - ) - gateway_interface: GatewayInterfaceMode = Field( - GatewayInterfaceMode.INHERIT, - description="Bifrost API surface: inherit org default, LiteLLM shim, or native OpenAI-compatible.", - ) - gateway_base_url: Optional[str] = Field( - None, - max_length=512, - description="Optional per-credential Bifrost/gateway base URL override.", - ) - gateway_auth_header: Optional[str] = Field( - None, - max_length=64, - description="Auth header name for Bifrost (default x-bf-vk).", - ) - gateway_auth_secret_env: Optional[str] = Field( - None, - max_length=128, - description="Environment variable name holding the gateway auth secret.", - ) - gateway_auth_secret: Optional[str] = Field( - None, - description="Inline gateway auth secret (encrypted at rest). Alternative to env var.", - ) - gateway_extra_headers: Optional[Dict[str, str]] = Field( - None, - description="Arbitrary HTTP headers sent with gateway-routed LiteLLM calls.", - ) - enabled_models: Optional[List[str]] = Field( - None, - description=( - "Allowlisted model names for this credential. " - "Null or empty means all catalog models for the provider." - ), - ) - is_default: Optional[bool] = Field( - None, - description=( - "Mark this credential as the default for the (org, provider). " - "If omitted and no default exists yet, this row becomes the default." - ), - ) - endpoint_url: Optional[str] = Field( - None, - description="Provider endpoint URL (required for Azure OpenAI).", - ) - - @field_validator("api_key") - @classmethod - def validate_api_key(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("endpoint_url") - @classmethod - def validate_endpoint_url(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_model") - @classmethod - def validate_gateway_model(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_base_url") - @classmethod - def validate_gateway_base_url(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_auth_header") - @classmethod - def validate_gateway_auth_header(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - if not trimmed: - return None - if len(trimmed) > 64 or any(ch.isspace() for ch in trimmed): - raise ValueError("gateway_auth_header must be a single non-empty header name.") - return trimmed - - @field_validator("gateway_auth_secret_env") - @classmethod - def validate_gateway_auth_secret_env(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - if not trimmed: - return None - if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", trimmed): - raise ValueError( - "gateway_auth_secret_env must be a valid environment variable name." - ) - return trimmed - - @field_validator("gateway_auth_secret") - @classmethod - def validate_gateway_auth_secret(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_extra_headers") - @classmethod - def validate_gateway_extra_headers(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, str]]: - return _validate_gateway_extra_headers(v) - - @field_validator("enabled_models") - @classmethod - def validate_enabled_models_create(cls, v: Optional[List[str]]) -> Optional[List[str]]: - if v is None: - return None - seen: set[str] = set() - out: list[str] = [] - for item in v: - name = str(item).strip() - if not name or name in seen: - continue - seen.add(name) - out.append(name) - return out or None - - -class AIProviderUpdate(BaseModel): - """Schema for updating an AI Provider.""" - api_key: Optional[str] = Field(None, min_length=1) - name: Optional[str] = None - endpoint_url: Optional[str] = None - is_active: Optional[bool] = None - routing_mode: Optional[CredentialRoutingMode] = None - gateway_model: Optional[str] = Field(None, min_length=1, max_length=255) - gateway_interface: Optional[GatewayInterfaceMode] = None - gateway_base_url: Optional[str] = Field(None, max_length=512) - gateway_auth_header: Optional[str] = Field(None, max_length=64) - gateway_auth_secret_env: Optional[str] = Field(None, max_length=128) - gateway_auth_secret: Optional[str] = None - clear_gateway_auth_secret: bool = False - gateway_extra_headers: Optional[Dict[str, str]] = None - enabled_models: Optional[List[str]] = None - - @field_validator("gateway_model") - @classmethod - def validate_gateway_model(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_base_url") - @classmethod - def validate_gateway_base_url_update(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_auth_header") - @classmethod - def validate_gateway_auth_header_update(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - if not trimmed: - return None - if len(trimmed) > 64 or any(ch.isspace() for ch in trimmed): - raise ValueError("gateway_auth_header must be a single non-empty header name.") - return trimmed - - @field_validator("gateway_auth_secret_env") - @classmethod - def validate_gateway_auth_secret_env_update(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - if not trimmed: - return None - if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", trimmed): - raise ValueError( - "gateway_auth_secret_env must be a valid environment variable name." - ) - return trimmed - - @field_validator("gateway_auth_secret") - @classmethod - def validate_gateway_auth_secret_update(cls, v: Optional[str]) -> Optional[str]: - if v is None: - return v - trimmed = v.strip() - return trimmed or None - - @field_validator("gateway_extra_headers") - @classmethod - def validate_gateway_extra_headers_update( - cls, v: Optional[Dict[str, Any]] - ) -> Optional[Dict[str, str]]: - return _validate_gateway_extra_headers(v) - - @field_validator("enabled_models") - @classmethod - def validate_enabled_models_update(cls, v: Optional[List[str]]) -> Optional[List[str]]: - if v is None: - return None - seen: set[str] = set() - out: list[str] = [] - for item in v: - name = str(item).strip() - if not name or name in seen: - continue - seen.add(name) - out.append(name) - return out or None - - -class AIProviderResponse(BaseModel): - """Schema for AI Provider response.""" - id: UUID - provider: ModelProvider - api_key: Optional[str] = None # Will be None in response for security - name: Optional[str] - endpoint_url: Optional[str] = None - is_active: bool - is_default: bool = False - routing_mode: CredentialRoutingMode = CredentialRoutingMode.INHERIT - gateway_model: Optional[str] = None - gateway_interface: GatewayInterfaceMode = GatewayInterfaceMode.INHERIT - gateway_base_url: Optional[str] = None - gateway_auth_header: Optional[str] = None - gateway_auth_secret_env: Optional[str] = None - has_gateway_auth_secret: bool = False - gateway_extra_headers: Optional[Dict[str, str]] = None - enabled_models: Optional[List[str]] = None - gateway_managed: bool = False - effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" - effective_gateway_interface: Literal["litellm_shim", "native_openai"] = "litellm_shim" - created_at: datetime - updated_at: datetime - last_tested_at: Optional[datetime] - - @field_validator('provider', mode='before') - @classmethod - def convert_provider(cls, v): - """Convert string to ModelProvider (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return ModelProvider(v_lower) - except ValueError: - for enum_member in ModelProvider: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid ModelProvider value: {v}") - return v - - @field_validator('routing_mode', mode='before') - @classmethod - def convert_routing_mode(cls, v): - if v is None: - return CredentialRoutingMode.INHERIT - if isinstance(v, str): - try: - return CredentialRoutingMode(v.lower()) - except ValueError: - return CredentialRoutingMode.INHERIT - return v - - model_config = ConfigDict(from_attributes=True) - - -class LLMGenerationConfig(BaseModel): - """User-tunable LLM sampling / generation parameters.""" - - temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - max_tokens: Optional[int] = Field(None, gt=0) - top_p: Optional[float] = Field(None, ge=0.0, le=1.0) - top_k: Optional[int] = Field(None, ge=0) - frequency_penalty: Optional[float] = Field(None, ge=-2.0, le=2.0) - presence_penalty: Optional[float] = Field(None, ge=-2.0, le=2.0) - seed: Optional[int] = Field(None, ge=0) - - def to_dict(self) -> Dict[str, Any]: - """Return only explicitly set fields.""" - return self.model_dump(exclude_none=True) - - -# VoiceBundle Schemas -class VoiceBundleCreate(BaseModel): - """Schema for creating a VoiceBundle.""" - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = None - - # Bundle type: either STT+LLM+TTS or S2S - bundle_type: VoiceBundleType = Field(default=VoiceBundleType.STT_LLM_TTS) - - # STT Configuration - required for STT_LLM_TTS, optional for S2S - stt_provider: Optional[ModelProvider] = None - stt_model: Optional[str] = Field(None, min_length=1) - stt_credential_id: Optional[UUID] = Field( - None, - description=( - "Optional explicit AIProvider/Integration row id to use for STT. " - "When omitted the resolver picks the default credential for stt_provider." - ), - ) - - # LLM Configuration - required for STT_LLM_TTS, optional for S2S - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = Field(None, min_length=1) - llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - llm_max_tokens: Optional[int] = Field(None, gt=0) - llm_config: Optional[Dict[str, Any]] = None - llm_credential_id: Optional[UUID] = None - - # TTS Configuration - required for STT_LLM_TTS, optional for S2S - tts_provider: Optional[ModelProvider] = None - tts_model: Optional[str] = Field(None, min_length=1) - tts_voice: Optional[str] = None - tts_config: Optional[Dict[str, Any]] = None - tts_credential_id: Optional[UUID] = None - - # S2S Configuration - required for S2S, optional for STT_LLM_TTS - s2s_provider: Optional[ModelProvider] = None - s2s_model: Optional[str] = Field(None, min_length=1) - s2s_config: Optional[Dict[str, Any]] = None - s2s_credential_id: Optional[UUID] = None - - # Additional metadata - extra_metadata: Optional[Dict[str, Any]] = None - - @model_validator(mode='after') - def validate_bundle_configuration(self): - """Validate that required fields are provided based on bundle_type.""" - if self.bundle_type == VoiceBundleType.STT_LLM_TTS: - if not self.stt_provider or not self.stt_model: - raise ValueError('STT provider and model are required for STT_LLM_TTS bundle type') - if not self.llm_provider or not self.llm_model: - raise ValueError('LLM provider and model are required for STT_LLM_TTS bundle type') - if not self.tts_provider or not self.tts_model: - raise ValueError('TTS provider and model are required for STT_LLM_TTS bundle type') - elif self.bundle_type == VoiceBundleType.S2S: - if not self.s2s_provider or not self.s2s_model: - raise ValueError('S2S provider and model are required for S2S bundle type') - return self - - -class VoiceBundleUpdate(BaseModel): - """Schema for updating a VoiceBundle.""" - name: Optional[str] = Field(None, min_length=1, max_length=255) - description: Optional[str] = None - - # Bundle type - bundle_type: Optional[VoiceBundleType] = None - - # STT Configuration - stt_provider: Optional[ModelProvider] = None - stt_model: Optional[str] = Field(None, min_length=1) - stt_credential_id: Optional[UUID] = None - - # LLM Configuration - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = Field(None, min_length=1) - llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) - llm_max_tokens: Optional[int] = Field(None, gt=0) - llm_config: Optional[Dict[str, Any]] = None - llm_credential_id: Optional[UUID] = None - - # TTS Configuration - tts_provider: Optional[ModelProvider] = None - tts_model: Optional[str] = Field(None, min_length=1) - tts_voice: Optional[str] = None - tts_config: Optional[Dict[str, Any]] = None - tts_credential_id: Optional[UUID] = None - - # S2S Configuration - s2s_provider: Optional[ModelProvider] = None - s2s_model: Optional[str] = Field(None, min_length=1) - s2s_config: Optional[Dict[str, Any]] = None - s2s_credential_id: Optional[UUID] = None - - # Additional metadata - extra_metadata: Optional[Dict[str, Any]] = None - is_active: Optional[bool] = None - - -class VoiceBundleResponse(BaseModel): - """Schema for VoiceBundle response.""" - id: UUID - name: str - description: Optional[str] - - # Bundle type - can be string from DB or enum, validator handles conversion - bundle_type: VoiceBundleType - - @field_validator('bundle_type', mode='before') - @classmethod - def convert_bundle_type(cls, v): - """Convert string to VoiceBundleType enum if needed.""" - if isinstance(v, str): - try: - return VoiceBundleType(v) - except ValueError: - # Try to find by value - for enum_member in VoiceBundleType: - if enum_member.value == v: - return enum_member - raise ValueError(f"Invalid bundle_type value: {v}") - return v - - @field_validator('stt_provider', 'llm_provider', 'tts_provider', 's2s_provider', mode='before') - @classmethod - def convert_model_provider(cls, v): - """Convert string to ModelProvider enum if needed (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - # Try lowercase first (enum value) - v_lower = v.lower() - try: - return ModelProvider(v_lower) - except ValueError: - # Try to find by enum name (uppercase) - for enum_member in ModelProvider: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid ModelProvider value: {v}") - return v - - # STT Configuration - stt_provider: Optional[ModelProvider] - stt_model: Optional[str] - stt_credential_id: Optional[UUID] = None - - # LLM Configuration - llm_provider: Optional[ModelProvider] - llm_model: Optional[str] - llm_temperature: Optional[float] - llm_max_tokens: Optional[int] - llm_config: Optional[Dict[str, Any]] - llm_credential_id: Optional[UUID] = None - - # TTS Configuration - tts_provider: Optional[ModelProvider] - tts_model: Optional[str] - tts_voice: Optional[str] - tts_config: Optional[Dict[str, Any]] - tts_credential_id: Optional[UUID] = None - - # S2S Configuration - s2s_provider: Optional[ModelProvider] - s2s_model: Optional[str] - s2s_config: Optional[Dict[str, Any]] - s2s_credential_id: Optional[UUID] = None - - # Additional metadata - extra_metadata: Optional[Dict[str, Any]] - is_active: bool - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - model_config = ConfigDict(from_attributes=True) - - -# Test Agent Conversation Schemas -class TestAgentConversationCreate(BaseModel): - """Schema for creating a new test agent conversation.""" - agent_id: UUID - persona_id: UUID - scenario_id: UUID - voice_bundle_id: UUID - conversation_metadata: Optional[Dict[str, Any]] = None - - model_config = ConfigDict(json_schema_extra={ - "example": { - "agent_id": "123e4567-e89b-12d3-a456-426614174000", - "persona_id": "123e4567-e89b-12d3-a456-426614174001", - "scenario_id": "123e4567-e89b-12d3-a456-426614174002", - "voice_bundle_id": "123e4567-e89b-12d3-a456-426614174003" - } - }) - - -class TestAgentConversationUpdate(BaseModel): - """Schema for updating a test agent conversation.""" - status: Optional[str] = None - live_transcription: Optional[List[Dict[str, Any]]] = None - full_transcript: Optional[str] = None - conversation_metadata: Optional[Dict[str, Any]] = None - - -class TestAgentConversationResponse(BaseModel): - """Schema for test agent conversation response.""" - id: UUID - organization_id: UUID - agent_id: UUID - persona_id: UUID - scenario_id: UUID - voice_bundle_id: UUID - status: str - live_transcription: Optional[List[Dict[str, Any]]] - conversation_audio_key: Optional[str] - full_transcript: Optional[str] - started_at: datetime - ended_at: Optional[datetime] - duration_seconds: Optional[float] - conversation_metadata: Optional[Dict[str, Any]] - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - model_config = ConfigDict(from_attributes=True) - - -class ConversationTurn(BaseModel): - """Schema for a single conversation turn.""" - speaker: str # "test_agent" or "voice_agent" - text: str - timestamp: float # Time in seconds from start - audio_segment_key: Optional[str] = None # S3 key for this segment's audio - - -# Conversation Evaluation Schemas -class ConversationEvaluationCreate(BaseModel): - """Schema for creating a conversation evaluation.""" - transcription_id: UUID - agent_id: UUID - llm_provider: Optional[ModelProvider] = ModelProvider.OPENAI - llm_model: Optional[str] = "gpt-4o" - - model_config = ConfigDict(json_schema_extra={ - "example": { - "transcription_id": "123e4567-e89b-12d3-a456-426614174000", - "agent_id": "123e4567-e89b-12d3-a456-426614174001", - "llm_provider": "openai", - "llm_model": "gpt-4o" - } - }) - - -class ConversationEvaluationResponse(BaseModel): - """Schema for conversation evaluation response.""" - id: UUID - organization_id: UUID - transcription_id: UUID - agent_id: UUID - objective_achieved: bool - objective_achieved_reason: Optional[str] - additional_metrics: Optional[Dict[str, Any]] - overall_score: Optional[float] - llm_provider: Optional[ModelProvider] - llm_model: Optional[str] - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -# Evaluator Schemas -class EvaluatorCreate(BaseModel): - """Schema for creating an evaluator. Either provide agent_id+persona_id+scenario_id (standard) or metric_ids/custom_prompt (custom).""" - name: Optional[str] = None - agent_id: Optional[UUID] = None - persona_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - custom_prompt: Optional[str] = None - metric_ids: Optional[List[UUID]] = None - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - - -class EvaluatorUpdate(BaseModel): - """Schema for updating an evaluator.""" - name: Optional[str] = None - agent_id: Optional[UUID] = None - persona_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - custom_prompt: Optional[str] = None - metric_ids: Optional[List[UUID]] = None - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - - -class EvaluatorResponse(BaseModel): - """Schema for evaluator response.""" - id: UUID - evaluator_id: str - organization_id: UUID - name: Optional[str] = None - agent_id: Optional[UUID] = None - persona_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - custom_prompt: Optional[str] = None - metric_ids: Optional[List[UUID]] = None - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - @field_validator('llm_provider', mode='before') - @classmethod - def convert_llm_provider(cls, v): - """Convert string to ModelProvider (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return ModelProvider(v_lower) - except ValueError: - for enum_member in ModelProvider: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid ModelProvider value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -class EvaluatorBulkCreate(BaseModel): - """Schema for creating multiple evaluators at once.""" - name: Optional[str] = None - agent_id: UUID - scenario_id: UUID - persona_ids: List[UUID] - tags: Optional[List[str]] = None - - -class RunEvaluatorsRequest(BaseModel): - """Schema for running evaluators.""" - evaluator_ids: List[UUID] = Field(..., description="List of evaluator IDs to run") - - -class RunEvaluatorsResponse(BaseModel): - """Schema for run evaluators response.""" - task_ids: List[str] = Field(..., description="List of Celery task IDs for tracking") - evaluator_results: List["EvaluatorResultResponse"] = Field(default_factory=list, description="List of created evaluator results") - - model_config = ConfigDict( - from_attributes=True, - json_schema_extra={ - "example": { - "agent_id": "123e4567-e89b-12d3-a456-426614174000", - "scenario_id": "123e4567-e89b-12d3-a456-426614174002", - "persona_ids": [ - "123e4567-e89b-12d3-a456-426614174001", - "123e4567-e89b-12d3-a456-426614174003" - ], - "tags": ["test", "production"] - } - }, - ) - - -# Metric Schemas -SelectionMode = Literal["single_choice", "multi_label"] - - -MetricScope = Literal["workspace", "organization"] - -# Max length for metric rubric text (description / example) accepted by -# the API. DB columns are TEXT or unbounded VARCHAR; this cap is validation-only. -METRIC_RUBRIC_TEXT_MAX_LENGTH = 32_000 - - - -class EvaluatorSuiteCombinationResponse(BaseModel): - """One agent+persona+scenario combination inside a suite.""" - id: UUID - evaluator_id: str - scenario_id: Optional[UUID] = None - scenario_name: Optional[str] = None - scenario_description: Optional[str] = None - scenario_required_info: Optional[Any] = None - - - - -class EvaluatorSuiteCreate(BaseModel): - """Schema for creating an evaluator suite.""" - name: Optional[str] = None - agent_id: UUID - persona_id: UUID - scenario_ids: List[UUID] - metric_ids: Optional[List[UUID]] = None - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - default_runs_per_combination: int = 1 - - - - -class EvaluatorSuiteUpdate(BaseModel): - """Schema for updating an evaluator suite.""" - name: Optional[str] = None - tags: Optional[List[str]] = None - default_runs_per_combination: Optional[int] = None - llm_provider: Optional[ModelProvider] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - metric_ids: Optional[List[UUID]] = None - - - - -class EvaluatorSuiteResponse(BaseModel): - """Schema for evaluator suite response.""" - id: UUID - organization_id: UUID - name: Optional[str] = None - agent_id: UUID - persona_id: UUID - agent_name: Optional[str] = None - persona_name: Optional[str] = None - agent_call_type: Optional[str] = None - agent_call_medium: Optional[str] = None - metric_ids: Optional[List[str]] = None - llm_provider: Optional[str] = None - llm_model: Optional[str] = None - llm_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - default_runs_per_combination: int = 1 - round_robin_index: int = 0 - is_active: bool = False - agent_suite_count: int = 1 - combination_count: int = 0 - combinations: List[EvaluatorSuiteCombinationResponse] = Field(default_factory=list) - created_at: datetime - updated_at: datetime - created_by: Optional[str] = None - - - - -class EvaluatorSuiteAddScenariosRequest(BaseModel): - """Schema for adding scenarios to an existing suite.""" - scenario_ids: List[UUID] - - - - -class RunEvaluatorSuiteRequest(BaseModel): - """Schema for running all combinations in a suite.""" - runs_per_combination: Optional[int] = None - to_number: Optional[str] = None - from_number: Optional[str] = None - - - - -class RunEvaluatorSuiteResponse(BaseModel): - """Schema for suite run response.""" - total_runs: int - task_ids: List[str] = Field(default_factory=list) - evaluator_results: List["EvaluatorResultResponse"] = Field(default_factory=list) - phone_call_refs: List[str] = Field(default_factory=list) - - - - -class RunNextCombinationRequest(BaseModel): - """Schema for running the next round-robin combination.""" - from_number: Optional[str] = None - - - - -class RunNextCombinationResponse(BaseModel): - """Schema for round-robin run response.""" - evaluator_id: UUID - scenario_id: Optional[UUID] = None - scenario_name: str - combination_index: int - next_index: int - evaluator_result_id: Optional[UUID] = None - result_id: Optional[str] = None - task_id: Optional[str] = None - phone_call_ref: Optional[str] = None - call_short_id: Optional[str] = None - - - - -class ChooseNextCombinationResponse(BaseModel): - """Advance inbound round-robin without initiating a call or evaluation run.""" - evaluator_id: UUID - scenario_id: Optional[UUID] = None - scenario_name: str - combination_index: int - next_index: int - - -# Metric Schemas -SelectionMode = Literal["single_choice", "multi_label"] - - -MetricScope = Literal["workspace", "organization"] - -# Max length for metric rubric text (description / example) accepted by -# the API. DB columns are TEXT or unbounded VARCHAR; this cap is validation-only. -METRIC_RUBRIC_TEXT_MAX_LENGTH = 32_000 - - - -class MetricCreate(BaseModel): - """Schema for creating a metric. - - Hierarchy: - - ``parent_metric_id`` set => this is a child sub-metric. ``metric_type`` - is forced to ``boolean`` server-side; ``selection_mode`` must be None. - - ``selection_mode`` set => this is a parent category metric. - ``parent_metric_id`` must be None (max depth = 2). - - Scope: - - ``scope="workspace"`` (default) stamps the metric with the active - ``X-Workspace-Id`` so it only shows up inside that workspace. - - ``scope="organization"`` stamps ``workspace_id=NULL`` so the metric - is visible in every workspace of the org. Children always inherit - their parent's scope; setting ``scope`` on a child request body is - ignored server-side. - """ - name: str - description: Optional[str] = None - # Optional illustrative example surfaced alongside ``description`` - # in the LLM judge's rubric. Today this is mainly populated on - # child sub-labels (one example per categorization label) but - # standalone metrics may carry it too without a schema change. - example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - metric_type: MetricType = MetricType.RATING - metric_category: MetricCategory = MetricCategory.QUALITY - trigger: MetricTrigger = MetricTrigger.ALWAYS - enabled: bool = True - metric_origin: str = "custom" - supported_surfaces: List[str] = ["agent"] - enabled_surfaces: Optional[List[str]] = None - custom_data_type: Optional[str] = None - custom_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - capture_rationale: Optional[bool] = False - parent_metric_id: Optional[UUID] = None - selection_mode: Optional[SelectionMode] = None - # Only meaningful on multi_label parents; ignored everywhere else. - # When true, the LLM is invited during call-import evaluation to emit - # additional candidate sub-labels beyond the user-defined children. - allow_discovery: bool = False - # When true, this metric is a "transcript-compare judge": at - # call-import evaluation time the worker feeds BOTH the production - # transcript (``call_import_rows.transcript``, CSV-supplied) and - # the diarised transcript (``call_import_rows.diarised_transcript``, - # worker-produced) to the LLM as a labeled pair. The parent - # evaluation's ``transcript_source`` is ignored for these metrics. - # Mutually exclusive with ``parent_metric_id`` / ``selection_mode`` - # G�� comparison metrics stay standalone so the LLM grouping logic - # doesn't have to second-guess which prompt template to use within - # a hierarchy. (Parent-level keyword auto-detection in the worker - # still routes a categorisation parent through the comparison - # prompt without setting this flag.) - compare_transcripts: bool = False - # When ``"organization"``, the metric is stored with - # ``workspace_id=NULL`` so it surfaces in every workspace of the - # caller's org. Default ``"workspace"`` preserves the historical - # behavior of stamping the metric with the active ``X-Workspace-Id``. - # Ignored when ``parent_metric_id`` is set (children inherit the - # parent's scope unconditionally). - scope: MetricScope = "workspace" - - @model_validator(mode='after') - def validate_compare_transcripts_exclusions(self): - """Reject body combinations that don't make sense for a - transcript-compare judge. - - The Metric ORM column accepts the value; the validator just - prevents the user from accidentally requesting an incoherent - metric shape (e.g. "compare two transcripts but also live - inside a categorisation hierarchy" � different prompt - templates). - """ - if not self.compare_transcripts: - return self - if self.parent_metric_id is not None: - raise ValueError( - "Transcript-compare metrics must be standalone: " - "they cannot be a child sub-metric in this version." - ) - if self.selection_mode is not None: - raise ValueError( - "Transcript-compare metrics must be standalone: " - "they cannot own children (selection_mode must be " - "unset) in this version." - ) - return self - - model_config = ConfigDict(json_schema_extra={ - "example": { - "name": "Professionalism", - "description": "Measures the professional tone and behavior", - "metric_type": "rating", - "trigger": "always", - "enabled": True - } - }) - - -class MetricChildDraft(BaseModel): - """One child sub-metric in a parent + children atomic create body.""" - - name: str = Field(..., max_length=120) - description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - # Optional illustrative example for this label. Surfaced alongside - # ``description`` in the LLM judge's rubric so each label can carry - # both its definition AND a "what does this look like?" example. - example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - enabled: bool = True - capture_rationale: Optional[bool] = True - tags: Optional[List[str]] = None - - -class MetricCreateWithChildren(BaseModel): - """One-shot create body: a parent metric + N children, atomically. - - Children are persisted as full ``Metric`` rows with - ``parent_metric_id`` set to the new parent. ``metric_type`` on every - child is forced to ``boolean`` server-side regardless of what's - passed in the parent body. - """ - - name: str = Field(..., max_length=120) - description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - selection_mode: SelectionMode - metric_category: MetricCategory = MetricCategory.QUALITY - enabled: bool = True - supported_surfaces: List[str] = Field(default_factory=lambda: ["agent"]) - enabled_surfaces: Optional[List[str]] = None - tags: Optional[List[str]] = None - # When true on a multi_label parent, allow the LLM to emit candidate - # labels beyond the listed children at evaluation time. Validator - # rejects allow_discovery=True on single_choice parents. - allow_discovery: bool = False - # Parent-level "Enable LLM Rationale" toggle. When true the LLM - # judge emits a single rationale string at the parent level - # (children never carry rationales in hierarchical mode), which the - # table renders as the " - LLM Rationale" column. - capture_rationale: bool = False - children: List[MetricChildDraft] = Field( - default_factory=list, - description="Child sub-metric labels under this parent.", - ) - # See ``MetricCreate.scope``. Same semantics: ``"organization"`` - # creates the parent + all children with ``workspace_id=NULL`` so - # the whole category subtree is shared across every workspace in - # the org. - scope: MetricScope = "workspace" - - -class MetricUpdate(BaseModel): - """Schema for updating a metric.""" - name: Optional[str] = None - description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - # ``None`` here means "leave unchanged"; pass an empty string to - # clear a previously stored example. - example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - metric_type: Optional[MetricType] = None - trigger: Optional[MetricTrigger] = None - enabled: Optional[bool] = None - metric_origin: Optional[str] = None - supported_surfaces: Optional[List[str]] = None - enabled_surfaces: Optional[List[str]] = None - custom_data_type: Optional[str] = None - custom_config: Optional[Dict[str, Any]] = None - tags: Optional[List[str]] = None - metric_category: Optional[MetricCategory] = None - capture_rationale: Optional[bool] = None - selection_mode: Optional[SelectionMode] = None - allow_discovery: Optional[bool] = None - # See ``MetricCreate.compare_transcripts``. ``None`` here means - # "leave unchanged". The route layer enforces mutual exclusion - # against the row's existing ``parent_metric_id`` / - # ``selection_mode`` when this is set to True, because the patch - # body alone doesn't have enough context to validate cross-state. - compare_transcripts: Optional[bool] = None - - @model_validator(mode='after') - def validate_compare_transcripts_exclusions(self): - """Reject patch bodies that flip compare_transcripts on while - ALSO trying to set a conflicting field in the same request. - - Cross-state validation against the persisted row (e.g. "the - existing metric already has a parent") is done in the - update route since the schema doesn't have the row in hand. - """ - if self.compare_transcripts is not True: - return self - if self.selection_mode is not None: - raise ValueError( - "Transcript-compare metrics must be standalone: " - "selection_mode must be cleared before enabling " - "compare_transcripts." - ) - return self - - -class MetricResponse(BaseModel): - """Schema for metric response. - - ``children`` is populated for parent metrics (those with - ``selection_mode`` set) and is otherwise an empty list. The list is - built once at serialization time so callers get a single tree - structure without follow-up requests. - """ - id: UUID - organization_id: UUID - # ``None`` when the metric is org-shared (``scope == "organization"``). - # See the ORM ``Metric.workspace_id`` docstring. - workspace_id: Optional[UUID] = None - # Computed convenience field so the UI doesn't have to do - # ``workspace_id == null`` checks everywhere. Always one of - # ``"workspace"`` or ``"organization"``. - scope: MetricScope = "workspace" - name: str - description: Optional[str] - # Optional illustrative example. Populated mainly on categorization - # child labels but surfaced for every metric so the UI can render - # it uniformly without branching on parent/child shape. - example: Optional[str] = None - metric_type: MetricType - metric_category: MetricCategory = MetricCategory.QUALITY - trigger: MetricTrigger - enabled: bool - is_default: bool - metric_origin: str - supported_surfaces: List[str] - enabled_surfaces: List[str] - custom_data_type: Optional[str] - custom_config: Optional[Dict[str, Any]] - tags: Optional[List[str]] - capture_rationale: bool = False - parent_metric_id: Optional[UUID] = None - selection_mode: Optional[SelectionMode] = None - allow_discovery: bool = False - # See ``MetricCreate.compare_transcripts``. Surfaced so the UI can - # render a "Compare transcripts" badge in the metric picker and - # know to skip the run's transcript_source toggle for this metric. - compare_transcripts: bool = False - lifecycle: str = "active" - promoted_from_draft_at: Optional[datetime] = None - studio_notes: Optional[str] = None - children: List["MetricResponse"] = Field(default_factory=list) - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - @field_validator('metric_type', mode='before') - @classmethod - def convert_metric_type(cls, v): - """Convert string to MetricType (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return MetricType(v_lower) - except ValueError: - for enum_member in MetricType: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid MetricType value: {v}") - return v - - @field_validator('trigger', mode='before') - @classmethod - def convert_trigger(cls, v): - """Convert string to MetricTrigger (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return MetricTrigger(v_lower) - except ValueError: - for enum_member in MetricTrigger: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid MetricTrigger value: {v}") - return v - - @validator('supported_surfaces', 'enabled_surfaces', pre=True) - def normalize_surfaces(cls, v): - if v is None: - return [] - if isinstance(v, str): - return [v] - if isinstance(v, (list, tuple)): - return [str(item).lower() for item in v if item] - return [] - - @validator('metric_origin', pre=True) - def normalize_metric_origin(cls, v): - if v is None: - return "custom" - return str(v).lower() - - model_config = ConfigDict(from_attributes=True) - - -MetricResponse.model_rebuild() - - -class MetricDraftCreate(MetricCreate): - """Create a draft metric for Metrics Studio experimentation.""" - - studio_notes: Optional[str] = Field( - default=None, - description="Optional notes about what this draft is testing.", - ) - - -class MetricDraftCreateWithChildren(MetricCreateWithChildren): - """Atomically create a draft parent category metric plus its children.""" - - studio_notes: Optional[str] = Field( - default=None, - description="Optional notes about what this draft category is testing.", - ) - - -class MetricPromoteResponse(BaseModel): - """Response after promoting a draft metric to active.""" - - metric: MetricResponse - promoted_at: datetime - - -MetricStudioSourceKind = Literal[ - "call_import_row", "call_recording", "evaluator_result" -] - - -class MetricStudioSourceItem(BaseModel): - """One call source selected for a Studio run.""" - - source_kind: MetricStudioSourceKind - source_ref: str = Field( - ..., - min_length=1, - description="UUID for import rows / evaluator results; call_short_id for recordings.", - ) - display_label: Optional[str] = Field( - default=None, - max_length=512, - description="Optional UI label; resolved server-side when omitted.", - ) - - -class MetricStudioRunCreate(BaseModel): - """Request body for triggering a Metrics Studio evaluation run.""" - - metric_ids: List[UUID] = Field(..., min_length=1) - sources: List[MetricStudioSourceItem] = Field(..., min_length=1) - name: Optional[str] = Field(default=None, max_length=255) - transcript_source: Literal["production", "diarised"] = "diarised" - llm_provider: Optional[str] = Field(default=None, max_length=50) - llm_model: Optional[str] = Field(default=None, max_length=100) - llm_credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - metric_llm_overrides: Optional[Dict[str, Any]] = None - - -class MetricStudioRunRetryRequest(BaseModel): - """Retry failed or selected Studio run results.""" - - result_ids: Optional[List[UUID]] = Field( - default=None, - description="When omitted, retry all failed results in the run.", - ) - - -class MetricStudioRunResultResponse(BaseModel): - """Per-source result row for a Studio run.""" - - id: UUID - run_id: UUID - source_kind: str - source_ref: str - display_label: Optional[str] = None - source_metadata: Optional[Dict[str, Any]] = None - status: str - metric_scores: Dict[str, Any] = Field(default_factory=dict) - error_message: Optional[str] = None - started_at: Optional[datetime] = None - finished_at: Optional[datetime] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class MetricStudioRunResponse(BaseModel): - """Metrics Studio run summary.""" - - id: UUID - organization_id: UUID - workspace_id: UUID - name: Optional[str] = None - selected_metric_ids: List[str] = Field(default_factory=list) - selected_metric_groups: Optional[Dict[str, List[str]]] = None - transcript_source: str - llm_provider: Optional[str] = None - llm_model: Optional[str] = None - status: str - total_items: int - completed_items: int - failed_items: int - error_message: Optional[str] = None - started_at: Optional[datetime] = None - finished_at: Optional[datetime] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class MetricStudioRunListResponse(BaseModel): - items: List[MetricStudioRunResponse] - total: int - - -class MetricStudioRunResultListResponse(BaseModel): - items: List[MetricStudioRunResultResponse] - total: int - - -# Evaluator Result Schemas -class EvaluatorResultCreate(BaseModel): - """Schema for creating an evaluator result.""" - evaluator_id: UUID - agent_id: Optional[UUID] = None - persona_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - name: Optional[str] = None - duration_seconds: Optional[float] = None - audio_s3_key: Optional[str] = None - - -class EvaluatorResultCreateManual(BaseModel): - """Schema for manually creating an evaluator result from existing audio file.""" - evaluator_id: UUID - audio_s3_key: str - duration_seconds: Optional[float] = None - - -class EvaluatorResultUpdate(BaseModel): - """Schema for updating an evaluator result.""" - status: Optional[EvaluatorResultStatus] = None - transcription: Optional[str] = None - metric_scores: Optional[Dict[str, Any]] = None - error_message: Optional[str] = None - duration_seconds: Optional[float] = None - - - -class EvaluatorResultCounts(BaseModel): - """Rollup counts for evaluator result navigation.""" - - total: int = 0 - completed: int = 0 - failed: int = 0 - in_progress: int = 0 - last_run_at: Optional[datetime] = None - - - - -class EvaluatorResultsScenarioSummary(BaseModel): - scenario_id: UUID - scenario_name: str - counts: EvaluatorResultCounts - - - - -class EvaluatorResultsSuiteSummary(BaseModel): - suite_id: UUID - suite_name: Optional[str] = None - agent_id: UUID - persona_id: Optional[UUID] = None - counts: EvaluatorResultCounts - scenarios: Optional[List["EvaluatorResultsScenarioSummary"]] = None - - - - -class EvaluatorResultsAgentSummary(BaseModel): - agent_id: UUID - agent_name: str - counts: EvaluatorResultCounts - suites: Optional[List[EvaluatorResultsSuiteSummary]] = None - - - - -class EvaluatorResultsUnassignedSummary(BaseModel): - counts: EvaluatorResultCounts - recent_result_ids: List[str] = Field(default_factory=list) - - - - -class EvaluatorResultsOverviewResponse(BaseModel): - workspace_counts: EvaluatorResultCounts - agents: List[EvaluatorResultsAgentSummary] = Field(default_factory=list) - unassigned: EvaluatorResultsUnassignedSummary - - - - -class EvaluatorResultListResponse(BaseModel): - items: List["EvaluatorResultResponse"] - total: int - - - -class EvaluatorResultResponse(BaseModel): - """Schema for evaluator result response.""" - id: UUID - result_id: str - organization_id: UUID - evaluator_id: Optional[UUID] = None # Optional for playground test results - agent_id: Optional[UUID] = None # Nullable for custom evaluators - persona_id: Optional[UUID] = None # Optional for playground test results - scenario_id: Optional[UUID] = None # Optional for playground test results - name: Optional[str] = None # Optional for playground test results - timestamp: datetime - duration_seconds: Optional[float] - status: EvaluatorResultStatus - audio_s3_key: Optional[str] - transcription: Optional[str] - speaker_segments: Optional[List[Dict[str, Any]]] = None # [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] - metric_scores: Optional[Dict[str, Any]] - celery_task_id: Optional[str] - error_message: Optional[str] - - # Call tracking fields (for voice AI integrations) - call_event: Optional[str] = None - provider_call_id: Optional[str] = None - provider_platform: Optional[str] = None - call_data: Optional[Dict[str, Any]] = None # Full call details from provider - - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - # Related entities (optional, populated when requested) - agent: Optional[AgentResponse] = None - persona: Optional[PersonaResponse] = None - scenario: Optional[ScenarioResponse] = None - evaluator: Optional[EvaluatorResponse] = None - - @field_validator('status', mode='before') - @classmethod - def convert_status(cls, v): - """Convert string to EvaluatorResultStatus (handles uppercase DB values).""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return EvaluatorResultStatus(v_lower) - except ValueError: - for enum_member in EvaluatorResultStatus: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid EvaluatorResultStatus value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# ALERTING SCHEMAS -# ============================================ - -class AlertCreate(BaseModel): - """Schema for creating an alert.""" - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = None - - # Metric condition - metric_type: AlertMetricType = AlertMetricType.NUMBER_OF_CALLS - aggregation: AlertAggregation = AlertAggregation.SUM - operator: AlertOperator = AlertOperator.GREATER_THAN - threshold_value: float = Field(..., description="Threshold value for the alert") - time_window_minutes: int = Field(default=60, ge=1, description="Time window in minutes for aggregation") - - # Agent selection (null means all agents) - agent_ids: Optional[List[UUID]] = None - - # Notification settings - notify_frequency: AlertNotifyFrequency = AlertNotifyFrequency.IMMEDIATE - notify_emails: Optional[List[str]] = Field(default=None, description="List of email addresses to notify") - notify_webhooks: Optional[List[str]] = Field(default=None, description="List of webhook URLs (Slack, etc.)") - - model_config = ConfigDict(json_schema_extra={ - "example": { - "name": "High Call Volume Alert", - "description": "Alert when call volume exceeds threshold", - "metric_type": "number_of_calls", - "aggregation": "sum", - "operator": ">", - "threshold_value": 100, - "time_window_minutes": 60, - "agent_ids": None, - "notify_frequency": "immediate", - "notify_emails": ["admin@example.com"], - "notify_webhooks": ["https://hooks.slack.com/services/xxx"] - } - }) - - -class AlertUpdate(BaseModel): - """Schema for updating an alert.""" - name: Optional[str] = Field(None, min_length=1, max_length=255) - description: Optional[str] = None - - # Metric condition - metric_type: Optional[AlertMetricType] = None - aggregation: Optional[AlertAggregation] = None - operator: Optional[AlertOperator] = None - threshold_value: Optional[float] = None - time_window_minutes: Optional[int] = Field(default=None, ge=1) - - # Agent selection - agent_ids: Optional[List[UUID]] = None - - # Notification settings - notify_frequency: Optional[AlertNotifyFrequency] = None - notify_emails: Optional[List[str]] = None - notify_webhooks: Optional[List[str]] = None - - # Status - status: Optional[AlertStatus] = None - - -class AlertResponse(BaseModel): - """Schema for alert response.""" - id: UUID - organization_id: UUID - name: str - description: Optional[str] - - # Metric condition - metric_type: AlertMetricType - aggregation: AlertAggregation - operator: AlertOperator - threshold_value: float - time_window_minutes: int - - # Agent selection - agent_ids: Optional[List[UUID]] - - # Notification settings - notify_frequency: AlertNotifyFrequency - notify_emails: Optional[List[str]] - notify_webhooks: Optional[List[str]] - - # Status - status: AlertStatus - - # Metadata - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - @field_validator('metric_type', mode='before') - @classmethod - def convert_metric_type(cls, v): - """Convert string to AlertMetricType.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return AlertMetricType(v_lower) - except ValueError: - for enum_member in AlertMetricType: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertMetricType value: {v}") - return v - - @field_validator('aggregation', mode='before') - @classmethod - def convert_aggregation(cls, v): - """Convert string to AlertAggregation.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return AlertAggregation(v_lower) - except ValueError: - for enum_member in AlertAggregation: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertAggregation value: {v}") - return v - - @field_validator('operator', mode='before') - @classmethod - def convert_operator(cls, v): - """Convert string to AlertOperator.""" - if v is None: - return None - if isinstance(v, str): - try: - return AlertOperator(v) - except ValueError: - for enum_member in AlertOperator: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertOperator value: {v}") - return v - - @field_validator('notify_frequency', mode='before') - @classmethod - def convert_notify_frequency(cls, v): - """Convert string to AlertNotifyFrequency.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return AlertNotifyFrequency(v_lower) - except ValueError: - for enum_member in AlertNotifyFrequency: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertNotifyFrequency value: {v}") - return v - - @field_validator('status', mode='before') - @classmethod - def convert_status(cls, v): - """Convert string to AlertStatus.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return AlertStatus(v_lower) - except ValueError: - for enum_member in AlertStatus: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertStatus value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -class AlertHistoryResponse(BaseModel): - """Schema for alert history response.""" - id: UUID - organization_id: UUID - alert_id: UUID - - # Trigger information - triggered_at: datetime - triggered_value: float - threshold_value: float - - # Status - status: AlertHistoryStatus - - # Notification tracking - notified_at: Optional[datetime] - notification_details: Optional[Dict[str, Any]] - - # Resolution - acknowledged_at: Optional[datetime] - acknowledged_by: Optional[str] - resolved_at: Optional[datetime] - resolved_by: Optional[str] - resolution_notes: Optional[str] - - # Additional context - context_data: Optional[Dict[str, Any]] - - # Metadata - created_at: datetime - updated_at: datetime - - # Related alert info (optional) - alert: Optional[AlertResponse] = None - - @field_validator('status', mode='before') - @classmethod - def convert_status(cls, v): - """Convert string to AlertHistoryStatus.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return AlertHistoryStatus(v_lower) - except ValueError: - for enum_member in AlertHistoryStatus: - if enum_member.name == v or enum_member.value == v: - return enum_member - raise ValueError(f"Invalid AlertHistoryStatus value: {v}") - return v - - model_config = ConfigDict(from_attributes=True) - - -class AlertHistoryUpdate(BaseModel): - """Schema for updating alert history (acknowledge/resolve).""" - status: Optional[AlertHistoryStatus] = None - acknowledged_by: Optional[str] = None - resolved_by: Optional[str] = None - resolution_notes: Optional[str] = None - - -# ============================================ -# CRON JOB SCHEMAS -# ============================================ - -class CronJobCreate(BaseModel): - """Schema for creating a cron job.""" - name: str = Field(..., min_length=1, max_length=255) - cron_expression: str = Field(..., min_length=1, max_length=100, description="Cron expression (e.g., '0 9 * * 1-5')") - timezone: str = Field(default="UTC", max_length=100, description="Timezone for the cron schedule") - max_runs: int = Field(default=10, ge=1, le=1000, description="Maximum number of times to run") - evaluator_ids: Optional[List[UUID]] = Field( - None, - description="Evaluator IDs to trigger (expanded with evaluator_suite_ids when both are set).", - ) - evaluator_suite_ids: Optional[List[UUID]] = Field( - None, - description="Evaluator suite IDs whose combinations are expanded into evaluator_ids.", - ) - - model_config = ConfigDict(json_schema_extra={ - "example": { - "name": "Daily Evaluation Run", - "cron_expression": "0 9 * * 1-5", - "timezone": "America/New_York", - "max_runs": 100, - "evaluator_ids": ["123e4567-e89b-12d3-a456-426614174000"] - } - }) - - -class CronJobUpdate(BaseModel): - """Schema for updating a cron job.""" - name: Optional[str] = Field(None, min_length=1, max_length=255) - cron_expression: Optional[str] = Field(None, min_length=1, max_length=100) - timezone: Optional[str] = Field(None, max_length=100) - max_runs: Optional[int] = Field(None, ge=1, le=1000) - evaluator_ids: Optional[List[UUID]] = None - evaluator_suite_ids: Optional[List[UUID]] = None - status: Optional[CronJobStatus] = None - - -class CronJobResponse(BaseModel): - """Schema for cron job response.""" - id: UUID - organization_id: UUID - name: str - job_type: str = "evaluator_run" - is_system: bool = False - cron_expression: str - timezone: str - max_runs: int - current_runs: int - evaluator_ids: List[UUID] - status: CronJobStatus - next_run_at: Optional[datetime] - last_run_at: Optional[datetime] - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - @field_validator('status', mode='before') - @classmethod - def convert_status(cls, v): - """Convert string to CronJobStatus.""" - if v is None: - return None - if isinstance(v, str): - v_lower = v.lower() - try: - return CronJobStatus(v_lower) - except ValueError: - for enum_member in CronJobStatus: - if enum_member.value.lower() == v_lower: - return enum_member - raise ValueError(f"Invalid status: {v}") - return v - - @field_validator('evaluator_ids', mode='before') - @classmethod - def convert_evaluator_ids(cls, v): - """Convert evaluator_ids from JSON to list of UUIDs.""" - if v is None: - return [] - if isinstance(v, list): - return [UUID(str(id)) if not isinstance(id, UUID) else id for id in v] - return v - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# PROMPT PARTIAL SCHEMAS -# ============================================ - -class MetricPartialChild(BaseModel): - """One categorization label inside a metric partial.""" - - name: str = Field(..., min_length=1) - description: str = "" - example: str = "" - - -class MetricPartialContent(BaseModel): - """Structured JSON payload stored in metric partial ``content``.""" - - schema_version: int = 1 - metric_kind: Literal["single", "category"] - description: str = "" - children: Optional[List[MetricPartialChild]] = None - - -class PromptPartialCreate(BaseModel): - """Schema for creating a prompt partial.""" - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = None - content: str = Field(..., min_length=1) - tags: Optional[List[str]] = None - - -class PromptPartialUpdate(BaseModel): - """Schema for updating a prompt partial.""" - name: Optional[str] = Field(None, min_length=1, max_length=255) - description: Optional[str] = None - content: Optional[str] = Field(None, min_length=1) - tags: Optional[List[str]] = None - change_summary: Optional[str] = None - - -class PromptPartialVersionResponse(BaseModel): - """Schema for prompt partial version response.""" - id: UUID - prompt_partial_id: UUID - version: int - content: str - change_summary: Optional[str] - created_at: datetime - created_by: Optional[str] - - model_config = ConfigDict(from_attributes=True) - - -class AgentFlowNode(BaseModel): - """One step in an LLM-inferred agent logic flowchart.""" - - id: str - label: str - node_type: Literal["start", "decision", "action", "terminal"] = "action" - position_x: Optional[float] = None - position_y: Optional[float] = None - prompt_excerpt: Optional[str] = None - start_offset: Optional[int] = None - end_offset: Optional[int] = None - - -class AgentFlowEdge(BaseModel): - """Directed transition between two agent flow nodes.""" - - source: str - target: str - condition: Optional[str] = None - - -class AgentFlowNodeLayout(BaseModel): - id: str - position_x: float - position_y: float - - -class AgentFlowLayoutSaveRequest(BaseModel): - nodes: List[AgentFlowNodeLayout] = Field(default_factory=list) - - -class AgentFlowGraph(BaseModel): - """Aggregate flow diagram for an imported production agent prompt.""" - - nodes: List[AgentFlowNode] = Field(default_factory=list) - edges: List[AgentFlowEdge] = Field(default_factory=list) - generated_at: Optional[datetime] = None - provider: Optional[str] = None - model: Optional[str] = None - layout_saved_at: Optional[datetime] = None - prompt_content_hash: Optional[str] = None - mapping_error: Optional[str] = None - generation_error: Optional[str] = None - - -class PromptPartialResponse(BaseModel): - """Schema for prompt partial response.""" - id: UUID - organization_id: UUID - name: str - description: Optional[str] - content: str - tags: Optional[List[str]] - current_version: int - agent_flowchart: Optional[AgentFlowGraph] = None - agent_flowchart_status: Optional[str] = None - created_at: datetime - updated_at: datetime - created_by: Optional[str] - - model_config = ConfigDict(from_attributes=True) - - -class PromptPartialDetailResponse(PromptPartialResponse): - """Schema for prompt partial detail with versions.""" - versions: List[PromptPartialVersionResponse] = [] - - model_config = ConfigDict(from_attributes=True) - - -# ============================================ -# TELEPHONY SCHEMAS (provider-agnostic) -# ============================================ - - -class TelephonyIntegrationCreate(BaseModel): - """Schema for creating a telephony provider integration.""" - - provider: str = "plivo" - name: Optional[str] = None - auth_id: str - auth_token: str - verify_app_uuid: Optional[str] = None - voice_app_id: Optional[str] = None - sip_domain: Optional[str] = None - masking_config: Optional[Dict[str, Any]] = None - is_default: Optional[bool] = Field( - None, - description=( - "Mark this credential as the default for the (org, provider). " - "If omitted and no default exists yet, this row becomes the default." - ), - ) - - -class TelephonyIntegrationUpdate(BaseModel): - """Schema for partial updates to a telephony provider integration.""" - - id: Optional[UUID] = None - provider: Optional[str] = None - name: Optional[str] = None - auth_id: Optional[str] = None - auth_token: Optional[str] = None - verify_app_uuid: Optional[str] = None - voice_app_id: Optional[str] = None - sip_domain: Optional[str] = None - masking_config: Optional[Dict[str, Any]] = None - is_active: Optional[bool] = None - - -class TelephonyIntegrationResponse(BaseModel): - """Safe response model for telephony integration without secrets.""" - - id: UUID - organization_id: UUID - provider: str - name: Optional[str] = None - verify_app_uuid: Optional[str] - voice_app_id: Optional[str] - sip_domain: Optional[str] - masking_config: Optional[Dict[str, Any]] - is_active: bool - is_default: bool = False - last_tested_at: Optional[datetime] - created_at: datetime - updated_at: datetime - - class Config: - from_attributes = True - - -class TelephonyPhoneNumberResponse(BaseModel): - """Telephony phone number inventory response schema.""" - - id: UUID - phone_number: str - country_iso2: Optional[str] - region: Optional[str] - number_type: Optional[str] - capabilities: Optional[Dict[str, Any]] - is_masking_pool: bool - inbound_enabled: Optional[bool] = None - outbound_enabled: Optional[bool] = None - source: Optional[str] = None - agent_id: Optional[UUID] - linked_agent_name: Optional[str] = None - provider: Optional[str] = None - is_active: bool - created_at: datetime - - class Config: - from_attributes = True - - -class TelephonyDialTargetCreate(BaseModel): - """Schema for creating a saved outbound dial target.""" - phone_number: str - label: Optional[str] = None - - -class TelephonyDialTargetUpdate(BaseModel): - """Schema for updating a saved outbound dial target.""" - phone_number: Optional[str] = None - label: Optional[str] = None - - -class TelephonyDialTargetResponse(BaseModel): - """Schema for dial target response.""" - id: UUID - phone_number: str - label: Optional[str] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class TelephonyVerifyStartRequest(BaseModel): - """Request schema for starting voice OTP verification.""" - - phone_number: str - provider: str = "plivo" - - -class TelephonyVerifyStartResponse(BaseModel): - """Response schema for started voice OTP verification.""" - - session_id: UUID - provider_session_uuid: str - status: str - message: str - - -class TelephonyVerifyCheckRequest(BaseModel): - """Request schema for checking a submitted OTP code.""" - - session_id: UUID - otp_code: str - provider: str = "plivo" - - -class TelephonyVerifyCheckResponse(BaseModel): - """Response schema for OTP check status.""" - - verified: bool - status: str - message: str - - -class TelephonyMaskingSessionCreate(BaseModel): - """Request schema for creating a number masking session.""" - - party_a_number: str - party_b_number: str - provider: str = "plivo" - expires_in_minutes: Optional[int] = 60 - metadata: Optional[Dict[str, Any]] = None - provider: str = "plivo" - - -class TelephonyMaskingSessionResponse(BaseModel): - """Response schema for masking sessions.""" - - id: UUID - masked_number: str - party_a_number: str - party_b_number: str - status: str - expires_at: Optional[datetime] - created_at: datetime - - class Config: - from_attributes = True - - -class TelephonyOutboundCallRequest(BaseModel): - """Request schema for outbound call initiation.""" - - from_number: str - to_number: str - answer_url: Optional[str] = None - agent_id: Optional[UUID] = None - - -class TelephonyOutboundCallResponse(BaseModel): - """Response schema for outbound call initiation.""" - - provider_request_uuid: str - call_status: str - from_number: str - to_number: str - message: str - - -# --- Call Import Schemas --- - -class CallImportRowResponse(BaseModel): - """Single row within a call-import batch.""" - - id: UUID - row_index: int - # Renamed from ``external_call_id`` (DB column renamed in migration - # ``034_call_import_schemas``). Same data, same uniqueness rules. - conversation_id: str - recording_url: Optional[str] = None - recording_date: Optional[date] = None - # Production transcript: the value supplied via the CSV upload. - transcript: Optional[str] = None - transcript_source: Optional[str] = None - transcript_provider: Optional[str] = None - transcript_model: Optional[str] = None - transcript_status: Optional[str] = None - transcript_error: Optional[str] = None - transcribed_at: Optional[datetime] = None - # Diarised transcript: produced by the post-hoc diarisation - # worker. Independent of ``transcript`` so manual diarisation - # never overwrites the CSV-supplied production value. - diarised_transcript: Optional[str] = None - diarised_transcript_provider: Optional[str] = None - diarised_transcript_model: Optional[str] = None - diarised_transcript_status: Optional[str] = None - diarised_transcript_error: Optional[str] = None - diarised_at: Optional[datetime] = None - # LLM that turned the STT plain-text output into structured - # ``diarised_segments``. Surfaced in the row detail panel so - # reviewers can see "Diarised by openai/gpt-4o-mini" next to - # the swap toggle. NULL on rows diarised by the legacy pyannote - # worker (which has been removed). - diarised_llm_provider: Optional[str] = None - diarised_llm_model: Optional[str] = None - # The exact prompt the LLM diariser ran with. Persisted so a - # reviewer can copy it back into the modal and reproduce the - # turn layout against a different STT pass. - diarised_prompt: Optional[str] = None - # Structured speaker turns produced by the diarisation worker. Each - # entry is `{ "speaker": "agent"|"user"|"speaker_N", "text": str, - # "start": float, "end": float, "raw_speaker": "Speaker 1" }`. The - # plain ``diarised_transcript`` field above is a `: ` - # rendering of this list with ``diarised_speaker_swap`` applied. - diarised_segments: Optional[List[Dict[str, Any]]] = None - # When True the agent <-> user mapping in ``diarised_segments`` is - # inverted at render / export time. The worker writes the canonical - # mapping using the "first speaker is the agent" heuristic; the swap - # toggle lets reviewers correct that without re-running diarisation. - diarised_speaker_swap: bool = False - status: CallImportRowStatus - recording_s3_key: Optional[str] = None - recording_content_type: Optional[str] = None - recording_size_bytes: Optional[int] = None - error_message: Optional[str] = None - attempts: int - raw_columns: Optional[Dict[str, Any]] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -# --- Call Import Schema (Input Parameter definitions) --- - - -class CallImportSchemaParameterBase(BaseModel): - """A single typed parameter inside a Call Import schema. - - Used both in request bodies (create / update) and as the building - block of :class:`CallImportSchemaParameterResponse`. Names are - case-insensitive unique within their parent schema. - """ - - name: str = Field( - ..., - min_length=1, - max_length=255, - description=( - "Parameter name as it appears in the schema editor and the " - "upload mapping table. Must be unique within the schema " - "(case-insensitive)." - ), - ) - type: CallImportParameterType = Field( - ..., - description=( - "Parameter type. One of conversation_id / recording_url / " - "recording_date / transcript / text / number / boolean / " - "datetime / url. Exactly one parameter of type " - "'conversation_id' must be present; at most one each of " - "'recording_url', 'recording_date', and 'transcript'. " - "Only conversation_id is forced required." - ), - ) - description: Optional[str] = Field( - default=None, - max_length=2048, - description="Free-text help shown next to the parameter in the mapping UI.", - ) - is_required: bool = Field( - default=False, - description=( - "When True, the parameter must be mapped to a CSV column on " - "every upload. The ``conversation_id`` parameter is always " - "required and is force-set to True by the server." - ), - ) - - -class CallImportSchemaParameterCreate(CallImportSchemaParameterBase): - """Create payload for a single parameter (inside a schema CRUD body).""" - - -class CallImportSchemaParameterResponse(CallImportSchemaParameterBase): - """Response shape including the persisted id + ordering.""" - - id: UUID - ordering: int - - model_config = ConfigDict(from_attributes=True) - - -def _validate_schema_parameters( - parameters: List[CallImportSchemaParameterBase], -) -> List[CallImportSchemaParameterBase]: - """Apply the cross-parameter invariants shared by create + update.""" - - if not parameters: - raise ValueError("Schema must define at least one parameter.") - - seen_names: set[str] = set() - conv_count = 0 - recording_date_count = 0 - rec_url_count = 0 - transcript_count = 0 - for param in parameters: - norm = param.name.strip().lower() - if not norm: - raise ValueError("Parameter name must be non-empty.") - if norm in seen_names: - raise ValueError( - f"Duplicate parameter name '{param.name}' " - "(names must be unique within a schema)." - ) - seen_names.add(norm) - if param.type == CallImportParameterType.CONVERSATION_ID: - conv_count += 1 - elif param.type == CallImportParameterType.RECORDING_DATE: - recording_date_count += 1 - elif param.type == CallImportParameterType.RECORDING_URL: - rec_url_count += 1 - elif param.type == CallImportParameterType.TRANSCRIPT: - transcript_count += 1 - - if conv_count != 1: - raise ValueError( - "Schema must contain exactly one parameter of type " - "'conversation_id'." - ) - if rec_url_count > 1: - raise ValueError( - "Schema may contain at most one parameter of type " - "'recording_url'." - ) - if recording_date_count > 1: - raise ValueError( - "Schema may contain at most one parameter of type " - "'recording_date'." - ) - if transcript_count > 1: - raise ValueError( - "Schema may contain at most one parameter of type 'transcript'." - ) - return parameters - - -class CallImportSchemaCreate(BaseModel): - """Create body for a new call-import schema.""" - - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = Field(default=None, max_length=2048) - parameters: List[CallImportSchemaParameterCreate] = Field( - ..., - description=( - "Ordered list of parameters. Order is preserved; the server " - "stamps ``ordering`` from the list index." - ), - ) - - @model_validator(mode="after") - def _check_parameters(self): - _validate_schema_parameters(list(self.parameters)) - return self - - -class CallImportSchemaUpdate(BaseModel): - """Patch body for an existing schema (full parameter replacement).""" - - name: Optional[str] = Field(default=None, min_length=1, max_length=255) - description: Optional[str] = Field(default=None, max_length=2048) - parameters: Optional[List[CallImportSchemaParameterCreate]] = Field( - default=None, - description=( - "If provided, REPLACES the full set of parameters on the " - "schema. Omit to leave parameters untouched." - ), - ) - - @model_validator(mode="after") - def _check_parameters(self): - if self.parameters is not None: - _validate_schema_parameters(list(self.parameters)) - return self - - -class CallImportSchemaResponse(BaseModel): - """Read response for a single schema.""" - - id: UUID - organization_id: UUID - workspace_id: UUID - name: str - description: Optional[str] = None - parameters: List[CallImportSchemaParameterResponse] = Field(default_factory=list) - # How many CallImport batches reference this schema. Populated by the - # router when listing; defaults to 0 on detail responses where the - # caller doesn't need it. - usage_count: int = 0 - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class CallImportSchemaListResponse(BaseModel): - """Paginated list of schemas.""" - - items: List[CallImportSchemaResponse] = Field(default_factory=list) - total: int - - -class CallImportTagResponse(BaseModel): - """Tag attached to call import batches.""" - - id: UUID - name: str - color: Optional[str] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class CallImportTagCreate(BaseModel): - """Create a new call-import tag for the organization.""" - - name: str = Field(..., min_length=1, max_length=255) - color: Optional[str] = Field(None, max_length=32) - - -class CallImportTagUpdate(BaseModel): - """Partial update for a call-import tag.""" - - name: Optional[str] = Field(None, min_length=1, max_length=255) - color: Optional[str] = Field(None, max_length=32) - - -class CallImportPreviewSheet(BaseModel): - """One worksheet (or one CSV file synthesized as a single sheet).""" - - name: str = Field(..., description="Sheet name for xlsx; filename for csv.") - headers: List[str] = Field( - default_factory=list, - description="Column headers from the first non-empty row.", - ) - row_count: int = Field( - ..., - description="Approximate count of data rows (excluding the header row).", - ) - - -class CallImportSourceRowSkip(BaseModel): - """One source spreadsheet row skipped during parse (identity / recording URL).""" - - source_row: int = Field( - ..., - description="1-based row index in the source file (same semantics as parse errors).", - ) - reason: str = Field( - ..., - description=( - "Machine-readable skip reason, e.g. missing_conversation_id, " - "missing_recording_url, invalid_recording_url." - ), - ) - message: str = Field( - ..., - description="Human-readable explanation shown in the UI.", - ) - - -class CallImportResponse(BaseModel): - """Summary of a call-import batch.""" - - id: UUID - organization_id: UUID - workspace_id: UUID - # Provider is optional in the new staged flow (only resolved at the - # IMPORT stage). Stays populated for all post-import batches. - provider: Optional[str] = None - telephony_integration_id: Optional[UUID] = None - original_filename: Optional[str] = None - sheet_name: Optional[str] = None - dataset: Optional[str] = None - tags: List[CallImportTagResponse] = Field(default_factory=list) - # New schema-driven mapping. Empty on legacy batches; pre-schema - # batches keep their values in ``column_mapping`` / ``extra_columns`` - # / ``custom_column_mapping`` below for backwards-compatibility. - schema_id: Optional[UUID] = None - parameter_mapping: Dict[str, str] = Field(default_factory=dict) - column_mapping: Dict[str, Optional[str]] = Field(default_factory=dict) - extra_columns: List[str] = Field(default_factory=list) - custom_column_mapping: Dict[str, str] = Field(default_factory=dict) - # Persisted "drop these columns" decision captured at MAP time. - # Empty for legacy one-shot uploads where the value was ephemeral. - skipped_columns: List[str] = Field(default_factory=list) - source_row_skips: List[CallImportSourceRowSkip] = Field( - default_factory=list, - description=( - "Source rows skipped at parse time because of missing/invalid " - "conversation ID or recording URL." - ), - ) - # Source-file staging fields populated at UPLOAD time. ``None`` on - # legacy batches imported via the one-shot ``POST /upload`` endpoint. - source_s3_key: Optional[str] = None - source_format: Optional[str] = None - source_size_bytes: Optional[int] = None - source_content_type: Optional[str] = None - # Snapshot of the file's sheets + headers captured at UPLOAD time - # so the MAP UI can render without re-fetching the file from S3. - available_sheets: Optional[List[CallImportPreviewSheet]] = None - total_rows: int - completed_rows: int - failed_rows: int - status: CallImportStatus - error_message: Optional[str] = None - latest_evaluation_status: Optional[str] = Field( - None, - description=( - "Status of the most recent evaluation run for this batch, " - "when any evaluation exists." - ), - ) - created_at: datetime - updated_at: datetime - created_by_email: Optional[str] = None - last_updated_by_email: Optional[str] = None - - model_config = ConfigDict(from_attributes=True) - - -class CallImportDetailResponse(CallImportResponse): - """A call-import batch with its rows expanded. - - ``filtered_total_rows`` is only set when the caller passed a ``q`` - search term G�� it lets the UI paginate against the filtered subset - while still showing the unfiltered ``total_rows`` in the header. - - The ``diarised_*_rows`` counters aggregate - ``CallImportRow.diarised_transcript_status`` across the batch so the - UI can render a transcribe-and-diarise progress bar without paging - through every row. Rows that have never been touched by the - transcribe/diarise worker (``status='idle'``) are NOT counted here G�� - callers compute the idle bucket as - ``total_rows - (pending + running + completed + failed)``. - """ - - rows: List[CallImportRowResponse] = Field(default_factory=list) - filtered_total_rows: Optional[int] = None - diarised_pending_rows: int = 0 - diarised_running_rows: int = 0 - diarised_completed_rows: int = 0 - diarised_failed_rows: int = 0 - - -class CallImportListResponse(BaseModel): - """Paginated list of call-import batches.""" - - items: List[CallImportResponse] - total: int - page: int - page_size: int - - -class CallImportDispatchLimitSnapshot(BaseModel): - """Configured and live Redis in-flight caps for eval work.""" - - global_limit: int - global_inflight: int - global_at_capacity: bool - org_limit: int - org_inflight: int - org_at_capacity: bool - workspace_limit: int - job_limit: int - fair_dispatch_batch_size: int - - -class CallImportDispatchFairDispatchSnapshot(BaseModel): - """Fair-dispatch scheduler metadata from Redis.""" - - global_rr_cursor: int - dispatch_dedupe_active: bool - dispatch_queue: str - at_capacity_backoff_seconds: int - - -class CallImportDispatchEvaluationSnapshot(BaseModel): - """One in-flight evaluation run with row counters.""" - - evaluation_id: UUID - call_import_id: UUID - status: str - total_rows: int - pending_rows: int - running_rows: int - job_inflight: int - job_at_capacity: bool - - -class CallImportDispatchWorkspaceSnapshot(BaseModel): - """Per-workspace pending dispatch + slot usage.""" - - workspace_id: UUID - workspace_name: Optional[str] = None - workspace_slug: Optional[str] = None - inflight: int - inflight_at_capacity: bool - pending_dispatch_rows: int - pending_import_rows: int - eval_rr_cursor: int - active_evaluations: int - evaluations: List[CallImportDispatchEvaluationSnapshot] = Field( - default_factory=list - ) - - -class CallImportDispatchDiagnosticsResponse(BaseModel): - """Live operator snapshot for call-import eval fair dispatch.""" - - limits: CallImportDispatchLimitSnapshot - fair_dispatch: CallImportDispatchFairDispatchSnapshot - workspaces: List[CallImportDispatchWorkspaceSnapshot] - generated_at: datetime - - -class CallImportUploadResponse(BaseModel): - """Response returned right after a CSV is accepted.""" - - id: UUID - total_rows: int - status: CallImportStatus - dataset: Optional[str] = None - tags: List[CallImportTagResponse] = Field(default_factory=list) - message: str - - -class CallImportDeleteResponse(BaseModel): - """Response after a whole-batch call-import delete is accepted.""" - - id: UUID - status: Literal["accepted", "completed"] = Field( - ..., - description=( - "``accepted`` when teardown was queued to run asynchronously; " - "``completed`` when the batch was already removed." - ), - ) - - -class CallImportPreviewResponse(BaseModel): - """Sheets/headers extracted from an uploaded CSV or Excel workbook. - - The frontend uses this to drive the column-mapping UI without doing - its own parsing G�� keeps client and server in lockstep on quoted - fields, encodings, and Excel cell coercion. - """ - - format: str = Field(..., description="One of 'csv' or 'xlsx'.") - sheets: List[CallImportPreviewSheet] = Field(default_factory=list) - - -class CallImportUpdate(BaseModel): - """Partial update of a call-import batch.""" - - original_filename: Optional[str] = Field( - None, - description=( - "User-facing batch label shown in the UI. Pass an empty string to clear." - ), - ) - dataset: Optional[str] = Field( - None, - description=( - "Free-text dataset label. Pass an empty string to clear the dataset." - ), - ) - tag_ids: Optional[List[UUID]] = Field( - None, - description=( - "Replace the full set of tag assignments. Pass an empty list to clear all tags." - ), - ) - schema_id: Optional[UUID] = Field( - None, - description=( - "Reassign the Input Parameter schema. Only honoured while the " - "batch is in ``uploaded`` or ``mapped`` state; once the batch " - "has rows it's locked to its original schema." - ), - ) - - -class CallImportMappingUpdate(BaseModel): - """Mapping payload for the MAP stage (``PATCH /call-imports/{id}/mapping``). - - Idempotent: callers can submit this multiple times against an - ``uploaded`` or ``mapped`` batch. Validation re-runs against the - persisted ``available_sheets`` snapshot every time so the user can - correct mistakes without re-uploading the file. - """ - - schema_id: UUID = Field( - ..., - description=( - "Reusable Input Parameter schema this batch is mapped against. " - "Must belong to the active workspace." - ), - ) - sheet_name: Optional[str] = Field( - None, - description=( - "Worksheet to use when the staged source file is an Excel " - "workbook. REQUIRED for xlsx; ignored / rejected for CSV." - ), - ) - parameter_mapping: Dict[str, str] = Field( - default_factory=dict, - description=( - "``{schema_parameter_name: source_header}`` map covering every " - "required schema parameter." - ), - ) - skipped_columns: List[str] = Field( - default_factory=list, - description=( - "Source headers the uploader has explicitly skipped. Every " - "source header must be either mapped or appear here." - ), - ) - - -class CallImportStartRequest(BaseModel): - """Provider + credential picker for the IMPORT stage.""" - - provider: Optional[str] = Field( - default=None, - description=( - "Telephony provider key. Must match the " - "``telephony_integration_id``'s provider. Omit together with " - "``telephony_integration_id`` to download recordings directly " - "from CSV-supplied URLs without credentials." - ), - ) - telephony_integration_id: Optional[UUID] = Field( - default=None, - description=( - "Specific TelephonyIntegration credential row to use when " - "downloading recordings for this batch. Omit together with " - "``provider`` for direct-URL import." - ), - ) - - @model_validator(mode="after") - def validate_credential_mode(self) -> "CallImportStartRequest": - has_provider = bool((self.provider or "").strip()) - has_integration = self.telephony_integration_id is not None - if has_provider != has_integration: - raise ValueError( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL import." - ) - return self - - -# --- Call Import Evaluation Schemas --- - - -class CallImportEvaluationLLMOverride(BaseModel): - """Per-metric LLM override used on top of the run-level default. - - Any field left ``None`` falls back to the run-level value (which - itself falls back to the historical OpenAI/gpt-4o default). This - lets users pick a specific provider/model for a single metric (e.g. - a stronger Anthropic model for a tricky qualitative metric) without - re-typing the rest of the metrics in the run. - """ - - provider: Optional[str] = Field( - default=None, - max_length=50, - description="Override LLM provider key, e.g. 'openai' or 'anthropic'.", - ) - model: Optional[str] = Field( - default=None, - max_length=100, - description="Override LLM model name, e.g. 'gpt-4o' or 'claude-3-opus'.", - ) - credential_id: Optional[UUID] = Field( - default=None, - description="Optional AIProvider id when the org has multiple credentials.", - ) - llm_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Optional per-metric generation parameters (temperature, top_p, etc.).", - ) - - -CallImportEvaluationTranscriptSource = Literal["production", "diarised"] - - -class CallImportEvaluationCreate(BaseModel): - """Request body for triggering an evaluation over a call-import batch.""" - - metric_ids: List[UUID] = Field( - ..., - min_length=1, - description="Org Metric ids to score every completed row against.", - ) - name: Optional[str] = Field( - default=None, - max_length=255, - description=( - "Optional human-readable label for the run. Shown in the UI " - "instead of the UUID prefix." - ), - ) - transcript_sources: List[CallImportEvaluationTranscriptSource] = Field( - default_factory=lambda: ["diarised"], - min_length=1, - max_length=1, - description=( - "Which transcript to score against. ``'diarised'`` (default) " - "auto-diarises rows missing a diarised transcript then scores " - "``diarised_transcript``. ``'production'`` scores the CSV " - "``transcript`` column directly and skips diarisation." - ), - ) - - @field_validator("transcript_sources") - @classmethod - def _validate_transcript_sources( - cls, value: List[str] - ) -> List["CallImportEvaluationTranscriptSource"]: - allowed = {"production", "diarised"} - invalid = [src for src in value if src not in allowed] - if invalid: - raise ValueError( - "transcript_sources must be ['production'] or ['diarised'] " - "(received: " - + ", ".join(repr(src) for src in invalid) - + ")." - ) - return value # type: ignore[return-value] - # --- Run-level LLM config --- - llm_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "Run-level LLM provider key (e.g. 'openai', 'anthropic'). NULL " - "preserves the historical OpenAI/gpt-4o default." - ), - ) - llm_model: Optional[str] = Field( - default=None, - max_length=100, - description="Run-level LLM model name. Required when llm_provider is set.", - ) - llm_credential_id: Optional[UUID] = Field( - default=None, - description="Optional AIProvider row to pin for the run-level LLM.", - ) - llm_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Run-level LLM generation parameters (temperature, top_p, etc.).", - ) - metric_llm_overrides: Optional[ - Dict[str, CallImportEvaluationLLMOverride] - ] = Field( - default=None, - description=( - "Optional per-metric LLM overrides keyed by metric UUID. Each " - "entry overrides the run-level default for that metric only." - ), - ) - # --- Auto-transcribe / diarization hook --- - # Every diarised run auto-diarises rows that don't already have a - # diarised transcript. The flag stays on the schema so legacy API - # callers don't 400 immediately, but the route now requires - # ``stt_provider`` + ``stt_model`` on every run regardless of this - # value. - auto_transcribe: bool = Field( - default=True, - description=( - "Auto-diarise rows missing a diarised transcript before " - "evaluation. Defaults to true and is effectively required: " - "``stt_provider`` + ``stt_model`` are mandatory on every " - "evaluation run." - ), - ) - transcribe_overwrite: bool = Field( - default=False, - description=( - "When auto_transcribe is on, overwrite existing transcripts " - "instead of skipping rows that already have one." - ), - ) - transcribe_mode: Literal["stt_llm", "llm_only"] = Field( - default="stt_llm", - description=( - "Diarisation pipeline shape for the auto-transcribe step. " - "'stt_llm' (default) runs STT then an LLM diariser over the " - "resulting text G�� ``stt_provider`` + ``stt_model`` must be " - "provided. 'llm_only' skips STT and feeds the audio " - "directly to the multimodal ``diarization_llm_*`` model " - "along with ``diarization_prompt``; STT fields must be " - "omitted in that case." - ), - ) - stt_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "STT provider key, e.g. 'deepgram', 'openai'. Required when " - "``transcribe_mode='stt_llm'`` (the default); must be omitted " - "when ``transcribe_mode='llm_only'``." - ), - ) - stt_model: Optional[str] = Field( - default=None, - max_length=100, - description=( - "STT model name, e.g. 'nova-2', 'whisper-1'. Same presence " - "rules as ``stt_provider``." - ), - ) - stt_credential_id: Optional[UUID] = Field( - default=None, - description="Optional AIProvider/Integration row to pin for STT.", - ) - stt_language: Optional[str] = Field( - default=None, - max_length=20, - description="ISO language hint for the STT provider, e.g. 'en'.", - ) - # --- LLM diariser config (mirror of CallImportTranscribeRequest) --- - # Auto-diarised eval rows go through the same LLM-based diariser as - # the standalone Transcribe modal G�� the run remembers the provider / - # model / prompt so a follow-up retry can reproduce them without - # having to re-prompt the user. - diarization_llm_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "LLM provider for diarising STT output into agent/user " - "turns. Required when ``auto_transcribe`` is set (the worker " - "no longer falls back to pyannote)." - ), - ) - diarization_llm_model: Optional[str] = Field( - default=None, - max_length=100, - description="LLM model for the diariser.", - ) - diarization_llm_credential_id: Optional[UUID] = Field( - default=None, - description="Optional AIProvider row to pin for the diariser LLM.", - ) - diarization_prompt: Optional[str] = Field( - default=None, - max_length=10_000, - description=( - "Custom system prompt for the diariser LLM; falls back to " - "the canonical default when blank." - ), - ) - discover_new_metrics: bool = Field( - default=False, - description=( - "When true, the LLM is invited to propose net-new top-level " - "metrics (boolean / rating / category) observed in the " - "transcripts in addition to scoring the selected metrics. " - "Candidates surface in the Discovered metrics panel on the " - "evaluation detail Flow tab and can be promoted into real " - "standalone Metric rows. Defaults to false so existing " - "callers retain previous behaviour." - ), - ) - # Telephony credentials for unified pipeline (required when batch is mapped). - provider: Optional[str] = Field( - default=None, - description=( - "Telephony provider key. Required together with " - "``telephony_integration_id`` when starting evaluation " - "from a mapped batch. Omit both for direct-URL import." - ), - ) - telephony_integration_id: Optional[UUID] = Field( - default=None, - description=( - "TelephonyIntegration credential for recording fetch. " - "Required together with ``provider`` for credentialed import." - ), - ) - - @model_validator(mode="after") - def validate_telephony_credential_mode(self) -> "CallImportEvaluationCreate": - has_provider = bool((self.provider or "").strip()) - has_integration = self.telephony_integration_id is not None - if has_provider != has_integration: - raise ValueError( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL evaluation." - ) - return self - - -class CallImportEvaluationUpdate(BaseModel): - """Patch body for editing a previously-created evaluation run.""" - - name: Optional[str] = Field( - default=None, - max_length=255, - description="New name for the evaluation. Empty string clears it.", - ) - - -class CallImportEvaluationBulkDelete(BaseModel): - """Request body for deleting multiple evaluation runs in one call.""" - - evaluation_ids: List[UUID] = Field( - ..., - min_length=1, - description="Evaluation ids to delete.", - ) - - -class CallImportEvaluationRetryRequest(BaseModel): - """Body for retrying a subset (or all failed rows) of an evaluation run. - - ``eval_row_ids`` is optional: when ``None`` the retry applies to - every row in the run that is currently in the ``failed`` state. The - selection always intersects with the run's actual rows, so unknown - ids are silently skipped (and surfaced in the response's - ``skipped`` list with reason ``unknown``). - - The optional ``llm_*`` / ``metric_llm_overrides`` / ``stt_*`` fields - let the caller swap out the LLM or STT configuration that the - failed rows were originally evaluated with. When a field is left - ``None`` the run's existing value is preserved. When a field is - set, it is persisted onto the run (so a follow-up retry sees the - new value as the default) and used by the worker on the next - pass. Providing only one half of provider+model is rejected so - the worker never ends up with a half-configured run. - """ - - eval_row_ids: Optional[List[UUID]] = Field( - default=None, - description=( - "Restrict the retry to a specific subset of evaluation rows. " - "When omitted, every row with status='failed' in this run is " - "re-enqueued." - ), - ) - - # --- Metric-subset re-run --- - # When ``metric_ids`` is set, the retry recomputes ONLY those - # metrics instead of the whole row, and the new scores are merged - # into the existing ``metric_scores`` JSON (other metrics' - # previously-computed values are preserved). This is the path - # taken by the "Re-run metrics" UI in CallImportEvaluationDetail. - metric_ids: Optional[List[UUID]] = Field( - default=None, - description=( - "Restrict the retry to a specific subset of metrics. When " - "set, the worker recomputes only these metrics and merges " - "the new scores into the row's existing metric_scores " - "(other metrics' previous values are preserved). When " - "omitted, the row is fully re-scored as before. Every id " - "must already be present in the run's selected_metric_ids." - ), - ) - include_completed: bool = Field( - default=False, - description=( - "When True, rows whose status is currently 'completed' " - "become eligible for retry (otherwise only 'failed' rows " - "are picked up). Required when ``metric_ids`` is set on a " - "successful row, since otherwise the whole metric-subset " - "retry would be skipped as 'completed'." - ), - ) - - # --- LLM overrides --- - llm_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "Override the run-level LLM provider for this retry (and " - "future retries). Must be paired with ``llm_model``." - ), - ) - llm_model: Optional[str] = Field( - default=None, - max_length=100, - description=( - "Override the run-level LLM model. Must be paired with " - "``llm_provider``." - ), - ) - llm_credential_id: Optional[UUID] = Field( - default=None, - description=( - "Pin a specific AIProvider credential row for the LLM. " - "When omitted, the resolver falls back to the org default." - ), - ) - llm_config: Optional[Dict[str, Any]] = Field( - default=None, - description="Override run-level LLM generation parameters for this retry.", - ) - metric_llm_overrides: Optional[ - Dict[str, CallImportEvaluationLLMOverride] - ] = Field( - default=None, - description=( - "Replace the run's per-metric LLM overrides. When omitted, " - "the existing overrides are kept; when set, this dict " - "fully replaces them (pass an empty object to clear)." - ), - ) - - # --- STT overrides --- - stt_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "Override the run-level STT provider for this retry. Must " - "be paired with ``stt_model``. Only meaningful when the " - "run is configured for the diarised transcript source." - ), - ) - stt_model: Optional[str] = Field( - default=None, - max_length=100, - description="Override the run-level STT model.", - ) - stt_credential_id: Optional[UUID] = Field( - default=None, - description="Pin a specific credential row for the STT call.", - ) - # --- LLM diariser overrides --- - # When set, replace the run-stored diariser configuration for any - # rows that have to be re-diarised as part of the retry (i.e. - # ``transcribe_overwrite=True`` or the row never had a diarised - # transcript). Same provider+model pairing rule as STT. - diarization_llm_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "Override the run's diariser LLM provider. Must be paired " - "with ``diarization_llm_model``." - ), - ) - diarization_llm_model: Optional[str] = Field( - default=None, - max_length=100, - description="Override the run's diariser LLM model.", - ) - diarization_llm_credential_id: Optional[UUID] = Field( - default=None, - description="Pin a specific credential row for the diariser LLM.", - ) - diarization_prompt: Optional[str] = Field( - default=None, - max_length=10_000, - description=( - "Override the run's diariser prompt. Pass an empty string " - "to clear the override and fall back to the canonical " - "default; pass None to leave the existing value untouched." - ), - ) - transcribe_overwrite: bool = Field( - default=False, - description=( - "When True, wipe the diarised transcript on every retried " - "row's source CallImportRow so the (possibly new) STT runs " - "from scratch. When False, rows that already have a " - "diarised transcript skip diarisation and only re-evaluate." - ), - ) - transcribe_mode: Optional[Literal["stt_llm", "llm_only"]] = Field( - default=None, - description=( - "Override the run's diarisation pipeline mode for this retry. " - "``stt_llm`` runs STT then an LLM diariser; ``llm_only`` feeds " - "audio directly to a multimodal diariser LLM." - ), - ) - - # Telephony credentials for rows that must re-fetch recordings. - provider: Optional[str] = Field( - default=None, - description=( - "Override the batch's telephony provider for this retry pass. " - "Must be paired with ``telephony_integration_id``. Omit both " - "fields to keep the batch's existing pinned credentials." - ), - ) - telephony_integration_id: Optional[UUID] = Field( - default=None, - description=( - "Override the telephony credential used when re-fetching " - "recordings during this retry. Must be paired with " - "``provider``. Omit both to keep existing credentials; send " - "both as null for direct-URL retry." - ), - ) - - @model_validator(mode="after") - def validate_telephony_credential_mode(self) -> "CallImportEvaluationRetryRequest": - fields_set = self.model_fields_set - if ( - "provider" not in fields_set - and "telephony_integration_id" not in fields_set - ): - return self - has_provider = bool((self.provider or "").strip()) - has_integration = self.telephony_integration_id is not None - if has_provider != has_integration: - raise ValueError( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL retry." - ) - return self - - -class CallImportEvaluationRetrySkippedItem(BaseModel): - """One entry in the retry response's ``skipped`` list.""" - - eval_row_id: UUID - reason: str = Field( - ..., - description=( - "Why this row was not re-enqueued. Known values: " - "'unknown' (id not in this run), 'in_progress' " - "(status is pending/running), 'completed' (already " - "successful), 'source_row_missing'." - ), - ) - - -class CallImportEvaluationRetryResponse(BaseModel): - """Summary of a retry fan-out request.""" - - requeued: int = Field( - ..., - description="How many evaluation rows were reset and re-enqueued.", - ) - transcribe_requeued: int = Field( - default=0, - description=( - "Of those, how many were chained through a diarisation " - "task first because the diarised transcript was missing " - "(matches the auto-transcribe behavior of the create-run " - "endpoint)." - ), - ) - skipped: List[CallImportEvaluationRetrySkippedItem] = Field( - default_factory=list, - description="Rows the caller asked for that we did not re-enqueue.", - ) - - -class CallImportEvaluationBulkActionResponse(BaseModel): - """Acknowledgement for bulk cancel / force-fail requests accepted off-thread.""" - - accepted: bool = True - target_count: int = Field( - ..., - description="How many rows the background worker will process.", - ) - evaluation_id: UUID - - -class CallImportMetricSummary(BaseModel): - """Lightweight metric descriptor returned alongside an evaluation.""" - - id: UUID - name: str - metric_type: Optional[str] = None - description: Optional[str] = None - parent_metric_id: Optional[UUID] = None - selection_mode: Optional[SelectionMode] = None - # Surfaced so the Flow tab can decide whether to render the - # Discovered Labels panel next to a multi_label parent. Defaults to - # False to keep legacy clients (and standalone metrics) unaffected. - allow_discovery: bool = False - - model_config = ConfigDict(from_attributes=True) - - -class CallImportEvaluationResponse(BaseModel): - """Parent record describing one evaluation run over a batch.""" - - id: UUID - call_import_id: UUID - organization_id: UUID - name: Optional[str] = None - selected_metric_ids: List[UUID] = Field(default_factory=list) - # Parent UUID string -> [child UUID string]. Captured at run creation - # so the UI can rebuild the parent/child tree even after metrics are - # renamed or deleted. Empty / NULL = no hierarchy was used. - selected_metric_groups: Optional[Dict[str, List[str]]] = None - metrics: List[CallImportMetricSummary] = Field(default_factory=list) - status: str - total_rows: int - completed_rows: int - failed_rows: int - error_message: Optional[str] = None - llm_provider: Optional[str] = None - llm_model: Optional[str] = None - llm_credential_id: Optional[UUID] = None - llm_config: Optional[Dict[str, Any]] = None - metric_llm_overrides: Optional[Dict[str, Any]] = None - stt_provider: Optional[str] = None - stt_model: Optional[str] = None - stt_credential_id: Optional[UUID] = None - # Run-level LLM diariser config. Surfaced so the UI can show - # "Diarised via openai/gpt-4o-mini" on the evaluation header and - # pre-fill the retry modal with the previously-used prompt. - diarisation_llm_provider: Optional[str] = None - diarisation_llm_model: Optional[str] = None - diarisation_llm_credential_id: Optional[UUID] = None - diarisation_prompt: Optional[str] = None - # Diarisation pipeline shape this run was created with. ``stt_llm`` - # (default) is the legacy STT-then-LLM-diariser flow; ``llm_only`` - # means the audio was fed directly to a multimodal diariser LLM. - # Surfaced so the retry modal can preselect the right mode and the - # eval header can render "Diarised via LLM only (Gemini)" instead of - # an empty STT label. - transcribe_mode: Literal["stt_llm", "llm_only"] = "stt_llm" - # Which transcript column this run scored against. All current runs - # use diarised; legacy rows may still carry ``production``. - transcript_source: CallImportEvaluationTranscriptSource = "diarised" - # Sibling evaluation ids created in the same Run Evaluation request. - # Populated only on the POST response (and only when the user ticked - # both Production and Diarised in the modal G�� the backend creates - # one ``CallImportEvaluation`` per source and links them via this - # field so the frontend can deep-link to either run). Empty for all - # other reads. - sibling_evaluation_ids: List[UUID] = Field(default_factory=list) - expected_llm_calls_per_row: Optional[int] = Field( - None, - description=( - "Number of distinct LLM API calls made per evaluation row " - "(one per unique provider/model/config among selected metrics)." - ), - ) - started_at: Optional[datetime] = None - finished_at: Optional[datetime] = None - created_at: datetime - updated_at: datetime - created_by_email: Optional[str] = None - last_updated_by_email: Optional[str] = None - # Cached LLM-generated TLDR for the Visualizations tab. Lazily - # populated by ``POST /evaluations/{eval_id}/insights``; ``None`` - # for runs the user has not summarised yet. ``is_stale`` on the - # nested object is set by the route, not the model. - tldr_summary: Optional["EvaluationTldrSummary"] = None - # Cached LLM-generated user insights for External Audit PDF section 03. - user_insights: Optional["EvaluationUserInsightsState"] = None - # Cached per-metric failure clustering for internal diagnostics. - metric_clusters: Optional["EvaluationMetricClustersState"] = None - # True when the user opted into top-level metric discovery on the - # Run Evaluation modal. The frontend uses this to gate the - # "Discovered metrics" panel on the Flow tab. - discover_new_metrics: bool = False - bulk_operation: Optional[ - Literal["abort", "force_fail_pending", "retry"] - ] = Field( - default=None, - description=( - "When set, a bulk background operation (abort, force-fail pending, " - "or retry) is still running for this evaluation. Other mutating " - "actions are rejected until it completes." - ), - ) - - model_config = ConfigDict(from_attributes=True) - - -class CallImportEvaluationListResponse(BaseModel): - """Wrapper for listing evaluations on a single batch.""" - - items: List[CallImportEvaluationResponse] - total: int - - -class CallImportEvaluationRowResponse(BaseModel): - """Per-source-row evaluation output (one Metric set applied to one row). - - ``raw_columns``, ``recording_url`` and ``recording_s3_key`` come from - the parent ``CallImportRow`` so the row-detail panel can show the - full CSV row metadata + audio without a second round-trip. The UI - prefers ``recording_s3_key`` (resolved via a presigned URL) over - ``recording_url`` so playback uses our downloaded copy instead of - the raw provider URL, which is often expired/auth-gated. - """ - - id: UUID - evaluation_id: UUID - call_import_row_id: UUID - row_index: Optional[int] = None - # Renamed from ``external_call_id``; same value, mirrors the renamed - # ``call_import_rows.conversation_id`` column. - conversation_id: Optional[str] = None - transcript: Optional[str] = None - raw_columns: Optional[Dict[str, Any]] = None - recording_url: Optional[str] = None - recording_date: Optional[date] = None - recording_s3_key: Optional[str] = None - diarised_transcript_status: Optional[str] = None - diarised_transcript_error: Optional[str] = None - status: str - metric_scores: Dict[str, Any] = Field(default_factory=dict) - error_message: Optional[str] = None - started_at: Optional[datetime] = None - finished_at: Optional[datetime] = None - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class CallImportEvaluationRowListResponse(BaseModel): - """Paginated per-row evaluation results.""" - - items: List[CallImportEvaluationRowResponse] - total: int - page: int - page_size: int - - -class CallImportRowBulkDelete(BaseModel): - """Request body for deleting multiple rows from a call-import batch.""" - - row_ids: List[UUID] = Field( - ..., - min_length=1, - description="Row ids to delete (must belong to the same call import).", - ) - - -class CallImportRowBulkDeleteResponse(BaseModel): - """Response after a bulk-delete pass over ``CallImportRow`` rows.""" - - deleted: int = Field( - ..., - description="How many rows were actually removed (unknown ids are skipped).", - ) - status: Literal["completed", "accepted"] = Field( - default="completed", - description=( - "``accepted`` when deletion was queued to run asynchronously; " - "``completed`` when rows were removed before the response." - ), - ) - - -class CallImportRetryFailedRowsRequest(BaseModel): - """Optional credential override when re-enqueueing failed import rows.""" - - provider: Optional[str] = Field( - default=None, - description=( - "Telephony provider key for this retry pass. Omit together with " - "``telephony_integration_id`` to download from CSV recording URLs." - ), - ) - telephony_integration_id: Optional[UUID] = Field( - default=None, - description=( - "Telephony credential to use for this retry pass. Omit together " - "with ``provider`` for direct-URL retry." - ), - ) - - @model_validator(mode="after") - def validate_credential_mode(self) -> "CallImportRetryFailedRowsRequest": - has_provider = bool((self.provider or "").strip()) - has_integration = self.telephony_integration_id is not None - if has_provider != has_integration: - raise ValueError( - "provider and telephony_integration_id must both be provided " - "or both omitted for direct-URL retry." - ) - return self - - -class CallImportRetryFailedRowsResponse(BaseModel): - """Summary of a retry pass over failed call-import rows.""" - - requeued: int = Field( - ..., - description=( - "Rows reset to pending and successfully re-enqueued on the " - "``imports`` worker queue." - ), - ) - enqueue_failed: int = Field( - default=0, - description=( - "Rows that were eligible for retry but failed to enqueue again. " - "These rows are left in ``failed`` with an enqueue error." - ), - ) - skipped: int = Field( - default=0, - description=( - "Rows skipped because they were no longer in ``failed`` at retry " - "time (for example, already retried from another tab)." - ), - ) - - -# --- Diarization / Transcription request/response shapes --- - - -class CallImportTranscribeRequest(BaseModel): - """Body for kicking off diarization for one or many call-import rows. - - The same shape powers both the per-row endpoint (where ``row_ids`` - is ignored) and the batch-level endpoint. ``only_missing`` is the - safe default G�� rows with an existing transcript are skipped unless - ``overwrite_existing`` is set. - - Two modes are supported: - - * ``mode="stt_llm"`` (default) G�� the legacy two-stage pipeline: STT - produces plain text, an LLM splits it into agent/user turns using - ``diarization_prompt``. ``stt_provider`` and ``stt_model`` are - required in this mode. - * ``mode="llm_only"`` G�� skip STT entirely and hand the recording's - audio bytes to a multimodal chat model along with - ``diarization_prompt``. The model both transcribes and diarises in - a single pass. The STT fields are ignored (and must be omitted / - null). Only providers whose chat API accepts audio input (OpenAI - ``gpt-4o-audio-*``, Google Gemini ``1.5/2.0``) are usable; other - providers will surface a typed error on the row. - """ - - mode: Literal["stt_llm", "llm_only"] = Field( - default="stt_llm", - description=( - "Pipeline shape. 'stt_llm' (default) runs STT then an LLM " - "diariser over the resulting text. 'llm_only' skips STT and " - "feeds the raw audio to a multimodal LLM together with " - "``diarization_prompt`` for a single-pass transcribe + " - "diarise." - ), - ) - stt_provider: Optional[str] = Field( - default=None, - max_length=50, - description=( - "STT provider key, e.g. 'deepgram' or 'openai'. Required when " - "``mode='stt_llm'``; must be omitted when ``mode='llm_only'``." - ), - ) - stt_model: Optional[str] = Field( - default=None, - max_length=100, - description=( - "STT model name, e.g. 'nova-2' or 'whisper-1'. Required when " - "``mode='stt_llm'``; must be omitted when ``mode='llm_only'``." - ), - ) - credential_id: Optional[UUID] = Field( - default=None, - description="Optional AIProvider/Integration row to pin for this run.", - ) - language: Optional[str] = Field( - default=None, - max_length=20, - description="Optional ISO language hint, e.g. 'en'.", - ) - only_missing: bool = Field( - default=True, - description=( - "When true, rows with an existing transcript are skipped (the " - "default safe behavior)." - ), - ) - overwrite_existing: bool = Field( - default=False, - description=( - "When true, existing transcripts are replaced. Mutually " - "exclusive with only_missing." - ), - ) - row_ids: Optional[List[UUID]] = Field( - default=None, - description=( - "Restrict the run to a specific subset of rows. NULL = every " - "row in the import (subject to only_missing)." - ), - ) - # --- LLM diariser config --- - # In ``stt_llm`` mode diarisation runs as a *second* step: STT - # produces plain text, then this LLM splits it into agent/user - # turns. In ``llm_only`` mode this same LLM directly receives the - # audio and the prompt. Both fields are always mandatory because - # there is no longer a pyannote fallback and ``llm_only`` cannot - # function without an LLM either. - diarization_llm_provider: str = Field( - ..., - max_length=50, - description=( - "LLM provider that diarises the call. In ``stt_llm`` it sees " - "the STT text; in ``llm_only`` it sees the raw audio." - ), - ) - diarization_llm_model: str = Field( - ..., - max_length=100, - description=( - "LLM model name. In ``llm_only`` mode this must be a model " - "that accepts audio input (e.g. 'gpt-4o-audio-preview', " - "'gemini-1.5-pro')." - ), - ) - diarization_llm_credential_id: Optional[UUID] = Field( - default=None, - description=( - "Optional AIProvider row to pin for the diarisation LLM." - ), - ) - diarization_prompt: Optional[str] = Field( - default=None, - max_length=10_000, - description=( - "Operator-supplied system prompt for the diariser LLM. " - "When NULL/empty the worker uses the canonical default " - "(see ``GET /api/v1/call-imports/diarisation-prompt-default``)." - ), - ) - - @model_validator(mode="after") - def _validate_mode_fields(self) -> "CallImportTranscribeRequest": - """Enforce STT-field presence rules based on ``mode``. - - ``stt_llm`` (default) requires both STT fields G�� the worker - cannot diarise without a transcript. ``llm_only`` forbids them - so the API contract makes it clear that the audio is going - straight to the LLM; passing both would be ambiguous about - which path the worker should take. - """ - stt_provider = (self.stt_provider or "").strip() if self.stt_provider else None - stt_model = (self.stt_model or "").strip() if self.stt_model else None - if self.mode == "stt_llm": - if not stt_provider or not stt_model: - raise ValueError( - "stt_provider and stt_model are required when " - "mode='stt_llm'." - ) - else: # llm_only - if stt_provider or stt_model: - raise ValueError( - "stt_provider/stt_model must be omitted when " - "mode='llm_only'; the LLM consumes the audio " - "directly." - ) - return self - - -class CallImportDiarisationPromptDefaultResponse(BaseModel): - """Wrapper for the canonical diariser-prompt fetched by the modal.""" - - prompt: str = Field( - ..., - description=( - "The exact prompt the worker falls back to when the caller " - "leaves ``diarization_prompt`` blank. The frontend pre-fills " - "the textarea with this value so the operator can edit it." - ), - ) - - -class CallImportRowIdsResponse(BaseModel): - """Flat row-id list for cross-page bulk selection. - - Powers the "Select all M rows in this import" affordance on the - detail page G�� returning only ids keeps the payload tiny so the UI - can hold the full set in memory even for batches with thousands - of rows. The frontend then passes those ids straight to the - existing bulk-delete / bulk-transcribe endpoints. - """ - - ids: List[UUID] = Field( - default_factory=list, - description=( - "Every ``CallImportRow.id`` that matches the ``q`` and " - "``diarised_status`` filters (or every row when neither is " - "supplied), sorted by ``row_index``." - ), - ) - total: int = Field( - ..., - description=( - "Length of ``ids``. Sent explicitly so callers can show a " - "count without re-measuring the array." - ), - ) - - -class CallImportTranscribeResponse(BaseModel): - """Summary of a transcribe fan-out request.""" - - queued: int = Field( - ..., - description=( - "How many rows were enqueued for diarization. Skipped rows " - "(missing recording, transcript already present, etc.) are " - "not counted." - ), - ) - skipped_rows: int = Field( - default=0, - description="Rows excluded by only_missing or because they had no recording.", - ) - skipped_reason_counts: Dict[str, int] = Field( - default_factory=dict, - description="Per-reason breakdown of skipped rows for the UI to surface.", - ) - accepted: bool = Field( - default=False, - description=( - "When true, diarization setup was queued to a background worker " - "and ``queued`` reflects zero until the worker finishes enqueue." - ), - ) - - -class CallImportCancelDiarisationRequest(BaseModel): - """Body for the batch cancel-diarisation endpoint. - - Omit ``row_ids`` (or pass ``null``) to cancel every row in the - import whose ``diarised_transcript_status`` is currently - ``pending`` or ``running``. Pass an explicit list to scope the - cancel to a subset (e.g. the rows the operator selected in the - UI). - """ - - row_ids: Optional[List[UUID]] = Field( - default=None, - description=( - "Optional subset of CallImportRow UUIDs. ``None`` cancels " - "every pending / running diarisation in the import." - ), - ) - - -class CallImportCancelDiarisationResponse(BaseModel): - """Summary of a cancel-diarisation request. - - ``cancelled`` counts rows that were actively pending / running - when the cancel landed and got flipped to ``failed`` with a - "Cancelled by user" error. ``skipped`` counts rows that were - requested (or matched the implicit "all rows" filter) but were - not in a cancellable state G�� typically because they had already - finished or were never queued for diarisation in the first place. - """ - - cancelled: int = Field( - ..., - description=( - "Rows whose in-flight Celery task was revoked and whose " - "``diarised_transcript_status`` was flipped to ``failed`` " - "with a 'Cancelled by user' error message." - ), - ) - skipped: int = Field( - default=0, - description=( - "Rows that were requested but not in a cancellable state " - "(idle / completed / already failed)." - ), - ) - - -# --- Per-run aggregation / visualization payloads --- - - -class CallImportMetricHistogramBucket(BaseModel): - """One bin of a numeric metric histogram.""" - - x0: float - x1: float - count: int - - -class CallImportMetricValueCount(BaseModel): - """One row of a categorical metric's value frequency table.""" - - label: str - count: int - - -class CallImportMetricLabelPair(BaseModel): - """One unordered pair-count cell of a multi-label parent's - co-occurrence matrix. - - ``a`` and ``b`` are child label names; ``count`` is the number of - rows on which both labels fired together (intersection size). - Pairs are emitted with ``a < b`` lexicographically so the matrix - can be reconstructed without duplicates on the frontend. - """ - - a: str - b: str - count: int - - -class CallImportMetricAggregate(BaseModel): - """Per-metric aggregate computed from an evaluation run's rows. - - Numeric metrics return summary statistics + histogram buckets; - categorical / pass-fail / text metrics return the top value counts. - Both shapes can coexist if a metric mixes types G�� the UI prefers - histogram when present, falls back to value_counts otherwise. - """ - - metric_id: str - metric_name: str - metric_type: Optional[str] = None - metric_category: str = "quality" - # True when this aggregate represents a multi-label parent metric - # (selection_mode == "multi_label" with no parent_metric_id). For - # those, ``value_counts`` lists per-child label tallies and the - # rows scored != sum(value_counts.count). The UI uses this flag to - # force a horizontal bar layout (slices wouldn't sum to 100%) and - # to label the n-badge as rows scored, not label occurrences. - is_multi_label_parent: bool = False - count: int = 0 - skipped_count: int = 0 - error_count: int = 0 - # Numeric stats (None when no numeric values were observed) - mean: Optional[float] = None - median: Optional[float] = None - p25: Optional[float] = None - p75: Optional[float] = None - p95: Optional[float] = None - min: Optional[float] = None - max: Optional[float] = None - stddev: Optional[float] = None - histogram_buckets: List[CallImportMetricHistogramBucket] = Field( - default_factory=list - ) - value_counts: List[CallImportMetricValueCount] = Field(default_factory=list) - # Pairwise label intersections for multi-label parent metrics. - # Empty for everything else. The frontend reconstructs a square - # symmetric matrix from these unordered pairs and renders the - # co-occurrence heatmap chart type. - co_occurrence: List[CallImportMetricLabelPair] = Field(default_factory=list) - - -class MetricPeriodDelta(BaseModel): - """Week-over-week (or baseline-run) delta for one metric.""" - - label: str - detail: str - why: Optional[str] = None - - -class CallImportEvaluationAggregateResponse(BaseModel): - """Aggregated metric distributions for a single evaluation run.""" - - evaluation_id: UUID - total_rows: int - completed_rows: int - failed_rows: int - metrics: List[CallImportMetricAggregate] = Field(default_factory=list) - period_deltas: Dict[str, MetricPeriodDelta] = Field(default_factory=dict) - baseline_evaluation_id: Optional[UUID] = None - failure_policies_source: Optional[Literal["inferred", "user"]] = Field( - default=None, - description=( - "Whether flagged-rate semantics use user-confirmed failure policies " - "or inferred defaults from the Failure diagnostics flow." - ), - ) - - -# --- LLM-generated TLDR for the Visualizations tab --- - - - -class EvaluatorResultsAggregateResponse(BaseModel): - """Chart-friendly metric rollups for evaluator results in a suite or scenario scope.""" - - scope: str - suite_id: Optional[UUID] = None - agent_id: Optional[UUID] = None - scenario_id: Optional[UUID] = None - total_rows: int = 0 - completed_rows: int = 0 - failed_rows: int = 0 - metrics: List[CallImportMetricAggregate] = Field(default_factory=list) - - -# --- LLM-generated TLDR for the Visualizations tab --- - - - -class EvaluationTldrSummary(BaseModel): - """Cached LLM-generated narrative + bullet patterns for an eval run. - - Persisted on ``CallImportEvaluation.tldr_summary`` (JSONB) and - rendered above the per-metric charts. ``generated_at_completed_rows`` - is the snapshot of ``completed_rows`` at the time the summary was - written; the API compares it against the current count to flag - ``is_stale`` so the UI can prompt for a regenerate. - """ - - narrative: str - patterns: List[str] = Field(default_factory=list) - metric_insights: Dict[str, str] = Field(default_factory=dict) - generated_at: datetime - generated_at_completed_rows: int = 0 - provider: Optional[str] = None - model: Optional[str] = None - is_stale: bool = False - - -class EvaluationInsightsRequest(BaseModel): - """Body for ``POST /evaluations/{eval_id}/insights``. - - All fields are optional. When ``provider``/``model`` are unset the - backend resolves the org's first active OpenAI/Anthropic/Google - provider (mirroring the Prompt Partials AI-generate flow) so - callers that don't care can simply post ``{}``. - """ - - regenerate: bool = False - provider: Optional[str] = None - model: Optional[str] = Field(default=None, min_length=1) - credential_id: Optional[UUID] = None - max_llm_calls: Optional[int] = Field( - default=None, - ge=20, - le=500, - description=( - "Max LLM calls for user-insights sampling (extraction + synthesis). " - "Defaults to 200 when omitted." - ), - ) - - -class UserInsightCategory(BaseModel): - label: str - count: int - share_pct: float - - -class UserInsightEvidenceTurn(BaseModel): - speaker: str - text: str - - -class UserInsightEvidence(BaseModel): - conversation_id: Optional[str] = None - quote: str - turns: List[UserInsightEvidenceTurn] = Field(default_factory=list) - - -class EvaluationUserInsightItem(BaseModel): - id: str - title: str - categories: List[UserInsightCategory] = Field(default_factory=list) - observation: str - evidence: UserInsightEvidence - - -class EvaluationUserInsightsState(BaseModel): - """Cached map-reduce LLM user insights for an evaluation run.""" - - status: Literal["idle", "running", "completed", "failed"] = "idle" - insights: List[EvaluationUserInsightItem] = Field(default_factory=list) - overview: Optional[str] = None - generated_at: Optional[datetime] = None - generated_at_completed_rows: int = 0 - progress: Optional[Dict[str, int]] = None - provider: Optional[str] = None - model: Optional[str] = None - llm_calls_used: int = 0 - max_llm_calls: Optional[int] = None - error_message: Optional[str] = None - is_stale: bool = False - - -class EvaluationUserInsightsRequest(BaseModel): - """Body for ``POST /evaluations/{eval_id}/user-insights``.""" - - regenerate: bool = False - force: bool = False - provider: Optional[str] = None - model: Optional[str] = Field(default=None, min_length=1) - credential_id: Optional[UUID] = None - max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) - - -MetricClusterGapLabel = Literal[ - "LOGIC_GAP", - "UNDERSPEC", - "EXISTS_NO_TRIGGER", - "MISSING", -] - -FailurePolicyNumericOp = Literal["lt", "lte", "gt", "gte"] - - -class MetricFailurePolicy(BaseModel): - """Per-metric definition of which scores count as failures for this evaluation.""" - - metric_id: str - failure_values: List[str] = Field( - default_factory=list, - description="Normalized lowercase labels that count as failure (single-choice, enum, boolean-as-category).", - ) - failure_child_names: List[str] = Field( - default_factory=list, - description="Child label names that count as failure for multi_label parents.", - ) - numeric_rule: Optional[Dict[str, Any]] = Field( - default=None, - description='Numeric failure rule, e.g. {"op": "lt", "threshold": 0.5}.', - ) - - -class MetricFailurePolicyValueCount(BaseModel): - label: str - count: int = 0 - - -class MetricFailurePolicyMetricPreview(BaseModel): - metric_id: str - metric_name: str - metric_type: Optional[str] = None - selection_mode: Optional[str] = None - is_multi_label_parent: bool = False - value_counts: List[MetricFailurePolicyValueCount] = Field(default_factory=list) - child_names: List[str] = Field(default_factory=list) - row_count_by_value: Dict[str, int] = Field(default_factory=dict) - suggested_policy: MetricFailurePolicy - effective_policy: MetricFailurePolicy - - -class MetricFailurePoliciesResponse(BaseModel): - previews: List[MetricFailurePolicyMetricPreview] = Field(default_factory=list) - policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) - source: Literal["inferred", "user"] = "inferred" - updated_at: Optional[datetime] = None - - -class MetricFailurePoliciesSaveRequest(BaseModel): - policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) - source: Literal["user"] = "user" - - -class MetricClusterEvidenceTurn(BaseModel): - speaker: str - text: str - - -class MetricClusterEvidence(BaseModel): - conversation_id: Optional[str] = None - evaluation_row_id: Optional[UUID] = None - quote: str = "" - turns: List[MetricClusterEvidenceTurn] = Field(default_factory=list) - - -class MetricSubCluster(BaseModel): - label: str - count: int = 0 - share_pct: float = 0.0 - - -class MetricCluster(BaseModel): - id: str - label: str - gap_label: MetricClusterGapLabel - level: int = 1 - count: int = 0 - share_pct: float = 0.0 - sub_clusters: List[MetricSubCluster] = Field(default_factory=list) - observation: str = "" - failure_reason: str = "" - evidence: MetricClusterEvidence = Field(default_factory=MetricClusterEvidence) - is_discovered: bool = False - - -class MetricClusterGroup(BaseModel): - metric_id: str - metric_name: str - flagged_count: int = 0 - failure_reason: str = "" - clusters: List[MetricCluster] = Field(default_factory=list) - - -class DiscoveredProblemCluster(BaseModel): - id: str - label: str - gap_label: MetricClusterGapLabel - count: int = 0 - share_pct: float = 0.0 - observation: str = "" - failure_reason: str = "" - evidence: MetricClusterEvidence = Field(default_factory=MetricClusterEvidence) - - -class RcaRepeatedPatternRow(BaseModel): - metric_id: str - metric_name: str - top_rca_patterns: str = "" - evidence_share_pct: float = 0.0 - evidence_calls: int = 0 - evidence_cluster_count: int = 0 - failure_reason: str = "" - - -class RcaMetricHotspotRow(BaseModel): - metric_id: str - metric_name: str - description: str = "" - metric_rate_pct: float = 0.0 - flagged_calls: int = 0 - - -class RcaPromptAreaRow(BaseModel): - label: str - share_pct: float = 0.0 - gap_label: MetricClusterGapLabel - - -class MetricClustersRcaSummary(BaseModel): - total_clusters: int = 0 - total_clustered_instances: int = 0 - total_flagged_instances: int = 0 - analysed_calls: int = 0 - repeated_patterns: List[RcaRepeatedPatternRow] = Field(default_factory=list) - metric_hotspots: List[RcaMetricHotspotRow] = Field(default_factory=list) - prompt_areas: List[RcaPromptAreaRow] = Field(default_factory=list) - - -class EvaluationMetricClustersState(BaseModel): - """Cached per-metric failure clustering for internal diagnostics.""" - - status: Literal["idle", "running", "completed", "failed", "cancelled"] = "idle" - groups: List[MetricClusterGroup] = Field(default_factory=list) - discovered_problems: List[DiscoveredProblemCluster] = Field( - default_factory=list - ) - overview: Optional[str] = None - generated_at: Optional[datetime] = None - generated_at_completed_rows: int = 0 - progress: Optional[Dict[str, int]] = None - provider: Optional[str] = None - model: Optional[str] = None - llm_calls_used: int = 0 - max_llm_calls: Optional[int] = None - error_message: Optional[str] = None - is_stale: bool = False - selected_evaluation_row_ids: List[str] = Field( - default_factory=list, - description="Evaluation row IDs included in the last clustering run.", - ) - failure_policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) - failure_policies_source: Literal["inferred", "user"] = "inferred" - failure_policies_updated_at: Optional[datetime] = None - rca_summary: Optional[MetricClustersRcaSummary] = None - - -class MetricClusterEligibleRow(BaseModel): - """Completed evaluation row with at least one flagged quality metric.""" - - evaluation_row_id: UUID - conversation_id: Optional[str] = None - row_index: Optional[int] = None - flagged_metric_names: List[str] = Field(default_factory=list) - - -class MetricClusterEligibleRowsResponse(BaseModel): - items: List[MetricClusterEligibleRow] = Field(default_factory=list) - total: int = 0 - - -class PromptImprovementSuggestion(BaseModel): - """One LLM-generated prompt edit to address a failure cluster.""" - - id: str - metric_id: str - metric_name: str - cluster_id: str - cluster_label: str - gap_label: MetricClusterGapLabel - share_pct: float = 0.0 - priority: Literal["high", "medium", "low"] = "medium" - change_type: Literal["edit", "add"] = "add" - target_section: str = "" - anchor_excerpt: str = "" - current_gap: str = "" - suggested_text: str = "" - rationale: str = "" - flow_node_id: str = "" - flow_node_label: str = "" - - -class EvaluationPromptImprovementsState(BaseModel): - """Cached prompt improvement suggestions for an evaluation run.""" - - status: Literal["idle", "running", "completed", "failed"] = "idle" - imported_agent_id: Optional[str] = None - imported_agent_name: Optional[str] = None - suggestions: List[PromptImprovementSuggestion] = Field(default_factory=list) - overview: Optional[str] = None - generated_at: Optional[datetime] = None - generated_at_completed_rows: int = 0 - provider: Optional[str] = None - model: Optional[str] = None - error_message: Optional[str] = None - is_stale: bool = False - - -class EvaluationPromptImprovementsRequest(BaseModel): - """Body for ``POST /evaluations/{eval_id}/prompt-improvements``.""" - - imported_agent_id: UUID - regenerate: bool = False - force: bool = False - provider: Optional[str] = None - model: Optional[str] = None - credential_id: Optional[UUID] = None - - -class EvaluationMetricClustersRequest(BaseModel): - """Body for ``POST /evaluations/{eval_id}/metric-clusters``.""" - - regenerate: bool = False - force: bool = False - provider: Optional[str] = None - model: Optional[str] = Field(default=None, min_length=1) - credential_id: Optional[UUID] = None - max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) - evaluation_row_ids: Optional[List[UUID]] = Field( - default=None, - description=( - "Subset of completed evaluation row IDs to cluster. When omitted, " - "all completed rows with at least one flagged quality metric are used." - ), - ) - row_limit: Optional[int] = Field( - default=None, - ge=1, - description=( - "Use the first N eligible rows (by row order). Mutually exclusive " - "with evaluation_row_ids." - ), - ) - failure_policies: Optional[Dict[str, MetricFailurePolicy]] = Field( - default=None, - description="Per-metric failure policies confirmed in the cluster modal.", - ) - - -# Resolve the forward reference on ``CallImportEvaluationResponse`` -# (defined further up the file) now that ``EvaluationTldrSummary`` -# exists. Without this Pydantic raises at first ``.model_validate`` -# because the string annotation can't be evaluated. -CallImportEvaluationResponse.model_rebuild() - - -# --- Cross-run insights for a CallImport batch --- - - -class CallImportInsightsRunPoint(BaseModel): - """One run's mean for a metric, used to render trend lines.""" - - evaluation_id: UUID - name: Optional[str] = None - created_at: datetime - mean: Optional[float] = None - completed_rows: int = 0 - - -class CallImportInsightsMetric(BaseModel): - """Per-metric history across every evaluation run on this import.""" - - metric_id: str - metric_name: str - metric_type: Optional[str] = None - latest: Optional[CallImportMetricAggregate] = None - trend: List[CallImportInsightsRunPoint] = Field(default_factory=list) - - -class CallImportInsightsResponse(BaseModel): - """Aggregated cross-run signals for a single call-import batch.""" - - call_import_id: UUID - total_rows: int - rows_with_transcript: int - rows_without_transcript: int - transcript_source_counts: Dict[str, int] = Field(default_factory=dict) - evaluation_count: int = 0 - metrics: List[CallImportInsightsMetric] = Field(default_factory=list) - - -# --- Flow chart visualization for hierarchical metrics --- - - -class MetricFlowNode(BaseModel): - """One step in the LLM-inferred temporal flow for a parent metric. - - Represents a child sub-metric label. ``count`` is the number of rows - in the evaluation where this child appears anywhere in its - ``sequence`` array. ``is_terminal`` is set when the child is the - last entry in a meaningful fraction of those sequences. - - ``is_discovered`` is set when the node represents an LLM-discovered - candidate (parent has ``allow_discovery=true``) rather than a - user-defined child. The id of a discovered node is prefixed with - ``disc:`` so it can't collide with real child UUIDs. - """ - - id: str - label: str - count: int = 0 - is_terminal: bool = False - is_discovered: bool = False - - -class MetricFlowEdge(BaseModel): - """One directed transition between two children across all rows. - - ``count`` is the number of rows where ``source`` immediately - precedes ``target`` in the sequence. The synthetic ``START`` node - is used as the ``source`` for the first child in every sequence. - """ - - source: str - target: str - count: int = 0 - - -class MetricFlowResponse(BaseModel): - """Aggregate flow diagram payload for a single parent metric.""" - - parent_metric_id: str - parent_metric_name: str - selection_mode: Optional[SelectionMode] = None - nodes: List[MetricFlowNode] = Field(default_factory=list) - edges: List[MetricFlowEdge] = Field(default_factory=list) - total_rows: int = 0 - rows_with_sequence: int = 0 - - -class DiscoveredLabelItem(BaseModel): - """One LLM-discovered candidate sub-label aggregated across rows. - - ``key`` is the slugified label identifier (matches what appears in - ``sequence`` entries). ``count`` is the number of rows in the - evaluation that emitted this slug. ``sample_rationale`` is the - first non-empty rationale captured from any row (back-compat - field, identical to ``examples[0]`` when present). ``examples`` - holds up to 3 distinct rationales G�� the UI surfaces 2 of them as - ``Examples:`` in the rubric on Promote, with the third kept as - headroom in case the first is unhelpful. - """ - - key: str - name: str - description: Optional[str] = None - sample_rationale: Optional[str] = None - examples: List[str] = Field(default_factory=list, max_length=3) - count: int = 0 - - -class DiscoveredLabelsResponse(BaseModel): - """List of discovered candidate sub-labels for a parent metric.""" - - parent_metric_id: str - items: List[DiscoveredLabelItem] = Field(default_factory=list) - - -class DiscoveredLabelMergeRequest(BaseModel): - """Body for POST /evaluations/{eval_id}/discovered-labels/merge. - - Rewrites every row's ``metric_scores[parent_id].discovered_labels`` - entries whose key is ``from_key`` to use ``to_key`` instead, so the - user can collapse near-duplicate candidates ("On Hold" / "Customer - Put On Hold") into a single promoted child. - """ - - parent_metric_id: UUID - from_key: str = Field(..., min_length=1, max_length=120) - to_key: str = Field(..., min_length=1, max_length=120) - - -class DiscoveredLabelDeleteRequest(BaseModel): - """Body for POST /evaluations/{eval_id}/discovered-labels/delete. - - Strips a candidate sub-label from every row's - ``discovered_labels`` list AND from each row's ``sequence`` array, - then tombstones the slug at the evaluation level so workers - finishing later can't re-introduce it. Use for gibberish or - irrelevant candidates the LLM proposed; for near-duplicates that - you want to keep but unify, use the merge endpoint instead. - """ - - parent_metric_id: UUID - key: str = Field(..., min_length=1, max_length=120) - - -class PromoteDiscoveredChildRequest(BaseModel): - """Body for POST /metrics/{parent_id}/children/from-discovered. - - ``key`` is the slug under which the candidate is currently stored - on per-row ``metric_scores``. The newly-created child Metric's - name is normalized so ``slugify(name) == key``, which keeps every - already-scored row's ``sequence`` array resolvable against the - promoted child without a backfill. - """ - - key: str = Field(..., min_length=1, max_length=120) - name: str = Field(..., min_length=1, max_length=120) - description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - # Default True: when promoting a discovered label we want the new - # sub-metric to always capture rationales going forward, since the - # candidate was itself proposed *with* a rationale and the user - # almost always wants to see why future rows hit it. Explicit False - # keeps the original opt-in behavior available for callers that - # don't care about rationales. - capture_rationale: bool = True - - -# --- Discovered top-level metrics (per-evaluation discovery) --- -# -# Parallel to ``DiscoveredLabelItem`` / merge / delete / promote G�� but -# scoped to the evaluation as a whole, not to a parent category metric. -# Used by the "Discovered metrics" panel at the top of the evaluation -# detail Flow tab when ``CallImportEvaluation.discover_new_metrics`` -# is true. - - -# The promote endpoint accepts these three suggested types; "category" -# creates a parent (no children yet) that the user can later extend in -# the Metrics page. -DiscoveredMetricSuggestedType = Literal["boolean", "rating", "category"] - - -class DiscoveredMetricItem(BaseModel): - """One LLM-discovered candidate top-level metric aggregated across rows. - - Mirrors :class:`DiscoveredLabelItem` but at the evaluation level - (no ``parent_metric_id``). ``suggested_type`` is the LLM's guess at - the best representation; the promote flow lets the user override - it before creating the real :class:`Metric` row. - """ - - key: str - name: str - description: Optional[str] = None - suggested_type: DiscoveredMetricSuggestedType = "boolean" - sample_rationale: Optional[str] = None - examples: List[str] = Field(default_factory=list, max_length=3) - count: int = 0 - - -class DiscoveredMetricsResponse(BaseModel): - """List of discovered candidate top-level metrics for an evaluation.""" - - evaluation_id: UUID - items: List[DiscoveredMetricItem] = Field(default_factory=list) - - -class DiscoveredMetricMergeRequest(BaseModel): - """Body for POST /evaluations/{eval_id}/discovered-metrics/merge. - - Rewrites every row's ``metric_scores["__discovered_metrics__"]`` - entries whose key is ``from_key`` to use ``to_key`` instead, and - records the redirect in ``CallImportEvaluation.discovered_metric_aliases`` - so workers finishing later can't resurrect the merged-out slug. - """ - - from_key: str = Field(..., min_length=1, max_length=120) - to_key: str = Field(..., min_length=1, max_length=120) - - -class DiscoveredMetricDeleteRequest(BaseModel): - """Body for POST /evaluations/{eval_id}/discovered-metrics/delete. - - Strips a candidate from every row's - ``metric_scores["__discovered_metrics__"]`` list and tombstones - the slug at the evaluation level (empty-string alias) so workers - finishing later can't re-introduce it. - """ - - key: str = Field(..., min_length=1, max_length=120) - - -class PromoteDiscoveredMetricRequest(BaseModel): - """Body for POST /metrics/from-discovered. - - Creates a standalone :class:`Metric` (``parent_metric_id=None``) - from an LLM-discovered candidate. The new metric's name is - normalized so ``slugify(name) == key`` to keep already-scored row - payloads resolvable against the promoted metric. ``metric_type`` - selects how the new metric will be scored on future runs; - ``"category"`` creates a ``multi_label`` parent with no children - (the user adds children via the existing Metrics page). - """ - - key: str = Field(..., min_length=1, max_length=120) - name: str = Field(..., min_length=1, max_length=120) - description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) - metric_type: DiscoveredMetricSuggestedType = "boolean" - capture_rationale: bool = True - # Optional per-type config knobs passed through to ``Metric.custom_config``. - # For ``rating`` the frontend can supply {"min": 1, "max": 5}; for - # ``boolean`` / ``category`` the field is typically empty. - custom_config: Optional[Dict[str, Any]] = None - - -# --- Workspace Schemas --- - - -class WorkspaceBase(BaseModel): - """Shared fields for workspace create/update payloads.""" - - name: str = Field(..., min_length=1, max_length=255) - - -class WorkspaceCreate(WorkspaceBase): - """Body for POST /workspaces.""" - - # Optional: derived from name when omitted; uniqueness is per-org. - slug: Optional[str] = Field( - default=None, min_length=1, max_length=255 - ) - - -class WorkspaceUpdate(BaseModel): - """Body for PATCH /workspaces/{id} (rename and/or org-admin activation).""" - - name: Optional[str] = Field(default=None, min_length=1, max_length=255) - is_active: Optional[bool] = None - - -class WorkspaceResponse(BaseModel): - """Response schema for a single workspace.""" - - id: UUID - organization_id: UUID - name: str - slug: str - is_default: bool - is_active: bool = True - created_at: datetime - updated_at: datetime - role_id: Optional[UUID] = None - role_name: Optional[str] = None - capabilities: List[str] = Field(default_factory=list) - - model_config = ConfigDict(from_attributes=True) - - -class WorkspaceRoleBase(BaseModel): - name: str = Field(..., min_length=1, max_length=255) - description: Optional[str] = None - capabilities: List[str] = Field(default_factory=list) - - -class WorkspaceRoleCreate(WorkspaceRoleBase): - pass - - -class WorkspaceRoleUpdate(BaseModel): - name: Optional[str] = Field(default=None, min_length=1, max_length=255) - description: Optional[str] = None - capabilities: Optional[List[str]] = None - - -class WorkspaceRoleResponse(BaseModel): - id: UUID - organization_id: UUID - name: str - description: Optional[str] = None - capabilities: List[str] - is_system: bool - created_at: datetime - updated_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class WorkspaceMemberResponse(BaseModel): - id: UUID - workspace_id: UUID - user_id: UUID - role_id: UUID - role_name: str - user_email: str - user_name: Optional[str] = None - added_by_user_id: Optional[UUID] = None - created_at: datetime - - model_config = ConfigDict(from_attributes=True) - - -class WorkspaceMemberCreate(BaseModel): - user_id: UUID - role_id: UUID - - -class WorkspaceMemberUpdate(BaseModel): - role_id: UUID - - -class CapabilityInfoResponse(BaseModel): - key: str - label: str - - -class CapabilityDomainResponse(BaseModel): - key: str - label: str - capabilities: List[CapabilityInfoResponse] +"""Pydantic schemas for request/response validation.""" + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator, validator +import re +from typing import Optional, List, Dict, Any, Literal +from datetime import date, datetime +from uuid import UUID +from app.models.enums import ( + EvaluationType, EvaluationStatus, EvaluatorResultStatus, RoleEnum, InvitationStatus, + LanguageEnum, CallTypeEnum, CallMediumEnum, GenderEnum, AccentEnum, BackgroundNoiseEnum, + IntegrationPlatform, ModelProvider, CredentialRoutingMode, GatewayInterfaceMode, VoiceBundleType, TestAgentConversationStatus, + MetricType, MetricCategory, MetricTrigger, CallRecordingStatus, AlertMetricType, AlertAggregation, + AlertOperator, AlertNotifyFrequency, AlertStatus, AlertHistoryStatus, CronJobStatus, + CallImportStatus, CallImportRowStatus, CallImportParameterType, +) + + + +# Audio File Schemas +class AudioFileBase(BaseModel): + """Base audio file schema.""" + + filename: str + format: str + + +class AudioFileCreate(AudioFileBase): + """Schema for audio file creation.""" + + file_size: int + duration: Optional[float] = None + sample_rate: Optional[int] = None + channels: Optional[int] = None + + +class AudioFileResponse(AudioFileBase): + """Schema for audio file response.""" + + id: UUID + file_size: int + duration: Optional[float] = None + sample_rate: Optional[int] = None + channels: Optional[int] = None + uploaded_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# Evaluation Schemas +class EvaluationCreate(BaseModel): + """Schema for creating an evaluation.""" + + audio_id: UUID + reference_text: Optional[str] = None + evaluation_type: EvaluationType + model_name: Optional[str] = Field(None, description="Model to use for evaluation") + metrics: Optional[List[str]] = Field( + default=["wer", "latency"], description="Metrics to calculate" + ) + + @field_validator("metrics") + @classmethod + def validate_metrics(cls, v): + """Validate metrics list.""" + allowed_metrics = ["wer", "cer", "latency", "quality_score", "rtf"] + if v: + invalid = [m for m in v if m not in allowed_metrics] + if invalid: + raise ValueError(f"Invalid metrics: {invalid}") + return v + + +class EvaluationResponse(BaseModel): + """Schema for evaluation response.""" + + id: UUID + audio_id: UUID + reference_text: Optional[str] = None + evaluation_type: EvaluationType + model_name: Optional[str] = None + status: EvaluationStatus + metrics_requested: Optional[List[str]] = None + created_at: datetime + started_at: Optional[datetime] = None + completed_at: Optional[datetime] = None + error_message: Optional[str] = None + + model_config = ConfigDict(from_attributes=True) + + +class EvaluationStatusResponse(BaseModel): + """Schema for evaluation status response.""" + + id: UUID + status: EvaluationStatus + created_at: datetime + completed_at: Optional[datetime] = None + error_message: Optional[str] = None + + +# Evaluation Result Schemas +class EvaluationResultResponse(BaseModel): + """Schema for evaluation result response.""" + + evaluation_id: UUID + status: EvaluationStatus + transcript: Optional[str] = None + metrics: Dict[str, Any] + processing_time: Optional[float] = None + model_used: Optional[str] = None + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class MetricsResponse(BaseModel): + """Schema for metrics breakdown.""" + + evaluation_id: UUID + metrics: Dict[str, Any] + processing_time: Optional[float] = None + + +# Comparison Schema +class ComparisonRequest(BaseModel): + """Schema for comparing multiple evaluations.""" + + evaluation_ids: List[UUID] = Field(..., min_length=2, description="At least 2 evaluation IDs to compare") + + +class ComparisonResponse(BaseModel): + """Schema for comparison results.""" + + evaluations: List[EvaluationResultResponse] + comparison_metrics: Dict[str, Any] + + +# API Key Schemas +class APIKeyCreate(BaseModel): + """Schema for creating API key.""" + + name: Optional[str] = None + + +class APIKeyResponse(BaseModel): + """Schema for API key response.""" + + id: UUID + key: str + name: Optional[str] = None + is_active: bool + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# Generic Response Schemas +class MessageResponse(BaseModel): + """Generic message response.""" + + message: str + + +class ErrorResponse(BaseModel): + """Error response schema.""" + + detail: str + +# ============================================ +# VAIOPS SCHEMAS - Voice AI Ops +# ============================================ + +# Enums moved to enums.py + + +# Agent Schemas +class AgentCreate(BaseModel): + """Schema for creating a new agent""" + name: str = Field(..., min_length=1, max_length=255) + phone_number: Optional[str] = None + language: LanguageEnum = LanguageEnum.ENGLISH + description: str = Field(..., min_length=1) + call_type: CallTypeEnum = CallTypeEnum.OUTBOUND + call_medium: CallMediumEnum = CallMediumEnum.PHONE_CALL + telephony_phone_number_id: Optional[UUID] = None + voice_bundle_id: UUID = Field(..., description="Required voice bundle for test agent execution") + ai_provider_id: Optional[UUID] = None + voice_ai_integration_id: Optional[UUID] = None + voice_ai_agent_id: Optional[str] = None + provider_prompt: Optional[str] = None + silence_hangup_secs: int = Field( + default=15, + ge=0, + le=600, + description="End live calls after this many seconds of silence (0 disables)", + ) + + @field_validator('description') + @classmethod + def description_min_words(cls, v: str) -> str: + if len(v.split()) < 10: + raise ValueError('Description must be at least 10 words.') + return v + + @field_validator('phone_number') + @classmethod + def phone_number_format(cls, v: Optional[str]) -> Optional[str]: + if v is not None and v != '': + import re + if not re.fullmatch(r'[\d+]+', v): + raise ValueError('Phone number must contain only digits and the + character.') + return v + + @model_validator(mode='after') + def validate_phone_number(self): + """Ensure phone_number is provided when call_medium is phone_call""" + if self.call_medium == CallMediumEnum.PHONE_CALL and not self.phone_number: + raise ValueError('phone_number is required when call_medium is phone_call') + return self + + model_config = ConfigDict(json_schema_extra={ + "example": { + "name": "Customer Support Bot", + "phone_number": "+1234567890", + "language": "en", + "description": "A customer support bot that handles inquiries about orders, returns, and general questions", + "call_type": "outbound", + "voice_bundle_id": "123e4567-e89b-12d3-a456-426614174000", + "voice_ai_integration_id": "123e4567-e89b-12d3-a456-426614174001", + "voice_ai_agent_id": "agent_abc123" + } + }) + + +class AgentUpdate(BaseModel): + """Schema for updating an agent""" + name: Optional[str] = None + phone_number: Optional[str] = None + language: Optional[LanguageEnum] = None + description: Optional[str] = None + call_type: Optional[CallTypeEnum] = None + call_medium: Optional[CallMediumEnum] = None + telephony_phone_number_id: Optional[UUID] = None + voice_bundle_id: Optional[UUID] = None + voice_ai_integration_id: Optional[UUID] = None + voice_ai_agent_id: Optional[str] = None + provider_prompt: Optional[str] = None + prompt_variables: Optional[Dict[str, str]] = None + silence_hangup_secs: Optional[int] = Field(default=None, ge=0, le=600) + + @model_validator(mode='after') + def validate_voice_config(self): + """Validate voice configuration - both voice_bundle_id and voice_ai_integration_id can be provided independently""" + voice_bundle = self.voice_bundle_id + voice_ai_integration = self.voice_ai_integration_id + + # If voice_ai_integration_id is provided, voice_ai_agent_id must also be provided + if voice_ai_integration and not self.voice_ai_agent_id: + raise ValueError('voice_ai_agent_id is required when voice_ai_integration_id is provided.') + + return self + + @model_validator(mode='after') + def validate_phone_number(self): + """Ensure phone_number is provided when call_medium is phone_call""" + # Only validate if call_medium is being set to phone_call + if self.call_medium == CallMediumEnum.PHONE_CALL and not self.phone_number: + # If phone_number is not being updated, we need to check existing value + # This will be handled in the route + pass + return self + + + +class PreviewIntegrationAgentPromptRequest(BaseModel): + """Fetch a provider agent prompt before an EfficientAI agent exists.""" + voice_ai_agent_id: str = Field(..., min_length=1) + + + + +class PreviewIntegrationAgentPromptResponse(BaseModel): + provider_prompt: str + + +class IntegrationVoiceAgentListItem(BaseModel): + id: str + name: str + + +class ListIntegrationVoiceAgentsResponse(BaseModel): + agents: List[IntegrationVoiceAgentListItem] + platform: str + cached: bool = False + truncated: bool = False + list_supported: bool = True + message: Optional[str] = None + + + + +class AgentPhoneAssignmentConflict(BaseModel): + """Another agent already owns this phone number.""" + agent_id: UUID + agent_name: str + phone_number: str + + + + +class AgentPhoneAssignmentCheckResponse(BaseModel): + """Result of checking whether a phone number is free to assign.""" + available: bool + phone_number: Optional[str] = None + conflict: Optional[AgentPhoneAssignmentConflict] = None + + + + +class TestPromptSectionResponse(BaseModel): + """One canonical section of a generated test agent prompt.""" + key: str + title: str + content: str + + + + +class GeneratedScenarioDraftResponse(BaseModel): + """LLM-generated scenario draft before persistence.""" + name: str + description: str + goal: Optional[str] = None + + + + +class GenerateTestPromptRequest(BaseModel): + """Stage 1: generate foundational test agent prompt from production prompt.""" + production_prompt: str = Field(..., min_length=1) + agent_name: str = Field(..., min_length=1, max_length=255) + language: Optional[str] = None + call_type: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + additional_context: Optional[str] = None + + + + +class GenerateTestPromptResponse(BaseModel): + sections: List[TestPromptSectionResponse] + test_agent_prompt: str + provider: str + model: str + + + + +class GenerateScenariosFromPromptRequest(BaseModel): + """Stage 2: generate scenario drafts from test agent prompt.""" + test_agent_prompt: str = Field(..., min_length=1) + agent_name: str = Field(..., min_length=1, max_length=255) + scenario_count: int = Field(default=5, ge=1, le=10) + language: Optional[str] = None + call_type: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + additional_context: Optional[str] = None + + + + +class GenerateScenariosFromPromptResponse(BaseModel): + scenarios: List[GeneratedScenarioDraftResponse] + provider: str + model: str + + + + +class GenerateTestSetupRequest(BaseModel): + """Convenience: run stage 1 then stage 2 sequentially.""" + production_prompt: str = Field(..., min_length=1) + agent_name: str = Field(..., min_length=1, max_length=255) + scenario_count: int = Field(default=5, ge=1, le=10) + language: Optional[str] = None + call_type: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + additional_context: Optional[str] = None + + + + +class GenerateTestSetupResponse(BaseModel): + sections: List[TestPromptSectionResponse] + test_agent_prompt: str + scenarios: List[GeneratedScenarioDraftResponse] + provider: str + model: str + + + +class AgentResponse(BaseModel): + """Schema for agent response""" + id: UUID + agent_id: Optional[str] = None + name: str + phone_number: Optional[str] = None + language: LanguageEnum + description: Optional[str] + call_type: CallTypeEnum + call_medium: CallMediumEnum + telephony_phone_number_id: Optional[UUID] = None + voice_bundle_id: Optional[UUID] + ai_provider_id: Optional[UUID] + voice_ai_integration_id: Optional[UUID] + voice_ai_agent_id: Optional[str] + provider_prompt: Optional[str] = None + prompt_variables: Optional[Dict[str, str]] = None + silence_hangup_secs: int = Field( + default=15, + ge=0, + le=600, + description="End live calls after this many seconds of silence (0 disables)", + ) + provider_prompt_synced_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + @field_validator('language', mode='before') + @classmethod + def convert_language(cls, v): + """Convert string to LanguageEnum (handles uppercase DB values like ENGLISH -> en).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + # Map old uppercase names to new values + language_map = {'english': 'en', 'spanish': 'es', 'french': 'fr', 'german': 'de', + 'chinese': 'zh', 'japanese': 'ja', 'hindi': 'hi', 'arabic': 'ar'} + if v_lower in language_map: + return LanguageEnum(language_map[v_lower]) + try: + return LanguageEnum(v_lower) + except ValueError: + for enum_member in LanguageEnum: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid LanguageEnum value: {v}") + return v + + @field_validator('call_type', mode='before') + @classmethod + def convert_call_type(cls, v): + """Convert string to CallTypeEnum (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return CallTypeEnum(v_lower) + except ValueError: + for enum_member in CallTypeEnum: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid CallTypeEnum value: {v}") + return v + + @field_validator('call_medium', mode='before') + @classmethod + def convert_call_medium(cls, v): + """Convert string to CallMediumEnum (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return CallMediumEnum(v_lower) + except ValueError: + for enum_member in CallMediumEnum: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid CallMediumEnum value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +# Persona Schemas +class PersonaCreate(BaseModel): + """Schema for creating a new persona (TTS provider-tied voice identity)""" + name: str = Field(..., min_length=1, max_length=255) + gender: GenderEnum = GenderEnum.NEUTRAL + tts_provider: Optional[str] = None + tts_voice_id: Optional[str] = None + tts_voice_name: Optional[str] = None + is_custom: bool = False + description: Optional[str] = None + tts_config: Optional[Dict[str, Any]] = None + llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) + llm_max_tokens: Optional[int] = Field(None, gt=0, le=8192) + response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) + max_turns: Optional[int] = Field(None, ge=1, le=100) + allow_interruptions: Optional[bool] = None + + @model_validator(mode="after") + def validate_tts_config(self): + from app.services.personas.persona_tts_config import validate_persona_tts_config + + validate_persona_tts_config(self.tts_provider, self.tts_config) + return self + + +class PersonaUpdate(BaseModel): + """Schema for updating a persona""" + name: Optional[str] = None + gender: Optional[GenderEnum] = None + tts_provider: Optional[str] = None + tts_voice_id: Optional[str] = None + tts_voice_name: Optional[str] = None + is_custom: Optional[bool] = None + description: Optional[str] = None + tts_config: Optional[Dict[str, Any]] = None + llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) + llm_max_tokens: Optional[int] = Field(None, gt=0, le=8192) + response_delay_ms: Optional[int] = Field(None, ge=0, le=10000) + max_turns: Optional[int] = Field(None, ge=1, le=100) + allow_interruptions: Optional[bool] = None + + @model_validator(mode="after") + def validate_tts_config(self): + from app.services.personas.persona_tts_config import validate_persona_tts_config + + if self.tts_config is not None: + validate_persona_tts_config(self.tts_provider, self.tts_config) + return self + + +class PersonaResponse(BaseModel): + """Schema for persona response""" + id: UUID + name: str + gender: str + tts_provider: Optional[str] = None + tts_voice_id: Optional[str] = None + tts_voice_name: Optional[str] = None + is_custom: bool = False + description: Optional[str] = None + tts_config: Optional[Dict[str, Any]] = None + llm_temperature: Optional[float] = None + llm_max_tokens: Optional[int] = None + response_delay_ms: Optional[int] = None + max_turns: Optional[int] = None + allow_interruptions: Optional[bool] = None + created_at: datetime + updated_at: datetime + + @field_validator('gender', mode='before') + @classmethod + def convert_gender(cls, v): + if v is None: + return "neutral" + if isinstance(v, str): + return v.lower() + if hasattr(v, 'value'): + return v.value + return v + + model_config = ConfigDict(from_attributes=True) + + +class PersonaCloneRequest(BaseModel): + """Schema for cloning a persona""" + name: Optional[str] = None + + +# Scenario Schemas + +class AgentPromptSourcesResponse(BaseModel): + """Prompt texts from an agent that can seed a persona description.""" + agent_id: UUID + agent_name: str + test_agent_prompt: str + agent_prompt: str + + + + +class GeneratePersonaPromptRequest(BaseModel): + """Generate a persona caller prompt from an agent prompt via LLM.""" + agent_id: UUID + source: str = Field(default="auto", pattern="^(test_agent|agent|auto)$") + persona_name: Optional[str] = Field(None, max_length=255) + persona_gender: Optional[str] = None + additional_context: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + + + + +class GeneratePersonaPromptResponse(BaseModel): + persona_prompt: str + source_used: str + provider: str + model: str + + +# Scenario Schemas + +class ScenarioCreate(BaseModel): + """Schema for creating a new scenario""" + name: str = Field(..., min_length=1, max_length=255) + agent_id: Optional[UUID] = None + description: Optional[str] = None + required_info: Dict[str, str] = Field(default_factory=dict) + + +class ScenarioUpdate(BaseModel): + """Schema for updating a scenario""" + name: Optional[str] = None + agent_id: Optional[UUID] = None + description: Optional[str] = None + required_info: Optional[Dict[str, str]] = None + + +class ScenarioResponse(BaseModel): + """Schema for scenario response""" + id: UUID + name: str + agent_id: Optional[UUID] + description: Optional[str] + required_info: Dict[str, str] + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# IAM & USER SCHEMAS +# ============================================ + +# User Schemas +class UserCreate(BaseModel): + """Schema for creating a user.""" + email: str = Field(..., description="User email address") + name: Optional[str] = None + password: Optional[str] = None # Optional for invitation-based signup + + +class UserUpdate(BaseModel): + """Schema for updating user profile.""" + name: Optional[str] = None + first_name: Optional[str] = None + last_name: Optional[str] = None + email: Optional[str] = None + + +class UserResponse(BaseModel): + """Schema for user response.""" + id: UUID + email: str + name: Optional[str] + first_name: Optional[str] + last_name: Optional[str] + is_active: bool + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class OrganizationMemberResponse(BaseModel): + """Schema for organization member response.""" + id: UUID + user_id: UUID + organization_id: UUID + role: RoleEnum + joined_at: datetime + user: UserResponse # Include user details + + model_config = ConfigDict(from_attributes=True) + + +# Invitation Schemas +class InvitationCreate(BaseModel): + """Schema for creating an invitation.""" + email: str = Field(..., description="Email address of the user to invite") + role: RoleEnum = RoleEnum.READER + + +class InvitationResponse(BaseModel): + """Schema for invitation response.""" + id: UUID + organization_id: UUID + email: str + role: RoleEnum + status: InvitationStatus + expires_at: datetime + created_at: datetime + organization_name: Optional[str] = None # Include organization name + invite_path: Optional[str] = None + invite_url: Optional[str] = None + + @field_validator('role', mode='before') + @classmethod + def convert_role(cls, v): + """Convert string to RoleEnum (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return RoleEnum(v_lower) + except ValueError: + for enum_member in RoleEnum: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid RoleEnum value: {v}") + return v + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert string to InvitationStatus (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return InvitationStatus(v_lower) + except ValueError: + for enum_member in InvitationStatus: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid InvitationStatus value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +class InvitationUpdate(BaseModel): + """Schema for updating invitation (accept/decline).""" + token: str + + +class RoleUpdate(BaseModel): + """Schema for updating user role in organization.""" + role: RoleEnum + + +# Profile Schemas +class ProfileResponse(BaseModel): + """Schema for user profile response.""" + id: UUID + email: str + name: Optional[str] + first_name: Optional[str] + last_name: Optional[str] + created_at: datetime + organizations: List[dict] = Field(default_factory=list) # List of org memberships + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# INTEGRATION SCHEMAS +# ============================================ + +class IntegrationCreate(BaseModel): + """Schema for creating an integration.""" + platform: IntegrationPlatform + api_key: str = Field(..., description="Private API key for the platform") + public_key: Optional[str] = Field(None, description="Optional public API key (e.g. for Vapi)") + name: Optional[str] = Field(None, description="Optional friendly name for the integration") + routing_mode: CredentialRoutingMode = Field( + CredentialRoutingMode.INHERIT, + description="LLM routing preference: inherit org default, force gateway, or direct API key.", + ) + is_default: Optional[bool] = Field( + None, + description=( + "Mark this credential as the default for the (org, platform). " + "If omitted and no default exists yet, this row becomes the default." + ), + ) + + +class IntegrationUpdate(BaseModel): + """Schema for updating an integration.""" + name: Optional[str] = None + api_key: Optional[str] = None + public_key: Optional[str] = None + is_active: Optional[bool] = None + routing_mode: Optional[CredentialRoutingMode] = None + + +class IntegrationResponse(BaseModel): + """Schema for integration response.""" + id: UUID + organization_id: UUID + platform: IntegrationPlatform + name: Optional[str] + public_key: Optional[str] = None + is_active: bool + is_default: bool = False + routing_mode: CredentialRoutingMode = CredentialRoutingMode.INHERIT + effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" + created_at: datetime + updated_at: datetime + last_tested_at: Optional[datetime] = None + # Note: api_key is NOT included in response for security + + @field_validator('platform', mode='before') + @classmethod + def convert_platform(cls, v): + """Convert string to IntegrationPlatform enum if needed (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + # Try lowercase first (enum value) + v_lower = v.lower() + try: + return IntegrationPlatform(v_lower) + except ValueError: + # Try to find by enum name (uppercase) + for enum_member in IntegrationPlatform: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid IntegrationPlatform value: {v}") + return v + + @field_validator('routing_mode', mode='before') + @classmethod + def convert_routing_mode(cls, v): + if v is None: + return CredentialRoutingMode.INHERIT + if isinstance(v, str): + try: + return CredentialRoutingMode(v.lower()) + except ValueError: + return CredentialRoutingMode.INHERIT + return v + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# DATA SOURCES SCHEMAS +# ============================================ + +class S3ConnectionTest(BaseModel): + """Schema for testing S3 connection.""" + bucket_name: str + region: str = "us-east-1" + access_key_id: str + secret_access_key: str + endpoint_url: Optional[str] = None + + +class S3ConnectionTestResponse(BaseModel): + """Schema for S3 connection test response.""" + success: bool + message: str + bucket_name: Optional[str] = None + + +class S3FileInfo(BaseModel): + """Schema for S3 file information.""" + key: str + filename: str + size: int + last_modified: str + + +class S3ListFilesResponse(BaseModel): + """Schema for listing S3 files response.""" + files: List[S3FileInfo] + total: int + prefix: Optional[str] = None + + +class S3FolderInfo(BaseModel): + """Schema for S3 folder information.""" + name: str + path: str + + +class S3BrowseResponse(BaseModel): + """Schema for browsing S3 folders within an organization.""" + folders: List[S3FolderInfo] + files: List[S3FileInfo] + current_path: str + organization_id: str + + +class S3UploadResponse(BaseModel): + """Schema for S3 upload response.""" + key: str + bucket: str + file_id: UUID + message: str + + +# AIProvider Schemas +_MAX_GATEWAY_EXTRA_HEADERS = 20 + + +def _validate_gateway_extra_headers( + value: Optional[Dict[str, Any]], +) -> Optional[Dict[str, str]]: + if value is None: + return None + if not isinstance(value, dict): + raise ValueError("gateway_extra_headers must be a JSON object of string keys and values.") + if len(value) > _MAX_GATEWAY_EXTRA_HEADERS: + raise ValueError( + f"gateway_extra_headers supports at most {_MAX_GATEWAY_EXTRA_HEADERS} headers." + ) + normalized: Dict[str, str] = {} + for raw_key, raw_val in value.items(): + key = str(raw_key).strip() + if not key: + raise ValueError("gateway_extra_headers keys must be non-empty strings.") + if len(key) > 64 or any(ch.isspace() for ch in key): + raise ValueError(f"Invalid gateway header name: {key!r}") + if raw_val is None: + raise ValueError(f"gateway_extra_headers[{key!r}] must be a string value.") + val = str(raw_val).strip() + if not val: + raise ValueError(f"gateway_extra_headers[{key!r}] must be a non-empty string.") + if len(val) > 1024 or "\n" in val or "\r" in val: + raise ValueError(f"gateway_extra_headers[{key!r}] value is invalid.") + normalized[key] = val + return normalized or None + + +class AIProviderCreate(BaseModel): + """Schema for creating an AI Provider.""" + provider: ModelProvider + api_key: Optional[str] = Field( + None, + description=( + "Provider API key. Optional when routing via gateway with " + "gateway-managed credentials (passthrough_provider_keys: false)." + ), + ) + name: Optional[str] = None + routing_mode: CredentialRoutingMode = Field( + CredentialRoutingMode.INHERIT, + description="LLM routing preference: inherit org default, force gateway, or direct API key.", + ) + gateway_model: Optional[str] = Field( + None, + min_length=1, + max_length=255, + description="Bifrost custom model ID sent when routing via gateway.", + ) + gateway_interface: GatewayInterfaceMode = Field( + GatewayInterfaceMode.INHERIT, + description="Bifrost API surface: inherit org default, LiteLLM shim, or native OpenAI-compatible.", + ) + gateway_base_url: Optional[str] = Field( + None, + max_length=512, + description="Optional per-credential Bifrost/gateway base URL override.", + ) + gateway_auth_header: Optional[str] = Field( + None, + max_length=64, + description="Auth header name for Bifrost (default x-bf-vk).", + ) + gateway_auth_secret_env: Optional[str] = Field( + None, + max_length=128, + description="Environment variable name holding the gateway auth secret.", + ) + gateway_auth_secret: Optional[str] = Field( + None, + description="Inline gateway auth secret (encrypted at rest). Alternative to env var.", + ) + gateway_extra_headers: Optional[Dict[str, str]] = Field( + None, + description="Arbitrary HTTP headers sent with gateway-routed LiteLLM calls.", + ) + enabled_models: Optional[List[str]] = Field( + None, + description=( + "Allowlisted model names for this credential. " + "Null or empty means all catalog models for the provider." + ), + ) + is_default: Optional[bool] = Field( + None, + description=( + "Mark this credential as the default for the (org, provider). " + "If omitted and no default exists yet, this row becomes the default." + ), + ) + endpoint_url: Optional[str] = Field( + None, + description="Provider endpoint URL (required for Azure OpenAI).", + ) + + @field_validator("api_key") + @classmethod + def validate_api_key(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("endpoint_url") + @classmethod + def validate_endpoint_url(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_model") + @classmethod + def validate_gateway_model(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_base_url") + @classmethod + def validate_gateway_base_url(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_auth_header") + @classmethod + def validate_gateway_auth_header(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if len(trimmed) > 64 or any(ch.isspace() for ch in trimmed): + raise ValueError("gateway_auth_header must be a single non-empty header name.") + return trimmed + + @field_validator("gateway_auth_secret_env") + @classmethod + def validate_gateway_auth_secret_env(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", trimmed): + raise ValueError( + "gateway_auth_secret_env must be a valid environment variable name." + ) + return trimmed + + @field_validator("gateway_auth_secret") + @classmethod + def validate_gateway_auth_secret(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_extra_headers") + @classmethod + def validate_gateway_extra_headers(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, str]]: + return _validate_gateway_extra_headers(v) + + @field_validator("enabled_models") + @classmethod + def validate_enabled_models_create(cls, v: Optional[List[str]]) -> Optional[List[str]]: + if v is None: + return None + seen: set[str] = set() + out: list[str] = [] + for item in v: + name = str(item).strip() + if not name or name in seen: + continue + seen.add(name) + out.append(name) + return out or None + + +class AIProviderUpdate(BaseModel): + """Schema for updating an AI Provider.""" + api_key: Optional[str] = Field(None, min_length=1) + name: Optional[str] = None + endpoint_url: Optional[str] = None + is_active: Optional[bool] = None + routing_mode: Optional[CredentialRoutingMode] = None + gateway_model: Optional[str] = Field(None, min_length=1, max_length=255) + gateway_interface: Optional[GatewayInterfaceMode] = None + gateway_base_url: Optional[str] = Field(None, max_length=512) + gateway_auth_header: Optional[str] = Field(None, max_length=64) + gateway_auth_secret_env: Optional[str] = Field(None, max_length=128) + gateway_auth_secret: Optional[str] = None + clear_gateway_auth_secret: bool = False + gateway_extra_headers: Optional[Dict[str, str]] = None + enabled_models: Optional[List[str]] = None + + @field_validator("gateway_model") + @classmethod + def validate_gateway_model(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_base_url") + @classmethod + def validate_gateway_base_url_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_auth_header") + @classmethod + def validate_gateway_auth_header_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if len(trimmed) > 64 or any(ch.isspace() for ch in trimmed): + raise ValueError("gateway_auth_header must be a single non-empty header name.") + return trimmed + + @field_validator("gateway_auth_secret_env") + @classmethod + def validate_gateway_auth_secret_env_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + if not trimmed: + return None + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", trimmed): + raise ValueError( + "gateway_auth_secret_env must be a valid environment variable name." + ) + return trimmed + + @field_validator("gateway_auth_secret") + @classmethod + def validate_gateway_auth_secret_update(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + trimmed = v.strip() + return trimmed or None + + @field_validator("gateway_extra_headers") + @classmethod + def validate_gateway_extra_headers_update( + cls, v: Optional[Dict[str, Any]] + ) -> Optional[Dict[str, str]]: + return _validate_gateway_extra_headers(v) + + @field_validator("enabled_models") + @classmethod + def validate_enabled_models_update(cls, v: Optional[List[str]]) -> Optional[List[str]]: + if v is None: + return None + seen: set[str] = set() + out: list[str] = [] + for item in v: + name = str(item).strip() + if not name or name in seen: + continue + seen.add(name) + out.append(name) + return out or None + + +class AIProviderResponse(BaseModel): + """Schema for AI Provider response.""" + id: UUID + provider: ModelProvider + api_key: Optional[str] = None # Will be None in response for security + name: Optional[str] + endpoint_url: Optional[str] = None + is_active: bool + is_default: bool = False + routing_mode: CredentialRoutingMode = CredentialRoutingMode.INHERIT + gateway_model: Optional[str] = None + gateway_interface: GatewayInterfaceMode = GatewayInterfaceMode.INHERIT + gateway_base_url: Optional[str] = None + gateway_auth_header: Optional[str] = None + gateway_auth_secret_env: Optional[str] = None + has_gateway_auth_secret: bool = False + gateway_extra_headers: Optional[Dict[str, str]] = None + enabled_models: Optional[List[str]] = None + gateway_managed: bool = False + effective_routing: Literal["inherit", "direct", "gateway", "bifrost", "litellm_proxy"] = "inherit" + effective_gateway_interface: Literal["litellm_shim", "native_openai"] = "litellm_shim" + created_at: datetime + updated_at: datetime + last_tested_at: Optional[datetime] + + @field_validator('provider', mode='before') + @classmethod + def convert_provider(cls, v): + """Convert string to ModelProvider (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return ModelProvider(v_lower) + except ValueError: + for enum_member in ModelProvider: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid ModelProvider value: {v}") + return v + + @field_validator('routing_mode', mode='before') + @classmethod + def convert_routing_mode(cls, v): + if v is None: + return CredentialRoutingMode.INHERIT + if isinstance(v, str): + try: + return CredentialRoutingMode(v.lower()) + except ValueError: + return CredentialRoutingMode.INHERIT + return v + + model_config = ConfigDict(from_attributes=True) + + +class LLMGenerationConfig(BaseModel): + """User-tunable LLM sampling / generation parameters.""" + + temperature: Optional[float] = Field(None, ge=0.0, le=2.0) + max_tokens: Optional[int] = Field(None, gt=0) + top_p: Optional[float] = Field(None, ge=0.0, le=1.0) + top_k: Optional[int] = Field(None, ge=0) + frequency_penalty: Optional[float] = Field(None, ge=-2.0, le=2.0) + presence_penalty: Optional[float] = Field(None, ge=-2.0, le=2.0) + seed: Optional[int] = Field(None, ge=0) + + def to_dict(self) -> Dict[str, Any]: + """Return only explicitly set fields.""" + return self.model_dump(exclude_none=True) + + +# VoiceBundle Schemas +class VoiceBundleCreate(BaseModel): + """Schema for creating a VoiceBundle.""" + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None + + # Bundle type: either STT+LLM+TTS or S2S + bundle_type: VoiceBundleType = Field(default=VoiceBundleType.STT_LLM_TTS) + + # STT Configuration - required for STT_LLM_TTS, optional for S2S + stt_provider: Optional[ModelProvider] = None + stt_model: Optional[str] = Field(None, min_length=1) + stt_credential_id: Optional[UUID] = Field( + None, + description=( + "Optional explicit AIProvider/Integration row id to use for STT. " + "When omitted the resolver picks the default credential for stt_provider." + ), + ) + + # LLM Configuration - required for STT_LLM_TTS, optional for S2S + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = Field(None, min_length=1) + llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) + llm_max_tokens: Optional[int] = Field(None, gt=0) + llm_config: Optional[Dict[str, Any]] = None + llm_credential_id: Optional[UUID] = None + + # TTS Configuration - required for STT_LLM_TTS, optional for S2S + tts_provider: Optional[ModelProvider] = None + tts_model: Optional[str] = Field(None, min_length=1) + tts_voice: Optional[str] = None + tts_config: Optional[Dict[str, Any]] = None + tts_credential_id: Optional[UUID] = None + + # S2S Configuration - required for S2S, optional for STT_LLM_TTS + s2s_provider: Optional[ModelProvider] = None + s2s_model: Optional[str] = Field(None, min_length=1) + s2s_config: Optional[Dict[str, Any]] = None + s2s_credential_id: Optional[UUID] = None + + # Additional metadata + extra_metadata: Optional[Dict[str, Any]] = None + + @model_validator(mode='after') + def validate_bundle_configuration(self): + """Validate that required fields are provided based on bundle_type.""" + if self.bundle_type == VoiceBundleType.STT_LLM_TTS: + if not self.stt_provider or not self.stt_model: + raise ValueError('STT provider and model are required for STT_LLM_TTS bundle type') + if not self.llm_provider or not self.llm_model: + raise ValueError('LLM provider and model are required for STT_LLM_TTS bundle type') + if not self.tts_provider or not self.tts_model: + raise ValueError('TTS provider and model are required for STT_LLM_TTS bundle type') + elif self.bundle_type == VoiceBundleType.S2S: + if not self.s2s_provider or not self.s2s_model: + raise ValueError('S2S provider and model are required for S2S bundle type') + return self + + +class VoiceBundleUpdate(BaseModel): + """Schema for updating a VoiceBundle.""" + name: Optional[str] = Field(None, min_length=1, max_length=255) + description: Optional[str] = None + + # Bundle type + bundle_type: Optional[VoiceBundleType] = None + + # STT Configuration + stt_provider: Optional[ModelProvider] = None + stt_model: Optional[str] = Field(None, min_length=1) + stt_credential_id: Optional[UUID] = None + + # LLM Configuration + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = Field(None, min_length=1) + llm_temperature: Optional[float] = Field(None, ge=0.0, le=2.0) + llm_max_tokens: Optional[int] = Field(None, gt=0) + llm_config: Optional[Dict[str, Any]] = None + llm_credential_id: Optional[UUID] = None + + # TTS Configuration + tts_provider: Optional[ModelProvider] = None + tts_model: Optional[str] = Field(None, min_length=1) + tts_voice: Optional[str] = None + tts_config: Optional[Dict[str, Any]] = None + tts_credential_id: Optional[UUID] = None + + # S2S Configuration + s2s_provider: Optional[ModelProvider] = None + s2s_model: Optional[str] = Field(None, min_length=1) + s2s_config: Optional[Dict[str, Any]] = None + s2s_credential_id: Optional[UUID] = None + + # Additional metadata + extra_metadata: Optional[Dict[str, Any]] = None + is_active: Optional[bool] = None + + +class VoiceBundleResponse(BaseModel): + """Schema for VoiceBundle response.""" + id: UUID + name: str + description: Optional[str] + + # Bundle type - can be string from DB or enum, validator handles conversion + bundle_type: VoiceBundleType + + @field_validator('bundle_type', mode='before') + @classmethod + def convert_bundle_type(cls, v): + """Convert string to VoiceBundleType enum if needed.""" + if isinstance(v, str): + try: + return VoiceBundleType(v) + except ValueError: + # Try to find by value + for enum_member in VoiceBundleType: + if enum_member.value == v: + return enum_member + raise ValueError(f"Invalid bundle_type value: {v}") + return v + + @field_validator('stt_provider', 'llm_provider', 'tts_provider', 's2s_provider', mode='before') + @classmethod + def convert_model_provider(cls, v): + """Convert string to ModelProvider enum if needed (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + # Try lowercase first (enum value) + v_lower = v.lower() + try: + return ModelProvider(v_lower) + except ValueError: + # Try to find by enum name (uppercase) + for enum_member in ModelProvider: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid ModelProvider value: {v}") + return v + + # STT Configuration + stt_provider: Optional[ModelProvider] + stt_model: Optional[str] + stt_credential_id: Optional[UUID] = None + + # LLM Configuration + llm_provider: Optional[ModelProvider] + llm_model: Optional[str] + llm_temperature: Optional[float] + llm_max_tokens: Optional[int] + llm_config: Optional[Dict[str, Any]] + llm_credential_id: Optional[UUID] = None + + # TTS Configuration + tts_provider: Optional[ModelProvider] + tts_model: Optional[str] + tts_voice: Optional[str] + tts_config: Optional[Dict[str, Any]] + tts_credential_id: Optional[UUID] = None + + # S2S Configuration + s2s_provider: Optional[ModelProvider] + s2s_model: Optional[str] + s2s_config: Optional[Dict[str, Any]] + s2s_credential_id: Optional[UUID] = None + + # Additional metadata + extra_metadata: Optional[Dict[str, Any]] + is_active: bool + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + model_config = ConfigDict(from_attributes=True) + + +# Test Agent Conversation Schemas +class TestAgentConversationCreate(BaseModel): + """Schema for creating a new test agent conversation.""" + agent_id: UUID + persona_id: UUID + scenario_id: UUID + voice_bundle_id: UUID + conversation_metadata: Optional[Dict[str, Any]] = None + + model_config = ConfigDict(json_schema_extra={ + "example": { + "agent_id": "123e4567-e89b-12d3-a456-426614174000", + "persona_id": "123e4567-e89b-12d3-a456-426614174001", + "scenario_id": "123e4567-e89b-12d3-a456-426614174002", + "voice_bundle_id": "123e4567-e89b-12d3-a456-426614174003" + } + }) + + +class TestAgentConversationUpdate(BaseModel): + """Schema for updating a test agent conversation.""" + status: Optional[str] = None + live_transcription: Optional[List[Dict[str, Any]]] = None + full_transcript: Optional[str] = None + conversation_metadata: Optional[Dict[str, Any]] = None + + +class TestAgentConversationResponse(BaseModel): + """Schema for test agent conversation response.""" + id: UUID + organization_id: UUID + agent_id: UUID + persona_id: UUID + scenario_id: UUID + voice_bundle_id: UUID + status: str + live_transcription: Optional[List[Dict[str, Any]]] + conversation_audio_key: Optional[str] + full_transcript: Optional[str] + started_at: datetime + ended_at: Optional[datetime] + duration_seconds: Optional[float] + conversation_metadata: Optional[Dict[str, Any]] + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + model_config = ConfigDict(from_attributes=True) + + +class ConversationTurn(BaseModel): + """Schema for a single conversation turn.""" + speaker: str # "test_agent" or "voice_agent" + text: str + timestamp: float # Time in seconds from start + audio_segment_key: Optional[str] = None # S3 key for this segment's audio + + +# Conversation Evaluation Schemas +class ConversationEvaluationCreate(BaseModel): + """Schema for creating a conversation evaluation.""" + transcription_id: UUID + agent_id: UUID + llm_provider: Optional[ModelProvider] = ModelProvider.OPENAI + llm_model: Optional[str] = "gpt-4o" + + model_config = ConfigDict(json_schema_extra={ + "example": { + "transcription_id": "123e4567-e89b-12d3-a456-426614174000", + "agent_id": "123e4567-e89b-12d3-a456-426614174001", + "llm_provider": "openai", + "llm_model": "gpt-4o" + } + }) + + +class ConversationEvaluationResponse(BaseModel): + """Schema for conversation evaluation response.""" + id: UUID + organization_id: UUID + transcription_id: UUID + agent_id: UUID + objective_achieved: bool + objective_achieved_reason: Optional[str] + additional_metrics: Optional[Dict[str, Any]] + overall_score: Optional[float] + llm_provider: Optional[ModelProvider] + llm_model: Optional[str] + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# Evaluator Schemas +class EvaluatorCreate(BaseModel): + """Schema for creating an evaluator. Either provide agent_id+persona_id+scenario_id (standard) or metric_ids/custom_prompt (custom).""" + name: Optional[str] = None + agent_id: Optional[UUID] = None + persona_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + custom_prompt: Optional[str] = None + metric_ids: Optional[List[UUID]] = None + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + + +class EvaluatorUpdate(BaseModel): + """Schema for updating an evaluator.""" + name: Optional[str] = None + agent_id: Optional[UUID] = None + persona_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + custom_prompt: Optional[str] = None + metric_ids: Optional[List[UUID]] = None + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + + +class EvaluatorResponse(BaseModel): + """Schema for evaluator response.""" + id: UUID + evaluator_id: str + organization_id: UUID + name: Optional[str] = None + agent_id: Optional[UUID] = None + persona_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + custom_prompt: Optional[str] = None + metric_ids: Optional[List[UUID]] = None + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + @field_validator('llm_provider', mode='before') + @classmethod + def convert_llm_provider(cls, v): + """Convert string to ModelProvider (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return ModelProvider(v_lower) + except ValueError: + for enum_member in ModelProvider: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid ModelProvider value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +class EvaluatorBulkCreate(BaseModel): + """Schema for creating multiple evaluators at once.""" + name: Optional[str] = None + agent_id: UUID + scenario_id: UUID + persona_ids: List[UUID] + tags: Optional[List[str]] = None + + +class RunEvaluatorsRequest(BaseModel): + """Schema for running evaluators.""" + evaluator_ids: List[UUID] = Field(..., description="List of evaluator IDs to run") + + +class RunEvaluatorsResponse(BaseModel): + """Schema for run evaluators response.""" + task_ids: List[str] = Field(..., description="List of Celery task IDs for tracking") + evaluator_results: List["EvaluatorResultResponse"] = Field(default_factory=list, description="List of created evaluator results") + + model_config = ConfigDict( + from_attributes=True, + json_schema_extra={ + "example": { + "agent_id": "123e4567-e89b-12d3-a456-426614174000", + "scenario_id": "123e4567-e89b-12d3-a456-426614174002", + "persona_ids": [ + "123e4567-e89b-12d3-a456-426614174001", + "123e4567-e89b-12d3-a456-426614174003" + ], + "tags": ["test", "production"] + } + }, + ) + + +# Metric Schemas +SelectionMode = Literal["single_choice", "multi_label"] + + +MetricScope = Literal["workspace", "organization"] + +# Max length for metric rubric text (description / example) accepted by +# the API. DB columns are TEXT or unbounded VARCHAR; this cap is validation-only. +METRIC_RUBRIC_TEXT_MAX_LENGTH = 32_000 + + + +class EvaluatorSuiteCombinationResponse(BaseModel): + """One agent+persona+scenario combination inside a suite.""" + id: UUID + evaluator_id: str + scenario_id: Optional[UUID] = None + scenario_name: Optional[str] = None + scenario_description: Optional[str] = None + scenario_required_info: Optional[Any] = None + + + + +class EvaluatorSuiteCreate(BaseModel): + """Schema for creating an evaluator suite.""" + name: Optional[str] = None + agent_id: UUID + persona_id: UUID + scenario_ids: List[UUID] + metric_ids: Optional[List[UUID]] = None + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + default_runs_per_combination: int = 1 + + + + +class EvaluatorSuiteUpdate(BaseModel): + """Schema for updating an evaluator suite.""" + name: Optional[str] = None + tags: Optional[List[str]] = None + default_runs_per_combination: Optional[int] = None + llm_provider: Optional[ModelProvider] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + metric_ids: Optional[List[UUID]] = None + + + + +class EvaluatorSuiteResponse(BaseModel): + """Schema for evaluator suite response.""" + id: UUID + organization_id: UUID + name: Optional[str] = None + agent_id: UUID + persona_id: UUID + agent_name: Optional[str] = None + persona_name: Optional[str] = None + agent_call_type: Optional[str] = None + agent_call_medium: Optional[str] = None + metric_ids: Optional[List[str]] = None + llm_provider: Optional[str] = None + llm_model: Optional[str] = None + llm_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + default_runs_per_combination: int = 1 + round_robin_index: int = 0 + is_active: bool = False + agent_suite_count: int = 1 + combination_count: int = 0 + combinations: List[EvaluatorSuiteCombinationResponse] = Field(default_factory=list) + created_at: datetime + updated_at: datetime + created_by: Optional[str] = None + + + + +class EvaluatorSuiteAddScenariosRequest(BaseModel): + """Schema for adding scenarios to an existing suite.""" + scenario_ids: List[UUID] + + + + +class RunEvaluatorSuiteRequest(BaseModel): + """Schema for running all combinations in a suite.""" + runs_per_combination: Optional[int] = None + to_number: Optional[str] = None + from_number: Optional[str] = None + + + + +class RunEvaluatorSuiteResponse(BaseModel): + """Schema for suite run response.""" + total_runs: int + task_ids: List[str] = Field(default_factory=list) + evaluator_results: List["EvaluatorResultResponse"] = Field(default_factory=list) + phone_call_refs: List[str] = Field(default_factory=list) + + + + +class RunNextCombinationRequest(BaseModel): + """Schema for running the next round-robin combination.""" + from_number: Optional[str] = None + + + + +class RunNextCombinationResponse(BaseModel): + """Schema for round-robin run response.""" + evaluator_id: UUID + scenario_id: Optional[UUID] = None + scenario_name: str + combination_index: int + next_index: int + evaluator_result_id: Optional[UUID] = None + result_id: Optional[str] = None + task_id: Optional[str] = None + phone_call_ref: Optional[str] = None + call_short_id: Optional[str] = None + + + + +class ChooseNextCombinationResponse(BaseModel): + """Advance inbound round-robin without initiating a call or evaluation run.""" + evaluator_id: UUID + scenario_id: Optional[UUID] = None + scenario_name: str + combination_index: int + next_index: int + + +# Metric Schemas +SelectionMode = Literal["single_choice", "multi_label"] + + +MetricScope = Literal["workspace", "organization"] + +# Max length for metric rubric text (description / example) accepted by +# the API. DB columns are TEXT or unbounded VARCHAR; this cap is validation-only. +METRIC_RUBRIC_TEXT_MAX_LENGTH = 32_000 + + + +class MetricCreate(BaseModel): + """Schema for creating a metric. + + Hierarchy: + - ``parent_metric_id`` set => this is a child sub-metric. ``metric_type`` + is forced to ``boolean`` server-side; ``selection_mode`` must be None. + - ``selection_mode`` set => this is a parent category metric. + ``parent_metric_id`` must be None (max depth = 2). + + Scope: + - ``scope="workspace"`` (default) stamps the metric with the active + ``X-Workspace-Id`` so it only shows up inside that workspace. + - ``scope="organization"`` stamps ``workspace_id=NULL`` so the metric + is visible in every workspace of the org. Children always inherit + their parent's scope; setting ``scope`` on a child request body is + ignored server-side. + """ + name: str + description: Optional[str] = None + # Optional illustrative example surfaced alongside ``description`` + # in the LLM judge's rubric. Today this is mainly populated on + # child sub-labels (one example per categorization label) but + # standalone metrics may carry it too without a schema change. + example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + metric_type: MetricType = MetricType.RATING + metric_category: MetricCategory = MetricCategory.QUALITY + trigger: MetricTrigger = MetricTrigger.ALWAYS + enabled: bool = True + metric_origin: str = "custom" + supported_surfaces: List[str] = ["agent"] + enabled_surfaces: Optional[List[str]] = None + custom_data_type: Optional[str] = None + custom_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + capture_rationale: Optional[bool] = False + parent_metric_id: Optional[UUID] = None + selection_mode: Optional[SelectionMode] = None + # Only meaningful on multi_label parents; ignored everywhere else. + # When true, the LLM is invited during call-import evaluation to emit + # additional candidate sub-labels beyond the user-defined children. + allow_discovery: bool = False + # When true, this metric is a "transcript-compare judge": at + # call-import evaluation time the worker feeds BOTH the production + # transcript (``call_import_rows.transcript``, CSV-supplied) and + # the diarised transcript (``call_import_rows.diarised_transcript``, + # worker-produced) to the LLM as a labeled pair. The parent + # evaluation's ``transcript_source`` is ignored for these metrics. + # Mutually exclusive with ``parent_metric_id`` / ``selection_mode`` + # G�� comparison metrics stay standalone so the LLM grouping logic + # doesn't have to second-guess which prompt template to use within + # a hierarchy. (Parent-level keyword auto-detection in the worker + # still routes a categorisation parent through the comparison + # prompt without setting this flag.) + compare_transcripts: bool = False + # When ``"organization"``, the metric is stored with + # ``workspace_id=NULL`` so it surfaces in every workspace of the + # caller's org. Default ``"workspace"`` preserves the historical + # behavior of stamping the metric with the active ``X-Workspace-Id``. + # Ignored when ``parent_metric_id`` is set (children inherit the + # parent's scope unconditionally). + scope: MetricScope = "workspace" + + @model_validator(mode='after') + def validate_compare_transcripts_exclusions(self): + """Reject body combinations that don't make sense for a + transcript-compare judge. + + The Metric ORM column accepts the value; the validator just + prevents the user from accidentally requesting an incoherent + metric shape (e.g. "compare two transcripts but also live + inside a categorisation hierarchy" � different prompt + templates). + """ + if not self.compare_transcripts: + return self + if self.parent_metric_id is not None: + raise ValueError( + "Transcript-compare metrics must be standalone: " + "they cannot be a child sub-metric in this version." + ) + if self.selection_mode is not None: + raise ValueError( + "Transcript-compare metrics must be standalone: " + "they cannot own children (selection_mode must be " + "unset) in this version." + ) + return self + + model_config = ConfigDict(json_schema_extra={ + "example": { + "name": "Professionalism", + "description": "Measures the professional tone and behavior", + "metric_type": "rating", + "trigger": "always", + "enabled": True + } + }) + + +class MetricChildDraft(BaseModel): + """One child sub-metric in a parent + children atomic create body.""" + + name: str = Field(..., max_length=120) + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + # Optional illustrative example for this label. Surfaced alongside + # ``description`` in the LLM judge's rubric so each label can carry + # both its definition AND a "what does this look like?" example. + example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + enabled: bool = True + capture_rationale: Optional[bool] = True + tags: Optional[List[str]] = None + + +class MetricCreateWithChildren(BaseModel): + """One-shot create body: a parent metric + N children, atomically. + + Children are persisted as full ``Metric`` rows with + ``parent_metric_id`` set to the new parent. ``metric_type`` on every + child is forced to ``boolean`` server-side regardless of what's + passed in the parent body. + """ + + name: str = Field(..., max_length=120) + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + selection_mode: SelectionMode + metric_category: MetricCategory = MetricCategory.QUALITY + enabled: bool = True + supported_surfaces: List[str] = Field(default_factory=lambda: ["agent"]) + enabled_surfaces: Optional[List[str]] = None + tags: Optional[List[str]] = None + # When true on a multi_label parent, allow the LLM to emit candidate + # labels beyond the listed children at evaluation time. Validator + # rejects allow_discovery=True on single_choice parents. + allow_discovery: bool = False + # Parent-level "Enable LLM Rationale" toggle. When true the LLM + # judge emits a single rationale string at the parent level + # (children never carry rationales in hierarchical mode), which the + # table renders as the " - LLM Rationale" column. + capture_rationale: bool = False + children: List[MetricChildDraft] = Field( + default_factory=list, + description="Child sub-metric labels under this parent.", + ) + # See ``MetricCreate.scope``. Same semantics: ``"organization"`` + # creates the parent + all children with ``workspace_id=NULL`` so + # the whole category subtree is shared across every workspace in + # the org. + scope: MetricScope = "workspace" + + +class MetricUpdate(BaseModel): + """Schema for updating a metric.""" + name: Optional[str] = None + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + # ``None`` here means "leave unchanged"; pass an empty string to + # clear a previously stored example. + example: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + metric_type: Optional[MetricType] = None + trigger: Optional[MetricTrigger] = None + enabled: Optional[bool] = None + metric_origin: Optional[str] = None + supported_surfaces: Optional[List[str]] = None + enabled_surfaces: Optional[List[str]] = None + custom_data_type: Optional[str] = None + custom_config: Optional[Dict[str, Any]] = None + tags: Optional[List[str]] = None + metric_category: Optional[MetricCategory] = None + capture_rationale: Optional[bool] = None + selection_mode: Optional[SelectionMode] = None + allow_discovery: Optional[bool] = None + # See ``MetricCreate.compare_transcripts``. ``None`` here means + # "leave unchanged". The route layer enforces mutual exclusion + # against the row's existing ``parent_metric_id`` / + # ``selection_mode`` when this is set to True, because the patch + # body alone doesn't have enough context to validate cross-state. + compare_transcripts: Optional[bool] = None + + @model_validator(mode='after') + def validate_compare_transcripts_exclusions(self): + """Reject patch bodies that flip compare_transcripts on while + ALSO trying to set a conflicting field in the same request. + + Cross-state validation against the persisted row (e.g. "the + existing metric already has a parent") is done in the + update route since the schema doesn't have the row in hand. + """ + if self.compare_transcripts is not True: + return self + if self.selection_mode is not None: + raise ValueError( + "Transcript-compare metrics must be standalone: " + "selection_mode must be cleared before enabling " + "compare_transcripts." + ) + return self + + +class MetricResponse(BaseModel): + """Schema for metric response. + + ``children`` is populated for parent metrics (those with + ``selection_mode`` set) and is otherwise an empty list. The list is + built once at serialization time so callers get a single tree + structure without follow-up requests. + """ + id: UUID + organization_id: UUID + # ``None`` when the metric is org-shared (``scope == "organization"``). + # See the ORM ``Metric.workspace_id`` docstring. + workspace_id: Optional[UUID] = None + # Computed convenience field so the UI doesn't have to do + # ``workspace_id == null`` checks everywhere. Always one of + # ``"workspace"`` or ``"organization"``. + scope: MetricScope = "workspace" + name: str + description: Optional[str] + # Optional illustrative example. Populated mainly on categorization + # child labels but surfaced for every metric so the UI can render + # it uniformly without branching on parent/child shape. + example: Optional[str] = None + metric_type: MetricType + metric_category: MetricCategory = MetricCategory.QUALITY + trigger: MetricTrigger + enabled: bool + is_default: bool + metric_origin: str + supported_surfaces: List[str] + enabled_surfaces: List[str] + custom_data_type: Optional[str] + custom_config: Optional[Dict[str, Any]] + tags: Optional[List[str]] + capture_rationale: bool = False + parent_metric_id: Optional[UUID] = None + selection_mode: Optional[SelectionMode] = None + allow_discovery: bool = False + # See ``MetricCreate.compare_transcripts``. Surfaced so the UI can + # render a "Compare transcripts" badge in the metric picker and + # know to skip the run's transcript_source toggle for this metric. + compare_transcripts: bool = False + lifecycle: str = "active" + promoted_from_draft_at: Optional[datetime] = None + studio_notes: Optional[str] = None + children: List["MetricResponse"] = Field(default_factory=list) + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + @field_validator('metric_type', mode='before') + @classmethod + def convert_metric_type(cls, v): + """Convert string to MetricType (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return MetricType(v_lower) + except ValueError: + for enum_member in MetricType: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid MetricType value: {v}") + return v + + @field_validator('trigger', mode='before') + @classmethod + def convert_trigger(cls, v): + """Convert string to MetricTrigger (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return MetricTrigger(v_lower) + except ValueError: + for enum_member in MetricTrigger: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid MetricTrigger value: {v}") + return v + + @validator('supported_surfaces', 'enabled_surfaces', pre=True) + def normalize_surfaces(cls, v): + if v is None: + return [] + if isinstance(v, str): + return [v] + if isinstance(v, (list, tuple)): + return [str(item).lower() for item in v if item] + return [] + + @validator('metric_origin', pre=True) + def normalize_metric_origin(cls, v): + if v is None: + return "custom" + return str(v).lower() + + model_config = ConfigDict(from_attributes=True) + + +MetricResponse.model_rebuild() + + +class MetricDraftCreate(MetricCreate): + """Create a draft metric for Metrics Studio experimentation.""" + + studio_notes: Optional[str] = Field( + default=None, + description="Optional notes about what this draft is testing.", + ) + + +class MetricDraftCreateWithChildren(MetricCreateWithChildren): + """Atomically create a draft parent category metric plus its children.""" + + studio_notes: Optional[str] = Field( + default=None, + description="Optional notes about what this draft category is testing.", + ) + + +class MetricPromoteResponse(BaseModel): + """Response after promoting a draft metric to active.""" + + metric: MetricResponse + promoted_at: datetime + + +MetricStudioSourceKind = Literal[ + "call_import_row", "call_recording", "evaluator_result" +] + + +class MetricStudioSourceItem(BaseModel): + """One call source selected for a Studio run.""" + + source_kind: MetricStudioSourceKind + source_ref: str = Field( + ..., + min_length=1, + description="UUID for import rows / evaluator results; call_short_id for recordings.", + ) + display_label: Optional[str] = Field( + default=None, + max_length=512, + description="Optional UI label; resolved server-side when omitted.", + ) + + +class MetricStudioRunCreate(BaseModel): + """Request body for triggering a Metrics Studio evaluation run.""" + + metric_ids: List[UUID] = Field(..., min_length=1) + sources: List[MetricStudioSourceItem] = Field(..., min_length=1) + name: Optional[str] = Field(default=None, max_length=255) + transcript_source: Literal["production", "diarised"] = "diarised" + llm_provider: Optional[str] = Field(default=None, max_length=50) + llm_model: Optional[str] = Field(default=None, max_length=100) + llm_credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + metric_llm_overrides: Optional[Dict[str, Any]] = None + + +class MetricStudioRunRetryRequest(BaseModel): + """Retry failed or selected Studio run results.""" + + result_ids: Optional[List[UUID]] = Field( + default=None, + description="When omitted, retry all failed results in the run.", + ) + + +class MetricStudioRunResultResponse(BaseModel): + """Per-source result row for a Studio run.""" + + id: UUID + run_id: UUID + source_kind: str + source_ref: str + display_label: Optional[str] = None + source_metadata: Optional[Dict[str, Any]] = None + status: str + metric_scores: Dict[str, Any] = Field(default_factory=dict) + error_message: Optional[str] = None + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class MetricStudioRunResponse(BaseModel): + """Metrics Studio run summary.""" + + id: UUID + organization_id: UUID + workspace_id: UUID + name: Optional[str] = None + selected_metric_ids: List[str] = Field(default_factory=list) + selected_metric_groups: Optional[Dict[str, List[str]]] = None + transcript_source: str + llm_provider: Optional[str] = None + llm_model: Optional[str] = None + status: str + total_items: int + completed_items: int + failed_items: int + error_message: Optional[str] = None + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class MetricStudioRunListResponse(BaseModel): + items: List[MetricStudioRunResponse] + total: int + + +class MetricStudioRunResultListResponse(BaseModel): + items: List[MetricStudioRunResultResponse] + total: int + + +# Evaluator Result Schemas +class EvaluatorResultCreate(BaseModel): + """Schema for creating an evaluator result.""" + evaluator_id: UUID + agent_id: Optional[UUID] = None + persona_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + name: Optional[str] = None + duration_seconds: Optional[float] = None + audio_s3_key: Optional[str] = None + + +class EvaluatorResultCreateManual(BaseModel): + """Schema for manually creating an evaluator result from existing audio file.""" + evaluator_id: UUID + audio_s3_key: str + duration_seconds: Optional[float] = None + + +class EvaluatorResultUpdate(BaseModel): + """Schema for updating an evaluator result.""" + status: Optional[EvaluatorResultStatus] = None + transcription: Optional[str] = None + metric_scores: Optional[Dict[str, Any]] = None + error_message: Optional[str] = None + duration_seconds: Optional[float] = None + + + +class EvaluatorResultCounts(BaseModel): + """Rollup counts for evaluator result navigation.""" + + total: int = 0 + completed: int = 0 + failed: int = 0 + in_progress: int = 0 + last_run_at: Optional[datetime] = None + + + + +class EvaluatorResultsScenarioSummary(BaseModel): + scenario_id: UUID + scenario_name: str + counts: EvaluatorResultCounts + + + + +class EvaluatorResultsSuiteSummary(BaseModel): + suite_id: UUID + suite_name: Optional[str] = None + agent_id: UUID + persona_id: Optional[UUID] = None + counts: EvaluatorResultCounts + scenarios: Optional[List["EvaluatorResultsScenarioSummary"]] = None + + + + +class EvaluatorResultsAgentSummary(BaseModel): + agent_id: UUID + agent_name: str + counts: EvaluatorResultCounts + suites: Optional[List[EvaluatorResultsSuiteSummary]] = None + + + + +class EvaluatorResultsUnassignedSummary(BaseModel): + counts: EvaluatorResultCounts + recent_result_ids: List[str] = Field(default_factory=list) + + + + +class EvaluatorResultsOverviewResponse(BaseModel): + workspace_counts: EvaluatorResultCounts + agents: List[EvaluatorResultsAgentSummary] = Field(default_factory=list) + unassigned: EvaluatorResultsUnassignedSummary + + + + +class EvaluatorResultListResponse(BaseModel): + items: List["EvaluatorResultResponse"] + total: int + + + +class EvaluatorResultResponse(BaseModel): + """Schema for evaluator result response.""" + id: UUID + result_id: str + organization_id: UUID + evaluator_id: Optional[UUID] = None # Optional for playground test results + agent_id: Optional[UUID] = None # Nullable for custom evaluators + persona_id: Optional[UUID] = None # Optional for playground test results + scenario_id: Optional[UUID] = None # Optional for playground test results + name: Optional[str] = None # Optional for playground test results + timestamp: datetime + duration_seconds: Optional[float] + status: EvaluatorResultStatus + audio_s3_key: Optional[str] + transcription: Optional[str] + speaker_segments: Optional[List[Dict[str, Any]]] = None # [{"speaker": "Speaker 1", "text": "...", "start": 0.0, "end": 5.2}] + metric_scores: Optional[Dict[str, Any]] + celery_task_id: Optional[str] + error_message: Optional[str] + + # Call tracking fields (for voice AI integrations) + call_event: Optional[str] = None + provider_call_id: Optional[str] = None + provider_platform: Optional[str] = None + call_data: Optional[Dict[str, Any]] = None # Full call details from provider + synthetic_call_trace_id: Optional[UUID] = None + + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + # Related entities (optional, populated when requested) + agent: Optional[AgentResponse] = None + persona: Optional[PersonaResponse] = None + scenario: Optional[ScenarioResponse] = None + evaluator: Optional[EvaluatorResponse] = None + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert string to EvaluatorResultStatus (handles uppercase DB values).""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return EvaluatorResultStatus(v_lower) + except ValueError: + for enum_member in EvaluatorResultStatus: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid EvaluatorResultStatus value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# ALERTING SCHEMAS +# ============================================ + +class AlertCreate(BaseModel): + """Schema for creating an alert.""" + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None + + # Metric condition + metric_type: AlertMetricType = AlertMetricType.NUMBER_OF_CALLS + aggregation: AlertAggregation = AlertAggregation.SUM + operator: AlertOperator = AlertOperator.GREATER_THAN + threshold_value: float = Field(..., description="Threshold value for the alert") + time_window_minutes: int = Field(default=60, ge=1, description="Time window in minutes for aggregation") + + # Agent selection (null means all agents) + agent_ids: Optional[List[UUID]] = None + + # Notification settings + notify_frequency: AlertNotifyFrequency = AlertNotifyFrequency.IMMEDIATE + notify_emails: Optional[List[str]] = Field(default=None, description="List of email addresses to notify") + notify_webhooks: Optional[List[str]] = Field(default=None, description="List of webhook URLs (Slack, etc.)") + + model_config = ConfigDict(json_schema_extra={ + "example": { + "name": "High Call Volume Alert", + "description": "Alert when call volume exceeds threshold", + "metric_type": "number_of_calls", + "aggregation": "sum", + "operator": ">", + "threshold_value": 100, + "time_window_minutes": 60, + "agent_ids": None, + "notify_frequency": "immediate", + "notify_emails": ["admin@example.com"], + "notify_webhooks": ["https://hooks.slack.com/services/xxx"] + } + }) + + +class AlertUpdate(BaseModel): + """Schema for updating an alert.""" + name: Optional[str] = Field(None, min_length=1, max_length=255) + description: Optional[str] = None + + # Metric condition + metric_type: Optional[AlertMetricType] = None + aggregation: Optional[AlertAggregation] = None + operator: Optional[AlertOperator] = None + threshold_value: Optional[float] = None + time_window_minutes: Optional[int] = Field(default=None, ge=1) + + # Agent selection + agent_ids: Optional[List[UUID]] = None + + # Notification settings + notify_frequency: Optional[AlertNotifyFrequency] = None + notify_emails: Optional[List[str]] = None + notify_webhooks: Optional[List[str]] = None + + # Status + status: Optional[AlertStatus] = None + + +class AlertResponse(BaseModel): + """Schema for alert response.""" + id: UUID + organization_id: UUID + name: str + description: Optional[str] + + # Metric condition + metric_type: AlertMetricType + aggregation: AlertAggregation + operator: AlertOperator + threshold_value: float + time_window_minutes: int + + # Agent selection + agent_ids: Optional[List[UUID]] + + # Notification settings + notify_frequency: AlertNotifyFrequency + notify_emails: Optional[List[str]] + notify_webhooks: Optional[List[str]] + + # Status + status: AlertStatus + + # Metadata + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + @field_validator('metric_type', mode='before') + @classmethod + def convert_metric_type(cls, v): + """Convert string to AlertMetricType.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return AlertMetricType(v_lower) + except ValueError: + for enum_member in AlertMetricType: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertMetricType value: {v}") + return v + + @field_validator('aggregation', mode='before') + @classmethod + def convert_aggregation(cls, v): + """Convert string to AlertAggregation.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return AlertAggregation(v_lower) + except ValueError: + for enum_member in AlertAggregation: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertAggregation value: {v}") + return v + + @field_validator('operator', mode='before') + @classmethod + def convert_operator(cls, v): + """Convert string to AlertOperator.""" + if v is None: + return None + if isinstance(v, str): + try: + return AlertOperator(v) + except ValueError: + for enum_member in AlertOperator: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertOperator value: {v}") + return v + + @field_validator('notify_frequency', mode='before') + @classmethod + def convert_notify_frequency(cls, v): + """Convert string to AlertNotifyFrequency.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return AlertNotifyFrequency(v_lower) + except ValueError: + for enum_member in AlertNotifyFrequency: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertNotifyFrequency value: {v}") + return v + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert string to AlertStatus.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return AlertStatus(v_lower) + except ValueError: + for enum_member in AlertStatus: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertStatus value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +class AlertHistoryResponse(BaseModel): + """Schema for alert history response.""" + id: UUID + organization_id: UUID + alert_id: UUID + + # Trigger information + triggered_at: datetime + triggered_value: float + threshold_value: float + + # Status + status: AlertHistoryStatus + + # Notification tracking + notified_at: Optional[datetime] + notification_details: Optional[Dict[str, Any]] + + # Resolution + acknowledged_at: Optional[datetime] + acknowledged_by: Optional[str] + resolved_at: Optional[datetime] + resolved_by: Optional[str] + resolution_notes: Optional[str] + + # Additional context + context_data: Optional[Dict[str, Any]] + + # Metadata + created_at: datetime + updated_at: datetime + + # Related alert info (optional) + alert: Optional[AlertResponse] = None + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert string to AlertHistoryStatus.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return AlertHistoryStatus(v_lower) + except ValueError: + for enum_member in AlertHistoryStatus: + if enum_member.name == v or enum_member.value == v: + return enum_member + raise ValueError(f"Invalid AlertHistoryStatus value: {v}") + return v + + model_config = ConfigDict(from_attributes=True) + + +class AlertHistoryUpdate(BaseModel): + """Schema for updating alert history (acknowledge/resolve).""" + status: Optional[AlertHistoryStatus] = None + acknowledged_by: Optional[str] = None + resolved_by: Optional[str] = None + resolution_notes: Optional[str] = None + + +# ============================================ +# CRON JOB SCHEMAS +# ============================================ + +class CronJobCreate(BaseModel): + """Schema for creating a cron job.""" + name: str = Field(..., min_length=1, max_length=255) + cron_expression: str = Field(..., min_length=1, max_length=100, description="Cron expression (e.g., '0 9 * * 1-5')") + timezone: str = Field(default="UTC", max_length=100, description="Timezone for the cron schedule") + max_runs: int = Field(default=10, ge=1, le=1000, description="Maximum number of times to run") + evaluator_ids: Optional[List[UUID]] = Field( + None, + description="Evaluator IDs to trigger (expanded with evaluator_suite_ids when both are set).", + ) + evaluator_suite_ids: Optional[List[UUID]] = Field( + None, + description="Evaluator suite IDs whose combinations are expanded into evaluator_ids.", + ) + + model_config = ConfigDict(json_schema_extra={ + "example": { + "name": "Daily Evaluation Run", + "cron_expression": "0 9 * * 1-5", + "timezone": "America/New_York", + "max_runs": 100, + "evaluator_ids": ["123e4567-e89b-12d3-a456-426614174000"] + } + }) + + +class CronJobUpdate(BaseModel): + """Schema for updating a cron job.""" + name: Optional[str] = Field(None, min_length=1, max_length=255) + cron_expression: Optional[str] = Field(None, min_length=1, max_length=100) + timezone: Optional[str] = Field(None, max_length=100) + max_runs: Optional[int] = Field(None, ge=1, le=1000) + evaluator_ids: Optional[List[UUID]] = None + evaluator_suite_ids: Optional[List[UUID]] = None + status: Optional[CronJobStatus] = None + + +class CronJobResponse(BaseModel): + """Schema for cron job response.""" + id: UUID + organization_id: UUID + name: str + job_type: str = "evaluator_run" + is_system: bool = False + cron_expression: str + timezone: str + max_runs: int + current_runs: int + evaluator_ids: List[UUID] + status: CronJobStatus + next_run_at: Optional[datetime] + last_run_at: Optional[datetime] + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + @field_validator('status', mode='before') + @classmethod + def convert_status(cls, v): + """Convert string to CronJobStatus.""" + if v is None: + return None + if isinstance(v, str): + v_lower = v.lower() + try: + return CronJobStatus(v_lower) + except ValueError: + for enum_member in CronJobStatus: + if enum_member.value.lower() == v_lower: + return enum_member + raise ValueError(f"Invalid status: {v}") + return v + + @field_validator('evaluator_ids', mode='before') + @classmethod + def convert_evaluator_ids(cls, v): + """Convert evaluator_ids from JSON to list of UUIDs.""" + if v is None: + return [] + if isinstance(v, list): + return [UUID(str(id)) if not isinstance(id, UUID) else id for id in v] + return v + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# PROMPT PARTIAL SCHEMAS +# ============================================ + +class MetricPartialChild(BaseModel): + """One categorization label inside a metric partial.""" + + name: str = Field(..., min_length=1) + description: str = "" + example: str = "" + + +class MetricPartialContent(BaseModel): + """Structured JSON payload stored in metric partial ``content``.""" + + schema_version: int = 1 + metric_kind: Literal["single", "category"] + description: str = "" + children: Optional[List[MetricPartialChild]] = None + + +class PromptPartialCreate(BaseModel): + """Schema for creating a prompt partial.""" + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None + content: str = Field(..., min_length=1) + tags: Optional[List[str]] = None + + +class PromptPartialUpdate(BaseModel): + """Schema for updating a prompt partial.""" + name: Optional[str] = Field(None, min_length=1, max_length=255) + description: Optional[str] = None + content: Optional[str] = Field(None, min_length=1) + tags: Optional[List[str]] = None + change_summary: Optional[str] = None + + +class PromptPartialVersionResponse(BaseModel): + """Schema for prompt partial version response.""" + id: UUID + prompt_partial_id: UUID + version: int + content: str + change_summary: Optional[str] + created_at: datetime + created_by: Optional[str] + + model_config = ConfigDict(from_attributes=True) + + +class AgentFlowNode(BaseModel): + """One step in an LLM-inferred agent logic flowchart.""" + + id: str + label: str + node_type: Literal["start", "decision", "action", "terminal"] = "action" + position_x: Optional[float] = None + position_y: Optional[float] = None + prompt_excerpt: Optional[str] = None + start_offset: Optional[int] = None + end_offset: Optional[int] = None + + +class AgentFlowEdge(BaseModel): + """Directed transition between two agent flow nodes.""" + + source: str + target: str + condition: Optional[str] = None + + +class AgentFlowNodeLayout(BaseModel): + id: str + position_x: float + position_y: float + + +class AgentFlowLayoutSaveRequest(BaseModel): + nodes: List[AgentFlowNodeLayout] = Field(default_factory=list) + + +class AgentFlowGraph(BaseModel): + """Aggregate flow diagram for an imported production agent prompt.""" + + nodes: List[AgentFlowNode] = Field(default_factory=list) + edges: List[AgentFlowEdge] = Field(default_factory=list) + generated_at: Optional[datetime] = None + provider: Optional[str] = None + model: Optional[str] = None + layout_saved_at: Optional[datetime] = None + prompt_content_hash: Optional[str] = None + mapping_error: Optional[str] = None + generation_error: Optional[str] = None + + +class PromptPartialResponse(BaseModel): + """Schema for prompt partial response.""" + id: UUID + organization_id: UUID + name: str + description: Optional[str] + content: str + tags: Optional[List[str]] + current_version: int + agent_flowchart: Optional[AgentFlowGraph] = None + agent_flowchart_status: Optional[str] = None + created_at: datetime + updated_at: datetime + created_by: Optional[str] + + model_config = ConfigDict(from_attributes=True) + + +class PromptPartialDetailResponse(PromptPartialResponse): + """Schema for prompt partial detail with versions.""" + versions: List[PromptPartialVersionResponse] = [] + + model_config = ConfigDict(from_attributes=True) + + +# ============================================ +# TELEPHONY SCHEMAS (provider-agnostic) +# ============================================ + + +class TelephonyIntegrationCreate(BaseModel): + """Schema for creating a telephony provider integration.""" + + provider: str = "plivo" + name: Optional[str] = None + auth_id: str + auth_token: str + verify_app_uuid: Optional[str] = None + voice_app_id: Optional[str] = None + sip_domain: Optional[str] = None + masking_config: Optional[Dict[str, Any]] = None + is_default: Optional[bool] = Field( + None, + description=( + "Mark this credential as the default for the (org, provider). " + "If omitted and no default exists yet, this row becomes the default." + ), + ) + + +class TelephonyIntegrationUpdate(BaseModel): + """Schema for partial updates to a telephony provider integration.""" + + id: Optional[UUID] = None + provider: Optional[str] = None + name: Optional[str] = None + auth_id: Optional[str] = None + auth_token: Optional[str] = None + verify_app_uuid: Optional[str] = None + voice_app_id: Optional[str] = None + sip_domain: Optional[str] = None + masking_config: Optional[Dict[str, Any]] = None + is_active: Optional[bool] = None + + +class TelephonyIntegrationResponse(BaseModel): + """Safe response model for telephony integration without secrets.""" + + id: UUID + organization_id: UUID + provider: str + name: Optional[str] = None + verify_app_uuid: Optional[str] + voice_app_id: Optional[str] + sip_domain: Optional[str] + masking_config: Optional[Dict[str, Any]] + is_active: bool + is_default: bool = False + last_tested_at: Optional[datetime] + created_at: datetime + updated_at: datetime + + class Config: + from_attributes = True + + +class TelephonyPhoneNumberResponse(BaseModel): + """Telephony phone number inventory response schema.""" + + id: UUID + phone_number: str + country_iso2: Optional[str] + region: Optional[str] + number_type: Optional[str] + capabilities: Optional[Dict[str, Any]] + is_masking_pool: bool + inbound_enabled: Optional[bool] = None + outbound_enabled: Optional[bool] = None + source: Optional[str] = None + agent_id: Optional[UUID] + linked_agent_name: Optional[str] = None + provider: Optional[str] = None + is_active: bool + created_at: datetime + + class Config: + from_attributes = True + + +class TelephonyDialTargetCreate(BaseModel): + """Schema for creating a saved outbound dial target.""" + phone_number: str + label: Optional[str] = None + + +class TelephonyDialTargetUpdate(BaseModel): + """Schema for updating a saved outbound dial target.""" + phone_number: Optional[str] = None + label: Optional[str] = None + + +class TelephonyDialTargetResponse(BaseModel): + """Schema for dial target response.""" + id: UUID + phone_number: str + label: Optional[str] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class TelephonyVerifyStartRequest(BaseModel): + """Request schema for starting voice OTP verification.""" + + phone_number: str + provider: str = "plivo" + + +class TelephonyVerifyStartResponse(BaseModel): + """Response schema for started voice OTP verification.""" + + session_id: UUID + provider_session_uuid: str + status: str + message: str + + +class TelephonyVerifyCheckRequest(BaseModel): + """Request schema for checking a submitted OTP code.""" + + session_id: UUID + otp_code: str + provider: str = "plivo" + + +class TelephonyVerifyCheckResponse(BaseModel): + """Response schema for OTP check status.""" + + verified: bool + status: str + message: str + + +class TelephonyMaskingSessionCreate(BaseModel): + """Request schema for creating a number masking session.""" + + party_a_number: str + party_b_number: str + provider: str = "plivo" + expires_in_minutes: Optional[int] = 60 + metadata: Optional[Dict[str, Any]] = None + provider: str = "plivo" + + +class TelephonyMaskingSessionResponse(BaseModel): + """Response schema for masking sessions.""" + + id: UUID + masked_number: str + party_a_number: str + party_b_number: str + status: str + expires_at: Optional[datetime] + created_at: datetime + + class Config: + from_attributes = True + + +class TelephonyOutboundCallRequest(BaseModel): + """Request schema for outbound call initiation.""" + + from_number: str + to_number: str + answer_url: Optional[str] = None + agent_id: Optional[UUID] = None + + +class TelephonyOutboundCallResponse(BaseModel): + """Response schema for outbound call initiation.""" + + provider_request_uuid: str + call_status: str + from_number: str + to_number: str + message: str + + +# --- Call Import Schemas --- + +class CallImportRowResponse(BaseModel): + """Single row within a call-import batch.""" + + id: UUID + row_index: int + # Renamed from ``external_call_id`` (DB column renamed in migration + # ``034_call_import_schemas``). Same data, same uniqueness rules. + conversation_id: str + recording_url: Optional[str] = None + recording_date: Optional[date] = None + # Production transcript: the value supplied via the CSV upload. + transcript: Optional[str] = None + transcript_source: Optional[str] = None + transcript_provider: Optional[str] = None + transcript_model: Optional[str] = None + transcript_status: Optional[str] = None + transcript_error: Optional[str] = None + transcribed_at: Optional[datetime] = None + # Diarised transcript: produced by the post-hoc diarisation + # worker. Independent of ``transcript`` so manual diarisation + # never overwrites the CSV-supplied production value. + diarised_transcript: Optional[str] = None + diarised_transcript_provider: Optional[str] = None + diarised_transcript_model: Optional[str] = None + diarised_transcript_status: Optional[str] = None + diarised_transcript_error: Optional[str] = None + diarised_at: Optional[datetime] = None + # LLM that turned the STT plain-text output into structured + # ``diarised_segments``. Surfaced in the row detail panel so + # reviewers can see "Diarised by openai/gpt-4o-mini" next to + # the swap toggle. NULL on rows diarised by the legacy pyannote + # worker (which has been removed). + diarised_llm_provider: Optional[str] = None + diarised_llm_model: Optional[str] = None + # The exact prompt the LLM diariser ran with. Persisted so a + # reviewer can copy it back into the modal and reproduce the + # turn layout against a different STT pass. + diarised_prompt: Optional[str] = None + # Structured speaker turns produced by the diarisation worker. Each + # entry is `{ "speaker": "agent"|"user"|"speaker_N", "text": str, + # "start": float, "end": float, "raw_speaker": "Speaker 1" }`. The + # plain ``diarised_transcript`` field above is a `: ` + # rendering of this list with ``diarised_speaker_swap`` applied. + diarised_segments: Optional[List[Dict[str, Any]]] = None + # When True the agent <-> user mapping in ``diarised_segments`` is + # inverted at render / export time. The worker writes the canonical + # mapping using the "first speaker is the agent" heuristic; the swap + # toggle lets reviewers correct that without re-running diarisation. + diarised_speaker_swap: bool = False + status: CallImportRowStatus + recording_s3_key: Optional[str] = None + recording_content_type: Optional[str] = None + recording_size_bytes: Optional[int] = None + error_message: Optional[str] = None + attempts: int + raw_columns: Optional[Dict[str, Any]] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +# --- Call Import Schema (Input Parameter definitions) --- + + +class CallImportSchemaParameterBase(BaseModel): + """A single typed parameter inside a Call Import schema. + + Used both in request bodies (create / update) and as the building + block of :class:`CallImportSchemaParameterResponse`. Names are + case-insensitive unique within their parent schema. + """ + + name: str = Field( + ..., + min_length=1, + max_length=255, + description=( + "Parameter name as it appears in the schema editor and the " + "upload mapping table. Must be unique within the schema " + "(case-insensitive)." + ), + ) + type: CallImportParameterType = Field( + ..., + description=( + "Parameter type. One of conversation_id / recording_url / " + "recording_date / transcript / text / number / boolean / " + "datetime / url. Exactly one parameter of type " + "'conversation_id' must be present; at most one each of " + "'recording_url', 'recording_date', and 'transcript'. " + "Only conversation_id is forced required." + ), + ) + description: Optional[str] = Field( + default=None, + max_length=2048, + description="Free-text help shown next to the parameter in the mapping UI.", + ) + is_required: bool = Field( + default=False, + description=( + "When True, the parameter must be mapped to a CSV column on " + "every upload. The ``conversation_id`` parameter is always " + "required and is force-set to True by the server." + ), + ) + + +class CallImportSchemaParameterCreate(CallImportSchemaParameterBase): + """Create payload for a single parameter (inside a schema CRUD body).""" + + +class CallImportSchemaParameterResponse(CallImportSchemaParameterBase): + """Response shape including the persisted id + ordering.""" + + id: UUID + ordering: int + + model_config = ConfigDict(from_attributes=True) + + +def _validate_schema_parameters( + parameters: List[CallImportSchemaParameterBase], +) -> List[CallImportSchemaParameterBase]: + """Apply the cross-parameter invariants shared by create + update.""" + + if not parameters: + raise ValueError("Schema must define at least one parameter.") + + seen_names: set[str] = set() + conv_count = 0 + recording_date_count = 0 + rec_url_count = 0 + transcript_count = 0 + for param in parameters: + norm = param.name.strip().lower() + if not norm: + raise ValueError("Parameter name must be non-empty.") + if norm in seen_names: + raise ValueError( + f"Duplicate parameter name '{param.name}' " + "(names must be unique within a schema)." + ) + seen_names.add(norm) + if param.type == CallImportParameterType.CONVERSATION_ID: + conv_count += 1 + elif param.type == CallImportParameterType.RECORDING_DATE: + recording_date_count += 1 + elif param.type == CallImportParameterType.RECORDING_URL: + rec_url_count += 1 + elif param.type == CallImportParameterType.TRANSCRIPT: + transcript_count += 1 + + if conv_count != 1: + raise ValueError( + "Schema must contain exactly one parameter of type " + "'conversation_id'." + ) + if rec_url_count > 1: + raise ValueError( + "Schema may contain at most one parameter of type " + "'recording_url'." + ) + if recording_date_count > 1: + raise ValueError( + "Schema may contain at most one parameter of type " + "'recording_date'." + ) + if transcript_count > 1: + raise ValueError( + "Schema may contain at most one parameter of type 'transcript'." + ) + return parameters + + +class CallImportSchemaCreate(BaseModel): + """Create body for a new call-import schema.""" + + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = Field(default=None, max_length=2048) + parameters: List[CallImportSchemaParameterCreate] = Field( + ..., + description=( + "Ordered list of parameters. Order is preserved; the server " + "stamps ``ordering`` from the list index." + ), + ) + + @model_validator(mode="after") + def _check_parameters(self): + _validate_schema_parameters(list(self.parameters)) + return self + + +class CallImportSchemaUpdate(BaseModel): + """Patch body for an existing schema (full parameter replacement).""" + + name: Optional[str] = Field(default=None, min_length=1, max_length=255) + description: Optional[str] = Field(default=None, max_length=2048) + parameters: Optional[List[CallImportSchemaParameterCreate]] = Field( + default=None, + description=( + "If provided, REPLACES the full set of parameters on the " + "schema. Omit to leave parameters untouched." + ), + ) + + @model_validator(mode="after") + def _check_parameters(self): + if self.parameters is not None: + _validate_schema_parameters(list(self.parameters)) + return self + + +class CallImportSchemaResponse(BaseModel): + """Read response for a single schema.""" + + id: UUID + organization_id: UUID + workspace_id: UUID + name: str + description: Optional[str] = None + parameters: List[CallImportSchemaParameterResponse] = Field(default_factory=list) + # How many CallImport batches reference this schema. Populated by the + # router when listing; defaults to 0 on detail responses where the + # caller doesn't need it. + usage_count: int = 0 + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class CallImportSchemaListResponse(BaseModel): + """Paginated list of schemas.""" + + items: List[CallImportSchemaResponse] = Field(default_factory=list) + total: int + + +class CallImportTagResponse(BaseModel): + """Tag attached to call import batches.""" + + id: UUID + name: str + color: Optional[str] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class CallImportTagCreate(BaseModel): + """Create a new call-import tag for the organization.""" + + name: str = Field(..., min_length=1, max_length=255) + color: Optional[str] = Field(None, max_length=32) + + +class CallImportTagUpdate(BaseModel): + """Partial update for a call-import tag.""" + + name: Optional[str] = Field(None, min_length=1, max_length=255) + color: Optional[str] = Field(None, max_length=32) + + +class CallImportPreviewSheet(BaseModel): + """One worksheet (or one CSV file synthesized as a single sheet).""" + + name: str = Field(..., description="Sheet name for xlsx; filename for csv.") + headers: List[str] = Field( + default_factory=list, + description="Column headers from the first non-empty row.", + ) + row_count: int = Field( + ..., + description="Approximate count of data rows (excluding the header row).", + ) + + +class CallImportSourceRowSkip(BaseModel): + """One source spreadsheet row skipped during parse (identity / recording URL).""" + + source_row: int = Field( + ..., + description="1-based row index in the source file (same semantics as parse errors).", + ) + reason: str = Field( + ..., + description=( + "Machine-readable skip reason, e.g. missing_conversation_id, " + "missing_recording_url, invalid_recording_url." + ), + ) + message: str = Field( + ..., + description="Human-readable explanation shown in the UI.", + ) + + +class CallImportResponse(BaseModel): + """Summary of a call-import batch.""" + + id: UUID + organization_id: UUID + workspace_id: UUID + # Provider is optional in the new staged flow (only resolved at the + # IMPORT stage). Stays populated for all post-import batches. + provider: Optional[str] = None + telephony_integration_id: Optional[UUID] = None + original_filename: Optional[str] = None + sheet_name: Optional[str] = None + dataset: Optional[str] = None + tags: List[CallImportTagResponse] = Field(default_factory=list) + # New schema-driven mapping. Empty on legacy batches; pre-schema + # batches keep their values in ``column_mapping`` / ``extra_columns`` + # / ``custom_column_mapping`` below for backwards-compatibility. + schema_id: Optional[UUID] = None + parameter_mapping: Dict[str, str] = Field(default_factory=dict) + column_mapping: Dict[str, Optional[str]] = Field(default_factory=dict) + extra_columns: List[str] = Field(default_factory=list) + custom_column_mapping: Dict[str, str] = Field(default_factory=dict) + # Persisted "drop these columns" decision captured at MAP time. + # Empty for legacy one-shot uploads where the value was ephemeral. + skipped_columns: List[str] = Field(default_factory=list) + source_row_skips: List[CallImportSourceRowSkip] = Field( + default_factory=list, + description=( + "Source rows skipped at parse time because of missing/invalid " + "conversation ID or recording URL." + ), + ) + # Source-file staging fields populated at UPLOAD time. ``None`` on + # legacy batches imported via the one-shot ``POST /upload`` endpoint. + source_s3_key: Optional[str] = None + source_format: Optional[str] = None + source_size_bytes: Optional[int] = None + source_content_type: Optional[str] = None + # Snapshot of the file's sheets + headers captured at UPLOAD time + # so the MAP UI can render without re-fetching the file from S3. + available_sheets: Optional[List[CallImportPreviewSheet]] = None + total_rows: int + completed_rows: int + failed_rows: int + status: CallImportStatus + error_message: Optional[str] = None + latest_evaluation_status: Optional[str] = Field( + None, + description=( + "Status of the most recent evaluation run for this batch, " + "when any evaluation exists." + ), + ) + created_at: datetime + updated_at: datetime + created_by_email: Optional[str] = None + last_updated_by_email: Optional[str] = None + + model_config = ConfigDict(from_attributes=True) + + +class CallImportDetailResponse(CallImportResponse): + """A call-import batch with its rows expanded. + + ``filtered_total_rows`` is only set when the caller passed a ``q`` + search term G�� it lets the UI paginate against the filtered subset + while still showing the unfiltered ``total_rows`` in the header. + + The ``diarised_*_rows`` counters aggregate + ``CallImportRow.diarised_transcript_status`` across the batch so the + UI can render a transcribe-and-diarise progress bar without paging + through every row. Rows that have never been touched by the + transcribe/diarise worker (``status='idle'``) are NOT counted here G�� + callers compute the idle bucket as + ``total_rows - (pending + running + completed + failed)``. + """ + + rows: List[CallImportRowResponse] = Field(default_factory=list) + filtered_total_rows: Optional[int] = None + diarised_pending_rows: int = 0 + diarised_running_rows: int = 0 + diarised_completed_rows: int = 0 + diarised_failed_rows: int = 0 + + +class CallImportListResponse(BaseModel): + """Paginated list of call-import batches.""" + + items: List[CallImportResponse] + total: int + page: int + page_size: int + + +class CallImportDispatchLimitSnapshot(BaseModel): + """Configured and live Redis in-flight caps for eval work.""" + + global_limit: int + global_inflight: int + global_at_capacity: bool + org_limit: int + org_inflight: int + org_at_capacity: bool + workspace_limit: int + job_limit: int + fair_dispatch_batch_size: int + + +class CallImportDispatchFairDispatchSnapshot(BaseModel): + """Fair-dispatch scheduler metadata from Redis.""" + + global_rr_cursor: int + dispatch_dedupe_active: bool + dispatch_queue: str + at_capacity_backoff_seconds: int + + +class CallImportDispatchEvaluationSnapshot(BaseModel): + """One in-flight evaluation run with row counters.""" + + evaluation_id: UUID + call_import_id: UUID + status: str + total_rows: int + pending_rows: int + running_rows: int + job_inflight: int + job_at_capacity: bool + + +class CallImportDispatchWorkspaceSnapshot(BaseModel): + """Per-workspace pending dispatch + slot usage.""" + + workspace_id: UUID + workspace_name: Optional[str] = None + workspace_slug: Optional[str] = None + inflight: int + inflight_at_capacity: bool + pending_dispatch_rows: int + pending_import_rows: int + eval_rr_cursor: int + active_evaluations: int + evaluations: List[CallImportDispatchEvaluationSnapshot] = Field( + default_factory=list + ) + + +class CallImportDispatchDiagnosticsResponse(BaseModel): + """Live operator snapshot for call-import eval fair dispatch.""" + + limits: CallImportDispatchLimitSnapshot + fair_dispatch: CallImportDispatchFairDispatchSnapshot + workspaces: List[CallImportDispatchWorkspaceSnapshot] + generated_at: datetime + + +class CallImportUploadResponse(BaseModel): + """Response returned right after a CSV is accepted.""" + + id: UUID + total_rows: int + status: CallImportStatus + dataset: Optional[str] = None + tags: List[CallImportTagResponse] = Field(default_factory=list) + message: str + + +class CallImportDeleteResponse(BaseModel): + """Response after a whole-batch call-import delete is accepted.""" + + id: UUID + status: Literal["accepted", "completed"] = Field( + ..., + description=( + "``accepted`` when teardown was queued to run asynchronously; " + "``completed`` when the batch was already removed." + ), + ) + + +class CallImportPreviewResponse(BaseModel): + """Sheets/headers extracted from an uploaded CSV or Excel workbook. + + The frontend uses this to drive the column-mapping UI without doing + its own parsing G�� keeps client and server in lockstep on quoted + fields, encodings, and Excel cell coercion. + """ + + format: str = Field(..., description="One of 'csv' or 'xlsx'.") + sheets: List[CallImportPreviewSheet] = Field(default_factory=list) + + +class CallImportUpdate(BaseModel): + """Partial update of a call-import batch.""" + + original_filename: Optional[str] = Field( + None, + description=( + "User-facing batch label shown in the UI. Pass an empty string to clear." + ), + ) + dataset: Optional[str] = Field( + None, + description=( + "Free-text dataset label. Pass an empty string to clear the dataset." + ), + ) + tag_ids: Optional[List[UUID]] = Field( + None, + description=( + "Replace the full set of tag assignments. Pass an empty list to clear all tags." + ), + ) + schema_id: Optional[UUID] = Field( + None, + description=( + "Reassign the Input Parameter schema. Only honoured while the " + "batch is in ``uploaded`` or ``mapped`` state; once the batch " + "has rows it's locked to its original schema." + ), + ) + + +class CallImportMappingUpdate(BaseModel): + """Mapping payload for the MAP stage (``PATCH /call-imports/{id}/mapping``). + + Idempotent: callers can submit this multiple times against an + ``uploaded`` or ``mapped`` batch. Validation re-runs against the + persisted ``available_sheets`` snapshot every time so the user can + correct mistakes without re-uploading the file. + """ + + schema_id: UUID = Field( + ..., + description=( + "Reusable Input Parameter schema this batch is mapped against. " + "Must belong to the active workspace." + ), + ) + sheet_name: Optional[str] = Field( + None, + description=( + "Worksheet to use when the staged source file is an Excel " + "workbook. REQUIRED for xlsx; ignored / rejected for CSV." + ), + ) + parameter_mapping: Dict[str, str] = Field( + default_factory=dict, + description=( + "``{schema_parameter_name: source_header}`` map covering every " + "required schema parameter." + ), + ) + skipped_columns: List[str] = Field( + default_factory=list, + description=( + "Source headers the uploader has explicitly skipped. Every " + "source header must be either mapped or appear here." + ), + ) + + +class CallImportStartRequest(BaseModel): + """Provider + credential picker for the IMPORT stage.""" + + provider: Optional[str] = Field( + default=None, + description=( + "Telephony provider key. Must match the " + "``telephony_integration_id``'s provider. Omit together with " + "``telephony_integration_id`` to download recordings directly " + "from CSV-supplied URLs without credentials." + ), + ) + telephony_integration_id: Optional[UUID] = Field( + default=None, + description=( + "Specific TelephonyIntegration credential row to use when " + "downloading recordings for this batch. Omit together with " + "``provider`` for direct-URL import." + ), + ) + + @model_validator(mode="after") + def validate_credential_mode(self) -> "CallImportStartRequest": + has_provider = bool((self.provider or "").strip()) + has_integration = self.telephony_integration_id is not None + if has_provider != has_integration: + raise ValueError( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL import." + ) + return self + + +# --- Call Import Evaluation Schemas --- + + +class CallImportEvaluationLLMOverride(BaseModel): + """Per-metric LLM override used on top of the run-level default. + + Any field left ``None`` falls back to the run-level value (which + itself falls back to the historical OpenAI/gpt-4o default). This + lets users pick a specific provider/model for a single metric (e.g. + a stronger Anthropic model for a tricky qualitative metric) without + re-typing the rest of the metrics in the run. + """ + + provider: Optional[str] = Field( + default=None, + max_length=50, + description="Override LLM provider key, e.g. 'openai' or 'anthropic'.", + ) + model: Optional[str] = Field( + default=None, + max_length=100, + description="Override LLM model name, e.g. 'gpt-4o' or 'claude-3-opus'.", + ) + credential_id: Optional[UUID] = Field( + default=None, + description="Optional AIProvider id when the org has multiple credentials.", + ) + llm_config: Optional[Dict[str, Any]] = Field( + default=None, + description="Optional per-metric generation parameters (temperature, top_p, etc.).", + ) + + +CallImportEvaluationTranscriptSource = Literal["production", "diarised"] + + +class CallImportEvaluationCreate(BaseModel): + """Request body for triggering an evaluation over a call-import batch.""" + + metric_ids: List[UUID] = Field( + ..., + min_length=1, + description="Org Metric ids to score every completed row against.", + ) + name: Optional[str] = Field( + default=None, + max_length=255, + description=( + "Optional human-readable label for the run. Shown in the UI " + "instead of the UUID prefix." + ), + ) + transcript_sources: List[CallImportEvaluationTranscriptSource] = Field( + default_factory=lambda: ["diarised"], + min_length=1, + max_length=1, + description=( + "Which transcript to score against. ``'diarised'`` (default) " + "auto-diarises rows missing a diarised transcript then scores " + "``diarised_transcript``. ``'production'`` scores the CSV " + "``transcript`` column directly and skips diarisation." + ), + ) + + @field_validator("transcript_sources") + @classmethod + def _validate_transcript_sources( + cls, value: List[str] + ) -> List["CallImportEvaluationTranscriptSource"]: + allowed = {"production", "diarised"} + invalid = [src for src in value if src not in allowed] + if invalid: + raise ValueError( + "transcript_sources must be ['production'] or ['diarised'] " + "(received: " + + ", ".join(repr(src) for src in invalid) + + ")." + ) + return value # type: ignore[return-value] + # --- Run-level LLM config --- + llm_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "Run-level LLM provider key (e.g. 'openai', 'anthropic'). NULL " + "preserves the historical OpenAI/gpt-4o default." + ), + ) + llm_model: Optional[str] = Field( + default=None, + max_length=100, + description="Run-level LLM model name. Required when llm_provider is set.", + ) + llm_credential_id: Optional[UUID] = Field( + default=None, + description="Optional AIProvider row to pin for the run-level LLM.", + ) + llm_config: Optional[Dict[str, Any]] = Field( + default=None, + description="Run-level LLM generation parameters (temperature, top_p, etc.).", + ) + metric_llm_overrides: Optional[ + Dict[str, CallImportEvaluationLLMOverride] + ] = Field( + default=None, + description=( + "Optional per-metric LLM overrides keyed by metric UUID. Each " + "entry overrides the run-level default for that metric only." + ), + ) + # --- Auto-transcribe / diarization hook --- + # Every diarised run auto-diarises rows that don't already have a + # diarised transcript. The flag stays on the schema so legacy API + # callers don't 400 immediately, but the route now requires + # ``stt_provider`` + ``stt_model`` on every run regardless of this + # value. + auto_transcribe: bool = Field( + default=True, + description=( + "Auto-diarise rows missing a diarised transcript before " + "evaluation. Defaults to true and is effectively required: " + "``stt_provider`` + ``stt_model`` are mandatory on every " + "evaluation run." + ), + ) + transcribe_overwrite: bool = Field( + default=False, + description=( + "When auto_transcribe is on, overwrite existing transcripts " + "instead of skipping rows that already have one." + ), + ) + transcribe_mode: Literal["stt_llm", "llm_only"] = Field( + default="stt_llm", + description=( + "Diarisation pipeline shape for the auto-transcribe step. " + "'stt_llm' (default) runs STT then an LLM diariser over the " + "resulting text G�� ``stt_provider`` + ``stt_model`` must be " + "provided. 'llm_only' skips STT and feeds the audio " + "directly to the multimodal ``diarization_llm_*`` model " + "along with ``diarization_prompt``; STT fields must be " + "omitted in that case." + ), + ) + stt_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "STT provider key, e.g. 'deepgram', 'openai'. Required when " + "``transcribe_mode='stt_llm'`` (the default); must be omitted " + "when ``transcribe_mode='llm_only'``." + ), + ) + stt_model: Optional[str] = Field( + default=None, + max_length=100, + description=( + "STT model name, e.g. 'nova-2', 'whisper-1'. Same presence " + "rules as ``stt_provider``." + ), + ) + stt_credential_id: Optional[UUID] = Field( + default=None, + description="Optional AIProvider/Integration row to pin for STT.", + ) + stt_language: Optional[str] = Field( + default=None, + max_length=20, + description="ISO language hint for the STT provider, e.g. 'en'.", + ) + # --- LLM diariser config (mirror of CallImportTranscribeRequest) --- + # Auto-diarised eval rows go through the same LLM-based diariser as + # the standalone Transcribe modal G�� the run remembers the provider / + # model / prompt so a follow-up retry can reproduce them without + # having to re-prompt the user. + diarization_llm_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "LLM provider for diarising STT output into agent/user " + "turns. Required when ``auto_transcribe`` is set (the worker " + "no longer falls back to pyannote)." + ), + ) + diarization_llm_model: Optional[str] = Field( + default=None, + max_length=100, + description="LLM model for the diariser.", + ) + diarization_llm_credential_id: Optional[UUID] = Field( + default=None, + description="Optional AIProvider row to pin for the diariser LLM.", + ) + diarization_prompt: Optional[str] = Field( + default=None, + max_length=10_000, + description=( + "Custom system prompt for the diariser LLM; falls back to " + "the canonical default when blank." + ), + ) + discover_new_metrics: bool = Field( + default=False, + description=( + "When true, the LLM is invited to propose net-new top-level " + "metrics (boolean / rating / category) observed in the " + "transcripts in addition to scoring the selected metrics. " + "Candidates surface in the Discovered metrics panel on the " + "evaluation detail Flow tab and can be promoted into real " + "standalone Metric rows. Defaults to false so existing " + "callers retain previous behaviour." + ), + ) + # Telephony credentials for unified pipeline (required when batch is mapped). + provider: Optional[str] = Field( + default=None, + description=( + "Telephony provider key. Required together with " + "``telephony_integration_id`` when starting evaluation " + "from a mapped batch. Omit both for direct-URL import." + ), + ) + telephony_integration_id: Optional[UUID] = Field( + default=None, + description=( + "TelephonyIntegration credential for recording fetch. " + "Required together with ``provider`` for credentialed import." + ), + ) + + @model_validator(mode="after") + def validate_telephony_credential_mode(self) -> "CallImportEvaluationCreate": + has_provider = bool((self.provider or "").strip()) + has_integration = self.telephony_integration_id is not None + if has_provider != has_integration: + raise ValueError( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL evaluation." + ) + return self + + +class CallImportEvaluationUpdate(BaseModel): + """Patch body for editing a previously-created evaluation run.""" + + name: Optional[str] = Field( + default=None, + max_length=255, + description="New name for the evaluation. Empty string clears it.", + ) + + +class CallImportEvaluationBulkDelete(BaseModel): + """Request body for deleting multiple evaluation runs in one call.""" + + evaluation_ids: List[UUID] = Field( + ..., + min_length=1, + description="Evaluation ids to delete.", + ) + + +class CallImportEvaluationRetryRequest(BaseModel): + """Body for retrying a subset (or all failed rows) of an evaluation run. + + ``eval_row_ids`` is optional: when ``None`` the retry applies to + every row in the run that is currently in the ``failed`` state. The + selection always intersects with the run's actual rows, so unknown + ids are silently skipped (and surfaced in the response's + ``skipped`` list with reason ``unknown``). + + The optional ``llm_*`` / ``metric_llm_overrides`` / ``stt_*`` fields + let the caller swap out the LLM or STT configuration that the + failed rows were originally evaluated with. When a field is left + ``None`` the run's existing value is preserved. When a field is + set, it is persisted onto the run (so a follow-up retry sees the + new value as the default) and used by the worker on the next + pass. Providing only one half of provider+model is rejected so + the worker never ends up with a half-configured run. + """ + + eval_row_ids: Optional[List[UUID]] = Field( + default=None, + description=( + "Restrict the retry to a specific subset of evaluation rows. " + "When omitted, every row with status='failed' in this run is " + "re-enqueued." + ), + ) + + # --- Metric-subset re-run --- + # When ``metric_ids`` is set, the retry recomputes ONLY those + # metrics instead of the whole row, and the new scores are merged + # into the existing ``metric_scores`` JSON (other metrics' + # previously-computed values are preserved). This is the path + # taken by the "Re-run metrics" UI in CallImportEvaluationDetail. + metric_ids: Optional[List[UUID]] = Field( + default=None, + description=( + "Restrict the retry to a specific subset of metrics. When " + "set, the worker recomputes only these metrics and merges " + "the new scores into the row's existing metric_scores " + "(other metrics' previous values are preserved). When " + "omitted, the row is fully re-scored as before. Every id " + "must already be present in the run's selected_metric_ids." + ), + ) + include_completed: bool = Field( + default=False, + description=( + "When True, rows whose status is currently 'completed' " + "become eligible for retry (otherwise only 'failed' rows " + "are picked up). Required when ``metric_ids`` is set on a " + "successful row, since otherwise the whole metric-subset " + "retry would be skipped as 'completed'." + ), + ) + + # --- LLM overrides --- + llm_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "Override the run-level LLM provider for this retry (and " + "future retries). Must be paired with ``llm_model``." + ), + ) + llm_model: Optional[str] = Field( + default=None, + max_length=100, + description=( + "Override the run-level LLM model. Must be paired with " + "``llm_provider``." + ), + ) + llm_credential_id: Optional[UUID] = Field( + default=None, + description=( + "Pin a specific AIProvider credential row for the LLM. " + "When omitted, the resolver falls back to the org default." + ), + ) + llm_config: Optional[Dict[str, Any]] = Field( + default=None, + description="Override run-level LLM generation parameters for this retry.", + ) + metric_llm_overrides: Optional[ + Dict[str, CallImportEvaluationLLMOverride] + ] = Field( + default=None, + description=( + "Replace the run's per-metric LLM overrides. When omitted, " + "the existing overrides are kept; when set, this dict " + "fully replaces them (pass an empty object to clear)." + ), + ) + + # --- STT overrides --- + stt_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "Override the run-level STT provider for this retry. Must " + "be paired with ``stt_model``. Only meaningful when the " + "run is configured for the diarised transcript source." + ), + ) + stt_model: Optional[str] = Field( + default=None, + max_length=100, + description="Override the run-level STT model.", + ) + stt_credential_id: Optional[UUID] = Field( + default=None, + description="Pin a specific credential row for the STT call.", + ) + # --- LLM diariser overrides --- + # When set, replace the run-stored diariser configuration for any + # rows that have to be re-diarised as part of the retry (i.e. + # ``transcribe_overwrite=True`` or the row never had a diarised + # transcript). Same provider+model pairing rule as STT. + diarization_llm_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "Override the run's diariser LLM provider. Must be paired " + "with ``diarization_llm_model``." + ), + ) + diarization_llm_model: Optional[str] = Field( + default=None, + max_length=100, + description="Override the run's diariser LLM model.", + ) + diarization_llm_credential_id: Optional[UUID] = Field( + default=None, + description="Pin a specific credential row for the diariser LLM.", + ) + diarization_prompt: Optional[str] = Field( + default=None, + max_length=10_000, + description=( + "Override the run's diariser prompt. Pass an empty string " + "to clear the override and fall back to the canonical " + "default; pass None to leave the existing value untouched." + ), + ) + transcribe_overwrite: bool = Field( + default=False, + description=( + "When True, wipe the diarised transcript on every retried " + "row's source CallImportRow so the (possibly new) STT runs " + "from scratch. When False, rows that already have a " + "diarised transcript skip diarisation and only re-evaluate." + ), + ) + transcribe_mode: Optional[Literal["stt_llm", "llm_only"]] = Field( + default=None, + description=( + "Override the run's diarisation pipeline mode for this retry. " + "``stt_llm`` runs STT then an LLM diariser; ``llm_only`` feeds " + "audio directly to a multimodal diariser LLM." + ), + ) + + # Telephony credentials for rows that must re-fetch recordings. + provider: Optional[str] = Field( + default=None, + description=( + "Override the batch's telephony provider for this retry pass. " + "Must be paired with ``telephony_integration_id``. Omit both " + "fields to keep the batch's existing pinned credentials." + ), + ) + telephony_integration_id: Optional[UUID] = Field( + default=None, + description=( + "Override the telephony credential used when re-fetching " + "recordings during this retry. Must be paired with " + "``provider``. Omit both to keep existing credentials; send " + "both as null for direct-URL retry." + ), + ) + + @model_validator(mode="after") + def validate_telephony_credential_mode(self) -> "CallImportEvaluationRetryRequest": + fields_set = self.model_fields_set + if ( + "provider" not in fields_set + and "telephony_integration_id" not in fields_set + ): + return self + has_provider = bool((self.provider or "").strip()) + has_integration = self.telephony_integration_id is not None + if has_provider != has_integration: + raise ValueError( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL retry." + ) + return self + + +class CallImportEvaluationRetrySkippedItem(BaseModel): + """One entry in the retry response's ``skipped`` list.""" + + eval_row_id: UUID + reason: str = Field( + ..., + description=( + "Why this row was not re-enqueued. Known values: " + "'unknown' (id not in this run), 'in_progress' " + "(status is pending/running), 'completed' (already " + "successful), 'source_row_missing'." + ), + ) + + +class CallImportEvaluationRetryResponse(BaseModel): + """Summary of a retry fan-out request.""" + + requeued: int = Field( + ..., + description="How many evaluation rows were reset and re-enqueued.", + ) + transcribe_requeued: int = Field( + default=0, + description=( + "Of those, how many were chained through a diarisation " + "task first because the diarised transcript was missing " + "(matches the auto-transcribe behavior of the create-run " + "endpoint)." + ), + ) + skipped: List[CallImportEvaluationRetrySkippedItem] = Field( + default_factory=list, + description="Rows the caller asked for that we did not re-enqueue.", + ) + + +class CallImportEvaluationBulkActionResponse(BaseModel): + """Acknowledgement for bulk cancel / force-fail requests accepted off-thread.""" + + accepted: bool = True + target_count: int = Field( + ..., + description="How many rows the background worker will process.", + ) + evaluation_id: UUID + + +class CallImportMetricSummary(BaseModel): + """Lightweight metric descriptor returned alongside an evaluation.""" + + id: UUID + name: str + metric_type: Optional[str] = None + description: Optional[str] = None + parent_metric_id: Optional[UUID] = None + selection_mode: Optional[SelectionMode] = None + # Surfaced so the Flow tab can decide whether to render the + # Discovered Labels panel next to a multi_label parent. Defaults to + # False to keep legacy clients (and standalone metrics) unaffected. + allow_discovery: bool = False + + model_config = ConfigDict(from_attributes=True) + + +class CallImportEvaluationResponse(BaseModel): + """Parent record describing one evaluation run over a batch.""" + + id: UUID + call_import_id: UUID + organization_id: UUID + name: Optional[str] = None + selected_metric_ids: List[UUID] = Field(default_factory=list) + # Parent UUID string -> [child UUID string]. Captured at run creation + # so the UI can rebuild the parent/child tree even after metrics are + # renamed or deleted. Empty / NULL = no hierarchy was used. + selected_metric_groups: Optional[Dict[str, List[str]]] = None + metrics: List[CallImportMetricSummary] = Field(default_factory=list) + status: str + total_rows: int + completed_rows: int + failed_rows: int + error_message: Optional[str] = None + llm_provider: Optional[str] = None + llm_model: Optional[str] = None + llm_credential_id: Optional[UUID] = None + llm_config: Optional[Dict[str, Any]] = None + metric_llm_overrides: Optional[Dict[str, Any]] = None + stt_provider: Optional[str] = None + stt_model: Optional[str] = None + stt_credential_id: Optional[UUID] = None + # Run-level LLM diariser config. Surfaced so the UI can show + # "Diarised via openai/gpt-4o-mini" on the evaluation header and + # pre-fill the retry modal with the previously-used prompt. + diarisation_llm_provider: Optional[str] = None + diarisation_llm_model: Optional[str] = None + diarisation_llm_credential_id: Optional[UUID] = None + diarisation_prompt: Optional[str] = None + # Diarisation pipeline shape this run was created with. ``stt_llm`` + # (default) is the legacy STT-then-LLM-diariser flow; ``llm_only`` + # means the audio was fed directly to a multimodal diariser LLM. + # Surfaced so the retry modal can preselect the right mode and the + # eval header can render "Diarised via LLM only (Gemini)" instead of + # an empty STT label. + transcribe_mode: Literal["stt_llm", "llm_only"] = "stt_llm" + # Which transcript column this run scored against. All current runs + # use diarised; legacy rows may still carry ``production``. + transcript_source: CallImportEvaluationTranscriptSource = "diarised" + # Sibling evaluation ids created in the same Run Evaluation request. + # Populated only on the POST response (and only when the user ticked + # both Production and Diarised in the modal G�� the backend creates + # one ``CallImportEvaluation`` per source and links them via this + # field so the frontend can deep-link to either run). Empty for all + # other reads. + sibling_evaluation_ids: List[UUID] = Field(default_factory=list) + expected_llm_calls_per_row: Optional[int] = Field( + None, + description=( + "Number of distinct LLM API calls made per evaluation row " + "(one per unique provider/model/config among selected metrics)." + ), + ) + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + created_by_email: Optional[str] = None + last_updated_by_email: Optional[str] = None + # Cached LLM-generated TLDR for the Visualizations tab. Lazily + # populated by ``POST /evaluations/{eval_id}/insights``; ``None`` + # for runs the user has not summarised yet. ``is_stale`` on the + # nested object is set by the route, not the model. + tldr_summary: Optional["EvaluationTldrSummary"] = None + # Cached LLM-generated user insights for External Audit PDF section 03. + user_insights: Optional["EvaluationUserInsightsState"] = None + # Cached per-metric failure clustering for internal diagnostics. + metric_clusters: Optional["EvaluationMetricClustersState"] = None + # True when the user opted into top-level metric discovery on the + # Run Evaluation modal. The frontend uses this to gate the + # "Discovered metrics" panel on the Flow tab. + discover_new_metrics: bool = False + bulk_operation: Optional[ + Literal["abort", "force_fail_pending", "retry"] + ] = Field( + default=None, + description=( + "When set, a bulk background operation (abort, force-fail pending, " + "or retry) is still running for this evaluation. Other mutating " + "actions are rejected until it completes." + ), + ) + + model_config = ConfigDict(from_attributes=True) + + +class CallImportEvaluationListResponse(BaseModel): + """Wrapper for listing evaluations on a single batch.""" + + items: List[CallImportEvaluationResponse] + total: int + + +class CallImportEvaluationRowResponse(BaseModel): + """Per-source-row evaluation output (one Metric set applied to one row). + + ``raw_columns``, ``recording_url`` and ``recording_s3_key`` come from + the parent ``CallImportRow`` so the row-detail panel can show the + full CSV row metadata + audio without a second round-trip. The UI + prefers ``recording_s3_key`` (resolved via a presigned URL) over + ``recording_url`` so playback uses our downloaded copy instead of + the raw provider URL, which is often expired/auth-gated. + """ + + id: UUID + evaluation_id: UUID + call_import_row_id: UUID + row_index: Optional[int] = None + # Renamed from ``external_call_id``; same value, mirrors the renamed + # ``call_import_rows.conversation_id`` column. + conversation_id: Optional[str] = None + transcript: Optional[str] = None + raw_columns: Optional[Dict[str, Any]] = None + recording_url: Optional[str] = None + recording_date: Optional[date] = None + recording_s3_key: Optional[str] = None + diarised_transcript_status: Optional[str] = None + diarised_transcript_error: Optional[str] = None + status: str + metric_scores: Dict[str, Any] = Field(default_factory=dict) + error_message: Optional[str] = None + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class CallImportEvaluationRowListResponse(BaseModel): + """Paginated per-row evaluation results.""" + + items: List[CallImportEvaluationRowResponse] + total: int + page: int + page_size: int + + +class CallImportRowBulkDelete(BaseModel): + """Request body for deleting multiple rows from a call-import batch.""" + + row_ids: List[UUID] = Field( + ..., + min_length=1, + description="Row ids to delete (must belong to the same call import).", + ) + + +class CallImportRowBulkDeleteResponse(BaseModel): + """Response after a bulk-delete pass over ``CallImportRow`` rows.""" + + deleted: int = Field( + ..., + description="How many rows were actually removed (unknown ids are skipped).", + ) + status: Literal["completed", "accepted"] = Field( + default="completed", + description=( + "``accepted`` when deletion was queued to run asynchronously; " + "``completed`` when rows were removed before the response." + ), + ) + + +class CallImportRetryFailedRowsRequest(BaseModel): + """Optional credential override when re-enqueueing failed import rows.""" + + provider: Optional[str] = Field( + default=None, + description=( + "Telephony provider key for this retry pass. Omit together with " + "``telephony_integration_id`` to download from CSV recording URLs." + ), + ) + telephony_integration_id: Optional[UUID] = Field( + default=None, + description=( + "Telephony credential to use for this retry pass. Omit together " + "with ``provider`` for direct-URL retry." + ), + ) + + @model_validator(mode="after") + def validate_credential_mode(self) -> "CallImportRetryFailedRowsRequest": + has_provider = bool((self.provider or "").strip()) + has_integration = self.telephony_integration_id is not None + if has_provider != has_integration: + raise ValueError( + "provider and telephony_integration_id must both be provided " + "or both omitted for direct-URL retry." + ) + return self + + +class CallImportRetryFailedRowsResponse(BaseModel): + """Summary of a retry pass over failed call-import rows.""" + + requeued: int = Field( + ..., + description=( + "Rows reset to pending and successfully re-enqueued on the " + "``imports`` worker queue." + ), + ) + enqueue_failed: int = Field( + default=0, + description=( + "Rows that were eligible for retry but failed to enqueue again. " + "These rows are left in ``failed`` with an enqueue error." + ), + ) + skipped: int = Field( + default=0, + description=( + "Rows skipped because they were no longer in ``failed`` at retry " + "time (for example, already retried from another tab)." + ), + ) + + +# --- Diarization / Transcription request/response shapes --- + + +class CallImportTranscribeRequest(BaseModel): + """Body for kicking off diarization for one or many call-import rows. + + The same shape powers both the per-row endpoint (where ``row_ids`` + is ignored) and the batch-level endpoint. ``only_missing`` is the + safe default G�� rows with an existing transcript are skipped unless + ``overwrite_existing`` is set. + + Two modes are supported: + + * ``mode="stt_llm"`` (default) G�� the legacy two-stage pipeline: STT + produces plain text, an LLM splits it into agent/user turns using + ``diarization_prompt``. ``stt_provider`` and ``stt_model`` are + required in this mode. + * ``mode="llm_only"`` G�� skip STT entirely and hand the recording's + audio bytes to a multimodal chat model along with + ``diarization_prompt``. The model both transcribes and diarises in + a single pass. The STT fields are ignored (and must be omitted / + null). Only providers whose chat API accepts audio input (OpenAI + ``gpt-4o-audio-*``, Google Gemini ``1.5/2.0``) are usable; other + providers will surface a typed error on the row. + """ + + mode: Literal["stt_llm", "llm_only"] = Field( + default="stt_llm", + description=( + "Pipeline shape. 'stt_llm' (default) runs STT then an LLM " + "diariser over the resulting text. 'llm_only' skips STT and " + "feeds the raw audio to a multimodal LLM together with " + "``diarization_prompt`` for a single-pass transcribe + " + "diarise." + ), + ) + stt_provider: Optional[str] = Field( + default=None, + max_length=50, + description=( + "STT provider key, e.g. 'deepgram' or 'openai'. Required when " + "``mode='stt_llm'``; must be omitted when ``mode='llm_only'``." + ), + ) + stt_model: Optional[str] = Field( + default=None, + max_length=100, + description=( + "STT model name, e.g. 'nova-2' or 'whisper-1'. Required when " + "``mode='stt_llm'``; must be omitted when ``mode='llm_only'``." + ), + ) + credential_id: Optional[UUID] = Field( + default=None, + description="Optional AIProvider/Integration row to pin for this run.", + ) + language: Optional[str] = Field( + default=None, + max_length=20, + description="Optional ISO language hint, e.g. 'en'.", + ) + only_missing: bool = Field( + default=True, + description=( + "When true, rows with an existing transcript are skipped (the " + "default safe behavior)." + ), + ) + overwrite_existing: bool = Field( + default=False, + description=( + "When true, existing transcripts are replaced. Mutually " + "exclusive with only_missing." + ), + ) + row_ids: Optional[List[UUID]] = Field( + default=None, + description=( + "Restrict the run to a specific subset of rows. NULL = every " + "row in the import (subject to only_missing)." + ), + ) + # --- LLM diariser config --- + # In ``stt_llm`` mode diarisation runs as a *second* step: STT + # produces plain text, then this LLM splits it into agent/user + # turns. In ``llm_only`` mode this same LLM directly receives the + # audio and the prompt. Both fields are always mandatory because + # there is no longer a pyannote fallback and ``llm_only`` cannot + # function without an LLM either. + diarization_llm_provider: str = Field( + ..., + max_length=50, + description=( + "LLM provider that diarises the call. In ``stt_llm`` it sees " + "the STT text; in ``llm_only`` it sees the raw audio." + ), + ) + diarization_llm_model: str = Field( + ..., + max_length=100, + description=( + "LLM model name. In ``llm_only`` mode this must be a model " + "that accepts audio input (e.g. 'gpt-4o-audio-preview', " + "'gemini-1.5-pro')." + ), + ) + diarization_llm_credential_id: Optional[UUID] = Field( + default=None, + description=( + "Optional AIProvider row to pin for the diarisation LLM." + ), + ) + diarization_prompt: Optional[str] = Field( + default=None, + max_length=10_000, + description=( + "Operator-supplied system prompt for the diariser LLM. " + "When NULL/empty the worker uses the canonical default " + "(see ``GET /api/v1/call-imports/diarisation-prompt-default``)." + ), + ) + + @model_validator(mode="after") + def _validate_mode_fields(self) -> "CallImportTranscribeRequest": + """Enforce STT-field presence rules based on ``mode``. + + ``stt_llm`` (default) requires both STT fields G�� the worker + cannot diarise without a transcript. ``llm_only`` forbids them + so the API contract makes it clear that the audio is going + straight to the LLM; passing both would be ambiguous about + which path the worker should take. + """ + stt_provider = (self.stt_provider or "").strip() if self.stt_provider else None + stt_model = (self.stt_model or "").strip() if self.stt_model else None + if self.mode == "stt_llm": + if not stt_provider or not stt_model: + raise ValueError( + "stt_provider and stt_model are required when " + "mode='stt_llm'." + ) + else: # llm_only + if stt_provider or stt_model: + raise ValueError( + "stt_provider/stt_model must be omitted when " + "mode='llm_only'; the LLM consumes the audio " + "directly." + ) + return self + + +class CallImportDiarisationPromptDefaultResponse(BaseModel): + """Wrapper for the canonical diariser-prompt fetched by the modal.""" + + prompt: str = Field( + ..., + description=( + "The exact prompt the worker falls back to when the caller " + "leaves ``diarization_prompt`` blank. The frontend pre-fills " + "the textarea with this value so the operator can edit it." + ), + ) + + +class CallImportRowIdsResponse(BaseModel): + """Flat row-id list for cross-page bulk selection. + + Powers the "Select all M rows in this import" affordance on the + detail page G�� returning only ids keeps the payload tiny so the UI + can hold the full set in memory even for batches with thousands + of rows. The frontend then passes those ids straight to the + existing bulk-delete / bulk-transcribe endpoints. + """ + + ids: List[UUID] = Field( + default_factory=list, + description=( + "Every ``CallImportRow.id`` that matches the ``q`` and " + "``diarised_status`` filters (or every row when neither is " + "supplied), sorted by ``row_index``." + ), + ) + total: int = Field( + ..., + description=( + "Length of ``ids``. Sent explicitly so callers can show a " + "count without re-measuring the array." + ), + ) + + +class CallImportTranscribeResponse(BaseModel): + """Summary of a transcribe fan-out request.""" + + queued: int = Field( + ..., + description=( + "How many rows were enqueued for diarization. Skipped rows " + "(missing recording, transcript already present, etc.) are " + "not counted." + ), + ) + skipped_rows: int = Field( + default=0, + description="Rows excluded by only_missing or because they had no recording.", + ) + skipped_reason_counts: Dict[str, int] = Field( + default_factory=dict, + description="Per-reason breakdown of skipped rows for the UI to surface.", + ) + accepted: bool = Field( + default=False, + description=( + "When true, diarization setup was queued to a background worker " + "and ``queued`` reflects zero until the worker finishes enqueue." + ), + ) + + +class CallImportCancelDiarisationRequest(BaseModel): + """Body for the batch cancel-diarisation endpoint. + + Omit ``row_ids`` (or pass ``null``) to cancel every row in the + import whose ``diarised_transcript_status`` is currently + ``pending`` or ``running``. Pass an explicit list to scope the + cancel to a subset (e.g. the rows the operator selected in the + UI). + """ + + row_ids: Optional[List[UUID]] = Field( + default=None, + description=( + "Optional subset of CallImportRow UUIDs. ``None`` cancels " + "every pending / running diarisation in the import." + ), + ) + + +class CallImportCancelDiarisationResponse(BaseModel): + """Summary of a cancel-diarisation request. + + ``cancelled`` counts rows that were actively pending / running + when the cancel landed and got flipped to ``failed`` with a + "Cancelled by user" error. ``skipped`` counts rows that were + requested (or matched the implicit "all rows" filter) but were + not in a cancellable state G�� typically because they had already + finished or were never queued for diarisation in the first place. + """ + + cancelled: int = Field( + ..., + description=( + "Rows whose in-flight Celery task was revoked and whose " + "``diarised_transcript_status`` was flipped to ``failed`` " + "with a 'Cancelled by user' error message." + ), + ) + skipped: int = Field( + default=0, + description=( + "Rows that were requested but not in a cancellable state " + "(idle / completed / already failed)." + ), + ) + + +# --- Per-run aggregation / visualization payloads --- + + +class CallImportMetricHistogramBucket(BaseModel): + """One bin of a numeric metric histogram.""" + + x0: float + x1: float + count: int + + +class CallImportMetricValueCount(BaseModel): + """One row of a categorical metric's value frequency table.""" + + label: str + count: int + + +class CallImportMetricLabelPair(BaseModel): + """One unordered pair-count cell of a multi-label parent's + co-occurrence matrix. + + ``a`` and ``b`` are child label names; ``count`` is the number of + rows on which both labels fired together (intersection size). + Pairs are emitted with ``a < b`` lexicographically so the matrix + can be reconstructed without duplicates on the frontend. + """ + + a: str + b: str + count: int + + +class CallImportMetricAggregate(BaseModel): + """Per-metric aggregate computed from an evaluation run's rows. + + Numeric metrics return summary statistics + histogram buckets; + categorical / pass-fail / text metrics return the top value counts. + Both shapes can coexist if a metric mixes types G�� the UI prefers + histogram when present, falls back to value_counts otherwise. + """ + + metric_id: str + metric_name: str + metric_type: Optional[str] = None + metric_category: str = "quality" + # True when this aggregate represents a multi-label parent metric + # (selection_mode == "multi_label" with no parent_metric_id). For + # those, ``value_counts`` lists per-child label tallies and the + # rows scored != sum(value_counts.count). The UI uses this flag to + # force a horizontal bar layout (slices wouldn't sum to 100%) and + # to label the n-badge as rows scored, not label occurrences. + is_multi_label_parent: bool = False + count: int = 0 + skipped_count: int = 0 + error_count: int = 0 + # Numeric stats (None when no numeric values were observed) + mean: Optional[float] = None + median: Optional[float] = None + p25: Optional[float] = None + p75: Optional[float] = None + p95: Optional[float] = None + min: Optional[float] = None + max: Optional[float] = None + stddev: Optional[float] = None + histogram_buckets: List[CallImportMetricHistogramBucket] = Field( + default_factory=list + ) + value_counts: List[CallImportMetricValueCount] = Field(default_factory=list) + # Pairwise label intersections for multi-label parent metrics. + # Empty for everything else. The frontend reconstructs a square + # symmetric matrix from these unordered pairs and renders the + # co-occurrence heatmap chart type. + co_occurrence: List[CallImportMetricLabelPair] = Field(default_factory=list) + + +class MetricPeriodDelta(BaseModel): + """Week-over-week (or baseline-run) delta for one metric.""" + + label: str + detail: str + why: Optional[str] = None + + +class CallImportEvaluationAggregateResponse(BaseModel): + """Aggregated metric distributions for a single evaluation run.""" + + evaluation_id: UUID + total_rows: int + completed_rows: int + failed_rows: int + metrics: List[CallImportMetricAggregate] = Field(default_factory=list) + period_deltas: Dict[str, MetricPeriodDelta] = Field(default_factory=dict) + baseline_evaluation_id: Optional[UUID] = None + failure_policies_source: Optional[Literal["inferred", "user"]] = Field( + default=None, + description=( + "Whether flagged-rate semantics use user-confirmed failure policies " + "or inferred defaults from the Failure diagnostics flow." + ), + ) + + +# --- LLM-generated TLDR for the Visualizations tab --- + + + +class EvaluatorResultsAggregateResponse(BaseModel): + """Chart-friendly metric rollups for evaluator results in a suite or scenario scope.""" + + scope: str + suite_id: Optional[UUID] = None + agent_id: Optional[UUID] = None + scenario_id: Optional[UUID] = None + total_rows: int = 0 + completed_rows: int = 0 + failed_rows: int = 0 + metrics: List[CallImportMetricAggregate] = Field(default_factory=list) + + +# --- LLM-generated TLDR for the Visualizations tab --- + + + +class EvaluationTldrSummary(BaseModel): + """Cached LLM-generated narrative + bullet patterns for an eval run. + + Persisted on ``CallImportEvaluation.tldr_summary`` (JSONB) and + rendered above the per-metric charts. ``generated_at_completed_rows`` + is the snapshot of ``completed_rows`` at the time the summary was + written; the API compares it against the current count to flag + ``is_stale`` so the UI can prompt for a regenerate. + """ + + narrative: str + patterns: List[str] = Field(default_factory=list) + metric_insights: Dict[str, str] = Field(default_factory=dict) + generated_at: datetime + generated_at_completed_rows: int = 0 + provider: Optional[str] = None + model: Optional[str] = None + is_stale: bool = False + + +class EvaluationInsightsRequest(BaseModel): + """Body for ``POST /evaluations/{eval_id}/insights``. + + All fields are optional. When ``provider``/``model`` are unset the + backend resolves the org's first active OpenAI/Anthropic/Google + provider (mirroring the Prompt Partials AI-generate flow) so + callers that don't care can simply post ``{}``. + """ + + regenerate: bool = False + provider: Optional[str] = None + model: Optional[str] = Field(default=None, min_length=1) + credential_id: Optional[UUID] = None + max_llm_calls: Optional[int] = Field( + default=None, + ge=20, + le=500, + description=( + "Max LLM calls for user-insights sampling (extraction + synthesis). " + "Defaults to 200 when omitted." + ), + ) + + +class UserInsightCategory(BaseModel): + label: str + count: int + share_pct: float + + +class UserInsightEvidenceTurn(BaseModel): + speaker: str + text: str + + +class UserInsightEvidence(BaseModel): + conversation_id: Optional[str] = None + quote: str + turns: List[UserInsightEvidenceTurn] = Field(default_factory=list) + + +class EvaluationUserInsightItem(BaseModel): + id: str + title: str + categories: List[UserInsightCategory] = Field(default_factory=list) + observation: str + evidence: UserInsightEvidence + + +class EvaluationUserInsightsState(BaseModel): + """Cached map-reduce LLM user insights for an evaluation run.""" + + status: Literal["idle", "running", "completed", "failed"] = "idle" + insights: List[EvaluationUserInsightItem] = Field(default_factory=list) + overview: Optional[str] = None + generated_at: Optional[datetime] = None + generated_at_completed_rows: int = 0 + progress: Optional[Dict[str, int]] = None + provider: Optional[str] = None + model: Optional[str] = None + llm_calls_used: int = 0 + max_llm_calls: Optional[int] = None + error_message: Optional[str] = None + is_stale: bool = False + + +class EvaluationUserInsightsRequest(BaseModel): + """Body for ``POST /evaluations/{eval_id}/user-insights``.""" + + regenerate: bool = False + force: bool = False + provider: Optional[str] = None + model: Optional[str] = Field(default=None, min_length=1) + credential_id: Optional[UUID] = None + max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) + + +MetricClusterGapLabel = Literal[ + "LOGIC_GAP", + "UNDERSPEC", + "EXISTS_NO_TRIGGER", + "MISSING", +] + +FailurePolicyNumericOp = Literal["lt", "lte", "gt", "gte"] + + +class MetricFailurePolicy(BaseModel): + """Per-metric definition of which scores count as failures for this evaluation.""" + + metric_id: str + failure_values: List[str] = Field( + default_factory=list, + description="Normalized lowercase labels that count as failure (single-choice, enum, boolean-as-category).", + ) + failure_child_names: List[str] = Field( + default_factory=list, + description="Child label names that count as failure for multi_label parents.", + ) + numeric_rule: Optional[Dict[str, Any]] = Field( + default=None, + description='Numeric failure rule, e.g. {"op": "lt", "threshold": 0.5}.', + ) + + +class MetricFailurePolicyValueCount(BaseModel): + label: str + count: int = 0 + + +class MetricFailurePolicyMetricPreview(BaseModel): + metric_id: str + metric_name: str + metric_type: Optional[str] = None + selection_mode: Optional[str] = None + is_multi_label_parent: bool = False + value_counts: List[MetricFailurePolicyValueCount] = Field(default_factory=list) + child_names: List[str] = Field(default_factory=list) + row_count_by_value: Dict[str, int] = Field(default_factory=dict) + suggested_policy: MetricFailurePolicy + effective_policy: MetricFailurePolicy + + +class MetricFailurePoliciesResponse(BaseModel): + previews: List[MetricFailurePolicyMetricPreview] = Field(default_factory=list) + policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) + source: Literal["inferred", "user"] = "inferred" + updated_at: Optional[datetime] = None + + +class MetricFailurePoliciesSaveRequest(BaseModel): + policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) + source: Literal["user"] = "user" + + +class MetricClusterEvidenceTurn(BaseModel): + speaker: str + text: str + + +class MetricClusterEvidence(BaseModel): + conversation_id: Optional[str] = None + evaluation_row_id: Optional[UUID] = None + quote: str = "" + turns: List[MetricClusterEvidenceTurn] = Field(default_factory=list) + + +class MetricSubCluster(BaseModel): + label: str + count: int = 0 + share_pct: float = 0.0 + + +class MetricCluster(BaseModel): + id: str + label: str + gap_label: MetricClusterGapLabel + level: int = 1 + count: int = 0 + share_pct: float = 0.0 + sub_clusters: List[MetricSubCluster] = Field(default_factory=list) + observation: str = "" + failure_reason: str = "" + evidence: MetricClusterEvidence = Field(default_factory=MetricClusterEvidence) + is_discovered: bool = False + + +class MetricClusterGroup(BaseModel): + metric_id: str + metric_name: str + flagged_count: int = 0 + failure_reason: str = "" + clusters: List[MetricCluster] = Field(default_factory=list) + + +class DiscoveredProblemCluster(BaseModel): + id: str + label: str + gap_label: MetricClusterGapLabel + count: int = 0 + share_pct: float = 0.0 + observation: str = "" + failure_reason: str = "" + evidence: MetricClusterEvidence = Field(default_factory=MetricClusterEvidence) + + +class RcaRepeatedPatternRow(BaseModel): + metric_id: str + metric_name: str + top_rca_patterns: str = "" + evidence_share_pct: float = 0.0 + evidence_calls: int = 0 + evidence_cluster_count: int = 0 + failure_reason: str = "" + + +class RcaMetricHotspotRow(BaseModel): + metric_id: str + metric_name: str + description: str = "" + metric_rate_pct: float = 0.0 + flagged_calls: int = 0 + + +class RcaPromptAreaRow(BaseModel): + label: str + share_pct: float = 0.0 + gap_label: MetricClusterGapLabel + + +class MetricClustersRcaSummary(BaseModel): + total_clusters: int = 0 + total_clustered_instances: int = 0 + total_flagged_instances: int = 0 + analysed_calls: int = 0 + repeated_patterns: List[RcaRepeatedPatternRow] = Field(default_factory=list) + metric_hotspots: List[RcaMetricHotspotRow] = Field(default_factory=list) + prompt_areas: List[RcaPromptAreaRow] = Field(default_factory=list) + + +class EvaluationMetricClustersState(BaseModel): + """Cached per-metric failure clustering for internal diagnostics.""" + + status: Literal["idle", "running", "completed", "failed", "cancelled"] = "idle" + groups: List[MetricClusterGroup] = Field(default_factory=list) + discovered_problems: List[DiscoveredProblemCluster] = Field( + default_factory=list + ) + overview: Optional[str] = None + generated_at: Optional[datetime] = None + generated_at_completed_rows: int = 0 + progress: Optional[Dict[str, int]] = None + provider: Optional[str] = None + model: Optional[str] = None + llm_calls_used: int = 0 + max_llm_calls: Optional[int] = None + error_message: Optional[str] = None + is_stale: bool = False + selected_evaluation_row_ids: List[str] = Field( + default_factory=list, + description="Evaluation row IDs included in the last clustering run.", + ) + failure_policies: Dict[str, MetricFailurePolicy] = Field(default_factory=dict) + failure_policies_source: Literal["inferred", "user"] = "inferred" + failure_policies_updated_at: Optional[datetime] = None + rca_summary: Optional[MetricClustersRcaSummary] = None + + +class MetricClusterEligibleRow(BaseModel): + """Completed evaluation row with at least one flagged quality metric.""" + + evaluation_row_id: UUID + conversation_id: Optional[str] = None + row_index: Optional[int] = None + flagged_metric_names: List[str] = Field(default_factory=list) + + +class MetricClusterEligibleRowsResponse(BaseModel): + items: List[MetricClusterEligibleRow] = Field(default_factory=list) + total: int = 0 + + +class PromptImprovementSuggestion(BaseModel): + """One LLM-generated prompt edit to address a failure cluster.""" + + id: str + metric_id: str + metric_name: str + cluster_id: str + cluster_label: str + gap_label: MetricClusterGapLabel + share_pct: float = 0.0 + priority: Literal["high", "medium", "low"] = "medium" + change_type: Literal["edit", "add"] = "add" + target_section: str = "" + anchor_excerpt: str = "" + current_gap: str = "" + suggested_text: str = "" + rationale: str = "" + flow_node_id: str = "" + flow_node_label: str = "" + + +class EvaluationPromptImprovementsState(BaseModel): + """Cached prompt improvement suggestions for an evaluation run.""" + + status: Literal["idle", "running", "completed", "failed"] = "idle" + imported_agent_id: Optional[str] = None + imported_agent_name: Optional[str] = None + suggestions: List[PromptImprovementSuggestion] = Field(default_factory=list) + overview: Optional[str] = None + generated_at: Optional[datetime] = None + generated_at_completed_rows: int = 0 + provider: Optional[str] = None + model: Optional[str] = None + error_message: Optional[str] = None + is_stale: bool = False + + +class EvaluationPromptImprovementsRequest(BaseModel): + """Body for ``POST /evaluations/{eval_id}/prompt-improvements``.""" + + imported_agent_id: UUID + regenerate: bool = False + force: bool = False + provider: Optional[str] = None + model: Optional[str] = None + credential_id: Optional[UUID] = None + + +class EvaluationMetricClustersRequest(BaseModel): + """Body for ``POST /evaluations/{eval_id}/metric-clusters``.""" + + regenerate: bool = False + force: bool = False + provider: Optional[str] = None + model: Optional[str] = Field(default=None, min_length=1) + credential_id: Optional[UUID] = None + max_llm_calls: Optional[int] = Field(default=None, ge=20, le=500) + evaluation_row_ids: Optional[List[UUID]] = Field( + default=None, + description=( + "Subset of completed evaluation row IDs to cluster. When omitted, " + "all completed rows with at least one flagged quality metric are used." + ), + ) + row_limit: Optional[int] = Field( + default=None, + ge=1, + description=( + "Use the first N eligible rows (by row order). Mutually exclusive " + "with evaluation_row_ids." + ), + ) + failure_policies: Optional[Dict[str, MetricFailurePolicy]] = Field( + default=None, + description="Per-metric failure policies confirmed in the cluster modal.", + ) + + +# Resolve the forward reference on ``CallImportEvaluationResponse`` +# (defined further up the file) now that ``EvaluationTldrSummary`` +# exists. Without this Pydantic raises at first ``.model_validate`` +# because the string annotation can't be evaluated. +CallImportEvaluationResponse.model_rebuild() + + +# --- Cross-run insights for a CallImport batch --- + + +class CallImportInsightsRunPoint(BaseModel): + """One run's mean for a metric, used to render trend lines.""" + + evaluation_id: UUID + name: Optional[str] = None + created_at: datetime + mean: Optional[float] = None + completed_rows: int = 0 + + +class CallImportInsightsMetric(BaseModel): + """Per-metric history across every evaluation run on this import.""" + + metric_id: str + metric_name: str + metric_type: Optional[str] = None + latest: Optional[CallImportMetricAggregate] = None + trend: List[CallImportInsightsRunPoint] = Field(default_factory=list) + + +class CallImportInsightsResponse(BaseModel): + """Aggregated cross-run signals for a single call-import batch.""" + + call_import_id: UUID + total_rows: int + rows_with_transcript: int + rows_without_transcript: int + transcript_source_counts: Dict[str, int] = Field(default_factory=dict) + evaluation_count: int = 0 + metrics: List[CallImportInsightsMetric] = Field(default_factory=list) + + +# --- Flow chart visualization for hierarchical metrics --- + + +class MetricFlowNode(BaseModel): + """One step in the LLM-inferred temporal flow for a parent metric. + + Represents a child sub-metric label. ``count`` is the number of rows + in the evaluation where this child appears anywhere in its + ``sequence`` array. ``is_terminal`` is set when the child is the + last entry in a meaningful fraction of those sequences. + + ``is_discovered`` is set when the node represents an LLM-discovered + candidate (parent has ``allow_discovery=true``) rather than a + user-defined child. The id of a discovered node is prefixed with + ``disc:`` so it can't collide with real child UUIDs. + """ + + id: str + label: str + count: int = 0 + is_terminal: bool = False + is_discovered: bool = False + + +class MetricFlowEdge(BaseModel): + """One directed transition between two children across all rows. + + ``count`` is the number of rows where ``source`` immediately + precedes ``target`` in the sequence. The synthetic ``START`` node + is used as the ``source`` for the first child in every sequence. + """ + + source: str + target: str + count: int = 0 + + +class MetricFlowResponse(BaseModel): + """Aggregate flow diagram payload for a single parent metric.""" + + parent_metric_id: str + parent_metric_name: str + selection_mode: Optional[SelectionMode] = None + nodes: List[MetricFlowNode] = Field(default_factory=list) + edges: List[MetricFlowEdge] = Field(default_factory=list) + total_rows: int = 0 + rows_with_sequence: int = 0 + + +class DiscoveredLabelItem(BaseModel): + """One LLM-discovered candidate sub-label aggregated across rows. + + ``key`` is the slugified label identifier (matches what appears in + ``sequence`` entries). ``count`` is the number of rows in the + evaluation that emitted this slug. ``sample_rationale`` is the + first non-empty rationale captured from any row (back-compat + field, identical to ``examples[0]`` when present). ``examples`` + holds up to 3 distinct rationales G�� the UI surfaces 2 of them as + ``Examples:`` in the rubric on Promote, with the third kept as + headroom in case the first is unhelpful. + """ + + key: str + name: str + description: Optional[str] = None + sample_rationale: Optional[str] = None + examples: List[str] = Field(default_factory=list, max_length=3) + count: int = 0 + + +class DiscoveredLabelsResponse(BaseModel): + """List of discovered candidate sub-labels for a parent metric.""" + + parent_metric_id: str + items: List[DiscoveredLabelItem] = Field(default_factory=list) + + +class DiscoveredLabelMergeRequest(BaseModel): + """Body for POST /evaluations/{eval_id}/discovered-labels/merge. + + Rewrites every row's ``metric_scores[parent_id].discovered_labels`` + entries whose key is ``from_key`` to use ``to_key`` instead, so the + user can collapse near-duplicate candidates ("On Hold" / "Customer + Put On Hold") into a single promoted child. + """ + + parent_metric_id: UUID + from_key: str = Field(..., min_length=1, max_length=120) + to_key: str = Field(..., min_length=1, max_length=120) + + +class DiscoveredLabelDeleteRequest(BaseModel): + """Body for POST /evaluations/{eval_id}/discovered-labels/delete. + + Strips a candidate sub-label from every row's + ``discovered_labels`` list AND from each row's ``sequence`` array, + then tombstones the slug at the evaluation level so workers + finishing later can't re-introduce it. Use for gibberish or + irrelevant candidates the LLM proposed; for near-duplicates that + you want to keep but unify, use the merge endpoint instead. + """ + + parent_metric_id: UUID + key: str = Field(..., min_length=1, max_length=120) + + +class PromoteDiscoveredChildRequest(BaseModel): + """Body for POST /metrics/{parent_id}/children/from-discovered. + + ``key`` is the slug under which the candidate is currently stored + on per-row ``metric_scores``. The newly-created child Metric's + name is normalized so ``slugify(name) == key``, which keeps every + already-scored row's ``sequence`` array resolvable against the + promoted child without a backfill. + """ + + key: str = Field(..., min_length=1, max_length=120) + name: str = Field(..., min_length=1, max_length=120) + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + # Default True: when promoting a discovered label we want the new + # sub-metric to always capture rationales going forward, since the + # candidate was itself proposed *with* a rationale and the user + # almost always wants to see why future rows hit it. Explicit False + # keeps the original opt-in behavior available for callers that + # don't care about rationales. + capture_rationale: bool = True + + +# --- Discovered top-level metrics (per-evaluation discovery) --- +# +# Parallel to ``DiscoveredLabelItem`` / merge / delete / promote G�� but +# scoped to the evaluation as a whole, not to a parent category metric. +# Used by the "Discovered metrics" panel at the top of the evaluation +# detail Flow tab when ``CallImportEvaluation.discover_new_metrics`` +# is true. + + +# The promote endpoint accepts these three suggested types; "category" +# creates a parent (no children yet) that the user can later extend in +# the Metrics page. +DiscoveredMetricSuggestedType = Literal["boolean", "rating", "category"] + + +class DiscoveredMetricItem(BaseModel): + """One LLM-discovered candidate top-level metric aggregated across rows. + + Mirrors :class:`DiscoveredLabelItem` but at the evaluation level + (no ``parent_metric_id``). ``suggested_type`` is the LLM's guess at + the best representation; the promote flow lets the user override + it before creating the real :class:`Metric` row. + """ + + key: str + name: str + description: Optional[str] = None + suggested_type: DiscoveredMetricSuggestedType = "boolean" + sample_rationale: Optional[str] = None + examples: List[str] = Field(default_factory=list, max_length=3) + count: int = 0 + + +class DiscoveredMetricsResponse(BaseModel): + """List of discovered candidate top-level metrics for an evaluation.""" + + evaluation_id: UUID + items: List[DiscoveredMetricItem] = Field(default_factory=list) + + +class DiscoveredMetricMergeRequest(BaseModel): + """Body for POST /evaluations/{eval_id}/discovered-metrics/merge. + + Rewrites every row's ``metric_scores["__discovered_metrics__"]`` + entries whose key is ``from_key`` to use ``to_key`` instead, and + records the redirect in ``CallImportEvaluation.discovered_metric_aliases`` + so workers finishing later can't resurrect the merged-out slug. + """ + + from_key: str = Field(..., min_length=1, max_length=120) + to_key: str = Field(..., min_length=1, max_length=120) + + +class DiscoveredMetricDeleteRequest(BaseModel): + """Body for POST /evaluations/{eval_id}/discovered-metrics/delete. + + Strips a candidate from every row's + ``metric_scores["__discovered_metrics__"]`` list and tombstones + the slug at the evaluation level (empty-string alias) so workers + finishing later can't re-introduce it. + """ + + key: str = Field(..., min_length=1, max_length=120) + + +class PromoteDiscoveredMetricRequest(BaseModel): + """Body for POST /metrics/from-discovered. + + Creates a standalone :class:`Metric` (``parent_metric_id=None``) + from an LLM-discovered candidate. The new metric's name is + normalized so ``slugify(name) == key`` to keep already-scored row + payloads resolvable against the promoted metric. ``metric_type`` + selects how the new metric will be scored on future runs; + ``"category"`` creates a ``multi_label`` parent with no children + (the user adds children via the existing Metrics page). + """ + + key: str = Field(..., min_length=1, max_length=120) + name: str = Field(..., min_length=1, max_length=120) + description: Optional[str] = Field(default=None, max_length=METRIC_RUBRIC_TEXT_MAX_LENGTH) + metric_type: DiscoveredMetricSuggestedType = "boolean" + capture_rationale: bool = True + # Optional per-type config knobs passed through to ``Metric.custom_config``. + # For ``rating`` the frontend can supply {"min": 1, "max": 5}; for + # ``boolean`` / ``category`` the field is typically empty. + custom_config: Optional[Dict[str, Any]] = None + + +# --- Workspace Schemas --- + + +class WorkspaceBase(BaseModel): + """Shared fields for workspace create/update payloads.""" + + name: str = Field(..., min_length=1, max_length=255) + + +class WorkspaceCreate(WorkspaceBase): + """Body for POST /workspaces.""" + + # Optional: derived from name when omitted; uniqueness is per-org. + slug: Optional[str] = Field( + default=None, min_length=1, max_length=255 + ) + + +class WorkspaceUpdate(BaseModel): + """Body for PATCH /workspaces/{id} (rename and/or org-admin activation).""" + + name: Optional[str] = Field(default=None, min_length=1, max_length=255) + is_active: Optional[bool] = None + + +class WorkspaceResponse(BaseModel): + """Response schema for a single workspace.""" + + id: UUID + organization_id: UUID + name: str + slug: str + is_default: bool + is_active: bool = True + created_at: datetime + updated_at: datetime + role_id: Optional[UUID] = None + role_name: Optional[str] = None + capabilities: List[str] = Field(default_factory=list) + + model_config = ConfigDict(from_attributes=True) + + +class WorkspaceRoleBase(BaseModel): + name: str = Field(..., min_length=1, max_length=255) + description: Optional[str] = None + capabilities: List[str] = Field(default_factory=list) + + +class WorkspaceRoleCreate(WorkspaceRoleBase): + pass + + +class WorkspaceRoleUpdate(BaseModel): + name: Optional[str] = Field(default=None, min_length=1, max_length=255) + description: Optional[str] = None + capabilities: Optional[List[str]] = None + + +class WorkspaceRoleResponse(BaseModel): + id: UUID + organization_id: UUID + name: str + description: Optional[str] = None + capabilities: List[str] + is_system: bool + created_at: datetime + updated_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class WorkspaceMemberResponse(BaseModel): + id: UUID + workspace_id: UUID + user_id: UUID + role_id: UUID + role_name: str + user_email: str + user_name: Optional[str] = None + added_by_user_id: Optional[UUID] = None + created_at: datetime + + model_config = ConfigDict(from_attributes=True) + + +class WorkspaceMemberCreate(BaseModel): + user_id: UUID + role_id: UUID + + +class WorkspaceMemberUpdate(BaseModel): + role_id: UUID + + +class CapabilityInfoResponse(BaseModel): + key: str + label: str + + +class CapabilityDomainResponse(BaseModel): + key: str + label: str + capabilities: List[CapabilityInfoResponse] diff --git a/app/models/synthetic_trace_schemas.py b/app/models/synthetic_trace_schemas.py new file mode 100644 index 00000000..beddff14 --- /dev/null +++ b/app/models/synthetic_trace_schemas.py @@ -0,0 +1,150 @@ +"""Pydantic schemas for synthetic call traces.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, List, Optional +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field + + +class SyntheticTraceTurn(BaseModel): + turn_number: int + sut_response_latency_ms: Optional[float] = None + caller_stream_complete_at: Optional[float] = None + sut_speech_start_at: Optional[float] = None + sut_speech_stop_at: Optional[float] = None + talk_over: bool = False + stt_ttfb_ms: Optional[float] = None + llm_ttfb_ms: Optional[float] = None + tts_ttfb_ms: Optional[float] = None + s2s_ttfb_ms: Optional[float] = None + transcript: Optional[str] = None + extra: Dict[str, Any] = Field(default_factory=dict) + + +class OtelSpanRecord(BaseModel): + trace_id: str + span_id: str + parent_span_id: Optional[str] = None + name: str + start_time_unix_nano: Optional[int] = None + end_time_unix_nano: Optional[int] = None + attributes: Dict[str, Any] = Field(default_factory=dict) + events: List[Dict[str, Any]] = Field(default_factory=list) + + +class SyntheticCallTraceSummary(BaseModel): + id: UUID + evaluator_result_id: Optional[UUID] = None + agent_id: Optional[UUID] = None + call_short_id: Optional[str] = None + environment: str + transport: str + tier: str + status: str + started_at: datetime + ended_at: Optional[datetime] = None + turn_count: int = 0 + response_latency_p50_ms: Optional[float] = None + response_latency_p90_ms: Optional[float] = None + response_latency_p95_ms: Optional[float] = None + component_aggregates: Optional[Dict[str, Any]] = None + failure_flags: Optional[List[str]] = None + call_recording_id: Optional[UUID] = None + + model_config = ConfigDict(from_attributes=True) + + +class SyntheticCallTraceDetail(SyntheticCallTraceSummary): + turns: List[SyntheticTraceTurn] = Field(default_factory=list) + otel_spans: List[OtelSpanRecord] = Field(default_factory=list) + otel_trace_ids: List[str] = Field(default_factory=list) + pipeline_models: Dict[str, Dict[str, Optional[str]]] = Field(default_factory=dict) + + +class OtelCorrelationInfo(BaseModel): + evaluator_result_id: UUID + synthetic_call_trace_id: Optional[UUID] = None + call_short_id: Optional[str] = None + agent_id: Optional[UUID] = None + otlp_endpoint: str + suggested_env_vars: Dict[str, str] = Field(default_factory=dict) + suggested_span_attributes: Dict[str, str] = Field(default_factory=dict) + + +class OtlpIngestResponse(BaseModel): + accepted_spans: int + synthetic_call_trace_id: Optional[UUID] = None + correlated: bool = False + + +class SyntheticCallTraceListResponse(BaseModel): + items: List[SyntheticCallTraceSummary] + total: int + + +class OtlpSetupInfo(BaseModel): + """Pipecat WebRTC local setup for call trace ingest.""" + + otlp_endpoint: str + sessions_endpoint: str = "" + api_key_header: str = "X-API-Key" + workspace_header: str = "X-Workspace-Id" + one_time_env_vars: Dict[str, str] = Field(default_factory=dict) + setup_steps: List[Dict[str, str]] = Field(default_factory=list) + transport_options: Dict[str, str] = Field(default_factory=dict) + per_call_correlation: Dict[str, str] = Field(default_factory=dict) + suggested_span_resource_attributes: Dict[str, str] = Field(default_factory=dict) + pipecat_python_example: str = "" + + +VALID_TRACE_TRANSPORTS = ("webrtc", "websocket", "phone", "custom") + + +class TraceSessionCreateRequest(BaseModel): + evaluator_result_id: Optional[UUID] = None + agent_id: Optional[UUID] = None + transport: str = "websocket" + + +class TraceSessionOtelCorrelation(BaseModel): + otlp_endpoint: str + api_key_header: str = "X-API-Key" + suggested_env_vars: Dict[str, str] = Field(default_factory=dict) + suggested_otlp_headers: Dict[str, str] = Field(default_factory=dict) + suggested_span_attributes: Dict[str, str] = Field(default_factory=dict) + + +class TraceSessionResponse(BaseModel): + trace_id: UUID + call_short_id: str + workspace_id: UUID + transport: str + status: str + otel_correlation: TraceSessionOtelCorrelation + + +class TraceSessionCloseResponse(BaseModel): + trace_id: UUID + call_short_id: str + status: str + + +class JsonTraceSpanInput(BaseModel): + name: str + turn_number: int + ttfb_ms: Optional[float] = None + attributes: Dict[str, Any] = Field(default_factory=dict) + + +class JsonTraceIngestRequest(BaseModel): + call_short_id: str + spans: List[JsonTraceSpanInput] = Field(default_factory=list) + + +class JsonTraceIngestResponse(BaseModel): + accepted_spans: int + synthetic_call_trace_id: Optional[UUID] = None + correlated: bool = False diff --git a/app/services/billing/flexprice_service.py b/app/services/billing/flexprice_service.py index 80cb9bdd..a9471f56 100644 --- a/app/services/billing/flexprice_service.py +++ b/app/services/billing/flexprice_service.py @@ -4,6 +4,41 @@ Every event uses ``external_customer_id=str(organization.id)`` and a stable ``event_id`` for idempotency. ``properties`` should include ``workspace_id`` and ``feature`` (license key) when the surface is gated. + +Ingest **only when value is delivered** (completed), never on ``*_started`` / +``*_created`` / ``*_requested``. Event ``properties`` include billable fields +(``workspace_id``, ``feature``, ``quantity``, ``billable_minutes``) plus audit +IDs (``evaluation_id``, ``audio_seconds``, ``ui_surface``, etc.) for support — +audit fields are not used for Flexprice SUM/COUNT aggregation. Set ``ui_surface`` +only when multiple UI paths share one event (e.g. ``agents_talk`` vs +``agent_playground``); never wire meters to it. + +Billable events (wire plan usage charges to these meters only): + +- call_imports: ``call_import.batch_created`` (``quantity`` = rows imported), + ``call_import.evaluation_completed`` (``quantity`` = newly completed rows), + ``call_import.recording_minutes_billed`` (``quantity`` = ``billable_minutes``), + ``call_import.pdf_report_generated`` (``quantity`` = 1) +- agent_playground: ``playground.evaluation_completed`` (``quantity`` = ``billable_minutes`` + from call duration) **or** ``test_agent.conversation_ended`` (same minute rollup for + standalone test-agent sessions without playground eval) — never both for the same session +- voice_playground: ``tts.sample_synthesized``, ``tts.report_completed``, + ``blind_test.response_submitted`` +- evaluators: ``evaluator.run_completed`` (``quantity`` = 1) and + ``evaluator.recording_minutes_billed`` when the run has audio (``billable_minutes``) +- gepa_optimization: ``prompt_optimization.run_completed`` (``quantity`` = candidates) +- judge_alignment: ``judge_alignment.run_completed`` (``quantity`` = samples scored) +- metrics_ai_assist: ``metrics.ai_assist`` +- metric_studio: ``metric_studio.run_completed`` (``quantity`` = completed items) +- scenario_ai: ``scenario.ai_text_generated`` +- prompt_partials: ``prompt_partial.ai_assisted`` (``mode``: generate | improve | flowchart | flowchart_map) +- call_imports (add-ons): ``call_import.user_insights_generated``, + ``call_import.prompt_improvements_generated`` +- agent_playground (AI helpers): ``persona.prompt_generated``, ``agent.test_setup_generated`` + +Not ingested: ``*_started``, ``*_requested``, ``*_created``, ``observability.*``, +``playground.call_evaluated``, ``test_agent.conversation_started``, +``metric_studio.item_evaluated``, etc. """ from __future__ import annotations @@ -18,8 +53,19 @@ EVENT_SOURCE = "efficientai" FEATURE_CALL_IMPORTS = "call_imports" +FEATURE_AGENT_PLAYGROUND = "agent_playground" FEATURE_VOICE_PLAYGROUND = "voice_playground" FEATURE_GEPA = "gepa_optimization" +FEATURE_EVALUATORS = "evaluators" +FEATURE_JUDGE_ALIGNMENT = "judge_alignment" +FEATURE_METRICS_AI_ASSIST = "metrics_ai_assist" +FEATURE_METRIC_STUDIO = "metric_studio" +FEATURE_SCENARIO_AI = "scenario_ai" +FEATURE_PROMPT_PARTIALS = "prompt_partials" + +# Audit-only ui_surface values (never wire Flexprice meters to these). +UI_SURFACE_AGENTS_TALK = "agents_talk" +UI_SURFACE_AGENT_PLAYGROUND = "agent_playground" # Log once when metering is inactive so AWS/worker misconfig is obvious. _disabled_skip_logged = False @@ -32,16 +78,18 @@ TTS_REPORT_REQUESTED = "tts.report_requested" TTS_REPORT_COMPLETED = "tts.report_completed" CALL_IMPORT_BATCH_CREATED = "call_import.batch_created" -CALL_IMPORT_ROW_IMPORTED = "call_import.row_imported" CALL_IMPORT_EVALUATION_STARTED = "call_import.evaluation_started" CALL_IMPORT_EVALUATION_COMPLETED = "call_import.evaluation_completed" -CALL_IMPORT_EVALUATION_ROW_COMPLETED = "call_import.evaluation_row_completed" +CALL_IMPORT_RECORDING_MINUTES_BILLED = "call_import.recording_minutes_billed" +CALL_IMPORT_AUDIO_MINUTES_BILLED = CALL_IMPORT_RECORDING_MINUTES_BILLED +CALL_IMPORT_PDF_REPORT_GENERATED = "call_import.pdf_report_generated" PLAYGROUND_WEB_CALL_STARTED = "playground.web_call_started" PLAYGROUND_WEBSOCKET_SESSION_STARTED = "playground.websocket_session_started" PLAYGROUND_CALL_EVALUATED = "playground.call_evaluated" PLAYGROUND_EVALUATION_COMPLETED = "playground.evaluation_completed" EVALUATOR_RUN_REQUESTED = "evaluator.run_requested" EVALUATOR_RUN_COMPLETED = "evaluator.run_completed" +EVALUATOR_RECORDING_MINUTES_BILLED = "evaluator.recording_minutes_billed" EVALUATION_CREATED = "evaluation.created" EVALUATION_COMPLETED = "evaluation.completed" PROMPT_OPTIMIZATION_RUN_STARTED = "prompt_optimization.run_started" @@ -52,8 +100,28 @@ OBSERVABILITY_CALL_EVALUATED = "observability.call_evaluated" TEST_AGENT_CONVERSATION_STARTED = "test_agent.conversation_started" TEST_AGENT_CONVERSATION_ENDED = "test_agent.conversation_ended" -METRICS_LLM_ASSIST = "metrics.llm_assist" -CHAT_COMPLETION = "chat.completion" +METRICS_AI_ASSIST = "metrics.ai_assist" +METRIC_STUDIO_ITEM_EVALUATED = "metric_studio.item_evaluated" +METRIC_STUDIO_RUN_COMPLETED = "metric_studio.run_completed" +SCENARIO_AI_TEXT_GENERATED = "scenario.ai_text_generated" +PROMPT_PARTIAL_AI_ASSISTED = "prompt_partial.ai_assisted" +CALL_IMPORT_USER_INSIGHTS_GENERATED = "call_import.user_insights_generated" +CALL_IMPORT_PROMPT_IMPROVEMENTS_GENERATED = "call_import.prompt_improvements_generated" +PERSONA_PROMPT_GENERATED = "persona.prompt_generated" +AGENT_TEST_SETUP_GENERATED = "agent.test_setup_generated" +# Deprecated meters — never ingest (bill on completion events instead). +DEPRECATED_EVENT_NAMES = frozenset( + { + PLAYGROUND_CALL_EVALUATED, + PLAYGROUND_WEB_CALL_STARTED, + PLAYGROUND_WEBSOCKET_SESSION_STARTED, + TEST_AGENT_CONVERSATION_STARTED, + OBSERVABILITY_CALL_EVALUATED, + } +) +# Legacy aliases (Flexprice meters may still exist under old names) +METRICS_LLM_ASSIST = METRICS_AI_ASSIST +CHAT_COMPLETION = SCENARIO_AI_TEXT_GENERATED def _verbose_logging() -> bool: @@ -61,6 +129,14 @@ def _verbose_logging() -> bool: return os.getenv("FLEXPRICE_VERBOSE", "").lower() in {"1", "true", "yes"} +def _pytest_blocks_external_billing() -> bool: + """Block real Flexprice I/O during pytest unless explicitly opted in.""" + return ( + os.environ.get("EFFICIENTAI_PYTEST") == "1" + and os.environ.get("FLEXPRICE_TEST_ALLOW") != "1" + ) + + def _mask_api_key(api_key: Optional[str]) -> str: if not api_key: return "(missing)" @@ -121,6 +197,8 @@ def log_startup_status(*, component: str = "app") -> None: def _verify_connectivity() -> Optional[str]: """Best-effort reachability probe; returns error text or None when OK.""" + if _pytest_blocks_external_billing(): + return None try: import httpx @@ -176,6 +254,64 @@ def _coerce_properties(properties: Optional[dict[str, Any]]) -> dict[str, str]: return out +def _billable_minutes(duration_seconds: Optional[float]) -> int: + if duration_seconds is None: + return 1 + seconds = float(duration_seconds) + if seconds <= 0: + return 1 + import math + + return max(1, int(math.ceil(seconds / 60.0))) + + +def _billing_properties( + workspace_id: UUID, + feature: str, + *, + quantity: Optional[Union[int, float]] = None, + billable_minutes: Optional[int] = None, +) -> dict[str, Any]: + props: dict[str, Any] = {"workspace_id": workspace_id, "feature": feature} + if quantity is not None: + props["quantity"] = quantity + if billable_minutes is not None: + props["billable_minutes"] = billable_minutes + return props + + +def _event_properties( + workspace_id: UUID, + feature: str, + *, + quantity: Optional[Union[int, float]] = None, + billable_minutes: Optional[int] = None, + ui_surface: Optional[str] = None, + **audit: Any, +) -> dict[str, Any]: + """Billable fields plus optional audit metadata for support traceability.""" + props = _billing_properties( + workspace_id, + feature, + quantity=quantity, + billable_minutes=billable_minutes, + ) + surface = (str(ui_surface).strip() if ui_surface is not None else "") or None + if not surface and isinstance(audit.get("ui_surface"), str): + surface = audit["ui_surface"].strip() or None + if surface: + props["ui_surface"] = surface + for key, value in audit.items(): + if key == "ui_surface": + continue + if value is None: + continue + if isinstance(value, str) and not value.strip(): + continue + props[key] = value + return props + + def record_event( event_name: str, organization_id: UUID, @@ -190,6 +326,9 @@ def record_event( """ global _disabled_skip_logged + if _pytest_blocks_external_billing(): + return False + inactive_reason = disabled_reason() if inactive_reason: if not _disabled_skip_logged: @@ -208,6 +347,16 @@ def record_event( ) return False + if event_name in DEPRECATED_EVENT_NAMES: + if _verbose_logging(): + logger.info( + "Flexprice SKIP deprecated {} org={} event_id={}", + event_name, + organization_id, + event_id, + ) + return False + coerced = _coerce_properties(properties) quantity = coerced.get("quantity") @@ -254,6 +403,9 @@ def ensure_customer( email: Optional[str] = None, ) -> None: """Register an organization as a Flexprice customer. No-op when disabled.""" + if _pytest_blocks_external_billing(): + return + inactive_reason = disabled_reason() if inactive_reason: if _verbose_logging(): @@ -296,6 +448,89 @@ def ensure_customer( ) +def _subscription_inactive_reason() -> Optional[str]: + """Why auto-subscribe is off, or None when ensure_subscription may run.""" + inactive = disabled_reason() + if inactive: + return inactive + if not settings.FLEXPRICE_AUTO_SUBSCRIBE: + return "flexprice.auto_subscribe is false (or FLEXPRICE_AUTO_SUBSCRIBE unset)" + if not (settings.FLEXPRICE_DEFAULT_PLAN_ID or "").strip(): + return "flexprice.default_plan_id is unset (or FLEXPRICE_DEFAULT_PLAN_ID env missing)" + return None + + +def _has_active_subscription(client, *, organization_id: UUID, plan_id: str) -> bool: + response = client.subscriptions.query_subscription( + external_customer_id=str(organization_id), + plan_id=plan_id, + limit=1, + ) + items = getattr(response, "items", None) or [] + return len(items) > 0 + + +def ensure_subscription(organization_id: UUID) -> None: + """Assign the default SaaS plan when auto-subscribe is enabled. Never raises.""" + if _pytest_blocks_external_billing(): + return + + inactive_reason = _subscription_inactive_reason() + if inactive_reason: + if _verbose_logging(): + logger.info( + "Flexprice SKIP ensure_subscription org={} ({})", + organization_id, + inactive_reason, + ) + return + + plan_id = settings.FLEXPRICE_DEFAULT_PLAN_ID.strip() + try: + from flexprice import Flexprice + + with Flexprice( + server_url=settings.FLEXPRICE_API_HOST, + api_key_auth=settings.FLEXPRICE_API_KEY, + ) as client: + if _has_active_subscription(client, organization_id=organization_id, plan_id=plan_id): + logger.debug( + "Flexprice ensure_subscription already active org={} plan_id={}", + organization_id, + plan_id, + ) + return + + created = client.subscriptions.create_subscription( + billing_period=settings.FLEXPRICE_DEFAULT_BILLING_PERIOD, + currency=settings.FLEXPRICE_DEFAULT_CURRENCY, + plan_id=plan_id, + external_customer_id=str(organization_id), + subscription_status="active", + ) + logger.info( + "Flexprice ensure_subscription ok org={} plan_id={} subscription_id={}", + organization_id, + plan_id, + getattr(created, "id", None), + ) + except Exception as exc: + if _is_customer_already_exists(exc) or "already exist" in str(exc).lower(): + logger.debug( + "Flexprice ensure_subscription already exists org={} plan_id={}", + organization_id, + plan_id, + ) + return + logger.warning( + "Flexprice ensure_subscription FAILED org={} plan_id={} host={} error={}", + organization_id, + plan_id, + settings.FLEXPRICE_API_HOST, + exc, + ) + + # --- Voice playground --- @@ -306,17 +541,7 @@ def record_blind_test_share_created( workspace_id: UUID, comparison_id: UUID, ) -> None: - record_event( - BLIND_TEST_SHARE_CREATED, - organization_id, - share_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "share_id": share_id, - "comparison_id": comparison_id, - }, - ) + """Not ingested — bill on blind_test.response_submitted instead.""" def record_blind_test_response_submitted( @@ -331,13 +556,13 @@ def record_blind_test_response_submitted( BLIND_TEST_RESPONSE_SUBMITTED, organization_id, response_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "share_id": share_id, - "response_count": response_count, - "quantity": response_count, - }, + properties=_event_properties( + workspace_id, + FEATURE_VOICE_PLAYGROUND, + quantity=max(1, response_count), + share_id=share_id, + response_id=response_id, + ), ) @@ -348,17 +573,7 @@ def record_tts_generation_started( workspace_id: UUID, sample_count: int, ) -> None: - record_event( - TTS_GENERATION_STARTED, - organization_id, - comparison_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "comparison_id": comparison_id, - "sample_count": sample_count, - }, - ) + """Not ingested — bill on tts.sample_synthesized per completed sample.""" def record_tts_sample_synthesized( @@ -375,16 +590,16 @@ def record_tts_sample_synthesized( TTS_SAMPLE_SYNTHESIZED, organization_id, sample_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "comparison_id": comparison_id, - "sample_id": sample_id, - "provider": provider, - "side": side, - "duration_seconds": duration_seconds, - "quantity": 1, - }, + properties=_event_properties( + workspace_id, + FEATURE_VOICE_PLAYGROUND, + quantity=1, + comparison_id=comparison_id, + sample_id=sample_id, + provider=provider, + side=side, + duration_seconds=duration_seconds, + ), ) @@ -395,17 +610,7 @@ def record_tts_report_requested( workspace_id: UUID, comparison_id: UUID, ) -> None: - record_event( - TTS_REPORT_REQUESTED, - organization_id, - report_job_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "comparison_id": comparison_id, - "report_job_id": report_job_id, - }, - ) + """Not ingested — bill on tts.report_completed.""" def record_tts_report_completed( @@ -419,16 +624,17 @@ def record_tts_report_completed( TTS_REPORT_COMPLETED, organization_id, report_job_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_VOICE_PLAYGROUND, - "comparison_id": comparison_id, - "report_job_id": report_job_id, - }, + properties=_event_properties( + workspace_id, + FEATURE_VOICE_PLAYGROUND, + quantity=1, + comparison_id=comparison_id, + report_job_id=report_job_id, + ), ) -# --- Call imports --- +# --- Call imports (tracking complete: batch, eval lifecycle, audio minutes, PDF) --- def record_call_import_batch_created( @@ -444,21 +650,33 @@ def record_call_import_batch_created( CALL_IMPORT_BATCH_CREATED, organization_id, call_import_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_CALL_IMPORTS, - "call_import_id": call_import_id, - "total_rows": total_rows, - "quantity": total_rows, - "source": source, - "provider": provider, - }, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=max(0, total_rows), + call_import_id=call_import_id, + source=source, + provider=provider, + total_rows=total_rows, + ), ) # --- Call imports (evaluations) --- +def record_call_import_evaluation_started( + organization_id: UUID, + evaluation_id: UUID, + *, + workspace_id: UUID, + call_import_id: UUID, + total_rows: int, + metric_count: int = 0, +) -> None: + """Not ingested — bill on call_import.evaluation_completed.""" + + def record_call_import_evaluation_completed( organization_id: UUID, evaluation_id: UUID, @@ -467,23 +685,103 @@ def record_call_import_evaluation_completed( call_import_id: UUID, rows_billed: int, completed_total: int, + total_rows: int = 0, metric_count: int = 0, ) -> bool: - """Bill one pass of an evaluation run for newly completed rows.""" + """Bill one finished evaluation pass for newly completed rows (not per row).""" + if rows_billed <= 0: + return False return record_event( CALL_IMPORT_EVALUATION_COMPLETED, organization_id, f"{evaluation_id}:{completed_total}", - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_CALL_IMPORTS, - "call_import_id": call_import_id, - "evaluation_id": evaluation_id, - "rows_billed": rows_billed, - "completed_total": completed_total, - "metric_count": metric_count, - "quantity": rows_billed, - }, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=rows_billed, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + completed_total=completed_total, + total_rows=total_rows, + metric_count=metric_count, + rows_billed=rows_billed, + ), + ) + + +def record_call_import_recording_minutes_billed( + organization_id: UUID, + evaluation_row_id: UUID, + *, + workspace_id: UUID, + evaluation_id: UUID, + call_import_id: UUID, + audio_seconds: int, + billable_minutes: int, +) -> bool: + """Bill recording duration for one successfully evaluated call-import row.""" + if billable_minutes <= 0: + return False + return record_event( + CALL_IMPORT_RECORDING_MINUTES_BILLED, + organization_id, + evaluation_row_id, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=billable_minutes, + billable_minutes=billable_minutes, + evaluation_row_id=evaluation_row_id, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + audio_seconds=audio_seconds, + ), + ) + + +def record_call_import_audio_minutes_billed( + organization_id: UUID, + evaluation_row_id: UUID, + *, + workspace_id: UUID, + evaluation_id: UUID, + call_import_id: UUID, + audio_seconds: int, + billable_minutes: int, +) -> bool: + return record_call_import_recording_minutes_billed( + organization_id, + evaluation_row_id, + workspace_id=workspace_id, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + audio_seconds=audio_seconds, + billable_minutes=billable_minutes, + ) + + +def record_call_import_pdf_report_generated( + organization_id: UUID, + pdf_report_id: UUID, + *, + workspace_id: UUID, + evaluation_id: UUID, + call_import_id: UUID, + report_type: str, +) -> None: + record_event( + CALL_IMPORT_PDF_REPORT_GENERATED, + organization_id, + pdf_report_id, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=1, + pdf_report_id=pdf_report_id, + evaluation_id=evaluation_id, + call_import_id=call_import_id, + report_type=report_type, + ), ) @@ -497,16 +795,7 @@ def record_playground_web_call_started( workspace_id: UUID, agent_id: UUID, ) -> None: - record_event( - PLAYGROUND_WEB_CALL_STARTED, - organization_id, - call_short_id, - properties={ - "workspace_id": workspace_id, - "agent_id": agent_id, - "call_short_id": call_short_id, - }, - ) + """Not ingested — bill on playground.evaluation_completed.""" def record_playground_websocket_session_started( @@ -515,15 +804,7 @@ def record_playground_websocket_session_started( *, workspace_id: UUID, ) -> None: - record_event( - PLAYGROUND_WEBSOCKET_SESSION_STARTED, - organization_id, - call_short_id, - properties={ - "workspace_id": workspace_id, - "call_short_id": call_short_id, - }, - ) + """Not ingested — bill on playground.evaluation_completed.""" def record_playground_call_evaluated( @@ -535,18 +816,7 @@ def record_playground_call_evaluated( call_short_id: str, metric_count: int, ) -> None: - record_event( - PLAYGROUND_CALL_EVALUATED, - organization_id, - evaluation_attempt_id, - properties={ - "workspace_id": workspace_id, - "call_short_id": call_short_id, - "evaluator_result_id": evaluator_result_id, - "evaluation_attempt_id": evaluation_attempt_id, - "metric_count": metric_count, - }, - ) + """Not ingested — bill on playground.evaluation_completed.""" def record_playground_evaluation_completed( @@ -558,19 +828,25 @@ def record_playground_evaluation_completed( call_short_id: str, duration_seconds: Optional[float] = None, metric_count: int = 0, + ui_surface: Optional[str] = None, ) -> None: + """Bill playground voice/web call scoring. Pass ``ui_surface`` when known.""" + minutes = _billable_minutes(duration_seconds) record_event( PLAYGROUND_EVALUATION_COMPLETED, organization_id, evaluation_attempt_id, - properties={ - "workspace_id": workspace_id, - "call_short_id": call_short_id, - "evaluator_result_id": evaluator_result_id, - "evaluation_attempt_id": evaluation_attempt_id, - "duration_seconds": duration_seconds, - "metric_count": metric_count, - }, + properties=_event_properties( + workspace_id, + FEATURE_AGENT_PLAYGROUND, + quantity=minutes, + billable_minutes=minutes, + evaluator_result_id=evaluator_result_id, + call_short_id=call_short_id, + duration_seconds=duration_seconds, + metric_count=metric_count, + ui_surface=ui_surface, + ), ) @@ -584,15 +860,8 @@ def record_evaluator_run_requested( workspace_id: UUID, quantity: int, ) -> None: - record_event( - EVALUATOR_RUN_REQUESTED, - organization_id, - request_id, - properties={ - "workspace_id": workspace_id, - "quantity": quantity, - }, - ) + """Not ingested — bill on evaluator.run_completed when scoring finishes.""" + del organization_id, request_id, workspace_id, quantity def record_evaluator_run_completed( @@ -600,19 +869,52 @@ def record_evaluator_run_completed( result_id: str, *, workspace_id: UUID, - evaluator_id: UUID, + evaluator_id: Optional[UUID] = None, + evaluator_result_id: Optional[UUID] = None, call_count: int = 1, ) -> None: + del call_count record_event( EVALUATOR_RUN_COMPLETED, organization_id, result_id, - properties={ - "workspace_id": workspace_id, - "evaluator_id": evaluator_id, - "result_id": result_id, - "call_count": call_count, - }, + properties=_event_properties( + workspace_id, + FEATURE_EVALUATORS, + quantity=1, + result_id=result_id, + evaluator_id=evaluator_id, + evaluator_result_id=evaluator_result_id, + ), + ) + + +def record_evaluator_recording_minutes_billed( + organization_id: UUID, + evaluator_result_id: UUID, + *, + workspace_id: UUID, + duration_seconds: Optional[float] = None, +) -> bool: + """Bill audio duration for a completed evaluator run that includes a recording.""" + minutes = _billable_minutes(duration_seconds) + if minutes <= 0: + return False + return record_event( + EVALUATOR_RECORDING_MINUTES_BILLED, + organization_id, + evaluator_result_id, + properties=_event_properties( + workspace_id, + FEATURE_EVALUATORS, + quantity=minutes, + billable_minutes=minutes, + evaluator_result_id=evaluator_result_id, + duration_seconds=duration_seconds, + audio_seconds=int(round(float(duration_seconds))) + if duration_seconds is not None + else None, + ), ) @@ -627,16 +929,8 @@ def record_evaluation_created( audio_id: UUID, metrics_requested: int, ) -> None: - record_event( - EVALUATION_CREATED, - organization_id, - evaluation_id, - properties={ - "workspace_id": workspace_id, - "audio_id": audio_id, - "metrics_requested": metrics_requested, - }, - ) + """Not ingested — bill on evaluation.completed.""" + del organization_id, evaluation_id, workspace_id, audio_id, metrics_requested def record_evaluation_completed( @@ -649,7 +943,12 @@ def record_evaluation_completed( EVALUATION_COMPLETED, organization_id, evaluation_id, - properties={"workspace_id": workspace_id}, + properties=_event_properties( + workspace_id, + FEATURE_EVALUATORS, + quantity=1, + evaluation_id=evaluation_id, + ), ) @@ -664,18 +963,8 @@ def record_prompt_optimization_run_started( agent_id: UUID, max_metric_calls: Optional[int] = None, ) -> None: - record_event( - PROMPT_OPTIMIZATION_RUN_STARTED, - organization_id, - run_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_GEPA, - "run_id": run_id, - "agent_id": agent_id, - "max_metric_calls": max_metric_calls, - }, - ) + """Not ingested — bill on prompt_optimization.run_completed.""" + del organization_id, run_id, workspace_id, agent_id, max_metric_calls def record_prompt_optimization_run_completed( @@ -686,17 +975,19 @@ def record_prompt_optimization_run_completed( agent_id: UUID, candidates_count: int = 0, ) -> None: + billed = max(1, candidates_count) record_event( PROMPT_OPTIMIZATION_RUN_COMPLETED, organization_id, run_id, - properties={ - "workspace_id": workspace_id, - "feature": FEATURE_GEPA, - "run_id": run_id, - "agent_id": agent_id, - "candidates_count": candidates_count, - }, + properties=_event_properties( + workspace_id, + FEATURE_GEPA, + quantity=billed, + run_id=run_id, + agent_id=agent_id, + candidates_count=candidates_count, + ), ) @@ -711,17 +1002,8 @@ def record_judge_alignment_run_started( dataset_id: UUID, sample_count: int, ) -> None: - record_event( - JUDGE_ALIGNMENT_RUN_STARTED, - organization_id, - run_id, - properties={ - "workspace_id": workspace_id, - "run_id": run_id, - "dataset_id": dataset_id, - "sample_count": sample_count, - }, - ) + """Not ingested — bill on judge_alignment.run_completed.""" + del organization_id, run_id, workspace_id, dataset_id, sample_count def record_judge_alignment_run_completed( @@ -732,16 +1014,21 @@ def record_judge_alignment_run_completed( dataset_id: UUID, samples_scored: int, ) -> None: + billed = max(0, samples_scored) + if billed <= 0: + return record_event( JUDGE_ALIGNMENT_RUN_COMPLETED, organization_id, run_id, - properties={ - "workspace_id": workspace_id, - "run_id": run_id, - "dataset_id": dataset_id, - "samples_scored": samples_scored, - }, + properties=_event_properties( + workspace_id, + FEATURE_JUDGE_ALIGNMENT, + quantity=billed, + run_id=run_id, + dataset_id=dataset_id, + samples_scored=samples_scored, + ), ) @@ -755,16 +1042,8 @@ def record_observability_call_ingested( workspace_id: UUID, provider: Optional[str] = None, ) -> None: - record_event( - OBSERVABILITY_CALL_INGESTED, - organization_id, - call_short_id, - properties={ - "workspace_id": workspace_id, - "call_short_id": call_short_id, - "provider": provider, - }, - ) + """Not ingested — bill on observability.call_evaluated.""" + del organization_id, call_short_id, workspace_id, provider def record_observability_call_evaluated( @@ -773,15 +1052,8 @@ def record_observability_call_evaluated( *, workspace_id: UUID, ) -> None: - record_event( - OBSERVABILITY_CALL_EVALUATED, - organization_id, - call_short_id, - properties={ - "workspace_id": workspace_id, - "call_short_id": call_short_id, - }, - ) + """Not ingested — observability is not a billable product surface.""" + del organization_id, call_short_id, workspace_id # --- Test agents --- @@ -789,44 +1061,72 @@ def record_observability_call_evaluated( def record_test_agent_conversation_started( organization_id: UUID, - conversation_id: UUID, + conversation_id: Union[str, UUID], *, workspace_id: UUID, + result_id: Optional[str] = None, + agent_id: Optional[UUID] = None, + call_short_id: Optional[str] = None, ) -> None: - record_event( - TEST_AGENT_CONVERSATION_STARTED, - organization_id, - conversation_id, - properties={ - "workspace_id": workspace_id, - "conversation_id": conversation_id, - }, - ) + """Not ingested — bill on test_agent.conversation_ended.""" + del organization_id, conversation_id, workspace_id, result_id, agent_id, call_short_id def record_test_agent_conversation_ended( organization_id: UUID, - conversation_id: UUID, + conversation_id: Union[str, UUID], *, workspace_id: UUID, duration_seconds: Optional[float] = None, turn_count: int = 0, + result_id: Optional[str] = None, + agent_id: Optional[UUID] = None, + call_short_id: Optional[str] = None, ) -> None: + minutes = _billable_minutes(duration_seconds) record_event( TEST_AGENT_CONVERSATION_ENDED, organization_id, conversation_id, - properties={ - "workspace_id": workspace_id, - "conversation_id": conversation_id, - "duration_seconds": duration_seconds, - "turn_count": turn_count, - "quantity": duration_seconds or 1, - }, + properties=_event_properties( + workspace_id, + FEATURE_AGENT_PLAYGROUND, + quantity=minutes, + billable_minutes=minutes, + conversation_id=conversation_id, + duration_seconds=duration_seconds, + turn_count=turn_count, + result_id=result_id, + agent_id=agent_id, + call_short_id=call_short_id, + ), ) -# --- LLM assist --- +# --- Metrics AI assist (metric builder) --- + + +def record_metrics_ai_assist( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + mode: str, +) -> None: + if workspace_id is None: + return + record_event( + METRICS_AI_ASSIST, + organization_id, + request_id, + properties=_event_properties( + workspace_id, + FEATURE_METRICS_AI_ASSIST, + quantity=1, + request_id=request_id, + mode=mode, + ), + ) def record_metrics_llm_assist( @@ -836,14 +1136,96 @@ def record_metrics_llm_assist( workspace_id: Optional[UUID], mode: str, ) -> None: + record_metrics_ai_assist( + organization_id, + request_id, + workspace_id=workspace_id, + mode=mode, + ) + + +# --- Metric Studio --- + + +def record_metric_studio_item_evaluated( + organization_id: UUID, + result_row_id: UUID, + *, + workspace_id: UUID, + run_id: UUID, + source_kind: str, + source_ref: str, + metric_count: int = 0, +) -> None: + """Not ingested — bill on metric_studio.run_completed.""" + del ( + organization_id, + result_row_id, + workspace_id, + run_id, + source_kind, + source_ref, + metric_count, + ) + + +def record_metric_studio_run_completed( + organization_id: UUID, + run_id: UUID, + *, + workspace_id: UUID, + run_status: str, + total_items: int, + completed_items: int, + failed_items: int, +) -> None: + del run_status + billed = max(0, completed_items) + if billed <= 0: + return record_event( - METRICS_LLM_ASSIST, + METRIC_STUDIO_RUN_COMPLETED, + organization_id, + run_id, + properties=_event_properties( + workspace_id, + FEATURE_METRIC_STUDIO, + quantity=billed, + run_id=run_id, + total_items=total_items, + completed_items=completed_items, + failed_items=failed_items, + ), + ) + + +# --- Scenario / assistant AI text --- + + +def record_scenario_ai_text_generated( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + model: Optional[str] = None, + purpose: str = "scenario_description", + scenario_count: Optional[int] = None, +) -> None: + if workspace_id is None: + return + record_event( + SCENARIO_AI_TEXT_GENERATED, organization_id, request_id, - properties={ - "workspace_id": workspace_id, - "mode": mode, - }, + properties=_event_properties( + workspace_id, + FEATURE_SCENARIO_AI, + quantity=1, + request_id=request_id, + model=model, + purpose=purpose, + scenario_count=scenario_count, + ), ) @@ -853,14 +1235,152 @@ def record_chat_completion( *, workspace_id: Optional[UUID], model: Optional[str] = None, + purpose: str = "scenario_description", + scenario_count: Optional[int] = None, +) -> None: + record_scenario_ai_text_generated( + organization_id, + request_id, + workspace_id=workspace_id, + model=model, + purpose=purpose, + scenario_count=scenario_count, + ) + + +# --- Prompt partials AI assist --- + + +def record_prompt_partial_ai_assisted( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + mode: str, + partial_id: Optional[UUID] = None, + model: Optional[str] = None, +) -> None: + if workspace_id is None: + return + record_event( + PROMPT_PARTIAL_AI_ASSISTED, + organization_id, + request_id, + properties=_event_properties( + workspace_id, + FEATURE_PROMPT_PARTIALS, + quantity=1, + request_id=request_id, + mode=mode, + partial_id=partial_id, + model=model, + ), + ) + + +# --- Call import AI add-ons --- + + +def record_call_import_user_insights_generated( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + evaluation_id: UUID, +) -> None: + if workspace_id is None: + return + record_event( + CALL_IMPORT_USER_INSIGHTS_GENERATED, + organization_id, + request_id, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=1, + request_id=request_id, + evaluation_id=evaluation_id, + ), + ) + + +def record_call_import_prompt_improvements_generated( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + evaluation_id: UUID, + imported_agent_id: Optional[UUID] = None, +) -> None: + if workspace_id is None: + return + record_event( + CALL_IMPORT_PROMPT_IMPROVEMENTS_GENERATED, + organization_id, + request_id, + properties=_event_properties( + workspace_id, + FEATURE_CALL_IMPORTS, + quantity=1, + request_id=request_id, + evaluation_id=evaluation_id, + imported_agent_id=imported_agent_id, + ), + ) + + +# --- Agent / persona AI helpers --- + + +def record_persona_prompt_generated( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + agent_id: UUID, + model: Optional[str] = None, + source: Optional[str] = None, ) -> None: + if workspace_id is None: + return + record_event( + PERSONA_PROMPT_GENERATED, + organization_id, + request_id, + properties=_event_properties( + workspace_id, + FEATURE_AGENT_PLAYGROUND, + quantity=1, + request_id=request_id, + agent_id=agent_id, + model=model, + source=source, + ), + ) + + +def record_agent_test_setup_generated( + organization_id: UUID, + request_id: UUID, + *, + workspace_id: Optional[UUID], + purpose: str, + model: Optional[str] = None, + scenario_count: Optional[int] = None, +) -> None: + if workspace_id is None: + return record_event( - CHAT_COMPLETION, + AGENT_TEST_SETUP_GENERATED, organization_id, request_id, - properties={ - "workspace_id": workspace_id, - "model": model, - "quantity": 1, - }, + properties=_event_properties( + workspace_id, + FEATURE_AGENT_PLAYGROUND, + quantity=1, + request_id=request_id, + purpose=purpose, + model=model, + scenario_count=scenario_count, + ), ) diff --git a/app/services/evaluators/evaluator_phone_run_service.py b/app/services/evaluators/evaluator_phone_run_service.py index 6d94c825..5e30bfc4 100644 --- a/app/services/evaluators/evaluator_phone_run_service.py +++ b/app/services/evaluators/evaluator_phone_run_service.py @@ -1,175 +1,177 @@ -"""Phone-based evaluator suite runs (Vobiz outbound).""" - -from typing import List, Optional, Tuple -from uuid import UUID - -from fastapi import HTTPException -from sqlalchemy.orm import Session - -from app.models.database import Agent, Evaluator, Scenario -from app.models.schemas import EvaluatorResultResponse -from app.services.evaluators.evaluator_suite_service import generate_unique_result_id -from app.services.telephony.plivo_client import normalize_e164 - - -def initiate_phone_evaluator_call( - db: Session, - organization_id: UUID, - workspace_id: UUID, - evaluator: Evaluator, - agent: Agent, - to_number: str, - from_number: Optional[str] = None, -) -> Tuple[str, str, Optional[EvaluatorResultResponse]]: - """Place a Vobiz outbound call for one evaluator combination. - - Returns (call_ref, call_short_id, evaluator_result_response). - """ - import random - import string - - from app.models.database import CallRecording, CallRecordingSource, EvaluatorResult, EvaluatorResultStatus - from app.models.enums import CallRecordingStatus - from app.config import settings - from app.services.telephony.vobiz_outbound_pool import release_pool_slot, resolve_outbound_from_number - from app.services.telephony.vobiz_session import create_call_session - from app.services.telephony.vobiz_agent_context import vobiz_webhook_base_url - from app.workers.tasks.initiate_vobiz_outbound import initiate_vobiz_outbound_call_task - - scenario = db.query(Scenario).filter(Scenario.id == evaluator.scenario_id).first() - scenario_name = scenario.name if scenario else "Unknown Scenario" - - result_id = generate_unique_result_id(db) - evaluator_result = EvaluatorResult( - result_id=result_id, - organization_id=organization_id, - workspace_id=workspace_id, - evaluator_id=evaluator.id, - agent_id=evaluator.agent_id, - persona_id=evaluator.persona_id, - scenario_id=evaluator.scenario_id, - name=scenario_name, - status=EvaluatorResultStatus.QUEUED.value, - audio_s3_key=None, - ) - db.add(evaluator_result) - db.commit() - db.refresh(evaluator_result) - - try: - from_number_resolved, used_pool, provider = resolve_outbound_from_number( - db, - organization_id, - explicit_from_number=from_number, - ) - except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) from e - - if provider != "vobiz": - if used_pool: - release_pool_slot(organization_id) - raise HTTPException( - status_code=400, - detail=( - f"Phone evaluator outbound requires a Vobiz caller ID; " - f"resolved provider is {provider}." - ), - ) - - to_number_norm = normalize_e164(to_number) - persona_id = evaluator.persona_id - scenario_id = evaluator.scenario_id - evaluator_id = evaluator.id - - session = create_call_session( - agent_id=str(agent.id), - organization_id=str(organization_id), - direction="outbound", - from_number=from_number_resolved, - to_number=to_number_norm, - used_pool=used_pool, - persona_id=str(persona_id) if persona_id else None, - scenario_id=str(scenario_id) if scenario_id else None, - evaluator_id=str(evaluator_id), - ) - - base = vobiz_webhook_base_url() - answer_url = ( - f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/answer" - f"?call_ref={session.call_ref}" - ) - events_url = ( - f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/events" - f"?call_ref={session.call_ref}" - ) - recording_url = f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/recording-ready" - - call_short_id = "".join(random.choices(string.digits, k=6)) - recording = CallRecording( - organization_id=organization_id, - workspace_id=agent.workspace_id, - call_short_id=call_short_id, - status=CallRecordingStatus.PENDING, - source=CallRecordingSource.WEBHOOK, - call_event="outbound_initiated", - call_data={ - "call_ref": session.call_ref, - "call_short_id": call_short_id, - "recording_callback": recording_url, - "used_pool": used_pool, - "evaluator_id": str(evaluator_id), - "evaluator_result_id": str(evaluator_result.id), - "direction": "outbound", - "from_number": from_number_resolved, - "to_number": to_number_norm, - "live_transcript": [], - }, - provider_call_id=None, - provider_platform="vobiz", - agent_id=agent.id, - evaluator_result_id=evaluator_result.id, - ) - db.add(recording) - db.commit() - db.refresh(recording) - - initiate_vobiz_outbound_call_task.delay( - organization_id=str(organization_id), - call_ref=session.call_ref, - from_number=from_number_resolved, - to_number=to_number_norm, - answer_url=answer_url, - events_url=events_url, - used_pool=used_pool, - call_recording_id=str(recording.id), - ) - - return session.call_ref, call_short_id, EvaluatorResultResponse.model_validate(evaluator_result) - - -def run_phone_evaluator_batch( - db: Session, - organization_id: UUID, - workspace_id: UUID, - agent: Agent, - evaluators: List[Evaluator], - to_number: str, - from_number: Optional[str] = None, -) -> Tuple[List[str], List[EvaluatorResultResponse]]: - """Initiate phone calls for each evaluator in the list.""" - call_refs: List[str] = [] - results: List[EvaluatorResultResponse] = [] - for evaluator in evaluators: - call_ref, _short_id, result = initiate_phone_evaluator_call( - db, - organization_id, - workspace_id, - evaluator, - agent, - to_number, - from_number=from_number, - ) - call_refs.append(call_ref) - if result: - results.append(result) - return call_refs, results +"""Phone-based evaluator suite runs (Vobiz outbound).""" + +from typing import List, Optional, Tuple +from uuid import UUID + +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from app.models.database import Agent, Evaluator, Scenario +from app.models.schemas import EvaluatorResultResponse +from app.services.evaluators.evaluator_suite_service import generate_unique_result_id +from app.services.telephony.plivo_client import normalize_e164 + + +def initiate_phone_evaluator_call( + db: Session, + organization_id: UUID, + workspace_id: UUID, + evaluator: Evaluator, + agent: Agent, + to_number: str, + from_number: Optional[str] = None, +) -> Tuple[str, str, Optional[EvaluatorResultResponse]]: + """Place a Vobiz outbound call for one evaluator combination. + + Returns (call_ref, call_short_id, evaluator_result_response). + """ + from app.models.database import CallRecording, CallRecordingSource, EvaluatorResult, EvaluatorResultStatus + from app.models.enums import CallRecordingStatus + from app.config import settings + from app.services.telephony.vobiz_outbound_pool import release_pool_slot, resolve_outbound_from_number + from app.services.telephony.vobiz_session import create_call_session + from app.services.telephony.vobiz_agent_context import vobiz_webhook_base_url + from app.utils.call_recordings import generate_unique_call_short_id + from app.workers.tasks.initiate_vobiz_outbound import initiate_vobiz_outbound_call_task + + scenario = db.query(Scenario).filter(Scenario.id == evaluator.scenario_id).first() + scenario_name = scenario.name if scenario else "Unknown Scenario" + + result_id = generate_unique_result_id(db) + evaluator_result = EvaluatorResult( + result_id=result_id, + organization_id=organization_id, + workspace_id=workspace_id, + evaluator_id=evaluator.id, + agent_id=evaluator.agent_id, + persona_id=evaluator.persona_id, + scenario_id=evaluator.scenario_id, + name=scenario_name, + status=EvaluatorResultStatus.QUEUED.value, + audio_s3_key=None, + ) + db.add(evaluator_result) + db.commit() + db.refresh(evaluator_result) + + try: + from_number_resolved, used_pool, provider = resolve_outbound_from_number( + db, + organization_id, + explicit_from_number=from_number, + ) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) from e + + if provider != "vobiz": + if used_pool: + release_pool_slot(organization_id) + raise HTTPException( + status_code=400, + detail=( + f"Phone evaluator outbound requires a Vobiz caller ID; " + f"resolved provider is {provider}." + ), + ) + + to_number_norm = normalize_e164(to_number) + persona_id = evaluator.persona_id + scenario_id = evaluator.scenario_id + evaluator_id = evaluator.id + + session = create_call_session( + agent_id=str(agent.id), + organization_id=str(organization_id), + direction="outbound", + from_number=from_number_resolved, + to_number=to_number_norm, + used_pool=used_pool, + persona_id=str(persona_id) if persona_id else None, + scenario_id=str(scenario_id) if scenario_id else None, + evaluator_id=str(evaluator_id), + ) + + base = vobiz_webhook_base_url() + answer_url = ( + f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/answer" + f"?call_ref={session.call_ref}" + ) + events_url = ( + f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/events" + f"?call_ref={session.call_ref}" + ) + recording_url = f"{base}{settings.API_V1_PREFIX}/telephony/vobiz/webhooks/recording-ready" + + call_short_id = generate_unique_call_short_id(db) + recording = CallRecording( + organization_id=organization_id, + workspace_id=workspace_id, # active workspace from evaluator run + call_short_id=call_short_id, + status=CallRecordingStatus.PENDING, + source=CallRecordingSource.WEBHOOK, + call_event="outbound_initiated", + call_data={ + "call_ref": session.call_ref, + "call_short_id": call_short_id, + "recording_callback": recording_url, + "used_pool": used_pool, + "evaluator_id": str(evaluator_id), + "evaluator_result_id": str(evaluator_result.id), + "direction": "outbound", + "from_number": from_number_resolved, + "to_number": to_number_norm, + "live_transcript": [], + }, + provider_call_id=None, + provider_platform="vobiz", + agent_id=agent.id, + evaluator_result_id=evaluator_result.id, + ) + db.add(recording) + db.commit() + db.refresh(recording) + + from app.services.synthetic_traces.trace_service import open_trace_for_call_recording + + open_trace_for_call_recording(db, recording=recording, evaluator_result=evaluator_result) + + initiate_vobiz_outbound_call_task.delay( + organization_id=str(organization_id), + call_ref=session.call_ref, + from_number=from_number_resolved, + to_number=to_number_norm, + answer_url=answer_url, + events_url=events_url, + used_pool=used_pool, + call_recording_id=str(recording.id), + ) + + return session.call_ref, call_short_id, EvaluatorResultResponse.model_validate(evaluator_result) + + +def run_phone_evaluator_batch( + db: Session, + organization_id: UUID, + workspace_id: UUID, + agent: Agent, + evaluators: List[Evaluator], + to_number: str, + from_number: Optional[str] = None, +) -> Tuple[List[str], List[EvaluatorResultResponse]]: + """Initiate phone calls for each evaluator in the list.""" + call_refs: List[str] = [] + results: List[EvaluatorResultResponse] = [] + for evaluator in evaluators: + call_ref, _short_id, result = initiate_phone_evaluator_call( + db, + organization_id, + workspace_id, + evaluator, + agent, + to_number, + from_number=from_number, + ) + call_refs.append(call_ref) + if result: + results.append(result) + return call_refs, results diff --git a/app/services/media_urls.py b/app/services/media_urls.py index 6ef55adf..29117086 100644 --- a/app/services/media_urls.py +++ b/app/services/media_urls.py @@ -1,90 +1,96 @@ -"""URL helpers for routing live voice WebSockets to the media server.""" - -from __future__ import annotations - -from typing import Optional -from urllib.parse import quote - -from app.config import settings - - -def _normalize_ws_base(base: str) -> str: - base = base.rstrip("/") - if base.startswith("https://"): - return "wss://" + base[len("https://") :] - if base.startswith("http://"): - return "ws://" + base[len("http://") :] - if base.startswith("wss://") or base.startswith("ws://"): - return base - return f"wss://{base}" - - -def media_ws_base_url() -> Optional[str]: - """Explicit dedicated media server base (``MEDIA_WS_BASE_URL`` / config only).""" - base = (settings.MEDIA_WS_BASE_URL or "").strip() - if not base: - return None - return _normalize_ws_base(base) - - -def carrier_media_ws_base_url() -> Optional[str]: - """ - WebSocket base for carrier (Vobiz) answer XML. - - Uses ``MEDIA_WS_BASE_URL`` when set; otherwise reuses the telephony edge - webhook base (``vobiz_webhook_base_url`` / config) so a single public host - serves Vobiz webhooks and live audio. - """ - explicit = media_ws_base_url() - if explicit: - return explicit - try: - from app.services.telephony.vobiz_agent_context import vobiz_webhook_base_url - - return _normalize_ws_base(vobiz_webhook_base_url()) - except ValueError: - pass - webhook_base = (settings.VOBIZ_WEBHOOK_BASE_URL or settings.PLIVO_WEBHOOK_BASE_URL or "").strip() - if not webhook_base: - return None - return _normalize_ws_base(webhook_base) - - -def separate_media_server_configured() -> bool: - """True when live voice WebSockets should run on a dedicated media process.""" - return bool((settings.MEDIA_WS_BASE_URL or "").strip()) - - -def ws_base_from_http_host(host: str, *, scheme: str = "http") -> str: - """Build ws/wss base from an HTTP Host header (browser / reverse-proxy).""" - ws_scheme = "wss" if scheme == "https" else "ws" - return f"{ws_scheme}://{host.rstrip('/')}" - - -def build_voice_agent_ws_url( - *, - auth_query: str, - agent_id: Optional[str] = None, - persona_id: Optional[str] = None, - scenario_id: Optional[str] = None, - fallback_host: Optional[str] = None, - fallback_scheme: str = "http", -) -> str: - """Build the browser voice-agent WebSocket URL.""" - ws_base = media_ws_base_url() - if ws_base: - base = ws_base - elif fallback_host: - base = ws_base_from_http_host(fallback_host, scheme=fallback_scheme) - else: - base = f"ws://localhost:{settings.PORT}" - - query = auth_query - if agent_id: - query += f"&agent_id={quote(agent_id)}" - if persona_id: - query += f"&persona_id={quote(persona_id)}" - if scenario_id: - query += f"&scenario_id={quote(scenario_id)}" - - return f"{base}{settings.API_V1_PREFIX}/voice-agent/ws?{query}" +"""URL helpers for routing live voice WebSockets to the media server.""" + +from __future__ import annotations + +from typing import Optional +from urllib.parse import quote + +from app.config import settings + + +def _normalize_ws_base(base: str) -> str: + base = base.rstrip("/") + if base.startswith("https://"): + return "wss://" + base[len("https://") :] + if base.startswith("http://"): + return "ws://" + base[len("http://") :] + if base.startswith("wss://") or base.startswith("ws://"): + return base + return f"wss://{base}" + + +def media_ws_base_url() -> Optional[str]: + """Explicit dedicated media server base (``MEDIA_WS_BASE_URL`` / config only).""" + base = (settings.MEDIA_WS_BASE_URL or "").strip() + if not base: + return None + return _normalize_ws_base(base) + + +def carrier_media_ws_base_url() -> Optional[str]: + """ + WebSocket base for carrier (Vobiz) answer XML. + + Uses ``MEDIA_WS_BASE_URL`` when set; otherwise reuses the telephony edge + webhook base (``vobiz_webhook_base_url`` / config) so a single public host + serves Vobiz webhooks and live audio. + """ + explicit = media_ws_base_url() + if explicit: + return explicit + try: + from app.services.telephony.vobiz_agent_context import vobiz_webhook_base_url + + return _normalize_ws_base(vobiz_webhook_base_url()) + except ValueError: + pass + webhook_base = (settings.VOBIZ_WEBHOOK_BASE_URL or settings.PLIVO_WEBHOOK_BASE_URL or "").strip() + if not webhook_base: + return None + return _normalize_ws_base(webhook_base) + + +def separate_media_server_configured() -> bool: + """True when live voice WebSockets should run on a dedicated media process.""" + return bool((settings.MEDIA_WS_BASE_URL or "").strip()) + + +def ws_base_from_http_host(host: str, *, scheme: str = "http") -> str: + """Build ws/wss base from an HTTP Host header (browser / reverse-proxy).""" + ws_scheme = "wss" if scheme == "https" else "ws" + return f"{ws_scheme}://{host.rstrip('/')}" + + +def build_voice_agent_ws_url( + *, + auth_query: str, + agent_id: Optional[str] = None, + persona_id: Optional[str] = None, + scenario_id: Optional[str] = None, + call_short_id: Optional[str] = None, + ui_surface: Optional[str] = None, + fallback_host: Optional[str] = None, + fallback_scheme: str = "http", +) -> str: + """Build the browser voice-agent WebSocket URL.""" + ws_base = media_ws_base_url() + if ws_base: + base = ws_base + elif fallback_host: + base = ws_base_from_http_host(fallback_host, scheme=fallback_scheme) + else: + base = f"ws://localhost:{settings.PORT}" + + query = auth_query + if agent_id: + query += f"&agent_id={quote(agent_id)}" + if persona_id: + query += f"&persona_id={quote(persona_id)}" + if scenario_id: + query += f"&scenario_id={quote(scenario_id)}" + if ui_surface: + query += f"&ui_surface={quote(ui_surface, safe='')}" + if call_short_id: + query += f"&call_short_id={quote(call_short_id, safe='')}" + + return f"{base}{settings.API_V1_PREFIX}/voice-agent/ws?{query}" diff --git a/app/services/metric_studio/run_rollup.py b/app/services/metric_studio/run_rollup.py new file mode 100644 index 00000000..aee9015b --- /dev/null +++ b/app/services/metric_studio/run_rollup.py @@ -0,0 +1,61 @@ +"""Shared Metrics Studio run rollup + Flexprice emission.""" + +from __future__ import annotations + +from datetime import datetime, timezone + +from sqlalchemy.orm import Session + +from app.models.database import MetricStudioRun, MetricStudioRunResult + + +def _now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def rollup_metric_studio_run( + db: Session, + run: MetricStudioRun, + *, + emit_flexprice: bool = True, + commit: bool = True, +) -> None: + results = ( + db.query(MetricStudioRunResult) + .filter(MetricStudioRunResult.run_id == run.id) + .all() + ) + completed = sum(1 for r in results if r.status == "completed") + failed = sum(1 for r in results if r.status == "failed") + pending = sum(1 for r in results if r.status in {"pending", "running"}) + run.completed_items = completed + run.failed_items = failed + if pending: + run.status = "running" + elif failed and completed: + run.status = "partial" + run.finished_at = run.finished_at or _now_utc() + elif failed: + run.status = "failed" + run.finished_at = run.finished_at or _now_utc() + else: + run.status = "completed" + run.finished_at = run.finished_at or _now_utc() + + if commit: + db.commit() + else: + db.flush() + + if emit_flexprice and pending == 0 and run.finished_at is not None: + from app.services.billing.flexprice_service import record_metric_studio_run_completed + + record_metric_studio_run_completed( + run.organization_id, + run.id, + workspace_id=run.workspace_id, + run_status=run.status, + total_items=int(run.total_items or 0), + completed_items=completed, + failed_items=failed, + ) diff --git a/app/services/organization_provisioning.py b/app/services/organization_provisioning.py index a839ca67..847b8b03 100644 --- a/app/services/organization_provisioning.py +++ b/app/services/organization_provisioning.py @@ -7,7 +7,7 @@ from sqlalchemy.orm import Session from app.models.database import Workspace -from app.services.billing.flexprice_service import ensure_customer +from app.services.billing.flexprice_service import ensure_customer, ensure_subscription from app.services.workspace_rbac import ( backfill_org_workspace_memberships, ensure_creator_workspace_admin, @@ -62,3 +62,4 @@ def provision_billing_customer( ) -> None: """Register the org with Flexprice when billing is enabled (no-op otherwise).""" ensure_customer(organization_id, name=name, email=email) + ensure_subscription(organization_id) diff --git a/app/services/playground/post_call_processing.py b/app/services/playground/post_call_processing.py new file mode 100644 index 00000000..1d319f26 --- /dev/null +++ b/app/services/playground/post_call_processing.py @@ -0,0 +1,203 @@ +"""Atomic post-call processing for playground Voice AI poll tasks.""" +from __future__ import annotations + +from typing import Any, Optional +from uuid import UUID + +from loguru import logger +from sqlalchemy.orm import Session + +from app.models.database import CallRecording + +PLAYGROUND_CALL_DATA_PRESERVE_KEYS = ("ui_surface", "external_usage_recorded") + +_ENDED_STATUSES = frozenset( + {"ended", "completed", "failed", "end-of-call-report", "done"} +) + + +def call_metrics_indicate_ended(metrics: dict[str, Any]) -> bool: + status = str(metrics.get("call_status") or metrics.get("status") or "").lower() + end_timestamp = metrics.get("end_timestamp") or metrics.get("endedAt") + return bool(end_timestamp) or status in _ENDED_STATUSES + + +def provider_metrics_enriched(provider_platform: str, metrics: dict[str, Any]) -> bool: + """True when the provider payload has analysis and/or pipeline latency details.""" + plat = str(provider_platform or "").lower() + if plat == "vapi": + analysis = metrics.get("analysis") if isinstance(metrics.get("analysis"), dict) else {} + perf = (metrics.get("artifact") or {}).get("performanceMetrics") or {} + return bool(analysis.get("summary")) or bool(perf.get("turnLatencies")) + if plat == "retell": + return bool(metrics.get("call_analysis")) or bool(metrics.get("latency")) + if plat == "elevenlabs": + status = str(metrics.get("status") or "").lower() + return status in {"done", "completed"} or bool(metrics.get("conversation_turn_metrics")) + if plat == "smallest": + raw = metrics.get("raw_data") if isinstance(metrics.get("raw_data"), dict) else {} + return bool(raw.get("latencyStats")) or bool(metrics.get("transcript")) + return call_metrics_indicate_ended(metrics) + + +def merge_playground_call_data( + prev: Optional[dict[str, Any]], + new: dict[str, Any], +) -> dict[str, Any]: + """Keep internal audit fields when provider metrics replace call_data.""" + merged = dict(new) + if isinstance(prev, dict): + for key in PLAYGROUND_CALL_DATA_PRESERVE_KEYS: + if prev.get(key) is not None: + merged[key] = prev[key] + return merged + + +def _lock_call_recording(db: Session, call_recording_id: UUID) -> Optional[CallRecording]: + return ( + db.query(CallRecording) + .filter(CallRecording.id == call_recording_id) + .with_for_update() + .first() + ) + + +def record_playground_post_call_usage_once( + db: Session, + call_recording_id: UUID, + *, + provider_platform: str, + call_metrics: dict[str, Any], +) -> tuple[bool, dict[str, Any]]: + """ + Record external provider usage at most once per call recording. + + Returns (should_create_evaluator, updated_call_metrics). + """ + from app.services.usage.external_agent_usage import ( + apply_playground_provider_usage_from_call_data, + ) + + locked = _lock_call_recording(db, call_recording_id) + if not locked: + db.rollback() + return False, call_metrics + + if locked.evaluator_result_id: + db.rollback() + logger.info( + "[Poll Call Metrics] Skipping post-call processing — " + "evaluator result already exists" + ) + return False, call_metrics + + stored_data = locked.call_data if isinstance(locked.call_data, dict) else {} + metrics = dict(call_metrics) if isinstance(call_metrics, dict) else call_metrics + platform_key = str(provider_platform or "").lower() + + if stored_data.get("external_usage_recorded") and isinstance(metrics, dict): + metrics = merge_playground_call_data(stored_data, metrics) + locked.call_data = metrics + db.commit() + elif isinstance(metrics, dict): + try: + apply_playground_provider_usage_from_call_data( + organization_id=locked.organization_id, + workspace_id=locked.workspace_id, + agent_id=locked.agent_id, + provider_platform=platform_key, + call_short_id=locked.call_short_id, + call_data=metrics, + ) + except Exception: + db.rollback() + logger.exception( + "[Poll Call Metrics] Usage counters failed for " + f"call recording {call_recording_id}" + ) + return False, call_metrics + + metrics["external_usage_recorded"] = True + metrics = merge_playground_call_data(stored_data, metrics) + locked.call_data = metrics + try: + db.commit() + except Exception: + db.rollback() + logger.exception( + "[Poll Call Metrics] Failed to persist external_usage_recorded " + f"for call recording {call_recording_id}" + ) + return False, call_metrics + + return True, metrics + + +def claim_playground_evaluator_result_slot( + db: Session, + call_recording_id: UUID, + *, + provider_call_id: str | None = None, +) -> Optional[CallRecording]: + """ + Lock the call recording row and return it when evaluator creation may proceed. + + Caller must create/link EvaluatorResult and commit before the lock is released. + """ + locked = _lock_call_recording(db, call_recording_id) + if not locked: + db.rollback() + return None + + if locked.evaluator_result_id: + db.rollback() + call_ref = provider_call_id or locked.provider_call_id or str(call_recording_id) + logger.info( + f"[Poll Call Metrics] Skipping evaluator creation — " + f"another poll already created result for call {call_ref}" + ) + return None + + return locked + + +def persist_provider_call_metrics( + db: Session, + call_recording_id: UUID, + call_metrics: dict[str, Any], +) -> bool: + """Update stored provider call_data without creating an evaluator result.""" + from app.models.enums import CallRecordingStatus + + locked = _lock_call_recording(db, call_recording_id) + if not locked: + db.rollback() + return False + + prev_data = locked.call_data if isinstance(locked.call_data, dict) else {} + merged = merge_playground_call_data(prev_data, call_metrics) + locked.call_data = merged + locked.status = CallRecordingStatus.UPDATED + db.commit() + return True + + +def persist_provider_call_metrics( + db: Session, + call_recording_id: UUID, + call_metrics: dict[str, Any], +) -> bool: + """Update stored provider call_data without creating an evaluator result.""" + from app.models.enums import CallRecordingStatus + + locked = _lock_call_recording(db, call_recording_id) + if not locked: + db.rollback() + return False + + prev_data = locked.call_data if isinstance(locked.call_data, dict) else {} + merged = merge_playground_call_data(prev_data, call_metrics) + locked.call_data = merged + locked.status = CallRecordingStatus.UPDATED + db.commit() + return True diff --git a/app/services/playground/provider_call_logs.py b/app/services/playground/provider_call_logs.py new file mode 100644 index 00000000..ada52f81 --- /dev/null +++ b/app/services/playground/provider_call_logs.py @@ -0,0 +1,213 @@ +"""Fetch and normalize provider call logs for Vapi and Retell.""" + +from __future__ import annotations + +import gzip +import json +import re +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +import requests +from loguru import logger + +VAPI_API_URL = "https://api.vapi.ai" + + +def _fetch_bytes(url: str, *, headers: Optional[dict[str, str]] = None, timeout: int = 60) -> bytes: + response = requests.get(url, headers=headers or {}, timeout=timeout, allow_redirects=True) + response.raise_for_status() + return response.content + + +def _decode_log_payload(content: bytes) -> str: + if not content: + return "" + if content[:2] == b"\x1f\x8b": + try: + return gzip.decompress(content).decode("utf-8") + except OSError: + pass + for encoding in ("utf-8", "utf-8-sig", "latin-1"): + try: + return content.decode(encoding) + except UnicodeDecodeError: + continue + return content.decode("utf-8", errors="replace") + + +def _parse_json_lines(text: str) -> List[Dict[str, Any]]: + entries: List[Dict[str, Any]] = [] + for line in text.splitlines(): + stripped = line.strip() + if not stripped: + continue + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + entries.append({"message": stripped}) + continue + if isinstance(parsed, dict): + entries.append(parsed) + else: + entries.append({"message": str(parsed)}) + return entries + + +def _coerce_timestamp(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, (int, float)): + ms = float(value) + if ms > 1_000_000_000_000: + ms /= 1000.0 + return datetime.fromtimestamp(ms, tz=timezone.utc).isoformat() + if isinstance(value, str): + text = value.strip() + if not text: + return None + if text.isdigit(): + return _coerce_timestamp(int(text)) + try: + return datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(timezone.utc).isoformat() + except ValueError: + return text + return str(value) + + +def _normalize_category(value: Any) -> str: + text = str(value or "call").strip().lower() + if text in {"transcriber", "stt", "speech-to-text", "asr"}: + return "transcriber" + if text in {"voice", "tts", "speech", "synthesizer"}: + return "voice" + if text in {"llm", "model", "assistant"}: + return "llm" + if text in {"call", "system", "pipeline"}: + return "call" + return text or "call" + + +def _normalize_level(value: Any) -> str: + text = str(value or "info").strip().lower() + if text in {"warn", "warning"}: + return "warning" + if text in {"err", "error", "fatal"}: + return "error" + if text in {"debug", "trace"}: + return "debug" + return "info" + + +def _summary_from_raw(raw: Dict[str, Any]) -> str: + for key in ("message", "event", "name", "title", "type", "action", "status"): + value = raw.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + category = raw.get("category") or raw.get("component") + if isinstance(category, str) and category.strip(): + return category.strip() + return "log event" + + +def normalize_provider_log_entry(raw: Dict[str, Any]) -> Dict[str, Any]: + time_value = ( + raw.get("time") + or raw.get("timestamp") + or raw.get("createdAt") + or raw.get("created_at") + or raw.get("ts") + or raw.get("date") + ) + category = _normalize_category( + raw.get("category") + or raw.get("component") + or raw.get("service") + or raw.get("source") + or raw.get("type") + ) + return { + "time": _coerce_timestamp(time_value), + "level": _normalize_level(raw.get("level") or raw.get("severity") or raw.get("logLevel")), + "category": category, + "summary": _summary_from_raw(raw), + "raw": raw, + } + + +def fetch_vapi_call_logs( + *, + api_key: str, + provider_call_id: str, + call_data: Optional[Dict[str, Any]] = None, +) -> List[Dict[str, Any]]: + content: Optional[bytes] = None + call_data = call_data or {} + artifact = call_data.get("artifact") if isinstance(call_data.get("artifact"), dict) else {} + log_url = artifact.get("presignedLogUrl") or artifact.get("logUrl") + if log_url: + try: + content = _fetch_bytes(str(log_url)) + except Exception as exc: + logger.warning("[VapiLogs] Failed to fetch artifact log URL for %s: %s", provider_call_id, exc) + + if content is None: + response = requests.get( + f"{VAPI_API_URL}/call/{provider_call_id}/call-logs", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=60, + allow_redirects=True, + ) + response.raise_for_status() + content = response.content + + text = _decode_log_payload(content or b"") + return [normalize_provider_log_entry(entry) for entry in _parse_json_lines(text)] + + +_RETELL_LINE_RE = re.compile( + r"^(?P