diff --git a/EfficientAI-Docs/docs/getting-started/installation.md b/EfficientAI-Docs/docs/getting-started/installation.md index d91dd8be..a8094ccd 100644 --- a/EfficientAI-Docs/docs/getting-started/installation.md +++ b/EfficientAI-Docs/docs/getting-started/installation.md @@ -8,9 +8,9 @@ sidebar_position: 1 There are two ways to run the application: -## Method 1: Using Docker Compose +## Method 1: Using Docker Compose (Recommended) -Start all services: +Start all services with a single command: ```bash docker compose up -d @@ -18,19 +18,22 @@ docker compose up -d This will automatically: -- Build Docker images if they don't exist -- Build the frontend during the Docker build process +- **Pull pre-built images** from GitHub Container Registry (no build required!) - Start all services (database, Redis, API, worker) - Run database migrations automatically on startup -**Note**: If you make changes to the frontend or backend code, you may need to rebuild: +The first run will download ~4GB of images, which typically takes 1-2 minutes depending on your internet speed. + +### Using a Specific Version + +You can pin to a specific release version for stability: ```bash -# Rebuild and restart (forces rebuild even if image exists) -docker compose up -d --build +# Use a specific version +EFFICIENTAI_VERSION=1.0.0 docker compose up -d -# Or rebuild without cache for a clean build -docker compose build --no-cache api +# Or add to your .env file for persistence +echo "EFFICIENTAI_VERSION=1.0.0" >> .env docker compose up -d ``` @@ -57,10 +60,17 @@ docker compose exec api python scripts/create_api_key.py "My API Key" - Frontend: http://localhost:8000/ - API Docs: http://localhost:8000/docs -**Note**: The frontend is automatically built into the Docker image during the first `docker compose up -d` command. If you make frontend changes later, rebuild with: +### Building Locally (for development) + +If you want to build images locally instead of pulling pre-built ones (e.g., for development): ```bash +# Edit docker-compose.yml to uncomment the 'build' sections, then: docker compose up -d --build + +# Or rebuild without cache for a clean build +docker compose build --no-cache api worker +docker compose up -d ``` ## Method 2: Using Command Line (CLI) @@ -210,9 +220,34 @@ This will: **For Docker Compose**: - Docker and Docker Compose installed +- ~4GB disk space for pre-built images **For CLI**: - Python 3.11+ - Node.js 18+ and npm - PostgreSQL running (locally or remote) - Redis running (locally or remote) + +## Docker Images + +EfficientAI provides pre-built Docker images hosted on GitHub Container Registry: + +| Image | Description | Size | +|-------|-------------|------| +| `ghcr.io/efficientai-tech/efficientai-api` | API server + frontend | ~1.5GB | +| `ghcr.io/efficientai-tech/efficientai-worker` | Celery worker with ML models | ~4GB | + +### Available Tags + +- `latest` - Most recent build from main branch +- `x.y.z` - Specific version (e.g., `1.0.0`) +- `x.y` - Latest patch of a minor version (e.g., `1.0`) + +### Manual Pull (Optional) + +Images are pulled automatically by `docker compose up`, but you can pre-pull them: + +```bash +docker pull ghcr.io/efficientai-tech/efficientai-api:latest +docker pull ghcr.io/efficientai-tech/efficientai-worker:latest +``` diff --git a/README.md b/README.md index ccc102de..29014bb9 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ Test quality, measure performance, & ship with confidence. There are two ways to run the application: -### Method 1: Using Docker Compose +### Method 1: Using Docker Compose (Recommended) 1. **Start all services** ```bash @@ -53,18 +53,17 @@ There are two ways to run the application: ``` This will automatically: - - Build Docker images if they don't exist - - Build the frontend during the Docker build process + - Pull pre-built images from GitHub Container Registry (no build required!) - Start all services (database, Redis, API, worker) - Run database migrations automatically on startup - **Note:** If you make changes to the frontend or backend code, you may need to rebuild: + **Using a specific version:** ```bash - # Rebuild and restart (forces rebuild even if image exists) - docker compose up -d --build + # Pin to a specific release version + EFFICIENTAI_VERSION=1.0.0 docker compose up -d - # Or rebuild without cache for a clean build - docker compose build --no-cache api worker + # Or add to your .env file + echo "EFFICIENTAI_VERSION=1.0.0" >> .env docker compose up -d ``` @@ -81,6 +80,19 @@ There are two ways to run the application: - Frontend: http://localhost:8000/ - API Docs: http://localhost:8000/docs +#### Building Locally (for development) + +If you want to build images locally instead of pulling pre-built ones: + +```bash +# Edit docker-compose.yml to uncomment the 'build' sections, then: +docker compose up -d --build + +# Or rebuild without cache for a clean build +docker compose build --no-cache api worker +docker compose up -d +``` + ### Method 2: Using Command Line (CLI) 1. **Install the package** @@ -161,6 +173,7 @@ There are two ways to run the application: **For Docker Compose:** - Docker and Docker Compose installed +- ~4GB disk space for pre-built images **For CLI:** - Python 3.11+ diff --git a/app/api/v1/routes/agents.py b/app/api/v1/routes/agents.py index e737b3a8..b5859252 100644 --- a/app/api/v1/routes/agents.py +++ b/app/api/v1/routes/agents.py @@ -3,7 +3,7 @@ Complete CRUD operations for test agents """ from fastapi import APIRouter, Depends, HTTPException, status, Query -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, Response from sqlalchemy.orm import Session from typing import List, Optional from uuid import UUID @@ -147,6 +147,48 @@ def generate_unique_agent_id(db: Session) -> str: ) +def get_agent_dependencies(db: Session, organization_id: UUID, agent_uuid: UUID) -> dict: + """Return dependency counts that block non-force delete.""" + evaluators_count = db.query(Evaluator).filter( + Evaluator.agent_id == agent_uuid, + Evaluator.organization_id == organization_id, + ).count() + + evaluator_results_count = db.query(EvaluatorResult).filter( + EvaluatorResult.agent_id == agent_uuid, + EvaluatorResult.organization_id == organization_id, + ).count() + + call_recordings_count = db.query(CallRecording).filter( + CallRecording.agent_id == agent_uuid, + CallRecording.organization_id == organization_id, + ).count() + + conversation_evaluations_count = db.query(ConversationEvaluation).filter( + ConversationEvaluation.agent_id == agent_uuid, + ConversationEvaluation.organization_id == organization_id, + ).count() + + test_conversations_count = db.query(TestAgentConversation).filter( + TestAgentConversation.agent_id == agent_uuid, + TestAgentConversation.organization_id == organization_id, + ).count() + + dependencies = {} + if evaluators_count > 0: + dependencies["evaluators"] = evaluators_count + if evaluator_results_count > 0: + dependencies["evaluator_results"] = evaluator_results_count + if call_recordings_count > 0: + dependencies["call_recordings"] = call_recordings_count + if conversation_evaluations_count > 0: + dependencies["conversation_evaluations"] = conversation_evaluations_count + if test_conversations_count > 0: + dependencies["test_conversations"] = test_conversations_count + + return dependencies + + @router.post("", response_model=AgentResponse, status_code=status.HTTP_201_CREATED) async def create_agent( agent: AgentCreate, @@ -380,56 +422,20 @@ async def delete_agent( raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") agent_uuid = db_agent.id - - evaluators_count = db.query(Evaluator).filter( - Evaluator.agent_id == agent_uuid, - Evaluator.organization_id == organization_id, - ).count() - - evaluator_results_count = db.query(EvaluatorResult).filter( - EvaluatorResult.agent_id == agent_uuid, - EvaluatorResult.organization_id == organization_id, - ).count() - - call_recordings_count = db.query(CallRecording).filter( - CallRecording.agent_id == agent_uuid, - CallRecording.organization_id == organization_id, - ).count() - - conversation_evaluations_count = db.query(ConversationEvaluation).filter( - ConversationEvaluation.agent_id == agent_uuid, - ConversationEvaluation.organization_id == organization_id, - ).count() - - test_conversations_count = db.query(TestAgentConversation).filter( - TestAgentConversation.agent_id == agent_uuid, - TestAgentConversation.organization_id == organization_id, - ).count() - - dependencies = {} - if evaluators_count > 0: - dependencies["evaluators"] = evaluators_count - if evaluator_results_count > 0: - dependencies["evaluator_results"] = evaluator_results_count - if call_recordings_count > 0: - dependencies["call_recordings"] = call_recordings_count - if conversation_evaluations_count > 0: - dependencies["conversation_evaluations"] = conversation_evaluations_count - if test_conversations_count > 0: - dependencies["test_conversations"] = test_conversations_count + dependencies = get_agent_dependencies(db, organization_id, agent_uuid) if dependencies and not force: parts = [] - if evaluators_count > 0: - parts.append(f"{evaluators_count} evaluator(s)") - if evaluator_results_count > 0: - parts.append(f"{evaluator_results_count} evaluator result(s)") - if call_recordings_count > 0: - parts.append(f"{call_recordings_count} call recording(s)") - if conversation_evaluations_count > 0: - parts.append(f"{conversation_evaluations_count} conversation evaluation(s)") - if test_conversations_count > 0: - parts.append(f"{test_conversations_count} test conversation(s)") + if dependencies.get("evaluators"): + parts.append(f"{dependencies['evaluators']} evaluator(s)") + if dependencies.get("evaluator_results"): + parts.append(f"{dependencies['evaluator_results']} evaluator result(s)") + if dependencies.get("call_recordings"): + parts.append(f"{dependencies['call_recordings']} call recording(s)") + if dependencies.get("conversation_evaluations"): + parts.append(f"{dependencies['conversation_evaluations']} conversation evaluation(s)") + if dependencies.get("test_conversations"): + parts.append(f"{dependencies['test_conversations']} test conversation(s)") raise HTTPException( status_code=status.HTTP_409_CONFLICT, @@ -479,5 +485,40 @@ async def delete_agent( }, ) - return JSONResponse(status_code=204, content=None) + return Response(status_code=204) + + +@router.get("/{agent_id}/delete-impact") +async def get_agent_delete_impact( + agent_id: str, + organization_id: UUID = Depends(get_organization_id), + db: Session = Depends(get_db) +): + """Preview dependent records that would be affected by force delete.""" + try: + agent_uuid = UUID(agent_id) + db_agent = db.query(Agent).filter( + and_( + Agent.id == agent_uuid, + Agent.organization_id == organization_id + ) + ).first() + except ValueError: + db_agent = db.query(Agent).filter( + and_( + Agent.agent_id == agent_id, + Agent.organization_id == organization_id + ) + ).first() + + if not db_agent: + raise HTTPException(status_code=404, detail=f"Agent {agent_id} not found") + + dependencies = get_agent_dependencies(db, organization_id, db_agent.id) + return { + "agent_id": str(db_agent.id), + "agent_name": db_agent.name, + "dependencies": dependencies, + "can_delete_without_force": len(dependencies) == 0, + } diff --git a/app/api/v1/routes/conversation_evaluations.py b/app/api/v1/routes/conversation_evaluations.py index 73301d76..8414cfc0 100644 --- a/app/api/v1/routes/conversation_evaluations.py +++ b/app/api/v1/routes/conversation_evaluations.py @@ -85,8 +85,6 @@ async def create_conversation_evaluation( 2. A brief reason for your answer 3. Additional metrics about the conversation quality: - Professionalism (0-1 scale) - - Clarity (0-1 scale) - - Empathy (0-1 scale, if applicable) - Problem Resolution (0-1 scale, if applicable) - Overall Quality (0-1 scale) 4. An overall score (0.0 to 1.0) representing how well the agent performed @@ -97,8 +95,6 @@ async def create_conversation_evaluation( "objective_achieved_reason": "brief explanation", "additional_metrics": {{ "professionalism": 0.0-1.0, - "clarity": 0.0-1.0, - "empathy": 0.0-1.0, "problem_resolution": 0.0-1.0, "overall_quality": 0.0-1.0 }}, @@ -151,6 +147,9 @@ async def create_conversation_evaluation( objective_achieved = bool(evaluation_data.get("objective_achieved", False)) objective_achieved_reason = evaluation_data.get("objective_achieved_reason", "") additional_metrics = evaluation_data.get("additional_metrics", {}) + # Explicitly remove deprecated dimensions so they are not evaluated/stored. + additional_metrics.pop("clarity", None) + additional_metrics.pop("empathy", None) overall_score = float(evaluation_data.get("overall_score", 0.0)) # Calculate audio metrics if enabled and audio is available diff --git a/app/api/v1/routes/evaluators.py b/app/api/v1/routes/evaluators.py index a1d87f3d..dbcd60a7 100644 --- a/app/api/v1/routes/evaluators.py +++ b/app/api/v1/routes/evaluators.py @@ -1,7 +1,7 @@ """Evaluator routes.""" from fastapi import APIRouter, Depends, HTTPException, status, Query -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, Response from sqlalchemy.orm import Session from sqlalchemy import and_ from uuid import UUID @@ -198,6 +198,7 @@ def create_evaluators_bulk( evaluator = Evaluator( evaluator_id=evaluator_id, organization_id=organization_id, + name=bulk_data.name, agent_id=bulk_data.agent_id, persona_id=persona_id, scenario_id=bulk_data.scenario_id, @@ -334,11 +335,11 @@ def update_evaluator( @router.delete("/{evaluator_id}") def delete_evaluator( evaluator_id: str, - force: bool = Query(False, description="Force delete with all dependent records"), + force: bool = Query(False, description="Deprecated: evaluator deletion keeps dependent results"), organization_id: UUID = Depends(get_organization_id), db: Session = Depends(get_db), ): - """Delete an evaluator. Returns 409 if dependent records exist unless force=true.""" + """Delete an evaluator while preserving dependent evaluator results.""" try: evaluator_uuid = UUID(evaluator_id) evaluator = db.query(Evaluator).filter( @@ -365,21 +366,10 @@ def delete_evaluator( dependencies = {} if evaluator_results_count > 0: dependencies["evaluator_results"] = evaluator_results_count - - if dependencies and not force: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail={ - "message": f"Cannot delete evaluator. It has {evaluator_results_count} evaluator result(s).", - "dependencies": dependencies, - "hint": "Use force=true to delete this evaluator and all its dependent records.", - }, - ) - - if dependencies: + # Preserve historical results by detaching them from this evaluator. db.query(EvaluatorResult).filter( EvaluatorResult.evaluator_id == evaluator.id - ).delete(synchronize_session=False) + ).update({EvaluatorResult.evaluator_id: None}, synchronize_session=False) db.delete(evaluator) db.commit() @@ -388,12 +378,12 @@ def delete_evaluator( return JSONResponse( status_code=200, content={ - "message": "Evaluator and all dependent records deleted successfully.", - "deleted": dependencies, + "message": "Evaluator deleted successfully. Dependent evaluator results were preserved and detached.", + "detached": dependencies, }, ) - return JSONResponse(status_code=204, content=None) + return Response(status_code=204) @router.post("/run", response_model=RunEvaluatorsResponse, status_code=200) diff --git a/app/api/v1/routes/integrations.py b/app/api/v1/routes/integrations.py index 8323f1a9..76c2c4ea 100644 --- a/app/api/v1/routes/integrations.py +++ b/app/api/v1/routes/integrations.py @@ -11,10 +11,9 @@ from app.dependencies import get_db, get_organization_id, get_api_key from app.models.database import Integration, IntegrationPlatform, Agent from app.models.schemas import ( - IntegrationCreate, IntegrationUpdate, IntegrationResponse, MessageResponse + IntegrationCreate, IntegrationUpdate, IntegrationResponse ) from app.core.encryption import encrypt_api_key, decrypt_api_key -from app.services.voice_providers import get_voice_provider router = APIRouter(prefix="/integrations", tags=["Integrations"]) @@ -213,62 +212,6 @@ async def delete_integration( return JSONResponse(status_code=204, content=None) - -@router.post("/{integration_id}/test", response_model=MessageResponse) -async def test_integration( - integration_id: UUID, - organization_id: UUID = Depends(get_organization_id), - api_key: str = Depends(get_api_key), - db: Session = Depends(get_db) -): - """ - Test an integration by validating the API key. - Requires at least READER role. - """ - integration = db.query(Integration).filter( - Integration.id == integration_id, - Integration.organization_id == organization_id - ).first() - - if not integration: - raise HTTPException(status_code=404, detail="Integration not found") - - # Decrypt API key - try: - decrypted_api_key = decrypt_api_key(integration.api_key) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to decrypt API key: {str(e)}" - ) - - # Get the appropriate voice provider - try: - provider_class = get_voice_provider(integration.platform) - provider = provider_class(api_key=decrypted_api_key) - - # Test connection - provider.test_connection() - - except ValueError as e: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=str(e) - ) - except Exception as e: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail=f"Connection test failed: {str(e)}" - ) - - # Mark as tested - from datetime import datetime - integration.last_tested_at = datetime.utcnow() - db.commit() - - return {"message": f"Integration with {integration.platform} is valid"} - - @router.get("/{integration_id}/api-key") async def get_integration_api_key( integration_id: UUID, diff --git a/app/api/v1/routes/metrics.py b/app/api/v1/routes/metrics.py index ce36b127..96fd46e9 100644 --- a/app/api/v1/routes/metrics.py +++ b/app/api/v1/routes/metrics.py @@ -62,7 +62,8 @@ def list_metrics( ): """List all metrics for the organization.""" metrics = db.query(Metric).filter( - Metric.organization_id == organization_id + Metric.organization_id == organization_id, + ~Metric.name.in_(REMOVED_DEFAULT_METRICS), ).order_by(Metric.is_default.desc(), Metric.created_at.desc()).all() return metrics @@ -154,7 +155,9 @@ def update_metric( # Deprecated default metrics that can be deleted -DEPRECATED_DEFAULT_METRICS = {"Response Time", "Customer Satisfaction"} +DEPRECATED_DEFAULT_METRICS = {"Response Time", "Customer Satisfaction", "Clarity and Empathy"} +# Removed default metrics should no longer be listed/seeded/evaluated. +REMOVED_DEFAULT_METRICS = {"Clarity and Empathy"} @router.delete("/{metric_id}", status_code=204) @@ -204,13 +207,6 @@ def seed_default_metrics( "trigger": MetricTrigger.ALWAYS, "enabled": True, }, - { - "name": "Clarity and Empathy", - "description": "Evaluates the clarity of communication and empathetic responses", - "metric_type": MetricType.RATING, - "trigger": MetricTrigger.ALWAYS, - "enabled": True, - }, { "name": "Professionalism", "description": "Assesses the professional tone and behavior throughout the conversation", @@ -240,21 +236,21 @@ def seed_default_metrics( "description": "Cycle-to-cycle pitch period variation as percentage - indicates vocal stability. Lower values (< 1%) indicate stable voice.", "metric_type": MetricType.NUMBER, "trigger": MetricTrigger.ALWAYS, - "enabled": True, + "enabled": False, }, { "name": "Shimmer", "description": "Cycle-to-cycle amplitude variation as percentage - indicates voice quality. Lower values (< 3%) indicate consistent voice.", "metric_type": MetricType.NUMBER, "trigger": MetricTrigger.ALWAYS, - "enabled": True, + "enabled": False, }, { "name": "HNR", "description": "Harmonics-to-Noise Ratio in dB - indicates voice clarity. Higher values (> 20 dB) indicate cleaner voice with less breathiness.", "metric_type": MetricType.NUMBER, "trigger": MetricTrigger.ALWAYS, - "enabled": True, + "enabled": False, }, # ========================================================================= # AI Voice Metrics (ML models - human-likeness, emotion, consistency) @@ -332,6 +328,21 @@ def seed_default_metrics( ) db.add(metric) created_metrics.append(metric) + else: + # Keep default acoustic metric toggles aligned with product defaults. + if existing.enabled != metric_data["enabled"]: + existing.enabled = metric_data["enabled"] + + # Ensure removed defaults are disabled for existing orgs. + removed_metrics = db.query(Metric).filter( + and_( + Metric.organization_id == organization_id, + Metric.name.in_(REMOVED_DEFAULT_METRICS), + Metric.enabled == True, + ) + ).all() + for metric in removed_metrics: + metric.enabled = False db.commit() for metric in created_metrics: diff --git a/app/api/v1/routes/prompt_partials.py b/app/api/v1/routes/prompt_partials.py index 2b2fcdce..8d68d95f 100644 --- a/app/api/v1/routes/prompt_partials.py +++ b/app/api/v1/routes/prompt_partials.py @@ -3,7 +3,7 @@ CRUD operations with version history for reusable prompt templates. """ from fastapi import APIRouter, Depends, HTTPException, status, Query -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, Response from sqlalchemy.orm import Session from sqlalchemy.exc import IntegrityError, SQLAlchemyError from typing import List, Optional @@ -344,7 +344,7 @@ async def delete_prompt_partial( db.delete(partial) db.commit() - return JSONResponse(status_code=204, content=None) + return Response(status_code=204) @router.get("/{partial_id}/versions", response_model=List[PromptPartialVersionResponse]) diff --git a/app/api/v1/routes/scenarios.py b/app/api/v1/routes/scenarios.py index fd987aef..cce91b7f 100644 --- a/app/api/v1/routes/scenarios.py +++ b/app/api/v1/routes/scenarios.py @@ -3,13 +3,13 @@ Complete CRUD operations for test scenarios """ from fastapi import APIRouter, Depends, HTTPException, status, Query -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, Response from sqlalchemy.orm import Session from typing import List from uuid import UUID from app.dependencies import get_db, get_organization_id -from app.models.database import Scenario, Evaluator, EvaluatorResult, TestAgentConversation +from app.models.database import Scenario, Agent, Evaluator, EvaluatorResult, TestAgentConversation from app.models.schemas import ( ScenarioCreate, ScenarioUpdate, ScenarioResponse ) @@ -24,8 +24,17 @@ async def create_scenario( db: Session = Depends(get_db) ): """Create a new scenario""" + if scenario.agent_id is not None: + linked_agent = db.query(Agent).filter( + Agent.id == scenario.agent_id, + Agent.organization_id == organization_id, + ).first() + if not linked_agent: + raise HTTPException(status_code=404, detail=f"Agent {scenario.agent_id} not found") + db_scenario = Scenario( organization_id=organization_id, + agent_id=scenario.agent_id, name=scenario.name, description=scenario.description, required_info=scenario.required_info @@ -81,6 +90,14 @@ async def update_scenario( if not db_scenario: raise HTTPException(status_code=404, detail=f"Scenario {scenario_id} not found") + if scenario_update.agent_id is not None: + linked_agent = db.query(Agent).filter( + Agent.id == scenario_update.agent_id, + Agent.organization_id == organization_id, + ).first() + if not linked_agent: + raise HTTPException(status_code=404, detail=f"Agent {scenario_update.agent_id} not found") + update_data = scenario_update.dict(exclude_unset=True) for field, value in update_data.items(): setattr(db_scenario, field, value) @@ -174,5 +191,5 @@ async def delete_scenario( }, ) - return JSONResponse(status_code=204, content=None) + return Response(status_code=204) diff --git a/app/api/v1/routes/voice_playground.py b/app/api/v1/routes/voice_playground.py index 93b08ccf..e222563f 100644 --- a/app/api/v1/routes/voice_playground.py +++ b/app/api/v1/routes/voice_playground.py @@ -200,6 +200,7 @@ class CustomVoiceCreate(BaseModel): class CustomVoiceUpdate(BaseModel): + voice_id: Optional[str] = None name: Optional[str] = None gender: Optional[str] = None accent: Optional[str] = None @@ -409,6 +410,20 @@ async def update_custom_tts_voice( if not voice: raise HTTPException(404, "Custom voice not found") + if data.voice_id is not None: + cleaned_voice_id = data.voice_id.strip() + if not cleaned_voice_id: + raise HTTPException(400, "voice_id cannot be empty") + duplicate = db.query(CustomTTSVoice).filter( + CustomTTSVoice.organization_id == organization_id, + CustomTTSVoice.provider == voice.provider, + CustomTTSVoice.voice_id == cleaned_voice_id, + CustomTTSVoice.id != voice.id, + ).first() + if duplicate: + raise HTTPException(409, f"Custom voice already exists for provider '{voice.provider}' and voice_id '{cleaned_voice_id}'") + voice.voice_id = cleaned_voice_id + if data.name is not None: cleaned_name = data.name.strip() if not cleaned_name: diff --git a/app/migrations/012_add_agent_link_to_scenarios.py b/app/migrations/012_add_agent_link_to_scenarios.py new file mode 100644 index 00000000..d56d7529 --- /dev/null +++ b/app/migrations/012_add_agent_link_to_scenarios.py @@ -0,0 +1,54 @@ +""" +Migration: Add optional agent_id link to scenarios table. + +Allows scenarios to be loosely associated with agents. +If an agent is deleted, scenario.agent_id is set to NULL. +""" + +from sqlalchemy import text +from sqlalchemy.orm import Session + +description = "Add nullable agent_id to scenarios with ON DELETE SET NULL" + + +def upgrade(db: Session): + result = db.execute( + text( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_name = 'scenarios' + AND column_name = 'agent_id' + """ + ) + ) + + if result.fetchone() is not None: + print("Column agent_id already exists on scenarios, skipping...") + return + + db.execute( + text( + """ + ALTER TABLE scenarios + ADD COLUMN agent_id UUID REFERENCES agents(id) ON DELETE SET NULL + """ + ) + ) + + db.execute( + text( + """ + CREATE INDEX IF NOT EXISTS ix_scenarios_agent_id + ON scenarios(agent_id) + """ + ) + ) + + db.commit() + print("Added agent_id column to scenarios") + + +def downgrade(db: Session): + db.execute(text("ALTER TABLE scenarios DROP COLUMN IF EXISTS agent_id")) + db.commit() diff --git a/app/models/database.py b/app/models/database.py index 991510d7..6c2d6117 100644 --- a/app/models/database.py +++ b/app/models/database.py @@ -258,6 +258,7 @@ class Scenario(Base): id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False, index=True) + agent_id = Column(UUID(as_uuid=True), ForeignKey("agents.id", ondelete="SET NULL"), nullable=True, index=True) name = Column(String, nullable=False) description = Column(String) required_info = Column(JSON) diff --git a/app/models/schemas.py b/app/models/schemas.py index e18890e3..1f49beab 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -446,6 +446,7 @@ class PersonaCloneRequest(BaseModel): class ScenarioCreate(BaseModel): """Schema for creating a new scenario""" name: str = Field(..., min_length=1, max_length=255) + agent_id: Optional[UUID] = None description: Optional[str] = None required_info: Dict[str, str] = Field(default_factory=dict) @@ -453,6 +454,7 @@ class ScenarioCreate(BaseModel): class ScenarioUpdate(BaseModel): """Schema for updating a scenario""" name: Optional[str] = None + agent_id: Optional[UUID] = None description: Optional[str] = None required_info: Optional[Dict[str, str]] = None @@ -461,6 +463,7 @@ class ScenarioResponse(BaseModel): """Schema for scenario response""" id: UUID name: str + agent_id: Optional[UUID] description: Optional[str] required_info: Dict[str, str] created_at: datetime @@ -1075,6 +1078,7 @@ class Config: class EvaluatorBulkCreate(BaseModel): """Schema for creating multiple evaluators at once.""" + name: Optional[str] = None agent_id: UUID scenario_id: UUID persona_ids: List[UUID] diff --git a/app/services/evaluation_service.py b/app/services/evaluation_service.py index 90cd7445..97b1ae0c 100644 --- a/app/services/evaluation_service.py +++ b/app/services/evaluation_service.py @@ -1,6 +1,5 @@ """Core evaluation service for processing audio evaluations.""" -import whisper import time from datetime import datetime from typing import Optional, Dict, Any @@ -12,6 +11,23 @@ from app.services.audio_service import AudioService from app.core.exceptions import EvaluationNotFoundError, AudioFileNotFoundError +# Lazy import for whisper (optional dependency) +whisper = None + +def _get_whisper(): + """Lazy load whisper module.""" + global whisper + if whisper is None: + try: + import whisper as _whisper + whisper = _whisper + except ImportError: + raise ImportError( + "openai-whisper is not installed. Install it with: " + "pip install 'efficientai[local-whisper]' or use the OpenAI Whisper API instead." + ) + return whisper + audio_service = AudioService() @@ -36,7 +52,8 @@ def _load_model(self, model_name: Optional[str] = None) -> Any: if model_name not in self.model_cache: try: - model = whisper.load_model(model_name) + whisper_module = _get_whisper() + model = whisper_module.load_model(model_name) self.model_cache[model_name] = model except Exception as e: raise RuntimeError(f"Failed to load model {model_name}: {str(e)}") diff --git a/app/workers/celery_app.py b/app/workers/celery_app.py index 9955c997..5222204c 100644 --- a/app/workers/celery_app.py +++ b/app/workers/celery_app.py @@ -10,6 +10,9 @@ from uuid import UUID from loguru import logger +# Metrics that should never be evaluated. +REMOVED_EVALUATION_METRIC_NAMES = {"clarity and empathy"} + # Load config.yml if it exists (before using settings) # This ensures the Celery worker has the same configuration as the main app config_path = Path("config.yml") @@ -198,6 +201,10 @@ def provider_matches(db_provider, target_enum): Metric.organization_id == result.organization_id, Metric.enabled == True ).all() + enabled_metrics = [ + m for m in enabled_metrics + if (m.name or "").strip().lower() not in REMOVED_EVALUATION_METRIC_NAMES + ] # Step 5: Evaluate against enabled metrics using LLM metric_scores = {} diff --git a/docker-compose.yml b/docker-compose.yml index 7c3b2fdd..1401d453 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,11 @@ +# EfficientAI Docker Compose +# +# Usage: +# docker compose up # Uses latest images +# EFFICIENTAI_VERSION=1.0.0 docker compose up # Uses specific version +# +# For local development, uncomment the 'build' sections below. + services: db: image: postgres:15 @@ -29,7 +37,8 @@ services: api: # Pre-built image from GitHub Container Registry - image: ghcr.io/efficientai-tech/efficientai-api:latest + # Use EFFICIENTAI_VERSION env var to pin to a specific version (e.g., 1.0.0) + image: ghcr.io/efficientai-tech/efficientai-api:${EFFICIENTAI_VERSION:-latest} # For local development, uncomment below to build instead of pull: # build: # context: . @@ -60,7 +69,8 @@ services: worker: # Pre-built image from GitHub Container Registry - image: ghcr.io/efficientai-tech/efficientai-worker:latest + # Use EFFICIENTAI_VERSION env var to pin to a specific version (e.g., 1.0.0) + image: ghcr.io/efficientai-tech/efficientai-worker:${EFFICIENTAI_VERSION:-latest} # For local development, uncomment below to build instead of pull: # build: # context: . diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker index 35339d4f..aaa6098a 100644 --- a/docker/Dockerfile.worker +++ b/docker/Dockerfile.worker @@ -40,13 +40,14 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # ============================================================ # Download qualitative-voice models if that extra is installed +# Note: UTMOS uses torch.hub which can hit GitHub rate limits in CI +# We skip UTMOS preload in Docker (it will download on first use) but preload HuggingFace models RUN if [ "$PRELOAD_MODELS" = "true" ] && echo "$INSTALL_EXTRAS" | grep -q "qualitative-voice"; then \ - echo "Downloading UTMOS MOS predictor..." && \ - python -c "import torch; torch.hub.load('tarepan/SpeechMOS:v1.2.0', 'utmos22_strong', trust_repo=True); print('UTMOS cached')" && \ - echo "Downloading emotion classifier..." && \ + echo "Downloading emotion classifier from HuggingFace..." && \ python -c "from transformers import pipeline; pipeline('audio-classification', model='ehcalabres/wav2vec2-lg-xlsr-en-speech-emotion-recognition'); print('Emotion classifier cached')" && \ - echo "Downloading valence/arousal model..." && \ - python -c "from transformers import AutoProcessor, AutoModelForAudioClassification; AutoProcessor.from_pretrained('audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim'); AutoModelForAudioClassification.from_pretrained('audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim'); print('Valence/arousal cached')"; \ + echo "Downloading valence/arousal model from HuggingFace..." && \ + python -c "from transformers import AutoProcessor, AutoModelForAudioClassification; AutoProcessor.from_pretrained('audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim'); AutoModelForAudioClassification.from_pretrained('audeering/wav2vec2-large-robust-12-ft-emotion-msp-dim'); print('Valence/arousal cached')" && \ + echo "Note: UTMOS model will be downloaded on first use (torch.hub has GitHub rate limits in CI)"; \ fi # Download NeMo ASR model if that extra is installed diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index dd894c0c..89f5bfcf 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,38 +1,73 @@ import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom' import { useAuthStore } from './store/authStore' import { useLicenseStore } from './store/licenseStore' -import Login from './pages/Login' -import Dashboard from './pages/Dashboard' -import EvaluationDetail from './pages/EvaluationDetail' -import Playground from './components/Playground' import Layout from './components/Layout' -import Agents from './pages/Agents' -import Personas from './pages/Personas' -import Scenarios from './pages/Scenarios' -import IAM from './pages/IAM' -import Profile from './pages/Profile' -import Metrics from './pages/Metrics' -import Integrations from './pages/Integrations' -import DataSources from './pages/DataSources' -import VoiceBundles from './pages/VoiceBundles' -import EvaluateTestAgents from './pages/EvaluateTestAgents' -import MetricsManagement from './pages/MetricsManagement' -import Results from './pages/Results' -import EvaluatorResultDetail from './pages/EvaluatorResultDetail' -import AgentDetail from './pages/AgentDetail' -import Observability from './pages/Observability' -import ObservabilityCalls from './pages/ObservabilityCalls' -import ObservabilityCallDetail from './pages/ObservabilityCallDetail' -import CallRecordingDetail from './pages/CallRecordingDetail' -import TestAgentResultDetail from './pages/TestAgentResultDetail' -import Settings from './pages/Settings' -import Alerts from './pages/Alerts' -import AlertDetail from './pages/AlertDetail' -import AlertHistory from './pages/AlertHistory' -import CronJobs from './pages/CronJobs' -import VoicePlayground from './pages/VoicePlayground' -import EnterpriseUpgrade from './pages/EnterpriseUpgrade' -import PromptPartials from './pages/PromptPartials' + +// Auth +import Login from './pages/auth/Login' + +// Dashboard +import Dashboard from './pages/dashboard/Dashboard' + +// Prompt Partials +import PromptPartials from './pages/promptPartials/PromptPartials' + +// Agents +import Agents from './pages/agents/Agents' +import AgentDetail from './pages/agents/AgentDetail' + +// Personas +import Personas from './pages/personas/Personas' + +// Scenarios +import Scenarios from './pages/scenarios/Scenarios' + +// Metrics +import Metrics from './pages/metrics/Metrics' +import MetricsManagement from './pages/metrics/MetricsManagement' + +// Playground - Agent +import AgentPlayground from './pages/playground/agent/AgentPlayground' +import CallRecordingDetail from './pages/playground/agent/CallRecordingDetail' +import TestAgentResultDetail from './pages/playground/agent/TestAgentResultDetail' + +// Playground - Voice +import VoicePlayground from './pages/playground/voice/VoicePlayground' + +// Evaluators +import EvaluateTestAgents from './pages/evaluators/evaluators/EvaluateTestAgents' +import EvaluatorDetail from './pages/evaluators/evaluators/EvaluatorDetail' + +// Evaluator Results +import Results from './pages/evaluators/results/Results' +import EvaluatorResultDetail from './pages/evaluators/results/EvaluatorResultDetail' +import EvaluationDetail from './pages/evaluators/results/EvaluationDetail' + +// Observability +import Observability from './pages/observability/Observability' +import ObservabilityCalls from './pages/observability/ObservabilityCalls' +import ObservabilityCallDetail from './pages/observability/ObservabilityCallDetail' + +// Alerting +import Alerts from './pages/alerting/Alerts' +import AlertDetail from './pages/alerting/AlertDetail' +import AlertHistory from './pages/alerting/AlertHistory' + +// Configurations +import DataSources from './pages/configurations/DataSources' +import VoiceBundles from './pages/configurations/VoiceBundles' +import Integrations from './pages/configurations/Integrations' +import Settings from './pages/configurations/Settings' +import CronJobs from './pages/configurations/CronJobs' + +// IAM +import IAM from './pages/iam/IAM' + +// Profile +import Profile from './pages/profile/Profile' + +// Enterprise +import EnterpriseUpgrade from './pages/enterprise/EnterpriseUpgrade' function PrivateRoute({ children }: { children: React.ReactNode }) { @@ -78,7 +113,7 @@ function App() { > } /> } /> - } /> + } /> } /> } /> } /> @@ -90,6 +125,7 @@ function App() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/src/components/CreateEvaluationModal.tsx b/frontend/src/components/CreateEvaluationModal.tsx deleted file mode 100644 index 186693f7..00000000 --- a/frontend/src/components/CreateEvaluationModal.tsx +++ /dev/null @@ -1,194 +0,0 @@ -import { useState } from 'react' -import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' -import { useNavigate } from 'react-router-dom' -import { apiClient } from '../lib/api' -import { EvaluationType, AudioFile, Evaluation } from '../types/api' -import { X, Loader } from 'lucide-react' - -interface CreateEvaluationModalProps { - isOpen: boolean - onClose: () => void -} - -export default function CreateEvaluationModal({ - isOpen, - onClose, -}: CreateEvaluationModalProps) { - const navigate = useNavigate() - const queryClient = useQueryClient() - const [audioId, setAudioId] = useState('') - const [evaluationType, setEvaluationType] = useState(EvaluationType.ASR) - const [modelName, setModelName] = useState('base') - const [referenceText, setReferenceText] = useState('') - const [metrics, setMetrics] = useState(['wer', 'latency']) - - const { data: audioFiles } = useQuery({ - queryKey: ['audio', 'list'], - queryFn: () => apiClient.listAudio(), - enabled: isOpen, - }) - - const createMutation = useMutation({ - mutationFn: (data: any) => apiClient.createEvaluation(data), - onSuccess: (data: Evaluation) => { - queryClient.invalidateQueries({ queryKey: ['evaluations'] }) - onClose() - navigate(`/evaluations/${data.id}`) - }, - }) - - const handleSubmit = async (e: React.FormEvent) => { - e.preventDefault() - try { - await createMutation.mutateAsync({ - audio_id: audioId, - evaluation_type: evaluationType, - model_name: modelName || undefined, - reference_text: referenceText || undefined, - metrics: metrics.length > 0 ? metrics : undefined, - }) - } catch (error) { - // Error handled by mutation - } - } - - const toggleMetric = (metric: string) => { - if (metrics.includes(metric)) { - setMetrics(metrics.filter((m) => m !== metric)) - } else { - setMetrics([...metrics, metric]) - } - } - - if (!isOpen) return null - - return ( -
-
-
-
-
-

Create Evaluation

- -
- -
-
- - -
- -
- - -
- -
- - setModelName(e.target.value)} - placeholder="e.g., base, small, medium, large" - className="w-full px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-primary-500 focus:border-primary-500" - /> -
- -
- -