diff --git a/.jules/sentinel.md b/.jules/sentinel.md new file mode 100644 index 000000000..39b1f819a --- /dev/null +++ b/.jules/sentinel.md @@ -0,0 +1,4 @@ +## 2026-07-09 - Replace weak MD5 hashing with SHA-256 for caching +**Vulnerability:** Weak MD5 hashes were being used for generating cache keys and processing IDs across multiple backend services (e.g., `cache_service.py`, `database_optimizer.py`, etc.). +**Learning:** This repo frequently uses hashes for non-cryptographic purposes (caching and IDs). However, using MD5 triggers static analysis security warnings (like Bandit rules B324/B303) as the algorithm is vulnerable to collision attacks and considered insecure by modern cryptographic standards. +**Prevention:** Avoid using `hashlib.md5()` in the `src/` directory. Default to `hashlib.sha256()` even for non-cryptographic uses to maintain a secure baseline and comply with automated security policies. Archived scripts in `scripts/archive/` are explicitly excluded from this requirement. diff --git a/fix_json.py b/fix_json.py new file mode 100644 index 000000000..edc4179d7 --- /dev/null +++ b/fix_json.py @@ -0,0 +1,6 @@ +import re +with open('config/agent_network.json', 'r') as f: + c = f.read() + +# There are multiple conflict markers because git rebase/merge left them +# Let's completely clean up config/agent_network.json based on what we had done before. diff --git a/scripts/archive/videoprism_analyzer.py b/scripts/archive/videoprism_analyzer.py index 6b81d53d9..f2110d51f 100644 --- a/scripts/archive/videoprism_analyzer.py +++ b/scripts/archive/videoprism_analyzer.py @@ -229,7 +229,7 @@ async def _download_video(self, video_url: str) -> str: temp_dir.mkdir(exist_ok=True) # Generate unique filename - url_hash = hashlib.md5(video_url.encode()).hexdigest()[:8] + url_hash = hashlib.sha256(video_url.encode()).hexdigest()[:8] output_path = temp_dir / f"video_{url_hash}.%(ext)s" ydl_opts = { diff --git a/scripts/archive/youtube_innovation_learning_database.py b/scripts/archive/youtube_innovation_learning_database.py index 7172548fb..7631319e1 100644 --- a/scripts/archive/youtube_innovation_learning_database.py +++ b/scripts/archive/youtube_innovation_learning_database.py @@ -254,7 +254,7 @@ async def intake_youtube_video_with_innovation(self, args: Dict[str, Any]) -> Di # Extract video ID video_id = self._extract_video_id(video_url) - video_hash = hashlib.md5(video_url.encode()).hexdigest() + video_hash = hashlib.sha256(video_url.encode()).hexdigest() # Check if already processed with sqlite3.connect(self.db_path) as conn: @@ -700,7 +700,7 @@ def _extract_video_id(self, video_url: str) -> str: return video_url.split("youtu.be/")[1].split("?")[0] else: # Generate hash-based ID for non-YouTube URLs - return hashlib.md5(video_url.encode()).hexdigest()[:11] + return hashlib.sha256(video_url.encode()).hexdigest()[:11] async def _record_breakthrough(self, video_id: str, innovation_result: Dict[str, Any]) -> None: """Record breakthrough achievement""" diff --git a/scripts/nightly_audit_agent.py b/scripts/nightly_audit_agent.py index 7db77227f..7f00b591d 100644 --- a/scripts/nightly_audit_agent.py +++ b/scripts/nightly_audit_agent.py @@ -19,10 +19,11 @@ import asyncio import json import logging +import os import sys from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Dict +from typing import Any try: import orjson @@ -42,7 +43,6 @@ HealthStatus, get_health_monitoring_service, ) - from youtube_extension.backend.services.logging_service import get_logging_service from youtube_extension.backend.services.metrics_service import MetricsService except ImportError: # Print warning but don't fail immediately, allows dry-run in incomplete envs @@ -55,10 +55,54 @@ format='%(asctime)s - [AuditAgent] - %(levelname)s - %(message)s' ) logger = logging.getLogger(__name__) +SUPPORTS_LOAD_AVERAGE = hasattr(os, "getloadavg") + + +class FallbackActiveMeasurementService: + """Minimal active measurement collector used when MetricsService is unavailable.""" + + def __init__(self, log_dir: Path): + self.log_dir = log_dir + self.measurements = [] + + async def start_collection(self): + self.log_dir.mkdir(exist_ok=True) + + async def stop_collection(self): + return None + + async def get_system_metrics(self): + load_average = os.getloadavg()[0] if SUPPORTS_LOAD_AVERAGE else None + measurement = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "source": "fallback_active_measurement", + "load_average_1m": load_average, + } + self.measurements.append(measurement) + return measurement + + async def persist_metrics(self): + metrics_path = self.log_dir / "active_measurements.jsonl" + with open(metrics_path, "a") as f: + for measurement in self.measurements: + f.write(json.dumps(measurement) + "\n") + self.measurements.clear() + class AuditAgent: - def __init__(self, dry_run: bool = False): + def __init__( + self, + dry_run: bool = False, + lookback_hours: int = 72, + active_measurement: bool = False, + measurement_samples: int = 3, + measurement_interval: float = 1.0, + ): self.dry_run = dry_run + self.lookback_hours = max(1, lookback_hours) + self.active_measurement = active_measurement + self.measurement_samples = max(1, measurement_samples) + self.measurement_interval = max(0.0, measurement_interval) self.log_dir = Path("logs") self.log_dir.mkdir(exist_ok=True) self.report = [] @@ -89,6 +133,9 @@ async def run_audit(self): self._add_report_header(start_time) logger.info("Starting Nightly Audit...") + self.report.append(f"Analysis lookback: {self.lookback_hours} hours") + + await self._collect_active_measurements() # 1. Analysis Phase await self.analyze_phase() @@ -115,7 +162,7 @@ async def analyze_phase(self): # Check System Health await self._check_system_health() - # Scan Logs for Errors and Status Codes (Last 24h) + # Scan Logs for Errors and Status Codes await self._scan_logs() # Check Metrics for Latency @@ -151,8 +198,33 @@ async def _check_system_health(self): "details": str(e) }) + async def _collect_active_measurements(self): + """Collect live metric samples before analysis for more accurate output.""" + if not self.active_measurement: + return + + if not self.metrics_service: + self.metrics_service = FallbackActiveMeasurementService(self.log_dir) + + samples = self.measurement_samples + interval = self.measurement_interval + self.report.append(f"📏 ACTIVE MEASUREMENT: collecting {samples} live samples") + + try: + await self.metrics_service.start_collection() + for sample_index in range(samples): + await self.metrics_service.get_system_metrics() + if interval and sample_index < samples - 1: + await asyncio.sleep(interval) + + persist = getattr(self.metrics_service, "persist_metrics", None) + if persist: + await persist() + finally: + await self.metrics_service.stop_collection() + async def _scan_logs(self): - """Scan logs for recent critical failures and status codes > 400 (Last 24h)""" + """Scan logs for recent critical failures and status codes > 400.""" error_log_path = self.log_dir / "error_logs.jsonl" structured_log_path = self.log_dir / "structured_logs.jsonl" @@ -162,7 +234,7 @@ async def _scan_logs(self): logger.warning("No log files found to scan.") return - cutoff_time = datetime.now(timezone.utc) - timedelta(hours=24) + cutoff_time = datetime.now(timezone.utc) - timedelta(hours=self.lookback_hours) found_issues = [] for log_file in files_to_scan: @@ -170,7 +242,8 @@ async def _scan_logs(self): with open(log_file, 'rb') as f: for line in f: try: - if not line.strip(): continue + if not line.strip(): + continue if HAS_ORJSON: entry = orjson.loads(line) else: @@ -261,7 +334,7 @@ async def _check_latency_metrics(self): except Exception as e: logger.error(f"Error analyzing metrics: {e}") - async def first_principles_analysis(self, issue: Dict[str, Any]): + async def first_principles_analysis(self, issue: dict[str, Any]): """ Five Whys Interrogation """ @@ -412,9 +485,38 @@ def _generate_report_file(self, start_time): async def main(): parser = argparse.ArgumentParser(description="Jules Audit Agent") parser.add_argument("--dry-run", action="store_true", help="Simulate remediation actions") + parser.add_argument( + "--lookback-hours", + type=int, + default=72, + help="Hours of logs and metrics to scan (default: 72)", + ) + parser.add_argument( + "--active-measurement", + action="store_true", + help="Collect live metric samples before analysis", + ) + parser.add_argument( + "--measurement-samples", + type=int, + default=3, + help="Number of live metric samples to collect", + ) + parser.add_argument( + "--measurement-interval", + type=float, + default=1.0, + help="Seconds between live metric samples", + ) args = parser.parse_args() - agent = AuditAgent(dry_run=args.dry_run) + agent = AuditAgent( + dry_run=args.dry_run, + lookback_hours=args.lookback_hours, + active_measurement=args.active_measurement, + measurement_samples=args.measurement_samples, + measurement_interval=args.measurement_interval, + ) await agent.run_audit() if __name__ == "__main__": diff --git a/skills-lock.json b/skills-lock.json index 1bebbe487..6c927ac8f 100644 --- a/skills-lock.json +++ b/skills-lock.json @@ -1,205 +1,225 @@ { - "version": 1, - "skills": [ - { - "id": "firebase-ai-logic-basics", + "version": 2, + "emitted_events": [ + "pipeline.event", + "com.eventrelay.transcript.received", + "com.eventrelay.transcript.queued", + "com.eventrelay.transcript.completed", + "com.eventrelay.transcript.failed", + "com.eventrelay.video.received", + "com.eventrelay.pipeline.completed", + "com.eventrelay.pipeline.failed" + ], + "skills": { + "firebase-ai-logic-basics": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/firebase-ai-logic-basics/SKILL.md", - "computedHash": "c1e42edfaf46c3b2c240bc23413991948a8cc77b70dfddd2009e99c35db760eb" + "computedHash": "c1e42edfaf46c3b2c240bc23413991948a8cc77b70dfddd2009e99c35db760eb", + "subscribed_triggers": [] }, - { - "id": "firebase-app-hosting-basics", + "firebase-app-hosting-basics": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/firebase-app-hosting-basics/SKILL.md", - "computedHash": "7f0e0330510b4e6b06bcede472cebb183a491b8a0098f92d7563454c40d78050" + "computedHash": "7f0e0330510b4e6b06bcede472cebb183a491b8a0098f92d7563454c40d78050", + "subscribed_triggers": [] }, - { - "id": "firebase-auth-basics", + "firebase-auth-basics": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/firebase-auth-basics/SKILL.md", - "computedHash": "0d29bda451353a92c3b6048a943a46c28cee267ec2e3b148f6207630adba3d73" + "computedHash": "0d29bda451353a92c3b6048a943a46c28cee267ec2e3b148f6207630adba3d73", + "subscribed_triggers": [] }, - { - "id": "firebase-basics", + "firebase-basics": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/firebase-basics/SKILL.md", - "computedHash": "88fb9ee785fa7aaa74b2c662e53b2aca0b9ee4b67c84587ee017460f54b97471" + "computedHash": "88fb9ee785fa7aaa74b2c662e53b2aca0b9ee4b67c84587ee017460f54b97471", + "subscribed_triggers": [] }, - { - "id": "firebase-crashlytics", + "firebase-crashlytics": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/firebase-crashlytics/SKILL.md", - "computedHash": "2c2b5ad36eeea0910b2e335e84d678c6af75dad3ccf73033fcb7e5a8768cabbc" + "computedHash": "2c2b5ad36eeea0910b2e335e84d678c6af75dad3ccf73033fcb7e5a8768cabbc", + "subscribed_triggers": [] }, - { - "id": "firebase-data-connect", + "firebase-data-connect": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/firebase-data-connect-basics/SKILL.md", - "computedHash": "2dfebf7892b9b17f8022057be93a1b3c11438f2c0ce89e9d56ef7be16b7cdecd" + "computedHash": "2dfebf7892b9b17f8022057be93a1b3c11438f2c0ce89e9d56ef7be16b7cdecd", + "subscribed_triggers": [] }, - { - "id": "firebase-firestore", + "firebase-firestore": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/firebase-firestore/SKILL.md", - "computedHash": "09ce3baf45a8d2cd8f32dd48d436628d7d4ac04f24ad351bf3e352a81760ecf8" + "computedHash": "09ce3baf45a8d2cd8f32dd48d436628d7d4ac04f24ad351bf3e352a81760ecf8", + "subscribed_triggers": [] }, - { - "id": "firebase-hosting-basics", + "firebase-hosting-basics": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/firebase-hosting-basics/SKILL.md", - "computedHash": "fb86fd4035e8e6379931faeb443557ac6f2e43fde04b397433f287e69b6532a9" + "computedHash": "fb86fd4035e8e6379931faeb443557ac6f2e43fde04b397433f287e69b6532a9", + "subscribed_triggers": [] }, - { - "id": "firebase-remote-config-basics", + "firebase-remote-config-basics": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/firebase-remote-config-basics/SKILL.md", - "computedHash": "855963d0c979692811c8b0ea112aba94894ca4f538934268d33e7e4665e7412b" + "computedHash": "855963d0c979692811c8b0ea112aba94894ca4f538934268d33e7e4665e7412b", + "subscribed_triggers": [] }, - { - "id": "firebase-security-rules-auditor", + "firebase-security-rules-auditor": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/firebase-security-rules-auditor/SKILL.md", - "computedHash": "5a90e991bb9acfd3e43bfb570498dee60b9cef94cbb80cfb99257c7e4f61c1a0" + "computedHash": "5a90e991bb9acfd3e43bfb570498dee60b9cef94cbb80cfb99257c7e4f61c1a0", + "subscribed_triggers": [] }, - { - "id": "systematic-debugging", + "systematic-debugging": { "source": "obra/superpowers", "sourceType": "github", "skillPath": "skills/systematic-debugging/SKILL.md", - "computedHash": "7246fdd3a795fc3daff0af72044ca99bf836e4e6a46844742858786fdfb86488" + "computedHash": "7246fdd3a795fc3daff0af72044ca99bf836e4e6a46844742858786fdfb86488", + "subscribed_triggers": [] }, - { - "id": "test-driven-development", + "test-driven-development": { "source": "obra/superpowers", "sourceType": "github", "skillPath": "skills/test-driven-development/SKILL.md", - "computedHash": "126f1ebf6ccd414f42544f6e83d8cc5adb089e1108eaffb7c400701e37eecd9f" + "computedHash": "126f1ebf6ccd414f42544f6e83d8cc5adb089e1108eaffb7c400701e37eecd9f", + "subscribed_triggers": [] }, - { - "id": "vercel-react-best-practices", + "vercel-react-best-practices": { "source": "vercel-labs/agent-skills", "sourceType": "github", "skillPath": "skills/react-best-practices/SKILL.md", - "computedHash": "ca7b0c0c6e5f2750043f7f0cd72d16ac4e2abc48f9b5500d047a4b77a2506212" + "computedHash": "ca7b0c0c6e5f2750043f7f0cd72d16ac4e2abc48f9b5500d047a4b77a2506212", + "subscribed_triggers": [] }, - { - "id": "verification-before-completion", + "verification-before-completion": { "source": "obra/superpowers", "sourceType": "github", "skillPath": "skills/verification-before-completion/SKILL.md", - "computedHash": "9b446f0c7fe1cfb560b1d34439523b1a76d5f177290007b2c053a1c749a4a8ba" + "computedHash": "9b446f0c7fe1cfb560b1d34439523b1a76d5f177290007b2c053a1c749a4a8ba", + "subscribed_triggers": [] }, - { - "id": "xcode-project-setup", + "xcode-project-setup": { "source": "firebase/agent-skills", "sourceType": "github", "skillPath": "skills/xcode-project-setup/SKILL.md", - "computedHash": "65fc8ef640574e34cd315cef3a2e8ea6eb2d3b29d38eba18e1e749d812215161" + "computedHash": "65fc8ef640574e34cd315cef3a2e8ea6eb2d3b29d38eba18e1e749d812215161", + "subscribed_triggers": [] }, - { - "id": "content-generation", - "name": "Content Generation", - "version": "1.0.0", + "content-generation": { "source": "uvai-skills", - "entry_point": "src/skills/content_generation/main.py", + "sourceType": "local", + "skillPath": "src/skills/content_generation/main.py", + "className": "ContentGenerationSkill", + "version": "1.0.0", "triggers": [ - "video_published", - "manual" + "youtube.video.published" ], "dependencies": [ "gemini_service", "database_service" - ] + ], + "subscribed_triggers": [] }, - { - "id": "seo-optimizer", - "name": "SEO Optimizer", - "version": "1.0.0", + "seo-optimizer": { "source": "uvai-skills", - "entry_point": "src/skills/seo_optimizer/main.py", + "sourceType": "local", + "skillPath": "src/skills/seo_optimizer/main.py", + "className": "SEOOptimizerSkill", + "version": "1.0.0", "triggers": [ - "video_uploaded" + "youtube.video.uploaded" ], "dependencies": [ "gemini_service" - ] + ], + "subscribed_triggers": [] }, - { - "id": "social-scheduler", - "name": "Social Scheduler", - "version": "1.0.0", + "social-scheduler": { "source": "uvai-skills", - "entry_point": "src/skills/social_scheduler/main.py", + "sourceType": "local", + "skillPath": "src/skills/social_scheduler/main.py", + "className": "SocialSchedulerSkill", + "version": "1.0.0", "triggers": [ - "content_generated" + "ai.content.generated" ], "dependencies": [ + "gemini_service", "social_api_service" - ] + ], + "subscribed_triggers": [] }, - { - "id": "lead-scorer", - "name": "Lead Scorer", - "version": "1.0.0", + "lead-scorer": { "source": "uvai-skills", - "entry_point": "src/skills/lead_scorer/main.py", + "sourceType": "local", + "skillPath": "src/skills/lead_scorer/main.py", + "className": "LeadScorerSkill", + "version": "1.0.0", "triggers": [ - "analytics_updated" + "youtube.analytics.updated" ], "dependencies": [ "database_service" - ] + ], + "subscribed_triggers": [] }, - { - "id": "email-campaign", - "name": "Email Campaign", - "version": "1.0.0", + "email-campaign": { "source": "uvai-skills", - "entry_point": "src/skills/email_campaign/main.py", + "sourceType": "local", + "skillPath": "src/skills/email_campaign/main.py", + "className": "EmailCampaignSkill", + "version": "1.0.0", "triggers": [ - "lead_scored" + "crm.lead.scored" ], "dependencies": [ + "gemini_service", + "database_service", "email_service" - ] + ], + "subscribed_triggers": [] }, - { - "id": "analytics-dashboard", - "name": "Analytics Dashboard", - "version": "1.0.0", + "analytics-dashboard": { "source": "uvai-skills", - "entry_point": "src/skills/analytics_dashboard/main.py", + "sourceType": "local", + "skillPath": "src/skills/analytics_dashboard/main.py", + "className": "AnalyticsDashboardSkill", + "version": "1.0.0", "triggers": [ - "daily_cron" + "system.cron.daily" ], "dependencies": [ "database_service", "analytics_service" - ] + ], + "subscribed_triggers": [] }, - { - "id": "ab-testing", - "name": "A/B Testing", - "version": "1.0.0", + "ab-testing": { "source": "uvai-skills", - "entry_point": "src/skills/ab_testing/main.py", + "sourceType": "local", + "skillPath": "src/skills/ab_testing/main.py", + "className": "ABTestingSkill", + "version": "1.0.0", "triggers": [ - "video_uploaded" + "youtube.video.uploaded" ], "dependencies": [ "gemini_service", + "database_service", "analytics_service" - ] + ], + "subscribed_triggers": [] } - ] -} \ No newline at end of file + } +} diff --git a/src/agents/gemini_video_master_agent.py b/src/agents/gemini_video_master_agent.py index 0314fd429..53fc09892 100644 --- a/src/agents/gemini_video_master_agent.py +++ b/src/agents/gemini_video_master_agent.py @@ -1092,7 +1092,7 @@ async def _execute_with_gemini_text( @staticmethod def _build_gemini_generation_config( response_mime_type: str | None = None, - ) -> types.GenerateContentConfig: + ) -> "types.GenerateContentConfig": config_kwargs = { "max_output_tokens": int(os.getenv("GEMINI_MAX_OUTPUT_TOKENS", "16384")) } diff --git a/src/agents/llama_background_agent.py b/src/agents/llama_background_agent.py index f70d39347..a04a0282a 100644 --- a/src/agents/llama_background_agent.py +++ b/src/agents/llama_background_agent.py @@ -122,7 +122,6 @@ def _get_default_model_path(self) -> str: def _download_llama_model(self, models_dir: Path) -> str: """Download Llama 3.1 8B Instruct model from HuggingFace""" try: - import huggingface_hub from huggingface_hub import hf_hub_download # Optional HuggingFace token from environment diff --git a/src/agents/mcp_ecosystem_coordinator.py b/src/agents/mcp_ecosystem_coordinator.py index 65fabf888..226b964e5 100644 --- a/src/agents/mcp_ecosystem_coordinator.py +++ b/src/agents/mcp_ecosystem_coordinator.py @@ -4,26 +4,27 @@ Unified hub for coordinating all MCP servers in the YouTube extension ecosystem """ +from __future__ import annotations + import abc import asyncio +import importlib import json import logging import os -import subprocess -import sys from dataclasses import asdict -from typing import Any, Dict, List, Optional +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, List, Optional -from youtube_extension.processors.enhanced_extractor import ( - EnhancedVideoExtractor, - VideoContent, -) +if TYPE_CHECKING: + from youtube_extension.processors.enhanced_extractor import VideoContent # Add src/mcp to path for imports # REMOVED: sys.path.append removed logger = logging.getLogger(__name__) + class BaseMCPServer(abc.ABC): """Abstract base class for all MCP servers.""" @@ -53,8 +54,15 @@ class MCPVideoProcessorServer(BaseMCPServer): def __init__(self): super().__init__("video_processor", "video_processing") self.supported_formats = ["mp4", "webm", "avi"] - # Initialize the Unified Pipeline Extractor - self.extractor = EnhancedVideoExtractor() + self.extractor = None + try: + module = importlib.import_module( + "youtube_extension.processors.enhanced_extractor" + ) + EnhancedVideoExtractor = module.EnhancedVideoExtractor + self.extractor = EnhancedVideoExtractor() + except Exception as e: + logger.warning(f"Enhanced extractor unavailable: {e}") async def handle_request(self, request: dict) -> dict: """Process video processing requests.""" @@ -64,6 +72,8 @@ async def handle_request(self, request: dict) -> dict: if action == "process_video": logger.info(f"Processing video: {video_id}") try: + if self.extractor is None: + return {"status": "error", "message": "Video extractor unavailable"} # Use the Unified Pipeline (Gemini + Scoring) # Note: process_video expects a URL usually, but if ID is passed, we might need to construct URL # or ensure process_video handles IDs (it extracts ID from URL, so URL is safer) @@ -154,18 +164,22 @@ def get_capabilities(self) -> dict: async def health_check(self) -> dict: return {"status": "healthy", "server": self.name} + class MCPEcosystemCoordinator: """Coordinates multiple MCP servers, routing requests and managing capabilities.""" - def __init__(self): + def __init__(self, skill_registry: SkillRegistry | None = None): self.servers: dict[str, BaseMCPServer] = {} self.capabilities_map: dict[str, dict] = {} self.workflow_history: list[dict] = [] - self.skill_registry = SkillRegistry() + self.skill_registry = skill_registry or SkillRegistry() def list_skills(self, source: Optional[str] = None) -> List[Dict[str, Any]]: """Returns a list of discovered skills from the registry.""" - return self.skill_registry.list_skills(source=source) + skills = self.skill_registry.list_skills() + if source: + return [s for s in skills if s.get("source") == source] + return skills def register_server(self, server: BaseMCPServer) -> bool: """Registers an MCP server with the coordinator.""" @@ -186,7 +200,8 @@ async def discover_capabilities(self) -> dict: all_capabilities = { "total_servers": len(self.servers), "servers": {}, - "available_tools": [] + "available_tools": [], + "skills": self.skill_registry.list_skills(), } for name, caps in self.capabilities_map.items(): @@ -196,6 +211,18 @@ async def discover_capabilities(self) -> dict: return all_capabilities + async def invoke_skill( + self, skill_id: str, payload: dict[str, Any] + ) -> dict[str, Any]: + """Invoke a GTM skill via the class-based SkillRegistry. + + Thin delegate to ``SkillRegistry.invoke_skill``, which resolves the + skill class from ``skills-lock.json`` and runs it in-process with + dependency injection. The registry is the single invocation path used + by the tests and callers. + """ + return await self.skill_registry.invoke_skill(skill_id, payload) + async def dispatch_request(self, server_name: str, request: dict) -> dict: """Dispatches a request to the specified MCP server.""" server = self.servers.get(server_name) @@ -277,108 +304,217 @@ async def get_system_status(self) -> dict: return status -class SkillRegistry: - """Registry for discovering and invoking skills from skills-lock.json.""" - def __init__(self, lock_file: str = "skills-lock.json"): - self.lock_file = lock_file - self.skills: List[Dict[str, Any]] = [] +class SkillRegistry: + """Registry for discovering and invoking GTM skills from skills-lock.json. + + Reads skill definitions from the lock file and dynamically loads skill + classes for in-process execution, resolving each skill's declared + dependencies via the service container (dependency injection) rather than + spawning subprocesses. + """ + + _LOCK_FILE = "skills-lock.json" + + def __init__(self, lock_file_path: Optional[str] = None): + self._lock_path = Path( + lock_file_path + or os.environ.get("SKILLS_LOCK_PATH", "") + or self._find_lock_file() + ) + self._skills: dict[str, dict[str, Any]] = {} + self._instances: dict[str, Any] = {} self._load_skills() - def _load_skills(self): - """Loads skills from the lock file.""" - if not os.path.exists(self.lock_file): - logger.warning(f"Lock file {self.lock_file} not found.") - return + def _find_lock_file(self) -> str: + """Walk up from CWD or src/agents to find skills-lock.json.""" + candidates = [ + Path.cwd() / self._LOCK_FILE, + Path(__file__).resolve().parents[2] / self._LOCK_FILE, + Path(__file__).resolve().parents[3] / self._LOCK_FILE, + ] + for candidate in candidates: + if candidate.is_file(): + return str(candidate) + return self._LOCK_FILE + def _load_skills(self) -> None: + """Load GTM skill definitions from the lock file.""" try: - with open(self.lock_file, 'r') as f: + with open(self._lock_path) as f: data = json.load(f) - # Handle both list and dict formats for backward compatibility during transition - skills_data = data.get("skills", []) - if isinstance(skills_data, list): - self.skills = skills_data - elif isinstance(skills_data, dict): - # Convert dict format to list - self.skills = [] - for skill_id, skill_info in skills_data.items(): - skill_info["id"] = skill_id - self.skills.append(skill_info) - except Exception as e: - logger.error(f"Error loading skills from {self.lock_file}: {e}") + except (FileNotFoundError, json.JSONDecodeError) as e: + logger.warning("Could not load skills-lock.json: %s", e) + return - def list_skills(self, source: Optional[str] = None) -> List[Dict[str, Any]]: - """Returns a list of discovered skills, optionally filtered by source.""" + skills_data = data.get("skills", {}) + if isinstance(skills_data, list): + # Handle list format; only load entries that have a className + # so that _load_skill_instance() can instantiate them. + for skill in skills_data: + if ( + skill.get("source") == "uvai-skills" + and skill.get("className") + and skill.get("id") + ): + self._skills[skill["id"]] = skill + elif isinstance(skills_data, dict): + # Handle dict format; apply same guards as list branch + for skill_id, meta in skills_data.items(): + if ( + meta.get("source") == "uvai-skills" + and meta.get("sourceType") == "local" + and meta.get("className") + ): + self._skills[skill_id] = meta + + logger.info("Loaded %d GTM skills from %s", len(self._skills), self._lock_path) + + def _build_skill_metadata(self, skill_id: str, meta: dict[str, Any]) -> dict[str, Any]: + """Build a normalized metadata dict for a skill entry.""" + return { + "id": skill_id, + "name": meta.get("name") or skill_id.replace("-", " ").title(), + "class_name": meta.get("className", ""), + "version": meta.get("version", "0.0.0"), + "triggers": meta.get("triggers", []), + "dependencies": meta.get("dependencies", []), + "entry_point": meta.get("skillPath") or meta.get("entry_point", ""), + "source": meta.get("source", ""), + } + + def list_skills(self, source: Optional[str] = None) -> list[dict[str, Any]]: + """Return metadata for all registered GTM skills.""" + skills = [ + self._build_skill_metadata(skill_id, meta) + for skill_id, meta in self._skills.items() + ] if source: - return [s for s in self.skills if s.get("source") == source] - return self.skills - - def get_skill(self, skill_id: str) -> Optional[Dict[str, Any]]: - """Retrieves a skill by its ID.""" - for skill in self.skills: - if skill.get("id") == skill_id: - return skill - return None - - async def invoke_skill(self, skill_id: str, context: Dict[str, Any]) -> Dict[str, Any]: - """Invokes a skill by its ID with the given context.""" - skill = self.get_skill(skill_id) - if not skill: - return {"status": "error", "message": f"Skill '{skill_id}' not found"} - - entry_point = skill.get("entry_point") - if not entry_point or not os.path.exists(entry_point): - return {"status": "error", "message": f"Entry point '{entry_point}' not found for skill '{skill_id}'"} - - # Explicitly pass required env vars (Gemini CLI security update) - allowed_env_vars = [ - "GEMINI_API_KEY", - "OPENAI_API_KEY", - "YOUTUBE_API_KEY", - "DATABASE_URL", - "GITHUB_TOKEN", - "PYTHONPATH" + return [s for s in skills if self._skills[s["id"]].get("source") == source] + return skills + + def get_skill(self, skill_id: str) -> Optional[dict[str, Any]]: + """Get metadata for a specific skill.""" + meta = self._skills.get(skill_id) + if meta is None: + return None + return self._build_skill_metadata(skill_id, meta) + + def get_skills_for_trigger(self, event_type: str) -> list[dict[str, Any]]: + """Return all skills that match a given trigger event.""" + return [ + self._build_skill_metadata(skill_id, meta) + for skill_id, meta in self._skills.items() + if event_type in meta.get("triggers", []) ] - env = {k: os.environ[k] for k in allowed_env_vars if k in os.environ} - env["SKILL_CONTEXT"] = json.dumps(context) - # Ensure minimal system env if needed - if "PATH" in os.environ: - env["PATH"] = os.environ["PATH"] + def _load_skill_instance(self, skill_id: str) -> Any: + """Dynamically import and instantiate a skill class.""" + if skill_id in self._instances: + return self._instances[skill_id] + + meta = self._skills.get(skill_id) + if meta is None: + raise ValueError(f"Unknown skill: {skill_id}") + + skill_path = meta.get("skillPath") or meta.get("entry_point") + class_name = meta.get("className") + + if not skill_path: + raise ValueError(f"Skill {skill_id} has no skillPath or entry_point") + + if not class_name: + raise ValueError(f"Skill {skill_id} has no className") + # Convert file path to module path + module_path = skill_path.replace("/", ".").removesuffix(".py") + # Strip leading "src." if present since src is on sys.path + if module_path.startswith("src."): + module_path = module_path[4:] + + module = importlib.import_module(module_path) + skill_class = getattr(module, class_name) + + # Resolve dependencies from the service container. Imported lazily to + # avoid a circular import at module load time, and guarded so that a + # container import failure (e.g. a missing optional transitive dep) + # degrades to no injection instead of breaking every skill invocation. + dependencies: dict[str, Any] = {} try: - logger.info(f"🚀 Invoking skill '{skill_id}' via {entry_point}") - # Run the skill as a subprocess - process = await asyncio.to_thread( - subprocess.run, - [sys.executable, entry_point], - env=env, - capture_output=True, - text=True, - check=True + from youtube_extension.backend.containers.service_container import ( + get_service, ) + except Exception as e: + logger.warning( + "Service container unavailable; skipping DI for skill %s: %s", + skill_id, + e, + ) + get_service = None + + if get_service is not None: + for dep_name in meta.get("dependencies", []): + try: + dependencies[dep_name] = get_service(dep_name) + except Exception as e: + logger.warning("Failed to resolve dependency %s for skill %s: %s", dep_name, skill_id, e) + + instance = skill_class(dependencies=dependencies) + self._instances[skill_id] = instance + return instance + + def get_env_for_skill(self, skill_id: str) -> dict[str, str]: + """Get the explicit env vars to pass through to a skill subprocess. + + Implements MCP security requirement: do NOT rely on environment + inheritance; explicitly pass only required vars. + """ + meta = self._skills.get(skill_id) + if meta is None: + return {} + + # Map dependency names to env vars + dep_env_map: dict[str, list[str]] = { + "gemini_service": ["GEMINI_API_KEY"], + "database_service": ["DATABASE_URL"], + "openai_service": ["OPENAI_API_KEY"], + "social_api_service": ["SOCIAL_API_KEY"], + "email_service": ["EMAIL_API_KEY"], + "analytics_service": ["ANALYTICS_API_KEY"], + } - try: - result = json.loads(process.stdout) - return result - except json.JSONDecodeError: - return { - "status": "success", - "output": process.stdout.strip(), - "warning": "Output was not valid JSON" - } + env: dict[str, str] = {} + for dep in meta.get("dependencies", []): + for var in dep_env_map.get(dep, []): + val = os.environ.get(var) + if val is not None: + env[var] = val + return env + + async def invoke_skill( + self, skill_id: str, payload: dict[str, Any] + ) -> dict[str, Any]: + """Invoke a skill by ID with the given payload. + + Returns the skill result as a dictionary. + """ + try: + instance = self._load_skill_instance(skill_id) + except (ValueError, ImportError, AttributeError) as e: + logger.error("Failed to load skill %s: %s", skill_id, e) + return {"status": "error", "error": str(e)} - except subprocess.CalledProcessError as e: - logger.error(f"❌ Skill '{skill_id}' failed with exit code {e.returncode}") - logger.error(f"Stderr: {e.stderr}") + try: + result = await instance.execute(payload) return { - "status": "error", - "message": f"Skill execution failed: {str(e)}", - "stderr": e.stderr + "status": result.status, + "output": result.output, + "error": result.error, } except Exception as e: - logger.error(f"❌ Error invoking skill '{skill_id}': {e}") - return {"status": "error", "message": str(e)} + logger.error("Skill %s execution failed: %s", skill_id, e) + return {"status": "error", "error": str(e)} + # Example usage and testing async def main(): diff --git a/src/skills/__init__.py b/src/skills/__init__.py new file mode 100644 index 000000000..555d0737a --- /dev/null +++ b/src/skills/__init__.py @@ -0,0 +1,25 @@ +"""GTM Skills package for EventRelay agent orchestration. + +Skills provide go-to-market automation capabilities (content generation, +SEO optimization, social media scheduling, lead scoring, email campaigns, +analytics dashboards, and A/B testing) that extend EventRelay's video +pipeline into a full marketing automation platform. +""" + +from skills.content_generation.main import ContentGenerationSkill +from skills.seo_optimizer.main import SEOOptimizerSkill +from skills.social_scheduler.main import SocialSchedulerSkill +from skills.lead_scorer.main import LeadScorerSkill +from skills.email_campaign.main import EmailCampaignSkill +from skills.analytics_dashboard.main import AnalyticsDashboardSkill +from skills.ab_testing.main import ABTestingSkill + +__all__ = [ + "ContentGenerationSkill", + "SEOOptimizerSkill", + "SocialSchedulerSkill", + "LeadScorerSkill", + "EmailCampaignSkill", + "AnalyticsDashboardSkill", + "ABTestingSkill", +] diff --git a/src/skills/ab_testing/__init__.py b/src/skills/ab_testing/__init__.py new file mode 100644 index 000000000..0de3db26c --- /dev/null +++ b/src/skills/ab_testing/__init__.py @@ -0,0 +1 @@ +"""A/B Testing skill module.""" diff --git a/src/skills/ab_testing/main.py b/src/skills/ab_testing/main.py index 3fe03407a..9ef3badb2 100644 --- a/src/skills/ab_testing/main.py +++ b/src/skills/ab_testing/main.py @@ -1,22 +1,63 @@ -import os -import sys -import json +"""A/B Testing skill - runs A/B tests on thumbnails and titles.""" + +from __future__ import annotations + import logging +from typing import Any, Optional + +from skills.base import BaseSkill, SkillResult -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) -def main(): - skill_name = "ab-testing" - logger.info(f"Skill {skill_name} invoked") - context = os.getenv("SKILL_CONTEXT", "{}") - logger.info(f"Context: {context}") - gemini_key = os.getenv("GEMINI_API_KEY") - if gemini_key: - logger.info("GEMINI_API_KEY is present") - else: - logger.warning("GEMINI_API_KEY is missing") - print(json.dumps({"status": "success", "skill": skill_name})) - -if __name__ == "__main__": - main() + +class ABTestingSkill(BaseSkill): + """Run A/B tests on video thumbnails and titles.""" + + skill_id = "ab-testing" + name = "A/B Testing" + version = "1.0.0" + triggers = ["youtube.video.uploaded"] + required_env_vars = ["GEMINI_API_KEY", "DATABASE_URL"] + + def __init__(self, dependencies: Optional[dict[str, Any]] = None): + super().__init__(dependencies) + self.gemini = self.dependencies.get("gemini_service") + self.analytics = self.dependencies.get("analytics_service") + + async def execute(self, payload: dict[str, Any]) -> SkillResult: + """Create and manage an A/B test. + + Expected payload keys: + - video_id: str - the video to test + - test_type: str - "thumbnail" | "title" | "description" + - variants: list[dict] - the test variants + """ + video_id = payload.get("video_id") + if not video_id: + return SkillResult(status="error", error="Missing 'video_id' in payload") + + test_type = payload.get("test_type", "thumbnail") + variants = payload.get("variants", []) + + logger.info( + "Creating %s A/B test for video %s with %d variants", + test_type, + video_id, + len(variants), + ) + + if self.gemini: + logger.info("Using injected gemini_service for A/B testing") + if self.analytics: + logger.info("Using injected analytics_service for A/B testing") + + return SkillResult( + status="success", + output={ + "video_id": video_id, + "test_type": test_type, + "variant_count": len(variants), + "created": True, + "message": f"A/B test ({test_type}) created for video {video_id}", + }, + ) diff --git a/src/skills/analytics_dashboard/__init__.py b/src/skills/analytics_dashboard/__init__.py new file mode 100644 index 000000000..af1ccceb9 --- /dev/null +++ b/src/skills/analytics_dashboard/__init__.py @@ -0,0 +1 @@ +"""Analytics Dashboard skill module.""" diff --git a/src/skills/analytics_dashboard/main.py b/src/skills/analytics_dashboard/main.py index 4575bf2e4..044ead1d6 100644 --- a/src/skills/analytics_dashboard/main.py +++ b/src/skills/analytics_dashboard/main.py @@ -1,22 +1,57 @@ -import os -import sys -import json +"""Analytics Dashboard skill - aggregates metrics into dashboard data.""" + +from __future__ import annotations + import logging +from typing import Any, Optional + +from skills.base import BaseSkill, SkillResult -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) -def main(): - skill_name = "analytics-dashboard" - logger.info(f"Skill {skill_name} invoked") - context = os.getenv("SKILL_CONTEXT", "{}") - logger.info(f"Context: {context}") - gemini_key = os.getenv("GEMINI_API_KEY") - if gemini_key: - logger.info("GEMINI_API_KEY is present") - else: - logger.warning("GEMINI_API_KEY is missing") - print(json.dumps({"status": "success", "skill": skill_name})) - -if __name__ == "__main__": - main() + +class AnalyticsDashboardSkill(BaseSkill): + """Aggregate engagement and performance metrics into dashboard data.""" + + skill_id = "analytics-dashboard" + name = "Analytics Dashboard" + version = "1.0.0" + triggers = ["system.cron.daily"] + required_env_vars = ["DATABASE_URL"] + + def __init__(self, dependencies: Optional[dict[str, Any]] = None): + super().__init__(dependencies) + self.db = self.dependencies.get("database_service") + self.analytics = self.dependencies.get("analytics_service") + + async def execute(self, payload: dict[str, Any]) -> SkillResult: + """Aggregate analytics metrics. + + Expected payload keys: + - date_range: str - ISO date range ("2024-01-01/2024-01-31") + - metrics: list[str] - which metrics to aggregate (optional) + """ + date_range = payload.get("date_range") + if not date_range: + return SkillResult(status="error", error="Missing 'date_range' in payload") + + metrics = payload.get("metrics", ["views", "engagement", "conversions"]) + + logger.info( + "Aggregating %d metrics for range %s", len(metrics), date_range + ) + + if self.db: + logger.info("Using injected database_service for aggregation") + if self.analytics: + logger.info("Using injected analytics_service for aggregation") + + return SkillResult( + status="success", + output={ + "date_range": date_range, + "metrics_aggregated": metrics, + "generated": True, + "message": f"Dashboard data aggregated for {date_range}", + }, + ) diff --git a/src/skills/base.py b/src/skills/base.py new file mode 100644 index 000000000..7f771fce0 --- /dev/null +++ b/src/skills/base.py @@ -0,0 +1,71 @@ +"""Base class for all GTM skills.""" + +from __future__ import annotations + +import abc +import logging +import os +from dataclasses import dataclass, field +from typing import Any, Optional + +logger = logging.getLogger(__name__) + + +@dataclass +class SkillResult: + """Result returned by a skill execution.""" + + status: str # "success", "error", "skipped" + output: dict[str, Any] = field(default_factory=dict) + error: Optional[str] = None + + +class BaseSkill(abc.ABC): + """Abstract base class for GTM skills. + + Each skill must define: + - skill_id: unique identifier + - name: human-readable name + - version: semver version string + - triggers: list of event types that trigger this skill + - required_env_vars: env vars needed at runtime + """ + + skill_id: str + name: str + version: str + triggers: list[str] + required_env_vars: list[str] = [] + + def get_env(self) -> dict[str, str]: + """Collect required environment variables for subprocess pass-through. + + Returns only the vars that are set in the current process environment. + This implements the MCP environment pass-through requirement (no + reliance on environment inheritance). + """ + env: dict[str, str] = {} + for var in self.required_env_vars: + val = os.environ.get(var) + if val is not None: + env[var] = val + return env + + @abc.abstractmethod + async def execute(self, payload: dict[str, Any]) -> SkillResult: + """Execute the skill with the given payload.""" + ... + + def matches_trigger(self, event_type: str) -> bool: + """Check if this skill should be triggered by the given event.""" + return event_type in self.triggers + + def to_dict(self) -> dict[str, Any]: + """Serialize skill metadata.""" + return { + "id": self.skill_id, + "name": self.name, + "version": self.version, + "triggers": self.triggers, + "required_env_vars": self.required_env_vars, + } diff --git a/src/skills/content_generation/__init__.py b/src/skills/content_generation/__init__.py new file mode 100644 index 000000000..2a81fb73a --- /dev/null +++ b/src/skills/content_generation/__init__.py @@ -0,0 +1 @@ +"""Content Generation skill module.""" diff --git a/src/skills/content_generation/main.py b/src/skills/content_generation/main.py index b44beaf32..eec1c510b 100644 --- a/src/skills/content_generation/main.py +++ b/src/skills/content_generation/main.py @@ -1,22 +1,62 @@ -import os -import sys -import json +"""Content Generation skill - generates blog/social posts from video transcripts.""" + +from __future__ import annotations + import logging +from typing import Any, Optional + +from skills.base import BaseSkill, SkillResult -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) -def main(): - skill_name = "content-generation" - logger.info(f"Skill {skill_name} invoked") - context = os.getenv("SKILL_CONTEXT", "{}") - logger.info(f"Context: {context}") - gemini_key = os.getenv("GEMINI_API_KEY") - if gemini_key: - logger.info("GEMINI_API_KEY is present") - else: - logger.warning("GEMINI_API_KEY is missing") - print(json.dumps({"status": "success", "skill": skill_name})) - -if __name__ == "__main__": - main() + +class ContentGenerationSkill(BaseSkill): + """Generate blog posts and social media content from video transcripts.""" + + skill_id = "content-generation" + name = "Content Generation" + version = "1.0.0" + triggers = ["youtube.video.published"] + required_env_vars = ["GEMINI_API_KEY"] + + def __init__(self, dependencies: Optional[dict[str, Any]] = None): + super().__init__(dependencies) + self.gemini = self.dependencies.get("gemini_service") + self.db = self.dependencies.get("database_service") + + async def execute(self, payload: dict[str, Any]) -> SkillResult: + """Generate content from a video transcript. + + Expected payload keys: + - transcript: str - the video transcript text + - video_id: str - the source video identifier + - content_type: str - "blog" | "social" | "both" (default: "both") + """ + transcript = payload.get("transcript") + if not transcript: + return SkillResult(status="error", error="Missing 'transcript' in payload") + + video_id = payload.get("video_id", "unknown") + content_type = payload.get("content_type", "both") + + logger.info( + "Generating %s content for video %s (transcript length: %d)", + content_type, + video_id, + len(transcript), + ) + + if self.gemini: + logger.info("Using injected gemini_service for generation") + # In a real implementation, we would call self.gemini.process_text(...) here + + # Thin wrapper: actual AI generation will be wired in a future iteration + return SkillResult( + status="success", + output={ + "video_id": video_id, + "content_type": content_type, + "generated": True, + "message": f"Content generation queued for video {video_id}", + }, + ) diff --git a/src/skills/email_campaign/__init__.py b/src/skills/email_campaign/__init__.py new file mode 100644 index 000000000..9eaa0afc9 --- /dev/null +++ b/src/skills/email_campaign/__init__.py @@ -0,0 +1 @@ +"""Email Campaign skill module.""" diff --git a/src/skills/email_campaign/main.py b/src/skills/email_campaign/main.py index 4da431b5b..804112f08 100644 --- a/src/skills/email_campaign/main.py +++ b/src/skills/email_campaign/main.py @@ -1,22 +1,55 @@ -import os -import sys -import json +"""Email Campaign skill - generates and sends email sequences.""" + +from __future__ import annotations + import logging +from typing import Any, Optional + +from skills.base import BaseSkill, SkillResult -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) -def main(): - skill_name = "email-campaign" - logger.info(f"Skill {skill_name} invoked") - context = os.getenv("SKILL_CONTEXT", "{}") - logger.info(f"Context: {context}") - gemini_key = os.getenv("GEMINI_API_KEY") - if gemini_key: - logger.info("GEMINI_API_KEY is present") - else: - logger.warning("GEMINI_API_KEY is missing") - print(json.dumps({"status": "success", "skill": skill_name})) - -if __name__ == "__main__": - main() + +class EmailCampaignSkill(BaseSkill): + """Generate and dispatch email campaign sequences.""" + + skill_id = "email-campaign" + name = "Email Campaign" + version = "1.0.0" + triggers = ["crm.lead.scored"] + required_env_vars = ["GEMINI_API_KEY", "DATABASE_URL"] + + def __init__(self, dependencies: Optional[dict[str, Any]] = None): + super().__init__(dependencies) + self.email_service = self.dependencies.get("email_service") + + async def execute(self, payload: dict[str, Any]) -> SkillResult: + """Generate an email campaign sequence. + + Expected payload keys: + - lead_id: str - the target lead + - campaign_type: str - "nurture" | "onboarding" | "re-engagement" + - template_id: str - optional template override + """ + lead_id = payload.get("lead_id") + if not lead_id: + return SkillResult(status="error", error="Missing 'lead_id' in payload") + + campaign_type = payload.get("campaign_type", "nurture") + + logger.info( + "Generating %s email campaign for lead %s", campaign_type, lead_id + ) + + if self.email_service: + logger.info("Using injected email_service for campaign dispatch") + + return SkillResult( + status="success", + output={ + "lead_id": lead_id, + "campaign_type": campaign_type, + "generated": True, + "message": f"Email campaign ({campaign_type}) queued for lead {lead_id}", + }, + ) diff --git a/src/skills/lead_scorer/__init__.py b/src/skills/lead_scorer/__init__.py new file mode 100644 index 000000000..7b8e04248 --- /dev/null +++ b/src/skills/lead_scorer/__init__.py @@ -0,0 +1 @@ +"""Lead Scorer skill module.""" diff --git a/src/skills/lead_scorer/main.py b/src/skills/lead_scorer/main.py index fab562ee3..863c57c8e 100644 --- a/src/skills/lead_scorer/main.py +++ b/src/skills/lead_scorer/main.py @@ -1,22 +1,52 @@ -import os -import sys -import json +"""Lead Scorer skill - scores leads based on engagement signals.""" + +from __future__ import annotations + import logging +from typing import Any, Optional + +from skills.base import BaseSkill, SkillResult -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) -def main(): - skill_name = "lead-scorer" - logger.info(f"Skill {skill_name} invoked") - context = os.getenv("SKILL_CONTEXT", "{}") - logger.info(f"Context: {context}") - gemini_key = os.getenv("GEMINI_API_KEY") - if gemini_key: - logger.info("GEMINI_API_KEY is present") - else: - logger.warning("GEMINI_API_KEY is missing") - print(json.dumps({"status": "success", "skill": skill_name})) - -if __name__ == "__main__": - main() + +class LeadScorerSkill(BaseSkill): + """Score leads based on video engagement and interaction signals.""" + + skill_id = "lead-scorer" + name = "Lead Scorer" + version = "1.0.0" + triggers = ["youtube.analytics.updated"] + required_env_vars = ["DATABASE_URL"] + + def __init__(self, dependencies: Optional[dict[str, Any]] = None): + super().__init__(dependencies) + self.db = self.dependencies.get("database_service") + + async def execute(self, payload: dict[str, Any]) -> SkillResult: + """Score a lead based on engagement signals. + + Expected payload keys: + - lead_id: str - the lead identifier + - signals: dict - engagement signals (views, comments, shares, etc.) + """ + lead_id = payload.get("lead_id") + if not lead_id: + return SkillResult(status="error", error="Missing 'lead_id' in payload") + + signals = payload.get("signals", {}) + + logger.info("Scoring lead %s with %d signals", lead_id, len(signals)) + + if self.db: + logger.info("Using injected database_service for lead scoring") + + return SkillResult( + status="success", + output={ + "lead_id": lead_id, + "scored": True, + "signal_count": len(signals), + "message": f"Lead {lead_id} scoring queued", + }, + ) diff --git a/src/skills/seo_optimizer/__init__.py b/src/skills/seo_optimizer/__init__.py new file mode 100644 index 000000000..b25eb1538 --- /dev/null +++ b/src/skills/seo_optimizer/__init__.py @@ -0,0 +1 @@ +"""SEO Optimizer skill module.""" diff --git a/src/skills/seo_optimizer/main.py b/src/skills/seo_optimizer/main.py index e35a46044..884736232 100644 --- a/src/skills/seo_optimizer/main.py +++ b/src/skills/seo_optimizer/main.py @@ -1,22 +1,58 @@ -import os -import sys -import json +"""SEO Optimizer skill - optimizes video titles, descriptions, and tags.""" + +from __future__ import annotations + import logging +from typing import Any, Optional + +from skills.base import BaseSkill, SkillResult -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) -def main(): - skill_name = "seo-optimizer" - logger.info(f"Skill {skill_name} invoked") - context = os.getenv("SKILL_CONTEXT", "{}") - logger.info(f"Context: {context}") - gemini_key = os.getenv("GEMINI_API_KEY") - if gemini_key: - logger.info("GEMINI_API_KEY is present") - else: - logger.warning("GEMINI_API_KEY is missing") - print(json.dumps({"status": "success", "skill": skill_name})) - -if __name__ == "__main__": - main() + +class SEOOptimizerSkill(BaseSkill): + """Optimize video metadata for search engine discoverability.""" + + skill_id = "seo-optimizer" + name = "SEO Optimizer" + version = "1.0.0" + triggers = ["youtube.video.uploaded"] + required_env_vars = ["GEMINI_API_KEY"] + + def __init__(self, dependencies: Optional[dict[str, Any]] = None): + super().__init__(dependencies) + self.gemini = self.dependencies.get("gemini_service") + + async def execute(self, payload: dict[str, Any]) -> SkillResult: + """Optimize SEO metadata for a video. + + Expected payload keys: + - video_id: str - the video identifier + - title: str - current video title + - description: str - current description + - tags: list[str] - current tags + """ + video_id = payload.get("video_id") + if not video_id: + return SkillResult(status="error", error="Missing 'video_id' in payload") + + title = payload.get("title", "") + description = payload.get("description", "") + tags = payload.get("tags", []) + + logger.info("Optimizing SEO for video %s", video_id) + + if self.gemini: + logger.info("Using injected gemini_service for SEO optimization") + + return SkillResult( + status="success", + output={ + "video_id": video_id, + "optimized": True, + "original_title": title, + "original_description": description, + "original_tags": tags, + "message": f"SEO optimization queued for video {video_id}", + }, + ) diff --git a/src/skills/social_scheduler/__init__.py b/src/skills/social_scheduler/__init__.py new file mode 100644 index 000000000..27cd454e7 --- /dev/null +++ b/src/skills/social_scheduler/__init__.py @@ -0,0 +1 @@ +"""Social Scheduler skill module.""" diff --git a/src/skills/social_scheduler/main.py b/src/skills/social_scheduler/main.py index 1c7d73abd..f36212bfd 100644 --- a/src/skills/social_scheduler/main.py +++ b/src/skills/social_scheduler/main.py @@ -1,22 +1,58 @@ -import os -import sys -import json +"""Social Scheduler skill - schedules cross-platform social media posts.""" + +from __future__ import annotations + import logging +from typing import Any, Optional + +from skills.base import BaseSkill, SkillResult -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) -def main(): - skill_name = "social-scheduler" - logger.info(f"Skill {skill_name} invoked") - context = os.getenv("SKILL_CONTEXT", "{}") - logger.info(f"Context: {context}") - gemini_key = os.getenv("GEMINI_API_KEY") - if gemini_key: - logger.info("GEMINI_API_KEY is present") - else: - logger.warning("GEMINI_API_KEY is missing") - print(json.dumps({"status": "success", "skill": skill_name})) - -if __name__ == "__main__": - main() + +class SocialSchedulerSkill(BaseSkill): + """Schedule and publish social media posts across platforms.""" + + skill_id = "social-scheduler" + name = "Social Scheduler" + version = "1.0.0" + triggers = ["ai.content.generated"] + required_env_vars = ["GEMINI_API_KEY"] + + def __init__(self, dependencies: Optional[dict[str, Any]] = None): + super().__init__(dependencies) + self.social_api = self.dependencies.get("social_api_service") + + async def execute(self, payload: dict[str, Any]) -> SkillResult: + """Schedule social media posts. + + Expected payload keys: + - content: str - the content to post + - platforms: list[str] - target platforms (e.g. ["twitter", "linkedin"]) + - schedule_time: str - ISO 8601 timestamp (optional, defaults to now) + """ + content = payload.get("content") + if not content: + return SkillResult(status="error", error="Missing 'content' in payload") + + platforms = payload.get("platforms", ["twitter", "linkedin"]) + schedule_time = payload.get("schedule_time") + + logger.info( + "Scheduling post to %s (scheduled: %s)", + platforms, + schedule_time or "immediate", + ) + + if self.social_api: + logger.info("Using injected social_api_service for scheduling") + + return SkillResult( + status="success", + output={ + "platforms": platforms, + "scheduled": True, + "schedule_time": schedule_time, + "message": f"Posts scheduled for {len(platforms)} platform(s)", + }, + ) diff --git a/src/unified_ai_sdk/unified_ai_sdk.py b/src/unified_ai_sdk/unified_ai_sdk.py index 0d791c3b8..bae994d39 100644 --- a/src/unified_ai_sdk/unified_ai_sdk.py +++ b/src/unified_ai_sdk/unified_ai_sdk.py @@ -5,9 +5,11 @@ import asyncio import logging import os +import re +from collections.abc import Awaitable from dataclasses import dataclass, field from enum import Enum -from typing import Any, Callable, Awaitable +from typing import Any, Callable from .rate_limiter import ModelProvider, RateLimiter @@ -247,15 +249,28 @@ def _should_retry(self, exc: Exception) -> bool: # General network errors or specific string-based checks for Gemini exc_str = str(exc).lower() + # Match common status formats like "400 INVALID_ARGUMENT" or + # "response: 500" without treating incidental counts as statuses. + status_match = re.match(r"\s*(\d{3})\b", exc_str) or re.search( + r"\b(?:http(?: status)?|response|status(?:_code)?|code)\s*[:=]\s*(\d{3})\b", + exc_str, + ) + if status_match: + status_code = int(status_match.group(1)) + if status_code == 429 or status_code >= 500: + return True + if 400 <= status_code < 500: + return False + if "timeout" in exc_str or "deadline exceeded" in exc_str: return True - if "rate limit" in exc_str or "429" in exc_str: + if "rate limit" in exc_str: return True - if "internal server error" in exc_str or "500" in exc_str or "503" in exc_str: + if "internal server error" in exc_str: return True # Auth and Validation errors should not be retried - if any(term in exc_str for term in ["authentication", "unauthorized", "api_key", "invalid_request", "400", "401", "403"]): + if any(term in exc_str for term in ["authentication", "unauthorized", "api_key", "invalid_request"]): return False return True diff --git a/src/uvai/api/v1/services/issue_tracker.py b/src/uvai/api/v1/services/issue_tracker.py index 84d750ad5..c965dbe2f 100644 --- a/src/uvai/api/v1/services/issue_tracker.py +++ b/src/uvai/api/v1/services/issue_tracker.py @@ -299,7 +299,7 @@ async def track_issue( Returns the issue ID. """ async with self._lock: - error_signature = error_type or hashlib.md5(error_message.encode()).hexdigest()[:12] + error_signature = error_type or hashlib.sha256(error_message.encode()).hexdigest()[:12] # Check for recurrence existing_issue = self._detect_recurrence(error_signature, component) diff --git a/src/youtube_extension/backend/api/v1/router.py b/src/youtube_extension/backend/api/v1/router.py index 9b870a9ca..9e7107292 100644 --- a/src/youtube_extension/backend/api/v1/router.py +++ b/src/youtube_extension/backend/api/v1/router.py @@ -1304,11 +1304,28 @@ async def startup_event(): def _persist_video_job(job: VideoJobStatusResponse) -> None: + """Persist job state. Uses a background task for expensive serialization to avoid blocking.""" _video_jobs[job.job_id] = job + + def _sync_persist(): + try: + # model_dump(mode="json") can be slow for large results (Issue 5) + data = job.model_dump(mode="json") + get_job_store().save(job.job_id, data) + except Exception as exc: + logger.warning("Job persist failed for %s: %s", job.job_id, exc) + + # If we are in an async loop, offload serialization and I/O to a thread try: - get_job_store().save(job.job_id, job.model_dump(mode="json")) - except Exception as exc: - logger.warning("Job persist failed for %s: %s", job.job_id, exc) + loop = asyncio.get_running_loop() + if loop.is_running(): + asyncio.create_task(asyncio.to_thread(_sync_persist)) + return + except RuntimeError: + pass + + # Fallback to sync execution if no loop + _sync_persist() def _load_video_job(job_id: str) -> Optional[VideoJobStatusResponse]: diff --git a/src/youtube_extension/backend/services/cache_service.py b/src/youtube_extension/backend/services/cache_service.py index b64b69b0d..7f4f9e0a4 100644 --- a/src/youtube_extension/backend/services/cache_service.py +++ b/src/youtube_extension/backend/services/cache_service.py @@ -69,7 +69,7 @@ def __init__(self, cache_dir: str = None, enhanced_cache_dir: str = None): def _get_cache_key(self, video_url: str) -> str: """Generate cache key from video URL""" - return hashlib.md5(video_url.encode()).hexdigest()[:12] + return hashlib.sha256(video_url.encode()).hexdigest()[:12] def get_cached_result(self, video_url: str) -> Optional[dict[str, Any]]: """ diff --git a/src/youtube_extension/backend/services/database_optimizer.py b/src/youtube_extension/backend/services/database_optimizer.py index 556eb35e6..5fefc35a0 100644 --- a/src/youtube_extension/backend/services/database_optimizer.py +++ b/src/youtube_extension/backend/services/database_optimizer.py @@ -289,7 +289,7 @@ def _get_query_hash(self, query: str) -> str: normalized = re.sub(r"\b\d+\b", "?", normalized) # Replace numbers with ? normalized = re.sub(r"'[^']*'", "'?'", normalized) # Replace string literals - return hashlib.md5(normalized.encode()).hexdigest() + return hashlib.sha256(normalized.encode()).hexdigest() def _get_query_pattern(self, query: str) -> str: """Extract query pattern for analysis""" @@ -331,7 +331,7 @@ async def execute_query( # Check query cache first if use_cache: - cache_key = f"query:{query_hash}:{hashlib.md5(str(params).encode()).hexdigest() if params else 'no_params'}" + cache_key = f"query:{query_hash}:{hashlib.sha256(str(params).encode()).hexdigest() if params else 'no_params'}" cached_result = await cache_get(cache_key) if cached_result is not None: diff --git a/src/youtube_extension/backend/services/horizontal_scaling_system.py b/src/youtube_extension/backend/services/horizontal_scaling_system.py index f7795b836..07fb9d968 100644 --- a/src/youtube_extension/backend/services/horizontal_scaling_system.py +++ b/src/youtube_extension/backend/services/horizontal_scaling_system.py @@ -258,7 +258,7 @@ def _consistent_hash_selection(self, instances: list[ServiceInstance], request_m return self._performance_based_selection(instances) # Create hash key from request metadata - hash_key = hashlib.md5(json.dumps(request_metadata, sort_keys=True).encode()).hexdigest() + hash_key = hashlib.sha256(json.dumps(request_metadata, sort_keys=True).encode()).hexdigest() hash_value = int(hash_key[:8], 16) # Use first 8 chars # Select instance based on hash diff --git a/src/youtube_extension/backend/services/intelligent_cache.py b/src/youtube_extension/backend/services/intelligent_cache.py index 979c354ad..b8e4505dc 100644 --- a/src/youtube_extension/backend/services/intelligent_cache.py +++ b/src/youtube_extension/backend/services/intelligent_cache.py @@ -713,7 +713,7 @@ def cache_key(*args, **kwargs) -> str: key_parts = [str(arg) for arg in args] key_parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items())) key_string = ":".join(key_parts) - return hashlib.md5(key_string.encode()).hexdigest() + return hashlib.sha256(key_string.encode()).hexdigest() def cached(ttl: Optional[int] = None, tags: list[str] = None, key_prefix: str = ""): """Decorator for caching function results""" diff --git a/src/youtube_extension/backend/services/load_balancer.py b/src/youtube_extension/backend/services/load_balancer.py index 75eed514a..7163c2c32 100644 --- a/src/youtube_extension/backend/services/load_balancer.py +++ b/src/youtube_extension/backend/services/load_balancer.py @@ -325,7 +325,7 @@ def _select_service(self, services: list[ServiceInstance], request_data: dict[st elif self.algorithm == LoadBalancingAlgorithm.IP_HASH: if request_data and 'client_ip' in request_data: - hash_value = int(hashlib.md5(request_data['client_ip'].encode()).hexdigest(), 16) + hash_value = int(hashlib.sha256(request_data['client_ip'].encode()).hexdigest(), 16) return services[hash_value % len(services)] else: return random.choice(services) diff --git a/src/youtube_extension/backend/services/metrics_service.py b/src/youtube_extension/backend/services/metrics_service.py index b38318512..feefb7586 100644 --- a/src/youtube_extension/backend/services/metrics_service.py +++ b/src/youtube_extension/backend/services/metrics_service.py @@ -326,6 +326,10 @@ async def _persist_metrics(self) -> None: except Exception as e: logger.error(f"Failed to persist metrics: {e}") + async def persist_metrics(self) -> None: + """Persist metrics to disk.""" + await self._persist_metrics() + async def load_persisted_metrics(self) -> bool: """ Load previously persisted metrics. diff --git a/src/youtube_extension/backend/services/real_video_processor.py b/src/youtube_extension/backend/services/real_video_processor.py index 083d4f86a..d2d5ba909 100644 --- a/src/youtube_extension/backend/services/real_video_processor.py +++ b/src/youtube_extension/backend/services/real_video_processor.py @@ -64,7 +64,7 @@ def __init__(self): def _get_cache_key(self, video_url: str) -> str: """Generate cache key for video URL""" - return hashlib.md5(video_url.encode()).hexdigest()[:12] + return hashlib.sha256(video_url.encode()).hexdigest()[:12] def _get_cache_path(self, video_id: str) -> Path: """Get cache file path for video""" diff --git a/src/youtube_extension/core/mcp/server_registry.py b/src/youtube_extension/core/mcp/server_registry.py index ccd5a9e68..302155ec4 100644 --- a/src/youtube_extension/core/mcp/server_registry.py +++ b/src/youtube_extension/core/mcp/server_registry.py @@ -471,7 +471,7 @@ async def register_ai_server( name: str, endpoint: str, capabilities: list[ServerCapability] ) -> MCPServer: """Convenience function to register an AI server""" - server_id = f"ai-{name.lower().replace(' ', '-')}-{hashlib.md5(endpoint.encode()).hexdigest()[:8]}" + server_id = f"ai-{name.lower().replace(' ', '-')}-{hashlib.sha256(endpoint.encode()).hexdigest()[:8]}" return get_server_registry().register_server( id=server_id, name=name, endpoint=endpoint, capabilities=capabilities ) diff --git a/src/youtube_extension/mcp/enterprise_mcp_server.py b/src/youtube_extension/mcp/enterprise_mcp_server.py index d3c3ca139..57a73d3a6 100644 --- a/src/youtube_extension/mcp/enterprise_mcp_server.py +++ b/src/youtube_extension/mcp/enterprise_mcp_server.py @@ -476,7 +476,7 @@ async def extract_video_content_enterprise(arguments: dict) -> CallToolResult: ) # Check cache first - cache_key = f"video_content_{hashlib.md5(video_url.encode()).hexdigest()}" + cache_key = f"video_content_{hashlib.sha256(video_url.encode()).hexdigest()}" if cache_key in self.processing_cache and self.cache_ttl.get(cache_key, 0) > time.time(): self.metrics.record_counter("video_extraction.cache_hit") cached_result = self.processing_cache[cache_key] diff --git a/src/youtube_extension/processors/strategies.py b/src/youtube_extension/processors/strategies.py index 676b95736..e0a608b07 100644 --- a/src/youtube_extension/processors/strategies.py +++ b/src/youtube_extension/processors/strategies.py @@ -204,7 +204,7 @@ async def process_video( Process video with all optimizations enabled """ start_time = time.time() - processing_id = hashlib.md5(f"{video_url}_{time.time()}".encode()).hexdigest()[ + processing_id = hashlib.sha256(f"{video_url}_{time.time()}".encode()).hexdigest()[ :8 ] @@ -214,7 +214,7 @@ async def process_video( try: # Check cache first - cache_key = f"optimized_video:{hashlib.md5(video_url.encode()).hexdigest()}" + cache_key = f"optimized_video:{hashlib.sha256(video_url.encode()).hexdigest()}" cached_result = await cache_get(cache_key) if cached_result and self.config.get("enable_intelligent_caching", True): @@ -287,7 +287,7 @@ async def process_video( self._enhanced_strategy = EnhancedStrategy(self.config) start_time = time.time() - processing_id = hashlib.md5( + processing_id = hashlib.sha256( f"{video_url}_{time.time()}".encode() ).hexdigest()[:8] diff --git a/src/youtube_extension/services/pipeline_job_store.py b/src/youtube_extension/services/pipeline_job_store.py index 69ea7338a..e3b13928f 100644 --- a/src/youtube_extension/services/pipeline_job_store.py +++ b/src/youtube_extension/services/pipeline_job_store.py @@ -5,6 +5,7 @@ import json import logging import os +import tempfile from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional @@ -27,7 +28,27 @@ def _path(self, job_id: str) -> Path: def save(self, job_id: str, payload: dict[str, Any]) -> None: path = self._path(job_id) - path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") + data = json.dumps(payload, ensure_ascii=False, indent=2) + # Write atomically: serialize to a temp file in the same directory then + # os.replace() onto the target path. os.replace is atomic on POSIX and + # Windows, so concurrent persists and any concurrent reader + # (load()/list_recent()/expire_before()) never observe a truncated or + # partially written file. + fd, tmp_name = tempfile.mkstemp( + dir=str(path.parent), prefix=f".{path.stem}.", suffix=".tmp" + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(data) + fh.flush() + os.fsync(fh.fileno()) + os.replace(tmp_name, path) + except BaseException: + try: + os.unlink(tmp_name) + except OSError: + pass + raise def load(self, job_id: str) -> Optional[dict[str, Any]]: path = self._path(job_id) diff --git a/tests/test_skills_integration.py b/tests/test_skills_integration.py index 25207376f..646c22934 100644 --- a/tests/test_skills_integration.py +++ b/tests/test_skills_integration.py @@ -1,88 +1,359 @@ -import os +"""Integration tests for GTM skill discovery and invocation. + +Tests verify: +- SkillRegistry discovers all 7 GTM skills from skills-lock.json +- Skills can be invoked and return expected results +- Trigger-based skill matching works correctly +- Env var pass-through works without relying on inheritance +""" + +from __future__ import annotations + import json -import pytest -import asyncio -from unittest.mock import MagicMock, patch +import os import sys +import types +from pathlib import Path +from unittest.mock import patch + +import pytest -# Ensure src is in path -sys.path.append(os.path.join(os.getcwd(), "src")) +# Ensure src is on path for imports +_SRC = Path(__file__).resolve().parents[1] / "src" +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) -# Mock dependencies that cause issues during import -# Using MagicMock for packages needs __path__ to be set if they are used in imports -mock_google = MagicMock() -mock_google.__path__ = [] -sys.modules['google'] = mock_google +_REPO_ROOT = Path(__file__).resolve().parents[1] -mock_google_cloud = MagicMock() -mock_google_cloud.__path__ = [] -sys.modules['google.cloud'] = mock_google_cloud +# Avoid importing the full agents package (which pulls heavy deps like aiohttp). +# Instead, import the coordinator module directly. +_agents_pkg = sys.modules.get("agents") +if _agents_pkg is None: + _agents_pkg = types.ModuleType("agents") + _agents_pkg.__path__ = [str(_SRC / "agents")] # type: ignore[attr-defined] + _agents_pkg.__package__ = "agents" + sys.modules["agents"] = _agents_pkg -sys.modules['google.genai'] = MagicMock() -sys.modules['google.generativeai'] = MagicMock() -sys.modules['google.cloud.aiplatform'] = MagicMock() -sys.modules['vertexai'] = MagicMock() -sys.modules['vertexai.generative_models'] = MagicMock() +# Stub youtube_extension.processors to avoid pulling in heavy ML deps +for _mod_name in [ + "youtube_extension", + "youtube_extension.processors", + "youtube_extension.processors.enhanced_extractor", +]: + if _mod_name not in sys.modules: + _stub = types.ModuleType(_mod_name) + _stub.__path__ = [] # type: ignore[attr-defined] + _stub.__package__ = _mod_name + # Provide stub classes so the coordinator imports fine + if _mod_name == "youtube_extension.processors.enhanced_extractor": + _stub.EnhancedVideoExtractor = type("EnhancedVideoExtractor", (), {}) # type: ignore[attr-defined] + _stub.VideoContent = type("VideoContent", (), {}) # type: ignore[attr-defined] + sys.modules[_mod_name] = _stub -sys.modules['aiohttp'] = MagicMock() -sys.modules['pandas'] = MagicMock() -sys.modules['youtube_transcript_api'] = MagicMock() -sys.modules['youtube_extension.processors.enhanced_extractor'] = MagicMock() -sys.modules['youtube_extension.services.pipeline_audit_store'] = MagicMock() +# Now we can safely import just the coordinator module +from agents.mcp_ecosystem_coordinator import SkillRegistry # noqa: E402 + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +LOCK_FILE = str(_REPO_ROOT / "skills-lock.json") -# Import SkillRegistry after mocking -from agents.mcp_ecosystem_coordinator import SkillRegistry @pytest.fixture -def skill_registry(): - # Use the real skills-lock.json created during the task - return SkillRegistry(lock_file="skills-lock.json") - -def test_skill_discovery(skill_registry): - """Verify that all 7 GTM skills are discovered from skills-lock.json.""" - skills = skill_registry.list_skills(source="uvai-skills") - assert len(skills) == 7 - - expected_ids = [ - "content-generation", - "seo-optimizer", - "social-scheduler", - "lead-scorer", - "email-campaign", - "analytics-dashboard", - "ab-testing" - ] - - discovered_ids = [s["id"] for s in skills] - for skill_id in expected_ids: - assert skill_id in discovered_ids - -@pytest.mark.asyncio -async def test_skill_invocation(skill_registry): - """Verify that a skill can be invoked and returns the expected result.""" - # We use content-generation for testing invocation - skill_id = "content-generation" - context = {"video_id": "test_123", "transcript": "Hello world"} - - # We expect this to work because we created the thin wrapper main.py - result = await skill_registry.invoke_skill(skill_id, context) - - assert result["status"] == "success" - assert result["skill"] == skill_id - -@pytest.mark.asyncio -async def test_skill_invocation_env_vars(skill_registry): - """Verify that environment variables are passed (simulated).""" - with patch("subprocess.run") as mock_run: - mock_run.return_value.stdout = json.dumps({"status": "success"}) - mock_run.return_value.returncode = 0 - - os.environ["GEMINI_API_KEY"] = "test_key" - - await skill_registry.invoke_skill("content-generation", {}) - - # Check that the env passed to subprocess.run contains GEMINI_API_KEY - args, kwargs = mock_run.call_args - passed_env = kwargs.get("env", {}) - assert passed_env.get("GEMINI_API_KEY") == "test_key" - assert "SKILL_CONTEXT" in passed_env +def registry() -> SkillRegistry: + """Create a SkillRegistry pointed at the repo's skills-lock.json.""" + return SkillRegistry(lock_file_path=LOCK_FILE) + + +# --------------------------------------------------------------------------- +# Discovery tests +# --------------------------------------------------------------------------- + + +class TestSkillDiscovery: + """Verify that SkillRegistry can discover all 7 GTM skills.""" + + def test_list_skills_returns_seven(self, registry: SkillRegistry) -> None: + skills = registry.list_skills() + assert len(skills) == 7 + + def test_all_expected_skill_ids_present(self, registry: SkillRegistry) -> None: + skills = registry.list_skills() + skill_ids = {s["id"] for s in skills} + expected = { + "content-generation", + "seo-optimizer", + "social-scheduler", + "lead-scorer", + "email-campaign", + "analytics-dashboard", + "ab-testing", + } + assert skill_ids == expected + + def test_each_skill_has_required_metadata(self, registry: SkillRegistry) -> None: + skills = registry.list_skills() + for skill in skills: + assert "id" in skill + assert "name" in skill + assert "version" in skill + assert "triggers" in skill + assert "entry_point" in skill + assert isinstance(skill["triggers"], list) + assert len(skill["triggers"]) >= 1 + + def test_get_skill_by_id(self, registry: SkillRegistry) -> None: + skill = registry.get_skill("content-generation") + assert skill is not None + assert skill["id"] == "content-generation" + assert skill["name"] == "Content Generation" + assert skill["class_name"] == "ContentGenerationSkill" + assert skill["version"] == "1.0.0" + assert "youtube.video.published" in skill["triggers"] + + def test_get_nonexistent_skill_returns_none(self, registry: SkillRegistry) -> None: + assert registry.get_skill("nonexistent-skill") is None + + +# --------------------------------------------------------------------------- +# Trigger matching tests +# --------------------------------------------------------------------------- + + +class TestSkillTriggerMatching: + """Verify trigger-based skill discovery.""" + + def test_video_published_triggers_content_generation( + self, registry: SkillRegistry + ) -> None: + skills = registry.get_skills_for_trigger("youtube.video.published") + skill_ids = {s["id"] for s in skills} + assert "content-generation" in skill_ids + + def test_video_uploaded_triggers_seo_and_ab( + self, registry: SkillRegistry + ) -> None: + skills = registry.get_skills_for_trigger("youtube.video.uploaded") + skill_ids = {s["id"] for s in skills} + assert "seo-optimizer" in skill_ids + assert "ab-testing" in skill_ids + + def test_content_generated_triggers_social_scheduler( + self, registry: SkillRegistry + ) -> None: + skills = registry.get_skills_for_trigger("ai.content.generated") + skill_ids = {s["id"] for s in skills} + assert "social-scheduler" in skill_ids + + def test_analytics_updated_triggers_lead_scorer( + self, registry: SkillRegistry + ) -> None: + skills = registry.get_skills_for_trigger("youtube.analytics.updated") + skill_ids = {s["id"] for s in skills} + assert "lead-scorer" in skill_ids + + def test_lead_scored_triggers_email_campaign( + self, registry: SkillRegistry + ) -> None: + skills = registry.get_skills_for_trigger("crm.lead.scored") + skill_ids = {s["id"] for s in skills} + assert "email-campaign" in skill_ids + + def test_daily_cron_triggers_analytics_dashboard( + self, registry: SkillRegistry + ) -> None: + skills = registry.get_skills_for_trigger("system.cron.daily") + skill_ids = {s["id"] for s in skills} + assert "analytics-dashboard" in skill_ids + + def test_unknown_trigger_returns_empty(self, registry: SkillRegistry) -> None: + skills = registry.get_skills_for_trigger("unknown.event.type") + assert skills == [] + + +# --------------------------------------------------------------------------- +# Invocation tests +# --------------------------------------------------------------------------- + + +class TestSkillInvocation: + """Verify that skills can be invoked with payloads.""" + + @pytest.mark.asyncio + async def test_invoke_content_generation_success( + self, registry: SkillRegistry + ) -> None: + result = await registry.invoke_skill( + "content-generation", + {"transcript": "Hello world test transcript", "video_id": "auJzb1D-fag"}, + ) + assert result["status"] == "success" + assert result["output"]["video_id"] == "auJzb1D-fag" + assert result["output"]["generated"] is True + + @pytest.mark.asyncio + async def test_invoke_content_generation_missing_transcript( + self, registry: SkillRegistry + ) -> None: + result = await registry.invoke_skill( + "content-generation", + {"video_id": "auJzb1D-fag"}, + ) + assert result["status"] == "error" + assert "transcript" in (result.get("error") or "") + + @pytest.mark.asyncio + async def test_invoke_seo_optimizer_success( + self, registry: SkillRegistry + ) -> None: + result = await registry.invoke_skill( + "seo-optimizer", + {"video_id": "auJzb1D-fag", "title": "Test Video", "tags": ["ai"]}, + ) + assert result["status"] == "success" + assert result["output"]["optimized"] is True + + @pytest.mark.asyncio + async def test_invoke_social_scheduler_success( + self, registry: SkillRegistry + ) -> None: + result = await registry.invoke_skill( + "social-scheduler", + {"content": "Check out this video!", "platforms": ["twitter"]}, + ) + assert result["status"] == "success" + assert result["output"]["scheduled"] is True + + @pytest.mark.asyncio + async def test_invoke_lead_scorer_success( + self, registry: SkillRegistry + ) -> None: + result = await registry.invoke_skill( + "lead-scorer", + {"lead_id": "lead_001", "signals": {"views": 100, "comments": 5}}, + ) + assert result["status"] == "success" + assert result["output"]["lead_id"] == "lead_001" + + @pytest.mark.asyncio + async def test_invoke_email_campaign_success( + self, registry: SkillRegistry + ) -> None: + result = await registry.invoke_skill( + "email-campaign", + {"lead_id": "lead_001", "campaign_type": "nurture"}, + ) + assert result["status"] == "success" + assert result["output"]["campaign_type"] == "nurture" + + @pytest.mark.asyncio + async def test_invoke_analytics_dashboard_success( + self, registry: SkillRegistry + ) -> None: + result = await registry.invoke_skill( + "analytics-dashboard", + {"date_range": "2024-01-01/2024-01-31"}, + ) + assert result["status"] == "success" + assert result["output"]["generated"] is True + + @pytest.mark.asyncio + async def test_invoke_ab_testing_success( + self, registry: SkillRegistry + ) -> None: + result = await registry.invoke_skill( + "ab-testing", + { + "video_id": "auJzb1D-fag", + "test_type": "thumbnail", + "variants": [{"url": "thumb1.jpg"}, {"url": "thumb2.jpg"}], + }, + ) + assert result["status"] == "success" + assert result["output"]["variant_count"] == 2 + + @pytest.mark.asyncio + async def test_invoke_nonexistent_skill(self, registry: SkillRegistry) -> None: + result = await registry.invoke_skill("nonexistent", {"foo": "bar"}) + assert result["status"] == "error" + + +# --------------------------------------------------------------------------- +# MCP env pass-through tests +# --------------------------------------------------------------------------- + + +class TestEnvPassthrough: + """Verify explicit env var pass-through for skill subprocesses.""" + + def test_gemini_skill_gets_api_key(self, registry: SkillRegistry) -> None: + with patch.dict(os.environ, {"GEMINI_API_KEY": "test-key-123"}): + env = registry.get_env_for_skill("content-generation") + assert env["GEMINI_API_KEY"] == "test-key-123" + + def test_database_skill_gets_database_url( + self, registry: SkillRegistry + ) -> None: + with patch.dict(os.environ, {"DATABASE_URL": "sqlite:///test.db"}): + env = registry.get_env_for_skill("lead-scorer") + assert env["DATABASE_URL"] == "sqlite:///test.db" + + def test_multi_dep_skill_gets_both_vars(self, registry: SkillRegistry) -> None: + with patch.dict( + os.environ, + {"GEMINI_API_KEY": "gkey", "DATABASE_URL": "sqlite:///test.db"}, + ): + env = registry.get_env_for_skill("ab-testing") + assert env["GEMINI_API_KEY"] == "gkey" + assert env["DATABASE_URL"] == "sqlite:///test.db" + + def test_missing_env_var_not_included(self, registry: SkillRegistry) -> None: + with patch.dict(os.environ, {}, clear=True): + # Remove the vars if they exist + os.environ.pop("GEMINI_API_KEY", None) + os.environ.pop("DATABASE_URL", None) + env = registry.get_env_for_skill("content-generation") + assert "GEMINI_API_KEY" not in env + + def test_nonexistent_skill_env_empty(self, registry: SkillRegistry) -> None: + env = registry.get_env_for_skill("nonexistent") + assert env == {} + + +# --------------------------------------------------------------------------- +# Lock file validation +# --------------------------------------------------------------------------- + + +class TestSkillsLockFile: + """Verify skills-lock.json structure and validity.""" + + def test_lock_file_is_valid_json(self) -> None: + with open(LOCK_FILE) as f: + data = json.load(f) + assert "skills" in data + assert isinstance(data["skills"], dict) + + def test_lock_file_contains_gtm_skills(self) -> None: + with open(LOCK_FILE) as f: + data = json.load(f) + gtm_skills = { + k: v + for k, v in data["skills"].items() + if v.get("source") == "uvai-skills" + } + assert len(gtm_skills) == 7 + + def test_each_gtm_skill_has_required_fields(self) -> None: + with open(LOCK_FILE) as f: + data = json.load(f) + for skill_id, meta in data["skills"].items(): + if meta.get("source") != "uvai-skills": + continue + assert "skillPath" in meta, f"{skill_id} missing skillPath" + assert "className" in meta, f"{skill_id} missing className" + assert "version" in meta, f"{skill_id} missing version" + assert "triggers" in meta, f"{skill_id} missing triggers" + assert "dependencies" in meta, f"{skill_id} missing dependencies" diff --git a/tests/unit/test_cache_service.py b/tests/unit/test_cache_service.py index 3c3bd7eab..8d5cde9f6 100644 --- a/tests/unit/test_cache_service.py +++ b/tests/unit/test_cache_service.py @@ -65,8 +65,8 @@ def test_different_urls_different_keys(self, cache): k2 = cache._get_cache_key("https://www.youtube.com/watch?v=bbbbbbbbbbb") assert k1 != k2 - def test_matches_md5_prefix(self, cache): - expected = hashlib.md5(_VIDEO_URL.encode()).hexdigest()[:12] + def test_matches_sha256_prefix(self, cache): + expected = hashlib.sha256(_VIDEO_URL.encode()).hexdigest()[:12] assert cache._get_cache_key(_VIDEO_URL) == expected diff --git a/tests/unit/test_database_optimizer.py b/tests/unit/test_database_optimizer.py index ad880e12d..2744cbedc 100644 --- a/tests/unit/test_database_optimizer.py +++ b/tests/unit/test_database_optimizer.py @@ -1092,6 +1092,7 @@ async def test_health_check_error_response_has_error_key(self): class TestConvenienceFunctions: """Tests for module-level convenience functions""" + @pytest.mark.asyncio async def test_execute_optimized_query_delegates(self, tmp_path) -> None: from youtube_extension.backend.services import database_optimizer as _mod orig = _mod.query_optimizer.execute_query @@ -1102,6 +1103,7 @@ async def test_execute_optimized_query_delegates(self, tmp_path) -> None: finally: _mod.query_optimizer.execute_query = orig + @pytest.mark.asyncio async def test_execute_batch_delegates(self) -> None: from youtube_extension.backend.services import database_optimizer as _mod orig = _mod.query_optimizer.execute_batch_queries @@ -1112,6 +1114,7 @@ async def test_execute_batch_delegates(self) -> None: finally: _mod.query_optimizer.execute_batch_queries = orig + @pytest.mark.asyncio async def test_get_database_performance_report_delegates(self) -> None: from youtube_extension.backend.services import database_optimizer as _mod orig = _mod.query_optimizer.get_performance_report @@ -1122,6 +1125,7 @@ async def test_get_database_performance_report_delegates(self) -> None: finally: _mod.query_optimizer.get_performance_report = orig + @pytest.mark.asyncio async def test_get_database_health_status_delegates(self) -> None: from youtube_extension.backend.services import database_optimizer as _mod orig = _mod.health_monitor.run_health_check @@ -1132,6 +1136,7 @@ async def test_get_database_health_status_delegates(self) -> None: finally: _mod.health_monitor.run_health_check = orig + @pytest.mark.asyncio async def test_initialize_database_optimization_calls_initialize(self, tmp_path) -> None: from youtube_extension.backend.services import database_optimizer as _mod orig_init = _mod.connection_pool.initialize @@ -1151,6 +1156,7 @@ async def test_initialize_database_optimization_calls_initialize(self, tmp_path) _mod.connection_pool.get_connection = orig_get _mod.connection_pool.release_connection = orig_rel + @pytest.mark.asyncio async def test_shutdown_database_optimization_calls_close(self) -> None: from youtube_extension.backend.services import database_optimizer as _mod orig = _mod.connection_pool.close @@ -1172,6 +1178,7 @@ def _make_pool(self) -> MagicMock: pool.release_connection = AsyncMock() return pool + @pytest.mark.asyncio async def test_batch_executes_all_queries(self, tmp_path) -> None: import sqlite3 pool = DatabaseConnectionPool(f"sqlite:///{tmp_path}/test.db") @@ -1184,6 +1191,7 @@ async def test_batch_executes_all_queries(self, tmp_path) -> None: results = await optimizer.execute_batch_queries(queries) assert len(results) == 2 + @pytest.mark.asyncio async def test_batch_exception_propagated(self) -> None: pool = self._make_pool() pool.get_connection.side_effect = RuntimeError("No DB") @@ -1195,16 +1203,19 @@ async def test_batch_exception_propagated(self) -> None: class TestConnectionPoolInitialize: """DatabaseConnectionPool.initialize with different URL types""" + @pytest.mark.asyncio async def test_sqlite_file_creates_dir(self, tmp_path) -> None: db_path = tmp_path / "subdir" / "test.db" pool = DatabaseConnectionPool(f"sqlite:///{db_path}") await pool.initialize() # No error should occur + @pytest.mark.asyncio async def test_sqlite_memory_initializes(self) -> None: pool = DatabaseConnectionPool("sqlite:///:memory:") await pool.initialize() + @pytest.mark.asyncio async def test_haspg_false_uses_sqlite_path(self, tmp_path) -> None: from youtube_extension.backend.services import database_optimizer as _mod orig = _mod.HAS_POSTGRESQL diff --git a/tests/unit/test_intelligent_cache.py b/tests/unit/test_intelligent_cache.py index 7a78e3de1..a4f28a6a2 100644 --- a/tests/unit/test_intelligent_cache.py +++ b/tests/unit/test_intelligent_cache.py @@ -711,10 +711,11 @@ async def test_cache_key_kwargs_sorted(self): k2 = cache_key(b=2, a=1) assert k1 == k2 + @pytest.mark.asyncio async def test_cache_key_returns_hex_string(self): from youtube_extension.backend.services.intelligent_cache import cache_key k = cache_key("test") - assert len(k) == 32 + assert len(k) == 64 # sha256 hex digest (migrated from md5's 32) int(k, 16) # should not raise diff --git a/tests/unit/test_metrics_service.py b/tests/unit/test_metrics_service.py index cdd165746..1da7b4c1f 100644 --- a/tests/unit/test_metrics_service.py +++ b/tests/unit/test_metrics_service.py @@ -3,9 +3,7 @@ from __future__ import annotations import sys -import time import types -from collections import deque from datetime import datetime, timedelta from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch @@ -218,6 +216,19 @@ def test_custom_collection_interval(self, tmp_path, monkeypatch): svc = MetricsService(config={"collection_interval": 30}) assert svc.collection_interval == 30 + +class TestMetricsServicePersistMetrics: + @pytest.mark.asyncio + async def test_persist_metrics_writes_metrics_file(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + svc = MetricsService() + await svc.record_metric("audit.active_sample", 1.0) + + await svc.persist_metrics() + + assert svc.metrics_file.exists() + assert "audit.active_sample" in svc.metrics_file.read_text() + def test_custom_retention_period(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) svc = MetricsService(config={"retention_period": 7200}) diff --git a/tests/unit/test_nightly_audit_agent.py b/tests/unit/test_nightly_audit_agent.py new file mode 100644 index 000000000..5ef3c61a0 --- /dev/null +++ b/tests/unit/test_nightly_audit_agent.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import importlib.util +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + + +def _load_audit_module(): + repo_root = Path(__file__).resolve().parents[2] + module_path = repo_root / "scripts" / "nightly_audit_agent.py" + spec = importlib.util.spec_from_file_location("nightly_audit_agent", module_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +@pytest.mark.asyncio +async def test_scan_logs_uses_extended_72_hour_timeframe(tmp_path, monkeypatch): + module = _load_audit_module() + monkeypatch.chdir(tmp_path) + log_dir = tmp_path / "logs" + log_dir.mkdir() + timestamp = (datetime.now(timezone.utc) - timedelta(hours=48)).isoformat() + (log_dir / "structured_logs.jsonl").write_text( + f'{{"timestamp": "{timestamp}", "status_code": 503, "message": "stale outage"}}\n' + ) + + agent = module.AuditAgent(dry_run=True) + agent.health_service = None + await agent._scan_logs() + + assert agent.lookback_hours == 72 + assert any("stale outage" in issue["description"] for issue in agent.issues) + + +class _RecordingMetricsService: + def __init__(self): + self.started = False + self.stopped = False + self.persisted = False + self.samples = 0 + + async def start_collection(self): + self.started = True + + async def stop_collection(self): + self.stopped = True + + async def get_system_metrics(self): + self.samples += 1 + return {"timestamp": datetime.now(timezone.utc).isoformat()} + + async def persist_metrics(self): + self.persisted = True + + +@pytest.mark.asyncio +async def test_run_audit_collects_active_measurements_before_analysis(tmp_path, monkeypatch): + module = _load_audit_module() + monkeypatch.chdir(tmp_path) + + metrics_service = _RecordingMetricsService() + agent = module.AuditAgent( + dry_run=True, + active_measurement=True, + measurement_samples=2, + measurement_interval=0, + ) + agent.health_service = None + agent.metrics_service = metrics_service + + await agent.run_audit() + + assert metrics_service.started is True + assert metrics_service.samples == 2 + assert metrics_service.persisted is True + assert metrics_service.stopped is True + + +@pytest.mark.asyncio +async def test_active_measurement_uses_fallback_when_metrics_service_unavailable( + tmp_path, monkeypatch +): + module = _load_audit_module() + monkeypatch.chdir(tmp_path) + + agent = module.AuditAgent( + dry_run=True, + active_measurement=True, + measurement_samples=2, + measurement_interval=0, + ) + agent.metrics_service = None + + await agent._collect_active_measurements() + + metrics_file = tmp_path / "logs" / "active_measurements.jsonl" + lines = metrics_file.read_text().strip().splitlines() + assert len(lines) == 2 + assert any("ACTIVE MEASUREMENT" in line for line in agent.report) diff --git a/tests/unit/test_processors_strategies.py b/tests/unit/test_processors_strategies.py index 88fc1afe1..9e99cee1f 100644 --- a/tests/unit/test_processors_strategies.py +++ b/tests/unit/test_processors_strategies.py @@ -264,9 +264,10 @@ async def test_adds_optimization_metadata(self): assert result.get("optimization_applied") is True assert "processing_id" in result + @pytest.mark.asyncio async def test_cache_hit_increments_counter(self): opt = OptimizedStrategy() - cache_key = f"optimized_video:{__import__('hashlib').md5(_VALID_URL.encode()).hexdigest()}" + cache_key = f"optimized_video:{__import__('hashlib').sha256(_VALID_URL.encode()).hexdigest()}" _cache[cache_key] = {"cached": True} try: result = await opt.process_video(_VALID_URL) @@ -277,7 +278,7 @@ async def test_cache_hit_increments_counter(self): async def test_cache_disabled_skips_hit(self): opt = OptimizedStrategy({"enable_intelligent_caching": False}) - cache_key = f"optimized_video:{__import__('hashlib').md5(_VALID_URL.encode()).hexdigest()}" + cache_key = f"optimized_video:{__import__('hashlib').sha256(_VALID_URL.encode()).hexdigest()}" _cache[cache_key] = {"cached": True} try: await opt.process_video(_VALID_URL) diff --git a/tests/unit/test_unified_ai_sdk.py b/tests/unit/test_unified_ai_sdk.py index 310fca5bc..85c489f36 100644 --- a/tests/unit/test_unified_ai_sdk.py +++ b/tests/unit/test_unified_ai_sdk.py @@ -1,8 +1,9 @@ from __future__ import annotations -import pytest -from unittest.mock import AsyncMock, MagicMock, patch from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest import unified_ai_sdk.unified_ai_sdk as sdk_mod from unified_ai_sdk import AIRequest, ModelProvider, TaskType, UnifiedAISDK @@ -205,6 +206,41 @@ async def test_unified_request_does_not_retry_on_auth_error(self): assert result.success is False assert result.metadata["attempts"] == 1 + def test_should_retry_rejects_gemini_400_with_incidental_500(self): + sdk = UnifiedAISDK({"retry_attempts": 3, "retry_base_delay": 0}) + + assert ( + sdk._should_retry( + RuntimeError( + "400 INVALID_ARGUMENT: input token count 500 exceeds model limit" + ) + ) + is False + ) + + def test_should_retry_ignores_non_status_colon_numbers(self): + sdk = UnifiedAISDK({"retry_attempts": 3, "retry_base_delay": 0}) + + assert ( + sdk._should_retry( + RuntimeError("invalid_request: expected 3 items: 503 found") + ) + is False + ) + + @pytest.mark.parametrize( + "message", + [ + "500 INTERNAL: upstream unavailable", + "Response: 500 Internal Server Error", + "429 RESOURCE_EXHAUSTED: quota exceeded", + ], + ) + def test_should_retry_accepts_retryable_status_formats(self, message): + sdk = UnifiedAISDK({"retry_attempts": 3, "retry_base_delay": 0}) + + assert sdk._should_retry(RuntimeError(message)) is True + @pytest.mark.asyncio async def test_structured_output_support(self): sdk = UnifiedAISDK({"retry_attempts": 1}) diff --git a/tests/unit/test_v1_router_extended.py b/tests/unit/test_v1_router_extended.py index 0676ac76e..d64ec76b8 100644 --- a/tests/unit/test_v1_router_extended.py +++ b/tests/unit/test_v1_router_extended.py @@ -31,6 +31,8 @@ "shared", "shared.youtube", "uvai", + "psutil", + "aiohttp", "uvai.ml", "uvai.ml.client", "youtube_extension.services", @@ -103,6 +105,11 @@ def _stub_attr(mod_name: str, attr: str, value=None): # Provide stub for TranscriptActionWorkflow (only if module is a stub) _stub_attr("youtube_extension.services.workflows.transcript_action_workflow", "TranscriptActionWorkflow") +# Stub pipeline_job_store load to return None by default +_job_store = MagicMock() +_job_store.load.return_value = None +_stub_attr("youtube_extension.services.pipeline_job_store", "get_job_store", MagicMock(return_value=_job_store)) + # --------------------------------------------------------------------------- # Now import the router (it will use the stubs above) # --------------------------------------------------------------------------- @@ -1950,8 +1957,9 @@ def get_duration_seconds(m): assert result["async_processing"] is True assert "job_id" in result assert result["processing_transport"] == "local_background" - # asyncio.create_task should have been called for fallback - mock_ct.assert_called_once() + # asyncio.create_task should have been called for fallback (may also be called + # by _persist_video_job background serialization, so check at least once) + mock_ct.assert_called() async def test_queue_job_cloud_tasks_success(self): """CloudTasksQueueService succeeds → queued_transport = cloud_tasks."""