diff --git a/.docker/sandbox/install-browser-agent.sh b/.docker/sandbox/install-browser-agent.sh index abdfb156a..dc242b941 100644 --- a/.docker/sandbox/install-browser-agent.sh +++ b/.docker/sandbox/install-browser-agent.sh @@ -9,7 +9,7 @@ set -euo pipefail # 1. Worker image builds (Dockerfiles) to prebake browser automation. # 2. Worker setup compatibility fallbacks on older Linux images. -AGENT_BROWSER_VERSION="${AGENT_BROWSER_VERSION:-0.27.0}" +AGENT_BROWSER_VERSION="${AGENT_BROWSER_VERSION:-0.33.2}" AGENT_BROWSER_INSTALL_ROOT="${AGENT_BROWSER_INSTALL_ROOT:-/opt/agent-browser}" AGENT_BROWSER_EXECUTABLE_PATH="${AGENT_BROWSER_EXECUTABLE_PATH:-${AGENT_BROWSER_INSTALL_ROOT}/chrome}" AGENT_BROWSER_SAVED_CLI_PATH="${AGENT_BROWSER_SAVED_CLI_PATH:-${AGENT_BROWSER_INSTALL_ROOT}/.cli-path}" @@ -374,21 +374,35 @@ collect_cli_browser_args() { local i=0 AGENT_BROWSER_EXEC_ARGS=() AGENT_BROWSER_FORWARD_ARGS=() + # Default headless; a caller may opt a single invocation into headed mode + # (e.g. the feature-demo capture runner, so GPU-backed canvases present to + # the compositor). On displayless Linux agent-browser auto-starts Xvfb. + AGENT_BROWSER_WANT_HEADED=0 while [ "$i" -lt "${#args[@]}" ]; do local arg="${args[$i]}" case "$arg" in --headed) + AGENT_BROWSER_WANT_HEADED=1 + AGENT_BROWSER_FORWARD_ARGS+=("$arg") if [ "$i" -lt $(( ${#args[@]} - 1 )) ]; then case "${args[$((i + 1))]}" in true|false) i=$((i + 1)) + [ "${args[$i]}" = false ] && AGENT_BROWSER_WANT_HEADED=0 + AGENT_BROWSER_FORWARD_ARGS+=("${args[$i]}") ;; esac fi ;; + --headed=false) + AGENT_BROWSER_WANT_HEADED=0 + AGENT_BROWSER_FORWARD_ARGS+=("$arg") + ;; --headed=*) + AGENT_BROWSER_WANT_HEADED=1 + AGENT_BROWSER_FORWARD_ARGS+=("$arg") ;; --args) AGENT_BROWSER_FORWARD_ARGS+=("$arg") @@ -487,12 +501,21 @@ seed_preview_cookies() { mkdir -p "$CACHE_ROOT" resolve_cli_paths + # Seeding may be what launches the daemon, and the daemon's headed/headless + # mode is fixed at launch — so seed in the SAME mode the wrapped command + # requested, or a headless seed daemon would make a later `--headed` command + # a silent no-op. + local seed_headed=false + if [ "${AGENT_BROWSER_WANT_HEADED:-0}" = 1 ]; then + seed_headed=true + fi + for url in "${preview_urls[@]}"; do if [ -n "$bypass_value" ]; then - AGENT_BROWSER_HEADED=false "$AGENT_BROWSER_BIN" "${AGENT_BROWSER_PREFIX_ARGS[@]}" cookies set "$header_name" "$bypass_value" --url "$url" --secure --sameSite Lax >/dev/null + AGENT_BROWSER_HEADED="$seed_headed" "$AGENT_BROWSER_BIN" "${AGENT_BROWSER_PREFIX_ARGS[@]}" cookies set "$header_name" "$bypass_value" --url "$url" --secure --sameSite Lax >/dev/null fi - AGENT_BROWSER_HEADED=false "$AGENT_BROWSER_BIN" "${AGENT_BROWSER_PREFIX_ARGS[@]}" cookies set "$HIDE_PREVIEW_WIDGET_COOKIE" "1" --url "$url" --secure --sameSite Lax >/dev/null + AGENT_BROWSER_HEADED="$seed_headed" "$AGENT_BROWSER_BIN" "${AGENT_BROWSER_PREFIX_ARGS[@]}" cookies set "$HIDE_PREVIEW_WIDGET_COOKIE" "1" --url "$url" --secure --sameSite Lax >/dev/null done : > "$cache_file" @@ -512,7 +535,14 @@ if should_clear_seed_cache; then fi resolve_cli_paths -export AGENT_BROWSER_HEADED=false +# Preview-cookie seeding above always runs headless (inline env). The task +# command itself honors an explicit --headed opt-in; everything else stays +# headless as before. +if [ "${AGENT_BROWSER_WANT_HEADED:-0}" = 1 ]; then + export AGENT_BROWSER_HEADED=true +else + export AGENT_BROWSER_HEADED=false +fi exec "$AGENT_BROWSER_BIN" "${AGENT_BROWSER_FORWARD_ARGS[@]}" EOF_WRAPPER } diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 4b2032d4f..0975d6461 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -25,6 +25,7 @@ ".env.*", ".qdrant", "packages/cloud-agents/src/server/workflows/skills/standard/agent-browser", + "packages/cloud-agents/src/server/workflows/skills/standard/feature-demo", ".vscode", ".git", "**/*.{md,mdx,yml,yaml,html,htm,toml,svg,graphql,gql}" diff --git a/.oxlintrc.json b/.oxlintrc.json index 25500e9e1..21bc24926 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -19,6 +19,7 @@ "apps/docs/**", "apps/marketing/**", "packages/cloud-agents/src/server/workflows/skills/standard/agent-browser/**", + "packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/**", "apps/codex-acp/target/**", "**/next-env.d.ts", "deploy/**" diff --git a/apps/api/src/handlers/index.ts b/apps/api/src/handlers/index.ts index 31a99bd6f..3f5167350 100644 --- a/apps/api/src/handlers/index.ts +++ b/apps/api/src/handlers/index.ts @@ -26,6 +26,9 @@ export { mcpRouting } from './mcp/routing'; // inference gateway export { inference } from './inference'; +// narration tts +export { tts } from './tts'; + // task runs export { taskRunsRouter } from './task-runs'; diff --git a/apps/api/src/handlers/mcp/index.ts b/apps/api/src/handlers/mcp/index.ts index fb41599d1..b4382948c 100644 --- a/apps/api/src/handlers/mcp/index.ts +++ b/apps/api/src/handlers/mcp/index.ts @@ -4,7 +4,11 @@ import { areCuratedIntegrationsDisabled, isCustomMcpDisabled, } from '@roomote/env'; -import { isNativeMcpIntegration, MCP_INTEGRATIONS } from '@roomote/types'; +import { + isCredentialOnlyMcpIntegration, + isNativeMcpIntegration, + MCP_INTEGRATIONS, +} from '@roomote/types'; import type { Variables } from '../../types'; @@ -66,7 +70,9 @@ mcp.route('/vercel', vercelMcp); for (const integration of MCP_INTEGRATIONS.filter( (candidate) => - !isNativeMcpIntegration(candidate) && candidate.id !== 'linear', + !isNativeMcpIntegration(candidate) && + !isCredentialOnlyMcpIntegration(candidate) && + candidate.id !== 'linear', )) { mcp.route( `/${integration.id}`, diff --git a/apps/api/src/handlers/tts/__tests__/tts.test.ts b/apps/api/src/handlers/tts/__tests__/tts.test.ts new file mode 100644 index 000000000..bdb09e050 --- /dev/null +++ b/apps/api/src/handlers/tts/__tests__/tts.test.ts @@ -0,0 +1,193 @@ +import { Hono } from 'hono'; +import type { RunTokenContext } from '@roomote/types'; + +import type { Variables } from '../../../types'; + +const { mockResolveCredentials } = vi.hoisted(() => ({ + mockResolveCredentials: vi.fn(), +})); + +vi.mock('../connection', () => ({ + resolveElevenLabsCredentials: mockResolveCredentials, +})); + +import { tts } from '../index'; + +function createRunToken(): RunTokenContext { + return { + runId: 42, + userId: null, + principal: 'deployment', + tokenType: 'run', + version: 1, + }; +} + +function createApp( + authContext: Variables['authContext'], +): Hono<{ Variables: Variables }> { + const app = new Hono<{ Variables: Variables }>(); + + app.use('*', async (c, next) => { + c.set('authContext', authContext); + await next(); + }); + + app.route('/tts', tts); + return app; +} + +function narrationRequest(body: unknown): Request { + return new Request('http://localhost/tts/narration', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }); +} + +const originalFetch = globalThis.fetch; + +beforeEach(() => { + mockResolveCredentials.mockResolvedValue({ + apiKey: 'el-key', + voiceId: 'voice-1', + }); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + vi.restoreAllMocks(); +}); + +describe('POST /tts/narration', () => { + it('404s when the deployment has no ElevenLabs credentials', async () => { + mockResolveCredentials.mockResolvedValue(undefined); + + const response = await createApp(createRunToken()).request( + narrationRequest({ lines: ['hello'] }), + ); + + expect(response.status).toBe(404); + }); + + it('500s without leaking details when credential resolution throws', async () => { + mockResolveCredentials.mockRejectedValue(new Error('db exploded')); + + const response = await createApp(createRunToken()).request( + narrationRequest({ lines: ['hello'] }), + ); + + expect(response.status).toBe(500); + + const payload = (await response.json()) as { error: string }; + + expect(payload.error).not.toContain('db exploded'); + }); + + it('rejects non-run-token callers', async () => { + const response = await createApp(undefined).request( + narrationRequest({ lines: ['hello'] }), + ); + + expect(response.status).toBe(403); + }); + + it('rejects invalid line payloads', async () => { + const app = createApp(createRunToken()); + + for (const body of [ + {}, + { lines: [] }, + { lines: ['ok', 42] }, + { lines: [' '] }, + { lines: ['x'.repeat(1_201)] }, + { lines: Array.from({ length: 25 }, () => 'line') }, + ]) { + const response = await app.request(narrationRequest(body)); + + expect(response.status).toBe(400); + } + }); + + it('413s a body that exceeds the cap even without content-length', async () => { + const chunk = new TextEncoder().encode('x'.repeat(1024)); + const endlessBody = new ReadableStream({ + pull(controller) { + controller.enqueue(chunk); + }, + }); + + const response = await createApp(createRunToken()).request( + new Request('http://localhost/tts/narration', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: endlessBody, + // @ts-expect-error duplex is required for stream bodies in undici + duplex: 'half', + }), + ); + + expect(response.status).toBe(413); + }); + + it('synthesizes each line through ElevenLabs with the server-held key', async () => { + const fetchMock = vi.fn( + async () => new Response(Buffer.from('mp3-bytes'), { status: 200 }), + ); + globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch; + + const response = await createApp(createRunToken()).request( + narrationRequest({ lines: ['First line.', 'Second line.'] }), + ); + + expect(response.status).toBe(200); + + const payload = (await response.json()) as { + clips: { audioBase64: string }[]; + }; + + expect(payload.clips).toHaveLength(2); + expect( + Buffer.from(payload.clips[0]!.audioBase64, 'base64').toString(), + ).toBe('mp3-bytes'); + + expect(fetchMock).toHaveBeenCalledTimes(2); + + const [url, init] = fetchMock.mock.calls[0] as unknown as [ + string, + RequestInit, + ]; + + expect(url).toContain('https://api.elevenlabs.io/v1/text-to-speech/'); + expect(url).toContain('voice-1'); + + const headers = init.headers as Record; + + expect(headers['xi-api-key']).toBe('el-key'); + + const body = JSON.parse(String(init.body)) as { + model_id: string; + voice_settings: { stability: number }; + }; + + expect(body.model_id).toBe('eleven_multilingual_v2'); + expect(body.voice_settings.stability).toBe(0.35); + }); + + it('502s when the upstream synthesis fails', async () => { + globalThis.fetch = vi.fn( + async () => new Response('nope', { status: 401 }), + ) as unknown as typeof globalThis.fetch; + + const response = await createApp(createRunToken()).request( + narrationRequest({ lines: ['hello'] }), + ); + + expect(response.status).toBe(502); + + const payload = (await response.json()) as { error: string }; + + // The upstream error body must not leak through. + expect(payload.error).toBe('Narration synthesis failed'); + }); +}); diff --git a/apps/api/src/handlers/tts/connection.ts b/apps/api/src/handlers/tts/connection.ts new file mode 100644 index 000000000..fa3d0e06a --- /dev/null +++ b/apps/api/src/handlers/tts/connection.ts @@ -0,0 +1,67 @@ +import { areCuratedIntegrationsDisabled, Env } from '@roomote/env'; +import { + and, + db, + deploymentMcpEnablements, + eq, + isNull, + mcpConnections, +} from '@roomote/db/server'; +import { decrypt } from '@roomote/db/encryption'; +import { isMcpConnectionElevenLabsConfig } from '@roomote/types'; + +type ElevenLabsCredentials = { + apiKey: string; + voiceId: string; +}; + +/** + * Resolves narration TTS credentials for the deployment. + * + * The admin-configured ElevenLabs integration (Settings → Integrations) is + * the primary source; the `R_ELEVENLABS_*` environment variables remain an + * operator fallback. Returns undefined when neither is configured — the + * feature is off and the endpoint 404s. + * + * The integration is credential-only: this control-plane resolver is its + * single consumer, and the connection is deliberately excluded from agent + * MCP config delivery, so the key never travels toward a task sandbox. + */ +export async function resolveElevenLabsCredentials(): Promise< + ElevenLabsCredentials | undefined +> { + if (!areCuratedIntegrationsDisabled(Env.R_CURATED_INTEGRATIONS_DISABLED)) { + const connection = await db.query.mcpConnections.findFirst({ + where: and( + eq(mcpConnections.mcpId, 'elevenlabs'), + isNull(mcpConnections.userId), + eq(mcpConnections.enabled, true), + eq(mcpConnections.authStatus, 'authenticated'), + ), + }); + + if (connection && isMcpConnectionElevenLabsConfig(connection.authConfig)) { + const enablement = await db.query.deploymentMcpEnablements.findFirst({ + where: eq(deploymentMcpEnablements.mcpId, 'elevenlabs'), + }); + + if (enablement?.enabled !== false) { + const apiKey = decrypt(connection.authConfig.encryptedApiKey).trim(); + const voiceId = connection.authConfig.voiceId.trim(); + + if (apiKey && voiceId) { + return { apiKey, voiceId }; + } + } + } + } + + if (Env.R_ELEVENLABS_API_KEY && Env.R_ELEVENLABS_VOICE_ID) { + return { + apiKey: Env.R_ELEVENLABS_API_KEY, + voiceId: Env.R_ELEVENLABS_VOICE_ID, + }; + } + + return undefined; +} diff --git a/apps/api/src/handlers/tts/index.ts b/apps/api/src/handlers/tts/index.ts new file mode 100644 index 000000000..2e6916d9c --- /dev/null +++ b/apps/api/src/handlers/tts/index.ts @@ -0,0 +1,226 @@ +import { Hono } from 'hono'; +import type { RunTokenContext } from '@roomote/types'; + +import type { Variables } from '../../types'; +import { logHandlerError } from '../utils'; +import { resolveElevenLabsCredentials } from './connection'; + +/** + * Narration text-to-speech for task sandboxes, used by the feature-demo + * skill. The sandbox posts plain narration lines with its run-scoped token; + * the ElevenLabs key lives only on the control plane (`R_ELEVENLABS_API_KEY` + * is in `CONTROL_PLANE_ENV_VAR_NAMES`, so it is also stripped from sandbox + * env injection) and the synthesized audio streams back as base64 clips. + * + * Unset credentials mean the feature is off: the endpoint 404s and callers + * degrade to captions-only output. + */ + +type NarrationBody = { + lines?: unknown; +}; + +class RequestBodyTooLargeError extends Error {} + +function isRunTokenContext( + auth: Variables['authContext'], +): auth is RunTokenContext { + return Boolean(auth && 'runId' in auth); +} + +/** Narration is a handful of short spoken lines, not prose. */ +const MAX_NARRATION_LINES = 24; +const MAX_NARRATION_LINE_CHARS = 1_200; +const MAX_NARRATION_REQUEST_BYTES = 64 * 1024; + +/** + * multilingual_v2 gives voice clones a natural, conversational read at a + * natural pace (the more "expressive" models lean announcer-voice on + * professional clones and read too slowly). + */ +const ELEVENLABS_MODEL_ID = 'eleven_multilingual_v2'; +const ELEVENLABS_TTS_TIMEOUT_MS = 60_000; + +async function readRequestBodyBytes( + request: Request, + maxBytes: number, +): Promise { + if (!request.body) { + return new Uint8Array(); + } + + const reader = request.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + break; + } + + totalBytes += value.byteLength; + if (totalBytes > maxBytes) { + await reader.cancel().catch(() => undefined); + throw new RequestBodyTooLargeError(); + } + + chunks.push(value); + } + + const bodyBytes = new Uint8Array(totalBytes); + let offset = 0; + + for (const chunk of chunks) { + bodyBytes.set(chunk, offset); + offset += chunk.byteLength; + } + + return bodyBytes; +} + +function parseNarrationLines(body: NarrationBody): string[] | null { + if (!Array.isArray(body.lines)) { + return null; + } + + if (body.lines.length === 0 || body.lines.length > MAX_NARRATION_LINES) { + return null; + } + + const lines: string[] = []; + + for (const line of body.lines) { + if (typeof line !== 'string') { + return null; + } + + const trimmed = line.trim(); + + if (!trimmed || trimmed.length > MAX_NARRATION_LINE_CHARS) { + return null; + } + + lines.push(trimmed); + } + + return lines; +} + +async function synthesizeLine(options: { + apiKey: string; + voiceId: string; + text: string; +}): Promise { + const response = await fetch( + `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent( + options.voiceId, + )}?output_format=mp3_44100_128`, + { + method: 'POST', + headers: { + 'xi-api-key': options.apiKey, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + text: options.text, + model_id: ELEVENLABS_MODEL_ID, + voice_settings: { + stability: 0.35, + similarity_boost: 0.75, + style: 0.45, + }, + }), + signal: AbortSignal.timeout(ELEVENLABS_TTS_TIMEOUT_MS), + }, + ); + + if (!response.ok) { + throw new Error( + `ElevenLabs TTS request failed with status ${response.status}`, + ); + } + + return Buffer.from(await response.arrayBuffer()); +} + +export const tts = new Hono<{ Variables: Variables }>(); + +tts.post('/narration', async (c) => { + let credentials; + + try { + credentials = await resolveElevenLabsCredentials(); + } catch (error) { + logHandlerError('ttsNarration', error); + return c.json({ error: 'Narration synthesis failed' }, 500); + } + + // Not configured => the feature does not exist on this deployment. + if (!credentials) { + return c.json({ error: 'Narration TTS is not configured' }, 404); + } + + const { apiKey, voiceId } = credentials; + + const auth = c.get('authContext'); + + // The route policy already rejects non-run tokens; this guard keeps the + // handler honest if the policy table ever drifts. + if (!auth || !isRunTokenContext(auth)) { + return c.json({ error: 'Narration TTS requires a task run token' }, 403); + } + + let body: NarrationBody; + + try { + const bodyBytes = await readRequestBodyBytes( + c.req.raw, + MAX_NARRATION_REQUEST_BYTES, + ); + + body = JSON.parse(new TextDecoder().decode(bodyBytes)) as NarrationBody; + } catch (error) { + if (error instanceof RequestBodyTooLargeError) { + return c.json( + { + error: `Request body exceeds max size of ${MAX_NARRATION_REQUEST_BYTES} bytes`, + }, + 413, + ); + } + + return c.json({ error: 'Invalid request body' }, 400); + } + + const lines = parseNarrationLines(body); + + if (!lines) { + return c.json( + { + error: + `lines must be 1-${MAX_NARRATION_LINES} non-empty strings of at ` + + `most ${MAX_NARRATION_LINE_CHARS} characters`, + }, + 400, + ); + } + + const clips: { audioBase64: string }[] = []; + + // Sequential on purpose: ElevenLabs enforces per-key concurrency limits, + // and a narration is a handful of short lines. + for (const text of lines) { + try { + const audio = await synthesizeLine({ apiKey, voiceId, text }); + + clips.push({ audioBase64: audio.toString('base64') }); + } catch (error) { + logHandlerError('ttsNarration', error); + return c.json({ error: 'Narration synthesis failed' }, 502); + } + } + + return c.json({ clips }); +}); diff --git a/apps/api/src/route-policies.ts b/apps/api/src/route-policies.ts index aa063f590..da8aa1aa0 100644 --- a/apps/api/src/route-policies.ts +++ b/apps/api/src/route-policies.ts @@ -236,6 +236,24 @@ export const ROUTE_POLICY_RULES: readonly RoutePolicyRule[] = [ policy: 'task-token', }, + // Narration text-to-speech for task sandboxes: the ElevenLabs key is + // injected server-side and never enters the sandbox. The client-keyed + // limit is a global ceiling (server-to-server callers share one bucket) + // sized far above legitimate use — a demo narrates a handful of lines + // once — to blunt credit-drain from a leaked run token. + { + name: 'tts', + match: { type: 'prefix', path: '/api/tts' }, + policy: 'task-token', + rateLimits: [ + { + keySource: 'client', + limit: 60, + windowSeconds: 60, + }, + ], + }, + // Router-facing MCP endpoints: accept user auth tokens (LLM router // gathering context before a run exists) and task run tokens. { diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index b146268dc..c86b6b22b 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -47,6 +47,7 @@ import { discord, cloudDeploymentAccess, inference, + tts, mcp, mcpRouting, taskRunsRouter, @@ -209,6 +210,7 @@ export function createApiApp(): ApiApp { app.route('/api/internal/discord', discord); app.route('/api/internal/cloud', cloudDeploymentAccess); app.route('/api/inference', inference); + app.route('/api/tts', tts); app.route('/api/mcp', mcp); app.route('/api/mcp-routing', mcpRouting); app.route('/api/task-runs', taskRunsRouter); diff --git a/apps/controller/src/__tests__/RoomoteController.test.ts b/apps/controller/src/__tests__/RoomoteController.test.ts index 2ce64159e..67db6e1d6 100644 --- a/apps/controller/src/__tests__/RoomoteController.test.ts +++ b/apps/controller/src/__tests__/RoomoteController.test.ts @@ -30,7 +30,7 @@ const { DOCKER_WORKER_CPU_LIMIT: 2, DOCKER_WORKER_MEMORY_LIMIT: '4g', DOCKER_TASK_DAEMON_MEMORY_LIMIT: '8g', - DOCKER_WORKER_PIDS_LIMIT: 512, + DOCKER_WORKER_PIDS_LIMIT: 2048, DOCKER_WORKER_DISK_LIMIT: '20g', DOCKER_WORKER_ALLOW_UNBOUNDED_DISK: false, DOCKER_WORKER_LOG_MAX_SIZE: '10m', @@ -132,7 +132,7 @@ describe('RoomoteController', () => { mockEnv.DOCKER_WORKER_CPU_LIMIT = 2; mockEnv.DOCKER_WORKER_MEMORY_LIMIT = '4g'; mockEnv.DOCKER_TASK_DAEMON_MEMORY_LIMIT = '8g'; - mockEnv.DOCKER_WORKER_PIDS_LIMIT = 512; + mockEnv.DOCKER_WORKER_PIDS_LIMIT = 2048; mockEnv.DOCKER_WORKER_DISK_LIMIT = '20g'; mockEnv.DOCKER_WORKER_ALLOW_UNBOUNDED_DISK = false; mockEnv.DOCKER_WORKER_LOG_MAX_SIZE = '10m'; @@ -195,7 +195,7 @@ describe('RoomoteController', () => { cpuLimit: 2, memoryLimit: '4g', taskDaemonMemoryLimit: '8g', - pidsLimit: 512, + pidsLimit: 2048, diskLimit: '20g', allowUnboundedDisk: false, logMaxSize: '10m', diff --git a/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts b/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts index 7c7c23352..ef201034d 100644 --- a/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts +++ b/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts @@ -37,6 +37,7 @@ describe('buildDockerWorkerResourceArgs', () => { logMaxFiles: 3, }), ).toEqual([ + '--init', '--cpus', '2', '--memory', @@ -60,6 +61,18 @@ describe('buildDockerWorkerResourceArgs', () => { ]); }); + it('runs an init process to reap orphaned browser child processes', () => { + expect( + buildDockerWorkerResourceArgs({ + cpuLimit: 2, + memoryLimit: '4g', + pidsLimit: 512, + logMaxSize: '10m', + logMaxFiles: 3, + }), + ).toContain('--init'); + }); + it('omits the storage option when the driver cannot enforce a disk limit', () => { expect( buildDockerWorkerResourceArgs({ @@ -85,6 +98,7 @@ describe('buildDockerTaskDaemonResourceArgs', () => { }); expect(args).toContain('--pids-limit'); + expect(args).toContain('--init'); expect(args).toEqual( expect.arrayContaining(['--memory', '8g', '--memory-swap', '8g']), ); diff --git a/apps/controller/src/compute-providers/docker-sandbox-security.ts b/apps/controller/src/compute-providers/docker-sandbox-security.ts index 343c71bb8..4776f18ab 100644 --- a/apps/controller/src/compute-providers/docker-sandbox-security.ts +++ b/apps/controller/src/compute-providers/docker-sandbox-security.ts @@ -367,6 +367,12 @@ export function buildDockerWorkerResourceArgs( limits: DockerWorkerResourceLimits, ): string[] { return [ + // Run an init process (tini) as PID 1 so orphaned child processes are + // reaped. Browser-driving tasks (agent-browser daemon restarts) leak + // Chrome process trees whose defunct children otherwise accumulate until + // they exhaust --pids-limit, after which fresh Chrome launches fail with + // `pthread_create: Resource temporarily unavailable`. + '--init', '--cpus', String(limits.cpuLimit), '--memory', @@ -393,6 +399,8 @@ export function buildDockerTaskDaemonResourceArgs( limits: DockerWorkerResourceLimits, ): string[] { return [ + // Reap orphaned children (see buildDockerWorkerResourceArgs). + '--init', '--cpus', String(limits.cpuLimit), '--memory', diff --git a/apps/docs/cookbook/feature-demo-videos.mdx b/apps/docs/cookbook/feature-demo-videos.mdx new file mode 100644 index 000000000..db2f79328 --- /dev/null +++ b/apps/docs/cookbook/feature-demo-videos.mdx @@ -0,0 +1,62 @@ +--- +title: Record feature demo videos +description: Ask Roomote for a polished, narrated demo video of a feature, recorded live with zooms, captions, and an optional voice-over. +contributor: Matt Rubens +contributor_url: https://github.com/mrubens +contributor_company: Roomote +contributor_company_url: https://roomote.dev +--- + +## Overview + +Shipping a feature usually ends with someone screen-recording a shaky +walkthrough. Roomote can produce that video for you: it drives the product in +its sandbox browser, records a clean clip, and post-produces it with smooth +zoom-to-element moves, a synthetic cursor, click ripples, and captions. With +narration configured, the demo gets a voice-over that leads each visual beat. + +- **Trigger**: Ask in a task +- **Setup time**: None for captioned demos; 5 minutes for narration +- **Requires**: Nothing extra; admin access for narration +- **Serves**: Engineers, Product, Marketing + +## Ingredients + +- Any task where the feature is reachable: a running dev environment or a + public page +- Optional: an [ElevenLabs](https://elevenlabs.io) API key and voice ID for + narration + +## Steps + +1. In a task, ask for a demo, for example: `Make a demo video of the new + custom MCP servers settings page` or invoke `$feature-demo` directly. +2. Roomote plans the story beats, records the interaction live, and renders a + 16:9 video (ask for a vertical cut for short-form if you want one). +3. The finished video is uploaded as a task artifact and linked in the reply + (and embedded under `## Screencasts` when the task opens a PR). + +## Narration + +Captioned demos work out of the box. For a voice-over in your own (cloned) +voice, a deployment admin connects ElevenLabs under **Settings → +Integrations → ElevenLabs** with: + +- an ElevenLabs API key, scoped to **text-to-speech only** with a credit cap + (sufficient and recommended), and +- the voice ID to narrate with. + +The key is encrypted at rest and stays on the Roomote control plane: task +sandboxes send narration text to an authenticated Roomote endpoint and +receive audio back, so the provider credential never enters a sandbox and +agents get no ElevenLabs tools. (Operators self-hosting can alternatively set +the `R_ELEVENLABS_API_KEY` and `R_ELEVENLABS_VOICE_ID` environment variables; +the integration takes precedence when both are present.) + +## Tips + +- Narration lines come from the on-screen captions; ask for wording changes + the way you would edit copy ("make the second line more casual"). +- The narrative drives the pacing: each shot holds for as long as its line + takes to speak, voiced or not. +- Keep demos to 2-5 beats. Two shorter videos beat one long one. diff --git a/apps/docs/cookbook/index.mdx b/apps/docs/cookbook/index.mdx index 5d68fcc28..10e059573 100644 --- a/apps/docs/cookbook/index.mdx +++ b/apps/docs/cookbook/index.mdx @@ -16,6 +16,7 @@ and the quality of your output. | [Ease your team into cloud agents](/cookbook/ease-your-team-into-cloud-agents) | Build trust in Roomote through small, visible, low-risk team habits. | | [Evaluate outage impact](/cookbook/vendor-outage-triage) | Filter vendor status noise by comparing each incident with your real code, regions, and feature usage. | | [Fix CI failures](/cookbook/ci-failure-auto-fix) | Keep the build green by having Roomote verify and fix CI breakages automatically. | +| [Record feature demo videos](/cookbook/feature-demo-videos) | Ask Roomote for a polished, narrated demo video of a feature, recorded live with zooms, captions, and an optional voice-over. | | [Schedule maintenance](/cookbook/scheduled-housekeeping) | Turn flaky-test scans, feature-flag audits, and dependency reviews into recurring Roomote work. | | [Triage customer issues](/cookbook/support-channel) | Give support escalations a repeatable path through production evidence, data, and code. | {/* cookbook-recipes:end */} diff --git a/apps/docs/environment-variables.mdx b/apps/docs/environment-variables.mdx index ff27fb411..f7b000260 100644 --- a/apps/docs/environment-variables.mdx +++ b/apps/docs/environment-variables.mdx @@ -340,6 +340,8 @@ as per-task auth tokens or workspace paths. | `R_MICROSOFT_CLIENT_SECRET` | Microsoft sign-in | Microsoft OAuth client secret. | | `R_MICROSOFT_TENANT_ID` | Microsoft sign-in | Microsoft tenant ID. Also accepted as the Azure DevOps tenant ID fallback. | | `R_ALLOWED_EMAILS` | Optional | Comma-separated email allowlist for deployments that restrict sign-in by email. | +| `R_ELEVENLABS_API_KEY` | Optional | ElevenLabs API key for narrated feature-demo videos. The key stays on the control plane; sandboxes reach text-to-speech only through an authenticated Roomote endpoint. A key scoped to text-to-speech only is sufficient and recommended. | +| `R_ELEVENLABS_VOICE_ID` | Optional | ElevenLabs voice ID used for feature-demo narration. Required alongside the API key for narration to be available. | During Microsoft Teams setup, Roomote uses the Microsoft Entra app values for the Teams bot by default. Use **Show advanced config** after the Directory diff --git a/apps/web/src/components/settings/Integrations.test.tsx b/apps/web/src/components/settings/Integrations.test.tsx index 252e0c026..7d7ed0b70 100644 --- a/apps/web/src/components/settings/Integrations.test.tsx +++ b/apps/web/src/components/settings/Integrations.test.tsx @@ -38,6 +38,10 @@ const state = vi.hoisted(() => ({ granolaConnection: null as null | { authStatus?: string | null; }, + elevenLabsConnection: null as null | { + authStatus?: string | null; + voiceId?: string; + }, grafanaConnection: null as null | { authStatus?: string | null; baseUrl: string; @@ -95,6 +99,7 @@ const { mutations, selectMock } = vi.hoisted(() => ({ setDisabledTools: vi.fn(), saveAsanaConnection: vi.fn(), saveGranolaConnection: vi.fn(), + saveElevenLabsConnection: vi.fn(), saveGrafanaConnection: vi.fn(), saveSnowflakeConnection: vi.fn(), saveVercelConnection: vi.fn(), @@ -254,6 +259,14 @@ vi.mock('@/hooks/mcp-connections', () => ({ data: state.granolaConnection, isPending: false, }), + useSaveElevenLabsConnection: () => ({ + isPending: false, + mutate: mutations.saveElevenLabsConnection, + }), + useElevenLabsConnection: () => ({ + data: state.elevenLabsConnection, + isPending: false, + }), useSaveGrafanaConnection: () => ({ isPending: false, mutate: mutations.saveGrafanaConnection, @@ -821,6 +834,7 @@ describe('Integrations settings', () => { 'Asana', 'Better Stack', 'Braintrust', + 'ElevenLabs', 'Grafana', 'Granola', 'Jira', diff --git a/apps/web/src/components/settings/Integrations.tsx b/apps/web/src/components/settings/Integrations.tsx index cdec85843..ce6cf2cb0 100644 --- a/apps/web/src/components/settings/Integrations.tsx +++ b/apps/web/src/components/settings/Integrations.tsx @@ -25,11 +25,13 @@ import { useDisconnectMcp, useGrafanaConnection, useGranolaConnection, + useElevenLabsConnection, useDeploymentMcpEnablements, useMcpOauthReadiness, useSaveAsanaConnection, useSaveGrafanaConnection, useSaveGranolaConnection, + useSaveElevenLabsConnection, useSaveSnowflakeConnection, useSaveVercelConnection, useSetDeploymentMcpEnabled, @@ -48,6 +50,7 @@ import { saveAsanaConnectionSchema, saveGrafanaConnectionSchema, saveGranolaConnectionSchema, + saveElevenLabsConnectionSchema, saveSnowflakeConnectionSchema, saveVercelConnectionSchema, } from '@/types'; @@ -89,6 +92,8 @@ const DEEP_LINK_ENABLE_DESCRIPTIONS: Record = { 'Roomote will be able to inspect dashboards, alert rules, live alert state, annotations, and data sources.', granola: 'Roomote will use one deployment-wide Granola connection to browse meeting notes, transcripts, decisions, and action items.', + elevenlabs: + 'Roomote will use one deployment-wide ElevenLabs connection to narrate feature-demo videos. The key stays on the control plane; agents get no ElevenLabs tools.', github: 'Roomote will be able to inspect PRs, issues, and repository context.', jira: 'Roomote will be able to inspect Jira issues, workflows, and JQL search results.', @@ -173,6 +178,16 @@ type GranolaFormState = { apiKey: string; }; +type ElevenLabsFormState = { + apiKey: string; + voiceId: string; +}; + +type ElevenLabsConnectionData = { + authStatus?: 'pending' | 'authenticated' | 'error' | null; + voiceId?: string; +}; + type GrafanaFormState = { baseUrl: string; serviceAccountToken: string; @@ -215,6 +230,26 @@ function buildEmptyGranolaForm(): GranolaFormState { }; } +function buildEmptyElevenLabsForm(): ElevenLabsFormState { + return { + apiKey: '', + voiceId: '', + }; +} + +function buildElevenLabsForm( + connection: ElevenLabsConnectionData | null | undefined, +): ElevenLabsFormState { + if (!connection) { + return buildEmptyElevenLabsForm(); + } + + return { + apiKey: '', + voiceId: connection.voiceId ?? '', + }; +} + function buildEmptyGrafanaForm(): GrafanaFormState { return { baseUrl: '', @@ -291,6 +326,21 @@ function getGranolaFieldErrors( }; } +function getElevenLabsFieldErrors( + result: ReturnType, +): Partial> { + if (result.success) { + return {}; + } + + const fieldErrors = result.error.flatten().fieldErrors; + + return { + apiKey: fieldErrors.apiKey, + voiceId: fieldErrors.voiceId, + }; +} + function buildGrafanaForm( connection: GrafanaConnectionData | null | undefined, ): GrafanaFormState { @@ -411,7 +461,10 @@ function buildAdminConfiguredIntegrationItem({ } : undefined, secondaryAction: - canManageTools && enabled && integration.serverMode !== 'native' + canManageTools && + enabled && + integration.serverMode !== 'native' && + integration.serverMode !== 'credential_only' ? { label: 'Manage tools', ariaLabel: `Manage ${integration.name} tools`, @@ -808,6 +861,82 @@ function GranolaConnectionFields({ ); } +function ElevenLabsConnectionFields({ + form, + fieldErrors, + formError, + allowBlankApiKey, + onFieldChange, +}: { + form: ElevenLabsFormState; + fieldErrors: Partial>; + formError: string | null; + allowBlankApiKey: boolean; + onFieldChange: (field: keyof ElevenLabsFormState, value: string) => void; +}) { + const fieldClassName = + 'mt-2 w-full border-border/70 bg-background data-[invalid=true]:border-destructive'; + + return ( + <> +
+ + onFieldChange('apiKey', event.target.value)} + data-invalid={fieldErrors.apiKey ? 'true' : undefined} + className={fieldClassName} + autoCapitalize="off" + autoCorrect="off" + spellCheck={false} + data-1p-ignore + /> +

+ We recommend a key scoped to text-to-speech only, with a credit limit. + The key is used exclusively by this deployment's control plane to + narrate feature-demo videos; it is never sent to agents or task + sandboxes. +

+ {allowBlankApiKey ? ( +

+ Leave blank to keep the existing API key. +

+ ) : null} + {fieldErrors.apiKey ? ( +

{fieldErrors.apiKey[0]}

+ ) : null} +
+
+ + onFieldChange('voiceId', event.target.value)} + data-invalid={fieldErrors.voiceId ? 'true' : undefined} + className={fieldClassName} + autoCapitalize="off" + autoCorrect="off" + spellCheck={false} + /> +

+ The ElevenLabs voice used for narration. Find voice IDs in the + ElevenLabs voice library. +

+ {fieldErrors.voiceId ? ( +

{fieldErrors.voiceId[0]}

+ ) : null} +
+ {formError ? ( +

{formError}

+ ) : null} + + ); +} + function GrafanaConnectionFields({ form, fieldErrors, @@ -1007,6 +1136,16 @@ export function Integrations() { const [granolaForm, setGranolaForm] = useState( buildEmptyGranolaForm(), ); + const [isElevenLabsDialogOpen, setIsElevenLabsDialogOpen] = useState(false); + const [elevenLabsForm, setElevenLabsForm] = useState( + buildEmptyElevenLabsForm(), + ); + const [elevenLabsFieldErrors, setElevenLabsFieldErrors] = useState< + Partial> + >({}); + const [elevenLabsFormError, setElevenLabsFormError] = useState( + null, + ); const [granolaFieldErrors, setGranolaFieldErrors] = useState< Partial> >({}); @@ -1065,6 +1204,7 @@ export function Integrations() { const saveAsanaConnection = useSaveAsanaConnection(); const saveGrafanaConnection = useSaveGrafanaConnection(); const saveGranolaConnection = useSaveGranolaConnection(); + const saveElevenLabsConnection = useSaveElevenLabsConnection(); const saveSnowflakeConnection = useSaveSnowflakeConnection(); const saveVercelConnection = useSaveVercelConnection(); const asanaConnectionSummary = useMemo(() => { @@ -1091,6 +1231,18 @@ export function Integrations() { const granolaConnection = useGranolaConnection( isAdmin && (isGranolaConnected || isGranolaDialogOpen), ); + const elevenLabsConnectionSummary = useMemo(() => { + const connection = (userMcpConnections.data ?? []).find( + (entry) => entry.mcpId === 'elevenlabs', + ); + + return connection; + }, [userMcpConnections.data]); + const isElevenLabsConnected = + elevenLabsConnectionSummary?.authStatus === 'authenticated'; + const elevenLabsConnection = useElevenLabsConnection( + isAdmin && (isElevenLabsConnected || isElevenLabsDialogOpen), + ); const grafanaConnectionSummary = useMemo(() => { const connection = (userMcpConnections.data ?? []).find( (entry) => entry.mcpId === 'grafana', @@ -1158,6 +1310,44 @@ export function Integrations() { setGranolaForm(buildEmptyGranolaForm()); }, [granolaConnection.isPending, isGranolaConnected, isGranolaDialogOpen]); + useEffect(() => { + if (!isElevenLabsDialogOpen) { + return; + } + + if (elevenLabsConnection.isPending && isElevenLabsConnected) { + return; + } + + setElevenLabsFieldErrors({}); + setElevenLabsFormError(null); + setElevenLabsForm(buildElevenLabsForm(elevenLabsConnection.data)); + }, [ + elevenLabsConnection.data, + elevenLabsConnection.isPending, + isElevenLabsConnected, + isElevenLabsDialogOpen, + ]); + + useEffect(() => { + if (!isElevenLabsDialogOpen) { + return; + } + + if (elevenLabsConnection.isPending && isElevenLabsConnected) { + return; + } + + setElevenLabsFieldErrors({}); + setElevenLabsFormError(null); + setElevenLabsForm(buildElevenLabsForm(elevenLabsConnection.data)); + }, [ + elevenLabsConnection.data, + elevenLabsConnection.isPending, + isElevenLabsConnected, + isElevenLabsDialogOpen, + ]); + useEffect(() => { if (!isSnowflakeDialogOpen) { return; @@ -1392,6 +1582,27 @@ export function Integrations() { }); } + if (integration.id === 'elevenlabs') { + return buildAdminConfiguredIntegrationItem({ + integration, + connection: userConnectionMap.get(integration.id), + orgEnabled: orgEnablementMap.get(integration.id) ?? false, + highlightedIntegrationId, + savePending: saveElevenLabsConnection.isPending, + disconnectPending: disconnectMcp.isPending, + disconnectingMcpId: disconnectMcp.variables?.mcpId, + dialogOpen: isElevenLabsDialogOpen, + connectionPending: elevenLabsConnection.isPending, + canConfigure: isAdmin, + // Credential-only: no agent tools to manage. + canManageTools: false, + openDialog: () => setIsElevenLabsDialogOpen(true), + openToolDialog: () => openMcpToolDialog(integration), + disconnectIntegration: () => + disconnectAdminConfiguredIntegration(integration), + }); + } + if (integration.id === 'snowflake') { return buildAdminConfiguredIntegrationItem({ integration, @@ -1497,6 +1708,7 @@ export function Integrations() { isAdmin && enabled && integration.serverMode !== 'native' && + integration.serverMode !== 'credential_only' && (!isDeploymentScoped || isConnected) ? { label: 'Manage tools', @@ -1592,6 +1804,7 @@ export function Integrations() { disconnectMcp, grafanaConnection.isPending, granolaConnection.isPending, + elevenLabsConnection.isPending, linearInstallation.data, linearInstallation.isPending, linearOauthSetup.isPending, @@ -1601,10 +1814,12 @@ export function Integrations() { isAdmin, isGrafanaDialogOpen, isGranolaDialogOpen, + isElevenLabsDialogOpen, isLinearOauthSetupOpen, saveAsanaConnection.isPending, saveGrafanaConnection.isPending, saveGranolaConnection.isPending, + saveElevenLabsConnection.isPending, saveVercelConnection.isPending, deploymentEnablements.data, pathname, @@ -1721,6 +1936,21 @@ export function Integrations() { setGranolaFormError(null); }; + const handleElevenLabsFieldChange = ( + field: keyof ElevenLabsFormState, + value: string, + ) => { + setElevenLabsForm((current) => ({ ...current, [field]: value })); + setElevenLabsFieldErrors((current) => { + if (!current[field]) { + return current; + } + + return { ...current, [field]: undefined }; + }); + setElevenLabsFormError(null); + }; + const handleGrafanaFieldChange = ( field: keyof GrafanaFormState, value: string, @@ -1777,6 +2007,19 @@ export function Integrations() { setGranolaForm(buildEmptyGranolaForm()); }; + const handleElevenLabsDialogOpenChange = (open: boolean) => { + setIsElevenLabsDialogOpen(open); + + setElevenLabsFieldErrors({}); + setElevenLabsFormError(null); + + if (!open) { + return; + } + + setElevenLabsForm(buildElevenLabsForm(elevenLabsConnection.data)); + }; + const handleSnowflakeDialogOpenChange = (open: boolean) => { setIsSnowflakeDialogOpen(open); @@ -1894,6 +2137,43 @@ export function Integrations() { }); }; + const handleElevenLabsSubmit = (event: FormEvent) => { + event.preventDefault(); + + const parsed = saveElevenLabsConnectionSchema.safeParse({ + apiKey: elevenLabsForm.apiKey, + voiceId: elevenLabsForm.voiceId, + }); + if (!parsed.success) { + setElevenLabsFieldErrors(getElevenLabsFieldErrors(parsed)); + return; + } + + if (!isElevenLabsConnected && parsed.data.apiKey.length === 0) { + setElevenLabsFieldErrors({ + apiKey: ['API key is required'], + }); + return; + } + + setElevenLabsFieldErrors({}); + setElevenLabsFormError(null); + + saveElevenLabsConnection.mutate(parsed.data, { + onSuccess: () => { + toast.success( + isElevenLabsConnected + ? 'ElevenLabs connection updated for this deployment.' + : 'ElevenLabs connected for this deployment.', + ); + handleElevenLabsDialogOpenChange(false); + }, + onError: (error) => { + setElevenLabsFormError(error.message); + }, + }); + }; + const handleSnowflakeSubmit = (event: FormEvent) => { event.preventDefault(); @@ -2106,6 +2386,30 @@ export function Integrations() { onFieldChange={handleGranolaFieldChange} /> + + Store an ElevenLabs API key and voice ID for this deployment. The + key stays encrypted server-side and is used only by the control + plane to narrate feature-demo videos. + + } + onSubmit={handleElevenLabsSubmit} + > + + = { asana: siAsana, betterstack: siBetterstack, braintrust: siBraintrust, + elevenlabs: siElevenlabs, dependabot: siDependabot, datadog: siDatadog, discord: siDiscord, diff --git a/apps/web/src/hooks/mcp-connections/index.ts b/apps/web/src/hooks/mcp-connections/index.ts index dc07469c5..42902ab64 100644 --- a/apps/web/src/hooks/mcp-connections/index.ts +++ b/apps/web/src/hooks/mcp-connections/index.ts @@ -13,6 +13,8 @@ export { useAsanaConnection } from './useAsanaConnection'; export { useSaveAsanaConnection } from './useSaveAsanaConnection'; export { useGranolaConnection } from './useGranolaConnection'; export { useSaveGranolaConnection } from './useSaveGranolaConnection'; +export { useElevenLabsConnection } from './useElevenLabsConnection'; +export { useSaveElevenLabsConnection } from './useSaveElevenLabsConnection'; export { useSaveGrafanaConnection } from './useSaveGrafanaConnection'; export { useSaveSnowflakeConnection } from './useSaveSnowflakeConnection'; export { useSnowflakeConnection } from './useSnowflakeConnection'; diff --git a/apps/web/src/hooks/mcp-connections/useElevenLabsConnection.ts b/apps/web/src/hooks/mcp-connections/useElevenLabsConnection.ts new file mode 100644 index 000000000..1314897f4 --- /dev/null +++ b/apps/web/src/hooks/mcp-connections/useElevenLabsConnection.ts @@ -0,0 +1,14 @@ +'use client'; + +import { useQuery } from '@tanstack/react-query'; + +import { useTRPC } from '@/trpc/client'; + +export function useElevenLabsConnection(enabled = true) { + const trpc = useTRPC(); + + return useQuery({ + ...trpc.mcpConnections.elevenLabsConnection.queryOptions(), + enabled, + }); +} diff --git a/apps/web/src/hooks/mcp-connections/useSaveElevenLabsConnection.ts b/apps/web/src/hooks/mcp-connections/useSaveElevenLabsConnection.ts new file mode 100644 index 000000000..ea7fa5515 --- /dev/null +++ b/apps/web/src/hooks/mcp-connections/useSaveElevenLabsConnection.ts @@ -0,0 +1,26 @@ +'use client'; + +import { useMutation, useQueryClient } from '@tanstack/react-query'; + +import { useTRPC } from '@/trpc/client'; + +export function useSaveElevenLabsConnection() { + const trpc = useTRPC(); + const queryClient = useQueryClient(); + + return useMutation( + trpc.mcpConnections.saveElevenLabsConnection.mutationOptions({ + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: trpc.mcpConnections.deploymentEnablements.queryKey(), + }); + queryClient.invalidateQueries({ + queryKey: trpc.mcpConnections.userConnections.queryKey(), + }); + queryClient.invalidateQueries({ + queryKey: trpc.mcpConnections.elevenLabsConnection.queryKey(), + }); + }, + }), + ); +} diff --git a/apps/web/src/trpc/commands/mcp-connections/index.ts b/apps/web/src/trpc/commands/mcp-connections/index.ts index 12b3b73bc..8dc911f2f 100644 --- a/apps/web/src/trpc/commands/mcp-connections/index.ts +++ b/apps/web/src/trpc/commands/mcp-connections/index.ts @@ -18,6 +18,7 @@ import { type McpConnectionRole, isMcpConnectionAsanaConfig, isMcpConnectionGranolaConfig, + isMcpConnectionElevenLabsConfig, isMcpConnectionGrafanaConfig, isMcpConnectionSnowflakeConfig, isMcpConnectionVercelConfig, @@ -42,6 +43,7 @@ import { MCP_TOOL_CATALOG_REQUIRES_PERSONAL_CONNECTION } from '@/lib/mcp-tool-er import type { SaveAsanaConnectionInput, SaveGranolaConnectionInput, + SaveElevenLabsConnectionInput, SaveGrafanaConnectionInput, SaveSnowflakeConnectionInput, SaveVercelConnectionInput, @@ -776,6 +778,30 @@ export async function getGranolaConnectionCommand(auth: UserAuthSuccess) { }; } +export async function getElevenLabsConnectionCommand(auth: UserAuthSuccess) { + assertAdmin(auth); + + const connection = await db.query.mcpConnections.findFirst({ + where: and( + eq(mcpConnections.mcpId, 'elevenlabs'), + isNull(mcpConnections.userId), + ), + columns: { + authConfig: true, + authStatus: true, + }, + }); + + if (!connection || !isMcpConnectionElevenLabsConfig(connection.authConfig)) { + return null; + } + + return { + authStatus: connection.authStatus, + voiceId: connection.authConfig.voiceId, + }; +} + export async function getVercelConnectionCommand( auth: UserAuthSuccess, ): Promise { @@ -1146,6 +1172,91 @@ export async function saveGranolaConnectionCommand( }; } +export async function saveElevenLabsConnectionCommand( + auth: UserAuthSuccess, + input: SaveElevenLabsConnectionInput, +) { + assertAdmin(auth); + assertCuratedIntegrationsEnabled(); + + const existingConnection = await db.query.mcpConnections.findFirst({ + where: and( + eq(mcpConnections.mcpId, 'elevenlabs'), + isNull(mcpConnections.userId), + ), + columns: { + authConfig: true, + }, + }); + + const existingConfig = isMcpConnectionElevenLabsConfig( + existingConnection?.authConfig, + ) + ? existingConnection.authConfig + : null; + const nextEncryptedApiKey = + input.apiKey.length > 0 + ? encrypt(input.apiKey) + : existingConfig?.encryptedApiKey; + + if (!nextEncryptedApiKey) { + throw new Error( + 'ElevenLabs API key is required when no ElevenLabs key is already stored.', + ); + } + + const authConfig = { + type: 'elevenlabs' as const, + encryptedApiKey: nextEncryptedApiKey, + voiceId: input.voiceId, + }; + + await db + .insert(mcpConnections) + .values({ + userId: null, + mcpId: 'elevenlabs', + connectionRole: 'default', + authConfig, + enabled: true, + authStatus: 'authenticated', + }) + .onConflictDoUpdate({ + target: [ + mcpConnections.userId, + mcpConnections.mcpId, + mcpConnections.connectionRole, + ], + set: { + connectionRole: 'default', + authConfig, + enabled: true, + authStatus: 'authenticated', + updatedAt: new Date(), + }, + }); + + await db + .insert(deploymentMcpEnablements) + .values({ + mcpId: 'elevenlabs', + enabled: true, + enabledByUserId: auth.userId, + }) + .onConflictDoUpdate({ + target: [deploymentMcpEnablements.mcpId], + set: { + enabled: true, + enabledByUserId: auth.userId, + updatedAt: new Date(), + }, + }); + + return { + authStatus: 'authenticated' as const, + }; +} + export async function saveVercelConnectionCommand( auth: UserAuthSuccess, input: SaveVercelConnectionInput, diff --git a/apps/web/src/trpc/routers/_app.ts b/apps/web/src/trpc/routers/_app.ts index 2c61968fc..8c0f68e36 100644 --- a/apps/web/src/trpc/routers/_app.ts +++ b/apps/web/src/trpc/routers/_app.ts @@ -39,6 +39,7 @@ import { filterSchema, saveAsanaConnectionSchema, saveGranolaConnectionSchema, + saveElevenLabsConnectionSchema, saveGrafanaConnectionSchema, saveSnowflakeConnectionSchema, saveVercelConnectionSchema, @@ -199,12 +200,14 @@ import { getUserMcpConnectionsCommand, getAsanaConnectionCommand, getGranolaConnectionCommand, + getElevenLabsConnectionCommand, getGrafanaConnectionCommand, getSnowflakeConnectionCommand, getVercelConnectionCommand, listDeploymentMcpIntegrationToolsCommand, saveAsanaConnectionCommand, saveGranolaConnectionCommand, + saveElevenLabsConnectionCommand, saveGrafanaConnectionCommand, saveSnowflakeConnectionCommand, saveVercelConnectionCommand, @@ -1693,6 +1696,10 @@ export const appRouter = createRouter({ getGranolaConnectionCommand(auth), ), + elevenLabsConnection: protectedProcedure.query(({ ctx: { auth } }) => + getElevenLabsConnectionCommand(auth), + ), + grafanaConnection: protectedProcedure.query(({ ctx: { auth } }) => getGrafanaConnectionCommand(auth), ), @@ -1761,6 +1768,12 @@ export const appRouter = createRouter({ saveGranolaConnectionCommand(auth, input), ), + saveElevenLabsConnection: protectedProcedure + .input(saveElevenLabsConnectionSchema) + .mutation(({ ctx: { auth }, input }) => + saveElevenLabsConnectionCommand(auth, input), + ), + saveGrafanaConnection: protectedProcedure .input(saveGrafanaConnectionSchema) .mutation(({ ctx: { auth }, input }) => diff --git a/apps/web/src/types/mcp-connections.ts b/apps/web/src/types/mcp-connections.ts index b7fdc5514..b55620eab 100644 --- a/apps/web/src/types/mcp-connections.ts +++ b/apps/web/src/types/mcp-connections.ts @@ -44,6 +44,18 @@ export type SaveGranolaConnectionInput = z.infer< typeof saveGranolaConnectionSchema >; +export const saveElevenLabsConnectionSchema = z.object({ + apiKey: z.string().transform((value) => value.trim()), + voiceId: z + .string() + .transform((value) => value.trim()) + .pipe(z.string().min(1, 'Voice ID is required')), +}); + +export type SaveElevenLabsConnectionInput = z.infer< + typeof saveElevenLabsConnectionSchema +>; + export const saveVercelConnectionSchema = z.object({ accessToken: z.string().transform((value) => value.trim()), defaultTeamIdOrSlug: z diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index 05f7197d2..f1a2a8efa 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -8,7 +8,7 @@ FROM ubuntu:24.04 AS runtime # to their source. LABEL org.opencontainers.image.source=https://github.com/RooCodeInc/Roomote -ARG AGENT_BROWSER_VERSION=0.27.0 +ARG AGENT_BROWSER_VERSION=0.33.2 ARG OPENCODE_CLI_VERSION=1.18.10 ARG FFMPEG_INSTALLER_VERSION=1.1.0 # @ffprobe-installer (not ffprobe-static, which ships no linux/arm64 binary @@ -17,6 +17,10 @@ ARG FFPROBE_INSTALLER_VERSION=2.1.2 ARG GH_VERSION=2.81.0 ARG NODE_PTY_VERSION=1.1.0 ARG PM2_VERSION=7.0.1 +# Remotion mirrors chrome-headless-shell builds for both linux64 and +# linux-arm64; the bundled agent-browser Chrome cannot render Remotion +# compositions (Chrome 143 removed old-headless). +ARG REMOTION_VERSION=4.0.507 RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -122,6 +126,7 @@ ENV PATH=/home/roomote/.local/bin:/opt/mise/shims:${PATH} ENV AGENT_BROWSER_EXECUTABLE_PATH="/opt/agent-browser/chrome" ENV FFMPEG_EXECUTABLE_PATH="/opt/ffmpeg/bin/ffmpeg" ENV FFPROBE_EXECUTABLE_PATH="/opt/ffmpeg/bin/ffprobe" +ENV REMOTION_HEADLESS_SHELL_PATH="/opt/remotion/headless-shell" ENV ROOMOTE_BAKED_OPENCODE_CLI_VERSION=${OPENCODE_CLI_VERSION} COPY .docker/sandbox/install-browser-agent.sh /tmp/install-browser-agent.sh @@ -220,6 +225,45 @@ RUN sudo mkdir -p /opt/ffmpeg/bin \ && ln -sf "${FFPROBE_EXECUTABLE_PATH}" "$HOME/.local/bin/ffprobe" \ && printf '%s\n' "${FFMPEG_INSTALLER_VERSION}+${FFPROBE_INSTALLER_VERSION}" > /opt/ffmpeg/.installed +# Bake Remotion's chrome-headless-shell at a fixed path so renders never +# download a browser at runtime. `npx remotion browser ensure` fetches the +# matching linux64/linux-arm64 build into node_modules/.remotion; copy that +# tree to /opt/remotion and expose a stable arch-independent symlink at +# ${REMOTION_HEADLESS_SHELL_PATH} so consumers need no arch logic. The +# binary name differs per platform (headless_shell on linux-arm64, +# chrome-headless-shell on linux64), so it is resolved by search instead of +# assuming either name. +RUN sudo mkdir -p /opt/remotion \ + && sudo chown -R roomote:roomote /opt/remotion \ + && REMOTION_TMP="$(mktemp -d)" \ + && cd "$REMOTION_TMP" \ + && printf '%s\n' '{ "name": "remotion-browser-seed", "private": true }' > package.json \ + && npm install --no-save --no-package-lock \ + "@remotion/cli@${REMOTION_VERSION}" \ + "remotion@${REMOTION_VERSION}" \ + && npx remotion browser ensure \ + && cp -R node_modules/.remotion/chrome-headless-shell /opt/remotion/chrome-headless-shell \ + && REMOTION_SHELL="$(find /opt/remotion/chrome-headless-shell -type f \( -name headless_shell -o -name chrome-headless-shell \) | head -n 1)" \ + && test -n "$REMOTION_SHELL" \ + && test -x "$REMOTION_SHELL" \ + && ln -sf "$REMOTION_SHELL" "${REMOTION_HEADLESS_SHELL_PATH}" \ + && printf '%s\n' "${REMOTION_VERSION}" > /opt/remotion/.installed \ + && cd / \ + && rm -rf "$REMOTION_TMP" + +# Pre-install the feature-demo render project's dependencies (Remotion + +# React) so demos reuse them instead of running a multi-minute npm install on +# every render. Keyed off the skill's own pinned package.json (the single +# source of truth), so the baked modules cannot drift from what the render +# project expects. The skill reuses /opt/feature-demo/render/node_modules and +# falls back to npm install only when it is absent or an adapted composition +# adds dependencies. +COPY packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json /opt/feature-demo/render/package.json +RUN sudo chown -R roomote:roomote /opt/feature-demo \ + && cd /opt/feature-demo/render \ + && npm install --no-audit --no-fund \ + && printf '%s\n' "ready" > /opt/feature-demo/render/.installed + RUN npm install --prefix /sandbox --no-save --no-package-lock \ "node-pty@${NODE_PTY_VERSION}" \ "opencode-ai@${OPENCODE_CLI_VERSION}" \ diff --git a/apps/worker/src/commands/setup/legacy-runtime-tools.ts b/apps/worker/src/commands/setup/legacy-runtime-tools.ts index 5c5b1557c..b770ed7a7 100644 --- a/apps/worker/src/commands/setup/legacy-runtime-tools.ts +++ b/apps/worker/src/commands/setup/legacy-runtime-tools.ts @@ -10,7 +10,7 @@ import type { StartupLogger } from '../../logging'; import { isCommandAvailable } from './command-availability'; import { withAptLock } from './package-manager'; -const AGENT_BROWSER_VERSION = '0.27.0'; +const AGENT_BROWSER_VERSION = '0.33.2'; const AGENT_BROWSER_INSTALL_ROOT = '/opt/agent-browser'; diff --git a/apps/worker/src/commands/setup/setup-mcps.ts b/apps/worker/src/commands/setup/setup-mcps.ts index 51dcfc1cf..9ce1fcfbe 100644 --- a/apps/worker/src/commands/setup/setup-mcps.ts +++ b/apps/worker/src/commands/setup/setup-mcps.ts @@ -3,6 +3,7 @@ import * as path from 'node:path'; import { CUSTOM_MCP_PROXY_PATH_PREFIX, getMcpIntegrationUpstreamUrl, + isCredentialOnlyMcpIntegration, isReservedRuntimeMcpEnvVarName, isRoomoteNamespacedEnvVarName, MCP_INTEGRATIONS, @@ -170,7 +171,11 @@ function resolveConfigValues( function buildIntegrationProxyMap(): Map { const integrationConfigs: IntegrationProxyConfig[] = []; - for (const integration of MCP_INTEGRATIONS) { + // Credential-only integrations have no MCP server and are never delivered + // to sandboxes, so they get no proxy-path entry. + for (const integration of MCP_INTEGRATIONS.filter( + (candidate) => !isCredentialOnlyMcpIntegration(candidate), + )) { const upstreamUrl = getMcpIntegrationUpstreamUrl(integration); const upstream = upstreamUrl ? parseUrl(upstreamUrl) : null; diff --git a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts index 4d9e03d58..7117d94ff 100644 --- a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts +++ b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts @@ -25,6 +25,47 @@ describe('createProofRunnerAgentPrompt', () => { ); }); + it('sanctions only the digest-verified feature-demo capture runner', () => { + const digest = 'a'.repeat(64); + const prompt = createProofRunnerAgentPrompt( + 'http://127.0.0.1:3000', + digest, + ); + + expect(prompt).toContain( + 'the feature-demo capture runner staged by the parent at `/tmp/feature-demo/capture.mjs`', + ); + + // The staging path is shared/parent-writable, so the sanction closes the + // check/use race by reading the file exactly once and executing the same + // bytes it hashed (a single node process; no separate hash-then-run step, + // no re-open of the swappable path). + expect(prompt).toContain('reading the file exactly once'); + expect(prompt).toContain( + 'const b=readFileSync("/tmp/feature-demo/capture.mjs")', + ); + expect(prompt).toContain( + `createHash("sha256").update(b).digest("hex")!=="${digest}"`, + ); + expect(prompt).toContain( + 'await import("data:text/javascript;base64,"+b.toString("base64"))', + ); + expect(prompt).toContain('capture runner integrity mismatch'); + expect(prompt).toContain( + 'never fall back to `node /tmp/feature-demo/capture.mjs`', + ); + expect(prompt).toContain( + 'This exception covers exactly this verified-execution command and no other script.', + ); + }); + + it('omits the capture-runner sanction entirely without a digest', () => { + const prompt = createProofRunnerAgentPrompt('http://127.0.0.1:3000'); + + expect(prompt).not.toContain('/tmp/feature-demo/capture.mjs'); + expect(prompt).not.toContain('sanctioned exception'); + }); + it('returns artifact IDs and explicit parent sharing guidance', () => { const prompt = createProofRunnerAgentPrompt('http://127.0.0.1:3000'); diff --git a/apps/worker/src/run-task/agent-home.ts b/apps/worker/src/run-task/agent-home.ts index 2f2c92265..1f6021915 100644 --- a/apps/worker/src/run-task/agent-home.ts +++ b/apps/worker/src/run-task/agent-home.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import * as fs from 'node:fs'; import * as path from 'node:path'; @@ -1304,15 +1305,47 @@ function createArchitectAgentConfig(options: { }; } +/** + * Digest of the activated feature-demo capture runner, pinned into the + * proof-runner prompt so its sanctioned /tmp staging path (writable by any + * parent flow) cannot be used to smuggle a different script into the + * runner's browser sanction. Undefined (skill absent/unreadable) omits the + * sanction entirely. + */ +function resolveFeatureDemoCaptureRunnerDigest( + homeDir: string, +): string | undefined { + const runnerPath = path.join( + homeDir, + '.agents', + 'skills', + 'feature-demo', + 'capture', + 'capture.mjs', + ); + + try { + return createHash('sha256') + .update(fs.readFileSync(runnerPath)) + .digest('hex'); + } catch { + return undefined; + } +} + function createProofRunnerAgentConfig( browserTarget: string, + featureDemoCaptureRunnerSha256: string | undefined, ): Record { return { description: 'Delegated browser proof runner that captures and uploads screenshot and screencast proof from the sandbox-local browser surface.', mode: 'subagent', hidden: true, - prompt: createProofRunnerAgentPrompt(browserTarget), + prompt: createProofRunnerAgentPrompt( + browserTarget, + featureDemoCaptureRunnerSha256, + ), permission: { read: 'allow', list: 'allow', @@ -1960,7 +1993,10 @@ export function generateOpenCodeConfig({ if (proofBrowserTarget) { operatorAgent[ROOMOTE_OPENCODE_PROOF_RUNNER_AGENT_NAME] = - createProofRunnerAgentConfig(proofBrowserTarget); + createProofRunnerAgentConfig( + proofBrowserTarget, + resolveFeatureDemoCaptureRunnerDigest(homeDir), + ); const proofRunnerInstructionsPath = path.join( openCodeConfigDir, diff --git a/apps/worker/src/run-task/proof-runner-prompt.ts b/apps/worker/src/run-task/proof-runner-prompt.ts index 0c4786ef6..89b256c6a 100644 --- a/apps/worker/src/run-task/proof-runner-prompt.ts +++ b/apps/worker/src/run-task/proof-runner-prompt.ts @@ -4,8 +4,16 @@ export const ROOMOTE_OPENCODE_PROOF_RUNNER_AGENT_NAME = 'proof-runner'; * Builds the hidden proof-runner subagent prompt. The browser target is baked * in at config-generation time so the runner never depends on a staged * runtime handoff file for its capture configuration. + * + * `featureDemoCaptureRunnerSha256` is the digest of the activated + * feature-demo capture runner. The runner's staging path (/tmp) is writable + * by any parent flow, so the sanction binds to verified content, not the + * pathname; without a digest the sanction is omitted entirely. */ -export function createProofRunnerAgentPrompt(browserTarget: string): string { +export function createProofRunnerAgentPrompt( + browserTarget: string, + featureDemoCaptureRunnerSha256?: string, +): string { return [ 'You are the single delegated proof runner for this task.', '', @@ -27,6 +35,17 @@ export function createProofRunnerAgentPrompt(browserTarget: string): string { '- Before the first browser command, explicitly load the `agent-browser` skill or CLI-served guidance exactly once. If the OpenCode Skill tool is available, invoke the `agent-browser` skill. If it is not available, run `agent-browser skills get core --full` in the shell and treat that output as the browser usage guide.', '- `agent-browser` is a command-line executable, not an OpenCode tool or MCP tool. Invoke it with shell commands such as `agent-browser get url`; do not look for an internal tool named `agent-browser`.', '- `agent-browser` is the only allowed browser automation CLI. Do not use Playwright, browser DevTools, curl-only screenshot substitutes, or any other browser automation path.', + ...(featureDemoCaptureRunnerSha256 + ? [ + '- One sanctioned exception: the feature-demo capture runner staged by the parent at `/tmp/feature-demo/capture.mjs`. It is an `agent-browser` orchestrator — every browser action it performs goes through the `agent-browser` CLI — so running it (with the environment variables the brief specifies) is compliant browser work, not a disallowed automation path. The staging path is in shared, parent-writable `/tmp`, so a separate hash step then a `node ` step would leave the file swappable in between. Close that window by reading the file exactly once and executing the same bytes you hashed — run precisely this command (it hashes the loaded bytes and, only on an exact match, executes those same bytes, never re-opening the file):', + '', + ' ```', + ` SCRIPT=/tmp/feature-demo/demo-script.json OUT_DIR=/tmp/feature-demo/work node --input-type=module -e 'import{readFileSync}from"node:fs";import{createHash}from"node:crypto";const b=readFileSync("/tmp/feature-demo/capture.mjs");if(createHash("sha256").update(b).digest("hex")!=="${featureDemoCaptureRunnerSha256}"){console.error("capture runner integrity mismatch");process.exit(1)}await import("data:text/javascript;base64,"+b.toString("base64"))'`, + ' ```', + '', + ' On the `capture runner integrity mismatch` exit, or if the file is missing, report blocked with blocker type `capture runner integrity mismatch` and do not run any other command in its place. Adjust only the `SCRIPT`/`OUT_DIR` values per the brief; never change the read-hash-execute body, and never fall back to `node /tmp/feature-demo/capture.mjs`. This exception covers exactly this verified-execution command and no other script.', + ] + : []), '- If the `agent-browser` skill/guidance cannot be loaded, or if the `agent-browser` executable is unavailable, report blocked with blocker type `proof runtime unavailable` and describe the missing skill guidance or CLI explicitly.', '- Treat the browser target above as the primary proof surface. Preserve its hostname exactly; do not rewrite `localhost` to `127.0.0.1` or the reverse, and do not substitute another surface.', '- Before deep diagnostics, inspect the live state with `agent-browser get url`, `agent-browser get title`, or `agent-browser snapshot -i`.', diff --git a/knip.ts b/knip.ts index 7bee864c5..fe1d3a636 100644 --- a/knip.ts +++ b/knip.ts @@ -74,6 +74,9 @@ const config: KnipConfig = { 'packages/cloud-agents': { entry: ['evals/router/**/*.ts'], project: ['src/**/*.ts'], + // The feature-demo skill bundles a standalone Remotion render project + // that runs from a sandbox work dir, not from this package's graph. + ignore: ['src/server/workflows/skills/standard/feature-demo/**'], ignoreBinaries: [ 'evals/router/promptfooconfig.ts', 'evals/router/promptfooconfig.followup.ts', diff --git a/packages/cloud-agents/src/packaged-skill-invocations.ts b/packages/cloud-agents/src/packaged-skill-invocations.ts index 1f3501eee..6e523b1fe 100644 --- a/packages/cloud-agents/src/packaged-skill-invocations.ts +++ b/packages/cloud-agents/src/packaged-skill-invocations.ts @@ -20,6 +20,7 @@ const CORE_PACKAGED_SKILL_INVOCATIONS = [ 'issue-fixer', 'environment-setup', 'explain-repo-code', + 'feature-demo', 'fix-pr', 'github-management', 'implement-repo-change', diff --git a/packages/cloud-agents/src/server/mcp-self-setup/catalog.ts b/packages/cloud-agents/src/server/mcp-self-setup/catalog.ts index 3958d9a1e..b3be68b56 100644 --- a/packages/cloud-agents/src/server/mcp-self-setup/catalog.ts +++ b/packages/cloud-agents/src/server/mcp-self-setup/catalog.ts @@ -165,6 +165,13 @@ export const MCP_SETUP_INTEGRATION_METADATA: Record< 'Keep access read-only and limited by the Granola key configuration', ], }, + elevenlabs: { + capabilities: [ + 'Narrate feature-demo videos in a configured ElevenLabs voice', + 'Keep the API key on the control plane, never exposed to agents', + 'Configure once per deployment with a text-to-speech-scoped key', + ], + }, supermemory: { capabilities: [ 'Save important decisions and context as shared memories during tasks', diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts new file mode 100644 index 000000000..c7f9b134b --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -0,0 +1,163 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { PACKAGED_SKILL_INVOCATIONS } from '../../../packaged-skill-invocations'; + +const thisFilePath = fileURLToPath(import.meta.url); +const thisDirPath = path.dirname(thisFilePath); +const skillDirPath = path.resolve( + thisDirPath, + '../skills/standard/feature-demo', +); + +describe('feature-demo skill', () => { + const skillContent = fs.readFileSync( + path.join(skillDirPath, 'SKILL.md'), + 'utf8', + ); + + it('is registered for explicit invocation', () => { + expect(PACKAGED_SKILL_INVOCATIONS).toContain('feature-demo'); + }); + + it('ships the capture runner, pipeline scripts, and render project', () => { + for (const relativePath of [ + 'capture/capture.mjs', + 'scripts/build-narration.mjs', + 'scripts/fit-timing.mjs', + 'render/package.json', + 'render/src/index.ts', + 'render/src/Root.tsx', + 'render/src/FeatureDemo.tsx', + 'render/src/DemoStage.tsx', + 'render/src/presets.ts', + 'render/props/timeline.json', + 'render/props/narration.json', + ]) { + expect( + fs.existsSync(path.join(skillDirPath, relativePath)), + `${relativePath} must ship with the skill`, + ).toBe(true); + } + }); + + it('delegates demo planning to the advisor with parent-owned verification', () => { + expect(skillContent).toContain( + 'Delegate the creative plan to the `advisor` subagent with the Task tool', + ); + // The advisor cannot see the live page; the parent must verify selectors. + expect(skillContent).toContain( + "Treat the advisor's plan as internal guidance, not finished work", + ); + // Repository-backed surfaces verify against source; external public + // pages have no local source and no separate browser pre-flight (the + // proof-runner takes one brief per task), so they rely on capture's own + // loud per-selector failure within the allowed retry. + expect(skillContent).toContain('Repository-backed surface'); + expect(skillContent).toContain('External public page named by the user'); + expect(skillContent).toContain( + 'the runner resolves every selector live and fails loudly naming any that do not resolve', + ); + }); + + it('keeps browser work delegated to proof-runner', () => { + expect(skillContent).toContain( + "Browser automation is the proof-runner subagent's exclusive surface", + ); + expect(skillContent).toContain('proof runtime unavailable'); + expect(skillContent).toContain( + 'Never load or invoke `agent-browser` (or any other browser automation) from this skill', + ); + }); + + it('stages the capture runner at the sanctioned /tmp path for delegation', () => { + // Home-directory paths do not survive the delegation boundary, and the + // proof-runner prompt sanctions exactly this staged path. + expect(skillContent).toContain( + 'cp "$HOME/.agents/skills/feature-demo/capture/capture.mjs" /tmp/feature-demo/capture.mjs', + ); + expect(skillContent).toContain('/tmp/feature-demo/capture.mjs'); + // The parent hands off the paths; the proof-runner owns the integrity- + // verified node invocation, so the skill must not dictate a raw node run. + expect(skillContent).toContain( + 'Do not dictate the `node` invocation yourself; the proof-runner owns that.', + ); + }); + + it('keeps TTS provider keys out of the sandbox', () => { + expect(skillContent).toContain( + 'Never ask for, read, or handle TTS provider keys.', + ); + expect(skillContent).toContain('captions-only'); + + const narrationScript = fs.readFileSync( + path.join(skillDirPath, 'scripts/build-narration.mjs'), + 'utf8', + ); + + // The sandbox-side script talks to the control plane with the run token + // and must never reference the provider or its key directly. + expect(narrationScript).toContain('/api/tts/narration'); + expect(narrationScript).toContain('ROOMOTE_CLOUD_TOKEN'); + expect(narrationScript.toLowerCase()).not.toContain('elevenlabs.io'); + expect(narrationScript).not.toContain('xi-api-key'); + }); + + it('never commits pipeline outputs into the repository', () => { + expect(skillContent).toContain( + 'Never commit recordings, renders, node_modules, or props into the repository.', + ); + }); + + it('records headless by default with a per-script headed opt-in for GPU surfaces', () => { + const captureRunner = fs.readFileSync( + path.join(skillDirPath, 'capture/capture.mjs'), + 'utf8', + ); + + // Headless + frame ticker is the deterministic default; headed exists + // for WebGL/3D surfaces that never present to the headless compositor. + expect(captureRunner).toContain( + "const HEADED = script.headed === true || process.env.HEADED === '1'", + ); + expect(captureRunner).toContain("HEADED ? ['--headed', ...args] : args"); + expect(skillContent).toContain('"headed": true'); + }); + + it('frames the render project as an adaptable reference template', () => { + expect(skillContent).toContain('**reference template**'); + // Pin to the three skills relevant to editing a composition, not the + // full bundle. + expect(skillContent).toContain( + 'npx -y skills add remotion-dev/skills --skill remotion-markup --skill remotion-render --skill remotion-docs --yes', + ); + expect(skillContent).toContain( + 'The timeline JSON schema is the stable contract', + ); + }); + + it('renders with the baked headless shell and a runtime fallback', () => { + expect(skillContent).toContain( + '--browser-executable="${REMOTION_HEADLESS_SHELL_PATH:-/opt/remotion/headless-shell}"', + ); + expect(skillContent).toContain('npx remotion browser ensure'); + }); + + it('reuses the image-baked render dependencies instead of installing', () => { + // Fast path copies the baked node_modules; npm install is only the + // adapted-deps / old-snapshot fallback. + expect(skillContent).toContain( + 'cp -R /opt/feature-demo/render/node_modules /tmp/feature-demo/render/node_modules', + ); + expect(skillContent).toContain( + 'only if you adapted the composition to add dependencies', + ); + }); + + it('delivers through manage_artifacts with canonical URLs only', () => { + expect(skillContent).toContain('manage_artifacts'); + expect(skillContent).toContain('never invent URLs'); + expect(skillContent).toContain('## Screencasts'); + }); +}); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md new file mode 100644 index 000000000..069f65956 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -0,0 +1,147 @@ +--- +name: feature-demo +description: Produce a polished, Screen Studio-style demo video of a product feature — recorded live in the sandbox browser, with smooth zooms, cursor effects, captions, and optional voice-over narration. +--- + + +You produce short, polished feature-demo videos: a real recording of the +product driven live in the browser, post-produced with smooth zoom-to-element +moves, a synthetic cursor, click ripples, captions, and (when the deployment +has narration configured) a voice-over that leads each visual beat. + +Use this skill when the user asks for a demo video, walkthrough recording, +or promo clip of a feature. It is NOT for verification screencasts of a code +change — that is `capture-visual-proof`. + + + +The narrative drives the visuals. The demo is planned as a spoken story +first; narration is synthesized (or, captions-only, each line's speaking +time is estimated) BEFORE capture, and the capture runner conducts the +browser to it — each beat holds for exactly as long as its line takes to +speak, and clip start times are stamped at the moment each zoom lands. The +runner also logs, on the same clock, where the cursor is, when clicks land, +and the resolved rectangle of every focused element, so a zoom can never +drift from its element and nothing needs retiming afterwards. + +Pipeline: plan the narrative → author script → narrate → capture (browser, +delegated, paced to the narrative) → trim opening → render → verify → +upload. + +The bundled `render/` project is a **reference template**, not a fixed +pipeline stage: copy it to the work dir and adapt the copy freely (branding, +caption styling, layout, extra presets) when the demo calls for it. The +timeline JSON is the stable contract between capture and render — keep the +copy consuming it and any adaptation stays in sync with the recording. + +All work happens under `/tmp/feature-demo/work` (the work dir). Nothing from +this pipeline is ever committed to the repository. + + + + + +Scope and plan the demo (advisor) + +Delegate the creative plan to the `advisor` subagent with the Task tool — it runs the planning model at deeper reasoning and can read the repository to study the feature. Give it a complete brief: the user's request, the surface to record (URL), the relevant source paths for the feature's UI, the beat vocabulary and pacing constraints from step 2 verbatim, and the caption style rules. Ask it to return, as short final text: 2-5 story beats in order (each with the on-screen element to focus, a proposed resilient CSS selector from the source, a 3-9 word caption that doubles as the spoken narration line, and any click/type interactions the story needs), plus which preset(s) fit. A good demo is 10-25 seconds of motion; more beats than that dilutes it. +Treat the advisor's plan as internal guidance, not finished work: the advisor cannot run commands or see the live page, so its selectors are educated guesses from source. You own verification (step 2) and every artifact. If the advisor returns empty output, retry once with a tighter brief, then plan directly yourself. +Pick presets: `wide` (1920x1080) is the default; add `vertical` (1080x1920) only when the user wants a social/short-form cut. +If the surface needs the app running, make sure the environment is up first (dev server reachable) before capturing. + + + + +Author the demo script + +Write `/tmp/feature-demo/demo-script.json`: `{ "url": string, "viewport": { "w": 1280, "h": 800 }, "beats": [...] }`. Beat actions: +- `{ "a": "wait", "ms": n }` / `{ "a": "hold", "ms": n }` — let the page settle / linger. +- `{ "a": "scrollTo", "sel": css, "ms": n }` — scroll an element into view (the scroll shows in the recording). +- `{ "a": "focus", "sel": css, "scale": 1.3-1.9, "moveMs": ~700, "holdMs": 800-1200, "caption": "..." }` — glide the cursor to the element and zoom to it. Captions become the narration lines. +- `{ "a": "click", "sel": css, "holdMs": ~300 }` — real click with ripple. +- `{ "a": "type", "sel": css, "text": "..." }` — real typing. +- `{ "a": "reset", "ms": ~600 }` — pull back between beats (partial by design; pass `"full": true` only for the closing shot). +Keep the opening tight: one short `wait` for page load, then get moving. Dead opening seconds are trimmed automatically, but do not rely on it. +Write captions as short, conversational spoken lines (they double as the narration): contractions are good, symbols and long clauses are not. 3-9 words on screen; if the spoken wording should differ from the on-screen caption, put the spoken variants in `/tmp/feature-demo/lines.json` (array, one string per caption). +Selectors must be resilient: prefer ids, stable data attributes, or unique semantic tags over deep CSS chains. How you gain confidence before capture depends on the surface: +- Repository-backed surface (the app's own UI): grep the component/template source for each proposed selector and confirm it exists — cheap and worth doing every time. +- External public page named by the user: there is no local source to grep, and the browser is reachable only through the single capture delegation (the `proof-runner` takes one brief per task and keeps its configured surface, so a separate pre-flight is not available). Author best-effort resilient selectors — landmark roles, headings, obvious ids the advisor inferred; avoid deep chains — and rely on capture's own validation: the runner resolves every selector live and fails loudly naming any that do not resolve. Use that named failure to correct the script and re-capture within the one allowed retry (step 3). + + + + +Narrate first (the narrative drives the visuals) + +Synthesize the narration BEFORE capture: `SCRIPT=/tmp/feature-demo/demo-script.json node "$HOME/.agents/skills/feature-demo/scripts/build-narration.mjs"` (add `LINES=/tmp/feature-demo/lines.json` if spoken lines differ from captions). This posts the lines to the Roomote control plane, which holds the TTS credentials — no provider key exists in this sandbox, and you must never ask for one. It writes `/tmp/feature-demo/vo/*.mp3` and `/tmp/feature-demo/narration.json` with each line's measured duration; during capture, each beat then holds exactly as long as its line takes to speak, and clip start times are stamped as the zooms land. No retiming happens afterwards. +Exit code 3 means narration is not configured on this deployment: proceed captions-only — capture paces each captioned beat from the caption's estimated speaking time instead, so the demo still reads at narrative pace. Mention in the final report that voice-over is available if an admin connects ElevenLabs under Settings → Integrations. + + + + +Capture (delegated browser work) + +Stage the capture runner where the delegated runtime can see it — home-directory paths do not survive the delegation boundary, so always copy first: + +`mkdir -p /tmp/feature-demo && cp "$HOME/.agents/skills/feature-demo/capture/capture.mjs" /tmp/feature-demo/capture.mjs` + +The runner also reads `/tmp/feature-demo/narration.json` (written in step 3) on its own; captions-only runs simply will not have one. +Browser automation is the proof-runner subagent's exclusive surface — do not load `agent-browser` or run the capture yourself. Delegate with the Task tool to `proof-runner`, telling it the script path (`/tmp/feature-demo/demo-script.json`), the output dir (`/tmp/feature-demo/work`), and the staged runner path (`/tmp/feature-demo/capture.mjs`), and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. The staged runner is proof-runner's one sanctioned script exception — an agent-browser orchestrator that shells the `agent-browser` CLI for every browser action — and its own instructions define the exact integrity-verified command it must use to execute it. Do not dictate the `node` invocation yourself; the proof-runner owns that. +If the harness has no proof-runner registered, report a blocker (`proof runtime unavailable`) instead of driving the browser from this skill. +Expected outputs: `/tmp/feature-demo/work/recording.mp4` and `/tmp/feature-demo/work/timeline.json`. Verify both exist and that `ffprobe` reports a duration close to the timeline's `durationSeconds` (the runner itself fails loudly when the recording is much shorter than the interaction). One retry on failure; then report blocked with the runner's error. +The runner records headless by default with an imperceptible frame ticker, which is deterministic and captures at wall-clock rate on ordinary pages. For GPU-backed canvases (games, 3D, WebGL/WebGPU) — which never present frames to the headless compositor — set `"headed": true` at the top level of the demo script: the runner then records through `agent-browser --headed` (auto-Xvfb in the sandbox), stops any existing daemon first so the flag actually applies, and verifies the browser's real mode from its user agent — headless Chrome self-identifies as `HeadlessChrome` — hard-failing when headed was requested but a headless daemon won the launch. Headed recording on the current agent-browser can wedge the daemon on longer sessions, so use it only when the surface requires it and keep headed demos short. +If a recording comes back much shorter than the interaction, the runner fails loudly: for a WebGL/WebGPU-heavy surface, retry once with `"headed": true` if the first attempt was headless, and if it still fails report `webgl surface stalls recording` naming the surface. If `record stop` itself reports an ffmpeg error, the sandbox is likely a stale snapshot with an outdated runtime ffmpeg (`stale sandbox runtime`). + + + + +Trim the opening + +Run `WORK_DIR=/tmp/feature-demo/work node "$HOME/.agents/skills/feature-demo/scripts/fit-timing.mjs"`. Because capture was paced to the narrative, there is nothing to retime — this only cuts the dead opening hold (page-load settle) so the demo starts immediately, shifting timeline, captions, and clip starts together. Check its printed line schedule for sanity. + + + + +Render + +Assemble the render project in the work dir: +- `cp -R "$HOME/.agents/skills/feature-demo/render" /tmp/feature-demo/render` +- `cp /tmp/feature-demo/work/timeline.json /tmp/feature-demo/work/narration.json /tmp/feature-demo/render/props/` (skip narration.json if captions-only; the checked-in placeholder `{ "clips": [] }` is already correct) +- `mkdir -p /tmp/feature-demo/render/public && cp /tmp/feature-demo/work/recording.mp4 /tmp/feature-demo/render/public/` +- `cp -R /tmp/feature-demo/vo /tmp/feature-demo/render/public/vo` (narrated demos only; the mp3s were written next to the script in step 3) +Adapt the copied composition when the default look does not fit the request — different backdrop, caption treatment, brand colors, an extra preset, a layout change. Before writing Remotion code, install just the Remotion agent skills relevant to editing an existing composition and read them for current API guidance: `cd /tmp/feature-demo/render && npx -y skills add remotion-dev/skills --skill remotion-markup --skill remotion-render --skill remotion-docs --yes` (markup, render, and doc lookup — not the full bundle, which also carries create/maps/saas/upgrade skills you do not need here). Preserve the pieces that encode hard-won correctness unless you have a reason not to: the zoom transform with its counter-scaled cursor, the edge-clamp guard (only clamp an axis when the scaled window exceeds the canvas), and the timeline-driven keyframe interpolation. +Provide the dependencies. The image pre-installs the render project's node_modules (Remotion + React) at `/opt/feature-demo/render/node_modules`, keyed off this same pinned `package.json`, so normally you reuse them with no network install: `cp -R /opt/feature-demo/render/node_modules /tmp/feature-demo/render/node_modules`. Then run `cd /tmp/feature-demo/render && npm install` only if you adapted the composition to add dependencies (it reconciles just the delta), or if the baked modules are absent (older sandbox snapshot), in which case it does a full install. +Render with the image's baked headless shell: + +`npx remotion render src/index.ts Demo-wide out/demo-wide.mp4 --browser-executable="${REMOTION_HEADLESS_SHELL_PATH:-/opt/remotion/headless-shell}" --log=error` + +If that binary does not exist (older sandbox snapshot), run `npx remotion browser ensure` once and render without the flag. Repeat for `Demo-vertical` when the vertical preset was requested. + + + + +Verify honestly + +`ffprobe` the output: sane duration, a video stream, and an audio stream when narration was generated. +Extract 3-4 spread frames (`ffmpeg -ss -i out/demo-wide.mp4 -frames:v 1 frame.png`) and review each whole frame: zoom landed on the right element, cursor glued to it, caption legible and correctly timed, no backdrop showing through mid-zoom, no broken page state in shot. +A defect in the video is a blocker to report, not something to ship quietly. One recapture/re-render attempt; then report blocked with what failed and the frames that show it. + + + + +Deliver + +Upload the mp4 via `manage_artifacts` (`action: upload`, `type: general`) plus one representative keyframe PNG. Treat only the returned `artifactId`/`viewUrl`/`rawUrl` values as canonical — never invent URLs. +In a PR body, embed under `## Screencasts` using the existing convention: the keyframe image (its signed `rawUrl`) hyperlinked to the video's `viewUrl`, with a one-line caption. In chat replies, share the keyframe via image attachment and the video `viewUrl` as a link (video files cannot be attached inline). +End with a sharing note: what the demo shows, which presets were rendered, and whether it is narrated or captions-only. + + + + + + +Never load or invoke `agent-browser` (or any other browser automation) from this skill — capture is always delegated to the `proof-runner` subagent. +Never ask for, read, or handle TTS provider keys. Narration goes through the control-plane endpoint with the run token; a 404 there means captions-only. +All intermediate files live under `/tmp/feature-demo`. Never commit recordings, renders, node_modules, or props into the repository. +Adapt the work-dir copy of the render template, never the installed skill sources under `~/.agents/skills/feature-demo`. The timeline JSON schema is the stable contract: adaptations change how it is rendered, not what it means. +Report blockers honestly: a missing selector, a failed render, or a visibly broken frame is a blocker with evidence, not a reason to narrow the claim or ship a degraded video silently. +Voice, wording, and pacing choices belong to the user when they express them; defaults are: wide preset, captions as narration lines, conversational tone. + diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs new file mode 100644 index 000000000..25730c4ba --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -0,0 +1,389 @@ +// Capture runner: drives agent-browser through a demo script, records a +// cursorless WebM, and emits the timeline the Remotion renderer consumes. +// Runs inside the worker sandbox (agent-browser + ffmpeg present). +// +// Two ideas make the output polished: +// 1. The runner performs the real interactions AND logs, for the same clock, +// where the (synthetic) cursor is, when clicks land, and the resolved +// rect of each target — so a zoom can never drift from its element. +// 2. The NARRATIVE drives the visuals: each captioned beat holds for as long +// as its line takes to speak (the real clip duration when narration was +// synthesized before capture; an estimated speaking time for the caption +// text otherwise), and the runner stamps each clip's start at the moment +// its zoom actually lands. Nothing needs retiming afterwards. +// +// Usage: SCRIPT=/path/to/demo-script.json OUT_DIR=/tmp/feature-demo/work \ +// node capture.mjs +// Reads narration.json (written by build-narration.mjs before capture) from +// the script's directory when present. See SKILL.md for the schema. + +import { execFileSync } from 'node:child_process'; +import { dirname } from 'node:path'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + +const AB = process.env.AGENT_BROWSER_BIN || 'agent-browser'; +const OUT_DIR = process.env.OUT_DIR || '/tmp/feature-demo/work'; + +const SCRIPT_PATH = process.env.SCRIPT; + +if (!SCRIPT_PATH) { + console.error('Set SCRIPT to the demo-script JSON path.'); + process.exit(1); +} + +const script = JSON.parse(readFileSync(SCRIPT_PATH, 'utf8')); + +if (!script.url || !Array.isArray(script.beats)) { + console.error('Demo script must have { url, beats: [...] }.'); + process.exit(1); +} + +const VIEWPORT = script.viewport || { w: 1280, h: 800 }; + +// Narration manifest (pre-capture synthesis). Optional: without it the demo +// is captions-only and lines are paced by estimated speaking time. +const NARRATION_PATH = + process.env.NARRATION || `${dirname(SCRIPT_PATH)}/narration.json`; +const narration = existsSync(NARRATION_PATH) + ? JSON.parse(readFileSync(NARRATION_PATH, 'utf8')) + : null; + +const captionedBeatCount = script.beats.filter((b) => b.caption).length; + +if (narration && narration.clips.length !== captionedBeatCount) { + console.error( + `narration.json has ${narration.clips.length} clips but the script has ` + + `${captionedBeatCount} captioned beats; they must match 1:1 in order.`, + ); + process.exit(1); +} + +// The voice starts just before its zoom lands, then speaks over the hold. +const VOICE_LEAD = 0.4; +// Breathing room after a line ends before the next beat's motion begins. +const LINE_GAP = 0.35; + +// Estimated speaking time for captions-only pacing: ~2.8 words/second, +// clamped so terse labels still get a beat and long lines don't stall. +function estimateSpokenSeconds(text) { + const words = String(text).trim().split(/\s+/).length; + return Math.min(6, Math.max(1.8, words / 2.8)); +} + +// Between focus beats the camera pulls back only partially; pogo-ing to full +// wide between every zoom reads as jumpy. The final reset goes fully wide. +const GLIDE_SCALE = 1.18; + +// Headless by default: with the frame ticker it records deterministically at +// wall-clock rate on ordinary pages. Headed mode (auto-Xvfb) exists for +// GPU-backed canvases (games, 3D, WebGL/WebGPU) that never present to the +// headless compositor — opt in per demo with `"headed": true` in the script +// (or HEADED=1). Note: current agent-browser headed recording can wedge the +// daemon on longer sessions; use it only when the surface requires it. +const HEADED = script.headed === true || process.env.HEADED === '1'; + +const ab = (...args) => + execFileSync(AB, HEADED ? ['--headed', ...args] : args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + +const sleep = (ms) => execFileSync('sleep', [String(ms / 1000)]); + +let t0 = 0; +const now = () => (Date.now() - t0) / 1000; // seconds since record start + +function rect(sel) { + // The selector travels into the page-eval as base64 so no selector bytes + // are ever interpolated into code; the alphabet check makes that a hard + // guarantee rather than an encoding assumption. + const selB64 = Buffer.from(String(sel), 'utf8').toString('base64'); + if (!/^[A-Za-z0-9+/=]*$/.test(selB64)) { + throw new Error(`unencodable selector: ${sel}`); + } + const js = + `(function(){var e=document.querySelector(atob("${selB64}"));` + + `if(!e)return null;var r=e.getBoundingClientRect();` + + `return{x:r.x,y:r.y,w:r.width,h:r.height};})()`; + const out = ab('eval', js).trim(); + const r = JSON.parse(out); + if (!r) throw new Error(`element not found: ${sel}`); + return r; +} + +const centerNorm = (r) => ({ + x: (r.x + r.w / 2) / VIEWPORT.w, + y: (r.y + r.h / 2) / VIEWPORT.h, +}); + +const timeline = { + video: { path: 'recording.mp4', width: VIEWPORT.w, height: VIEWPORT.h }, + fps: 30, + durationSeconds: 0, + scaleKeys: [{ t: 0, v: 1 }], + focalKeys: [{ t: 0, v: { x: 0.5, y: 0.5 } }], + cursorKeys: [{ t: 0, v: { x: 0.5, y: 1.1 } }], + clicks: [], + captions: [], +}; + +let cur = { scale: 1, focal: { x: 0.5, y: 0.5 } }; +// Narrative pacing state: which line is next, and when the previous one +// finishes, so consecutive lines never overlap even if beats land early. +let lineIndex = 0; +let prevLineEnd = 0; +// Track the cursor so motion can be bracketed by a hold key: the renderer +// eases between consecutive keys, so without a hold at motion start the +// synthetic cursor would drift toward the next target through every wait, +// hold, and scroll in between while the real mouse is stationary. +let curCursor = { x: 0.5, y: 1.1 }; +const pushScale = (t, v) => timeline.scaleKeys.push({ t, v }); +const pushFocal = (t, v) => timeline.focalKeys.push({ t, v }); +const pushCursorMove = (startT, endT, target) => { + timeline.cursorKeys.push({ t: startT, v: curCursor }); + timeline.cursorKeys.push({ t: endT, v: target }); + curCursor = target; +}; + +// Headless (default) only: headless capture emits frames only on visual +// damage and stamps them without wall-clock gaps, so a static surface +// collapses into a sub-second video; an imperceptible 2px dot re-painting +// every animation frame keeps frames flowing at wall-clock rate. Headed +// mode presents at wall-clock rate on its own — verified on a fully static +// page — and skips the ticker. +const TICKER_JS = + '(function(){var d=document.createElement("div");' + + 'd.style.cssText="position:fixed;left:0;bottom:0;width:2px;height:2px;' + + 'z-index:2147483647;pointer-events:none;background:#000;opacity:0.01";' + + 'document.body.appendChild(d);var f=0;' + + '(function t(){d.style.opacity=(f++%2)?"0.02":"0.01";' + + 'requestAnimationFrame(t)})()})()'; + +async function run() { + mkdirSync(OUT_DIR, { recursive: true }); + // Fully stop any existing agent-browser daemon before recording. `--headed` + // (and other launch options) only take effect when the daemon launches the + // browser; against an already-running daemon agent-browser prints + // "--headed ignored: daemon already running" and records headless. `close` + // (alias quit/exit) tears the daemon down; `close --all` alone only clears + // sessions. This also gives `record start` a clean slate so the beats and + // the recorder share one page. + try { + ab('close'); + } catch { + // no active daemon is fine + } + ab('set', 'viewport', String(VIEWPORT.w), String(VIEWPORT.h)); + ab('record', 'start', `${OUT_DIR}/raw.webm`, script.url); + + // Verify the browser's actual mode positively rather than parsing daemon + // warnings (the wrapper's cookie seeding or our own viewport call may have + // launched the daemon first, which makes "--headed ignored" ambiguous). + // Headless Chrome self-identifies in its user agent. + if (HEADED) { + const ua = ab('eval', 'navigator.userAgent'); + if (/HeadlessChrome/i.test(ua)) { + throw new Error( + 'headed capture was requested but the browser is running headless ' + + '(HeadlessChrome user agent) — an earlier browser step likely ' + + 'launched a headless daemon. Run `agent-browser close`, ensure ' + + 'nothing else drives the browser, and retry.', + ); + } + } + t0 = Date.now(); + if (!HEADED) { + ab('eval', TICKER_JS); + } + sleep(300); // let the first frames settle + + for (const beat of script.beats) { + if (beat.a === 'hold' || beat.a === 'wait') { + sleep(beat.ms); + continue; + } + + if (beat.a === 'scrollTo') { + ab('scrollintoview', beat.sel); + sleep(beat.ms ?? 650); // let the scroll settle (shows in the recording) + continue; + } + + if (beat.a === 'focus') { + const c = centerNorm(rect(beat.sel)); + const start = now(); + pushScale(start, cur.scale); + pushFocal(start, cur.focal); + // Real hover so the app's hover state shows under the synthetic cursor. + ab( + 'mouse', + 'move', + String((c.x * VIEWPORT.w) | 0), + String((c.y * VIEWPORT.h) | 0), + ); + sleep(beat.moveMs ?? 700); // pace the glide for the eased cursor + const end = now(); + pushScale(end, beat.scale ?? 1.5); + pushFocal(end, c); + pushCursorMove(start, end, c); + cur = { scale: beat.scale ?? 1.5, focal: c }; + + if (beat.caption) { + // The narrative drives the hold: the line starts just before the + // zoom lands and the beat holds until it has been fully spoken, + // plus breathing room. Script holdMs acts as a minimum only. + const lineSeconds = narration + ? narration.clips[lineIndex].durationSeconds + : estimateSpokenSeconds(beat.caption); + const lineStart = + Math.round(Math.max(0.1, prevLineEnd + 0.1, end - VOICE_LEAD) * 1000) / + 1000; + const lineEnd = lineStart + lineSeconds; + prevLineEnd = lineEnd; + + timeline.captions.push({ + start: lineStart, + end: Math.round((lineEnd + 0.25) * 1000) / 1000, + text: beat.caption, + }); + if (narration) { + narration.clips[lineIndex].startSeconds = lineStart; + } + lineIndex += 1; + + const holdSeconds = Math.max( + (beat.holdMs ?? 900) / 1000, + lineEnd + LINE_GAP - now(), + ); + sleep(holdSeconds * 1000); + } else { + sleep(beat.holdMs ?? 900); + } + continue; + } + + if (beat.a === 'click') { + const c = centerNorm(rect(beat.sel)); + const t = now(); + timeline.clicks.push({ t, at: c }); + // Usually the cursor is already here from a preceding focus; when it + // is not, give it a short bracketed hop instead of a slow drift. + if (curCursor.x !== c.x || curCursor.y !== c.y) { + pushCursorMove(Math.max(0, t - 0.25), t, c); + } + ab('click', beat.sel); + sleep(beat.holdMs ?? 300); + continue; + } + + if (beat.a === 'type') { + ab('type', beat.sel, beat.text); + continue; + } + + if (beat.a === 'reset') { + // Partial pull-back keeps the camera alive between beats; `full: true` + // (or the closing reset) returns to wide. + const target = beat.full ? 1 : GLIDE_SCALE; + const start = now(); + pushScale(start, cur.scale); + pushFocal(start, cur.focal); + sleep(beat.ms ?? 600); + const end = now(); + pushScale(end, target); + pushFocal(end, { x: 0.5, y: 0.5 }); + cur = { scale: target, focal: { x: 0.5, y: 0.5 } }; + continue; + } + + throw new Error(`unknown beat action: ${beat.a}`); + } + + // Close on a full wide shot even when the script forgot a final reset. + if (cur.scale !== 1) { + const start = now(); + pushScale(start, cur.scale); + pushFocal(start, cur.focal); + sleep(600); + const end = now(); + pushScale(end, 1); + pushFocal(end, { x: 0.5, y: 0.5 }); + } + + timeline.durationSeconds = now(); + ab('record', 'stop'); + try { + ab('close', '--all'); + } catch { + // best-effort; the recording is already on disk + } + + // Honest-state gate: a recording much shorter than the interaction means + // the screencast stalled (or frames stayed sparse despite the ticker) and + // the demo would be garbage. Fail loudly instead of shipping it. + const recordedSeconds = parseFloat( + execFileSync('ffprobe', [ + '-v', + 'error', + '-show_entries', + 'format=duration', + '-of', + 'default=nw=1:nk=1', + `${OUT_DIR}/raw.webm`, + ]) + .toString() + .trim(), + ); + + if ( + !Number.isFinite(recordedSeconds) || + recordedSeconds < timeline.durationSeconds * 0.75 + ) { + throw new Error( + `recording is ${recordedSeconds}s but the interaction ran ` + + `${timeline.durationSeconds.toFixed(2)}s — screencast frames were ` + + `sparse or the recorder stalled. If \`record stop\` also reported an ` + + `ffmpeg error, the sandbox runtime is likely a stale snapshot with ` + + `an outdated ffmpeg. Report this as a blocker; do not render.`, + ); + } + + // WebM (VP8/VP9) -> H.264 mp4 for the renderer. + execFileSync( + 'ffmpeg', + [ + '-y', + '-i', + `${OUT_DIR}/raw.webm`, + '-c:v', + 'libx264', + '-pix_fmt', + 'yuv420p', + '-preset', + 'veryfast', + `${OUT_DIR}/recording.mp4`, + ], + { stdio: ['ignore', 'ignore', 'pipe'] }, + ); + + writeFileSync(`${OUT_DIR}/timeline.json`, JSON.stringify(timeline, null, 2)); + if (narration) { + // Clip start times are now stamped at the moments their zooms landed; + // this manifest plus the vo/ mp3s next to the script is everything the + // renderer needs. + writeFileSync( + `${OUT_DIR}/narration.json`, + JSON.stringify(narration, null, 2), + ); + } + console.log( + `captured: duration=${timeline.durationSeconds.toFixed(2)}s ` + + `clicks=${timeline.clicks.length} captions=${timeline.captions.length} ` + + `pacing=${narration ? 'narrated' : 'captions-only (estimated)'}`, + ); +} + +run().catch((e) => { + console.error('capture failed:', e.message); + process.exit(1); +}); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md new file mode 100644 index 000000000..c9d1e58ac --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md @@ -0,0 +1,26 @@ +# feature-demo render template + +A reference Remotion composition for feature-demo videos. At demo time the +agent copies this directory into the sandbox work dir and may adapt the copy +(branding, caption styling, layout, presets) — see `../SKILL.md`. + +The stable contract is `props/timeline.json` (emitted by the capture runner) +and `props/narration.json` (emitted by the narration scripts): compositions +render those; adaptations change how they look, not what they mean. + +Worth preserving when adapting — each encodes a bug class found the hard way: + +- **Counter-scaled cursor/ripple** (`DemoStage.tsx`): cursor and ripple are + children of the zoom-transform container with a `1/S` counter-scale, so + they stay glued to page coordinates at constant size through any zoom. +- **Edge-clamp guard** (`DemoStage.tsx`): the translate that keeps backdrop + from showing behind the window must only clamp an axis when the scaled + window exceeds the canvas on that axis; clamping a smaller-than-canvas + window (vertical presets) shoves it into a corner. +- **Timeline-driven interpolation**: all motion eases between capture-emitted + keys. Do not invent motion that is not in the timeline — it will drift + from the recording. + +This directory is excluded from the repo's typecheck/lint/knip on purpose: +it is not part of the package graph and runs only from a work-dir copy with +its own pinned `npm install`. diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json new file mode 100644 index 000000000..a83125727 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json @@ -0,0 +1,12 @@ +{ + "name": "feature-demo-render", + "private": true, + "version": "1.0.0", + "description": "Remotion composition for Roomote feature-demo videos. Copied to a sandbox work dir at demo time; not a workspace package.", + "dependencies": { + "@remotion/cli": "4.0.507", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "remotion": "4.0.507" + } +} diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/narration.json b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/narration.json new file mode 100644 index 000000000..4f0573827 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/narration.json @@ -0,0 +1 @@ +{ "clips": [] } diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json new file mode 100644 index 000000000..5b7bbb061 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json @@ -0,0 +1,10 @@ +{ + "video": { "path": "recording.mp4", "width": 1280, "height": 800 }, + "fps": 30, + "durationSeconds": 1, + "scaleKeys": [{ "t": 0, "v": 1 }], + "focalKeys": [{ "t": 0, "v": { "x": 0.5, "y": 0.5 } }], + "cursorKeys": [{ "t": 0, "v": { "x": 0.5, "y": 1.1 } }], + "clicks": [], + "captions": [] +} diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx new file mode 100644 index 000000000..1153cc349 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx @@ -0,0 +1,225 @@ +import { OffthreadVideo, staticFile, useCurrentFrame } from 'remotion'; +import timeline from '../props/timeline.json'; +import type { CaptionStyle } from './presets'; + +export const FPS = timeline.fps; +export const DEMO_SECONDS = timeline.durationSeconds; + +type Vec = { x: number; y: number }; + +const smooth = (t: number) => t * t * (3 - 2 * t); +const clamp = (v: number, lo: number, hi: number) => + Math.max(lo, Math.min(hi, v)); + +function lerpNum(keys: { t: number; v: number }[], t: number): number { + if (t <= keys[0].t) return keys[0].v; + for (let i = 0; i < keys.length - 1; i++) { + const a = keys[i]; + const b = keys[i + 1]; + if (t >= a.t && t <= b.t) + return a.v + (b.v - a.v) * smooth((t - a.t) / Math.max(b.t - a.t, 1e-6)); + } + return keys[keys.length - 1].v; +} +function lerpVec(keys: { t: number; v: Vec }[], t: number): Vec { + if (t <= keys[0].t) return keys[0].v; + for (let i = 0; i < keys.length - 1; i++) { + const a = keys[i]; + const b = keys[i + 1]; + if (t >= a.t && t <= b.t) { + const p = smooth((t - a.t) / Math.max(b.t - a.t, 1e-6)); + return { x: a.v.x + (b.v.x - a.v.x) * p, y: a.v.y + (b.v.y - a.v.y) * p }; + } + } + return keys[keys.length - 1].v; +} + +const Cursor: React.FC<{ invScale: number }> = ({ invScale }) => ( +
+ + + +
+); + +// The polished demo "window": recorded video on a rounded panel, with the +// zoom/cursor/ripple/caption effects driven by the captured timeline. Laid +// out for whatever canvas size the preset asks for. +export const DemoStage: React.FC<{ + canvasW: number; + canvasH: number; + pad: number; + caption: CaptionStyle; + centerPull: number; + baseScale: number; +}> = ({ canvasW, canvasH, pad, caption, centerPull, baseScale }) => { + const frame = useCurrentFrame(); + const t = frame / FPS; + + const SHOT_W = timeline.video.width; + const SHOT_H = timeline.video.height; + const BASE_W = Math.round( + Math.min(canvasW - 2 * pad, ((canvasH - 2 * pad) * SHOT_W) / SHOT_H), + ); + const BASE_H = Math.round((BASE_W * SHOT_H) / SHOT_W); + const WIN_X = (canvasW - BASE_W) / 2; + const WIN_Y = (canvasH - BASE_H) / 2; + + const S = baseScale + (lerpNum(timeline.scaleKeys, t) - 1); + const focal = lerpVec(timeline.focalKeys, t); + const cursor = lerpVec(timeline.cursorKeys, t); + + const focal0x = WIN_X + focal.x * BASE_W; + const focal0y = WIN_Y + focal.y * BASE_H; + const k = clamp((S - 1) / 1.1, 0, 1) * centerPull; + const desiredTx = (canvasW / 2 - focal0x) * k; + const desiredTy = (canvasH / 2 - focal0y) * k; + // The edge-clamp keeps the backdrop from showing behind the window — but + // only makes sense when the scaled window is larger than the canvas on that + // axis (16:9 presets). For a vertical band the window is smaller than the + // canvas, so there is nothing to clamp: keep it centered instead of letting + // the clamp shove it into a corner. + const txMax = WIN_X + focal.x * BASE_W * (S - 1); + const txMin = canvasW - WIN_X - focal.x * BASE_W - BASE_W * (1 - focal.x) * S; + const tyMax = WIN_Y + focal.y * BASE_H * (S - 1); + const tyMin = canvasH - WIN_Y - focal.y * BASE_H - BASE_H * (1 - focal.y) * S; + const Tx = + BASE_W * S >= canvasW + ? clamp(desiredTx, Math.min(txMin, txMax), Math.max(txMin, txMax)) + : desiredTx; + const Ty = + BASE_H * S >= canvasH + ? clamp(desiredTy, Math.min(tyMin, tyMax), Math.max(tyMin, tyMax)) + : desiredTy; + const invScale = 1 / S; + + return ( + <> +
+
+ +
+ + {timeline.clicks.map((c, i) => { + const dt = t - c.t; + if (dt < 0 || dt > 0.6) return null; + const p = dt / 0.6; + const size = 12 + smooth(p) * 90; + return ( +
+ ); + })} + +
+ +
+
+ + {timeline.captions.map((cap, i) => { + const fadeIn = clamp((t - cap.start) / 0.25, 0, 1); + const fadeOut = clamp((cap.end - t) / 0.25, 0, 1); + const opacity = Math.min(fadeIn, fadeOut); + if (opacity <= 0) return null; + return ( +
+
+ {cap.text} +
+
+ ); + })} + + ); +}; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/FeatureDemo.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/FeatureDemo.tsx new file mode 100644 index 000000000..990dd6a07 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/FeatureDemo.tsx @@ -0,0 +1,44 @@ +import { AbsoluteFill, Audio, Sequence, staticFile } from 'remotion'; +import { DemoStage, FPS, DEMO_SECONDS } from './DemoStage'; +import { PRESETS, BACKDROP, type Preset } from './presets'; +import narration from '../props/narration.json'; + +export { FPS, DEMO_SECONDS }; + +// Narration can run slightly past the recorded motion; the composition holds +// on the last frame until it finishes. +const narrationEnd = narration.clips.reduce( + (m, c) => Math.max(m, c.startSeconds + c.durationSeconds), + 0, +); +export const TOTAL_SECONDS = Math.max(DEMO_SECONDS, narrationEnd + 0.4); +export const totalFrames = () => Math.round(TOTAL_SECONDS * FPS); + +export const FeatureDemo: React.FC<{ presetId: Preset['id'] }> = ({ + presetId, +}) => { + const preset = PRESETS[presetId]; + + return ( + + + + {narration.clips.map((c, i) => ( + + + ))} + + ); +}; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Root.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Root.tsx new file mode 100644 index 000000000..19dd1ce08 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Root.tsx @@ -0,0 +1,22 @@ +import { Composition } from 'remotion'; +import { FeatureDemo, FPS, totalFrames } from './FeatureDemo'; +import { PRESETS, type Preset } from './presets'; + +export const RemotionRoot: React.FC = () => { + return ( + <> + {(Object.values(PRESETS) as Preset[]).map((preset) => ( + + ))} + + ); +}; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/index.ts b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/index.ts new file mode 100644 index 000000000..91fa0f346 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/index.ts @@ -0,0 +1,4 @@ +import { registerRoot } from 'remotion'; +import { RemotionRoot } from './Root'; + +registerRoot(RemotionRoot); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/presets.ts b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/presets.ts new file mode 100644 index 000000000..d818a5b73 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/presets.ts @@ -0,0 +1,45 @@ +export type CaptionStyle = { + fontSize: number; + bottom: number; + maxWidthPct: number; +}; + +export type Preset = { + id: 'wide' | 'vertical'; + width: number; + height: number; + pad: number; + caption: CaptionStyle; + // How strongly a zoom pulls its focal point toward center. 0 keeps the + // window fixed and punches in place (right for vertical bands). + centerPull: number; + // Baseline zoom so a preset never fully zooms out (keeps a vertical band + // filled). Zooms add on top of it. + baseScale: number; +}; + +// One capture feeds every preset; presets differ only in framing and caption +// emphasis. No title cards or branding — the narration + captions carry it. +export const PRESETS: Record = { + wide: { + id: 'wide', + width: 1920, + height: 1080, + pad: 120, + caption: { fontSize: 34, bottom: 70, maxWidthPct: 74 }, + centerPull: 0.9, + baseScale: 1, + }, + vertical: { + id: 'vertical', + width: 1080, + height: 1920, + pad: 90, + caption: { fontSize: 46, bottom: 300, maxWidthPct: 86 }, + centerPull: 0, + baseScale: 1.25, + }, +}; + +export const BACKDROP = + 'radial-gradient(120% 120% at 20% 0%, #23305c 0%, #141726 55%, #0d0f18 100%)'; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs new file mode 100644 index 000000000..5e9267f89 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs @@ -0,0 +1,136 @@ +// Synthesize the demo's narrative BEFORE capture, so the narration drives +// the visuals: each line's real spoken duration paces its beat during +// recording, and no post-hoc retiming is needed. +// +// Reads the demo script's captioned beats (or LINES override), posts the +// lines to the Roomote control plane (which holds the TTS credentials — no +// provider key exists in this sandbox), writes vo/.mp3 next to the +// script, and emits narration.json with each clip's measured duration. +// startSeconds is stamped later by the capture runner at the moment each +// beat actually lands. +// +// Exit codes: 0 = narration written; 3 = TTS not configured on this +// deployment (callers proceed captions-only; capture then paces beats from +// estimated speaking time instead); 1 = real failure. +// +// Usage: SCRIPT=/tmp/feature-demo/demo-script.json node build-narration.mjs +// Optional: LINES=/path/to/lines.json (array of spoken lines, one per +// captioned beat, when spoken wording should differ from captions). + +import { execFileSync } from 'node:child_process'; +import { dirname } from 'node:path'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + +const SCRIPT_PATH = process.env.SCRIPT || '/tmp/feature-demo/demo-script.json'; +const OUT_DIR = dirname(SCRIPT_PATH); + +// Same base-URL/token resolution as the roomote MCP server config. +const rawBaseUrl = + process.env.ROOMOTE_PLATFORM_API_URL || + process.env.TRPC_URL || + 'http://localhost:13001'; +const baseUrl = rawBaseUrl.replace(/\/$/, ''); +const token = process.env.ROOMOTE_CLOUD_TOKEN || process.env.AUTH_TOKEN; + +if (!token) { + console.error('No run token in the environment (ROOMOTE_CLOUD_TOKEN).'); + process.exit(1); +} + +const script = JSON.parse(readFileSync(SCRIPT_PATH, 'utf8')); +const captions = (script.beats ?? []) + .filter((beat) => beat.caption) + .map((beat) => beat.caption); +const lines = process.env.LINES + ? JSON.parse(readFileSync(process.env.LINES, 'utf8')) + : captions; + +if (!Array.isArray(lines) || lines.length === 0) { + console.error('No narration lines (script has no captioned beats).'); + process.exit(1); +} + +if (lines.length !== captions.length) { + console.error( + `LINES has ${lines.length} entries but the script has ` + + `${captions.length} captioned beats; they must match 1:1 in order.`, + ); + process.exit(1); +} + +function ffprobeDuration(file) { + const out = execFileSync('ffprobe', [ + '-v', + 'error', + '-show_entries', + 'format=duration', + '-of', + 'default=nw=1:nk=1', + file, + ]); + return parseFloat(out.toString().trim()); +} + +const run = async () => { + const response = await fetch(`${baseUrl}/api/tts/narration`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ lines }), + }); + + if (response.status === 404) { + console.error( + 'Narration TTS is not configured on this deployment; ' + + 'proceed captions-only (capture paces beats from caption text).', + ); + process.exit(3); + } + + if (!response.ok) { + throw new Error(`TTS request failed with status ${response.status}`); + } + + const { clips } = await response.json(); + + if (!Array.isArray(clips) || clips.length !== lines.length) { + throw new Error('TTS response did not include one clip per line'); + } + + mkdirSync(`${OUT_DIR}/vo`, { recursive: true }); + + const manifest = []; + + for (let i = 0; i < clips.length; i++) { + const file = `vo/${i}.mp3`; + + writeFileSync( + `${OUT_DIR}/${file}`, + Buffer.from(clips[i].audioBase64, 'base64'), + ); + manifest.push({ + file, + startSeconds: 0, + durationSeconds: + Math.round(ffprobeDuration(`${OUT_DIR}/${file}`) * 1000) / 1000, + }); + } + + writeFileSync( + `${OUT_DIR}/narration.json`, + JSON.stringify({ clips: manifest }, null, 2), + ); + + const total = manifest.reduce((s, c) => s + c.durationSeconds, 0); + console.log( + `narration: ${manifest.length} clips (${total.toFixed(1)}s spoken) ` + + `ready in ${OUT_DIR}/vo — capture will pace beats to them`, + ); +}; + +run().catch((e) => { + console.error(e.message); + process.exit(1); +}); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs new file mode 100644 index 000000000..9636f1035 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs @@ -0,0 +1,93 @@ +// Post-capture trim. With narration synthesized BEFORE capture, the runner +// paces every beat to its line and stamps clip starts as they land, so no +// retiming is needed here — the only remaining job is cutting the dead +// opening hold (page-load settle) so the demo starts immediately. All key, +// caption, and clip times shift together with the video's startFrom. +// +// Usage: WORK_DIR=/tmp/feature-demo/work node fit-timing.mjs + +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; + +const WORK_DIR = process.env.WORK_DIR || '/tmp/feature-demo/work'; + +const INTRO_SECONDS = 1.2; // what survives of the opening hold + +const timelinePath = `${WORK_DIR}/timeline.json`; +const narrationPath = `${WORK_DIR}/narration.json`; +const timeline = JSON.parse(readFileSync(timelinePath, 'utf8')); +const narration = existsSync(narrationPath) + ? JSON.parse(readFileSync(narrationPath, 'utf8')) + : null; + +// The first visible action is the earlier of the first caption/line start +// and the first motion key after t=0. +const firstMotion = Math.min( + ...timeline.captions.map((c) => c.start), + ...timeline.scaleKeys.filter((k) => k.t > 0).map((k) => k.t), + Number.POSITIVE_INFINITY, +); +const trim = + Number.isFinite(firstMotion) && firstMotion > INTRO_SECONDS + 0.5 + ? firstMotion - INTRO_SECONDS + : 0; + +function shiftKeys(keys) { + const out = []; + for (const key of keys) { + const t = key.t - trim; + if (t < 0) { + // keep at most one pre-zero key as the t=0 anchor + const anchored = { ...key, t: 0 }; + if (out.length > 0 && out[out.length - 1].t === 0) { + out[out.length - 1] = anchored; + } else { + out.push(anchored); + } + } else { + out.push({ ...key, t: Math.round(t * 1000) / 1000 }); + } + } + return out; +} + +if (trim > 0) { + timeline.scaleKeys = shiftKeys(timeline.scaleKeys); + timeline.focalKeys = shiftKeys(timeline.focalKeys); + timeline.cursorKeys = shiftKeys(timeline.cursorKeys); + timeline.clicks = timeline.clicks + .filter((c) => c.t >= trim) + .map((c) => ({ ...c, t: Math.round((c.t - trim) * 1000) / 1000 })); + for (const cap of timeline.captions) { + cap.start = Math.max(0, Math.round((cap.start - trim) * 1000) / 1000); + cap.end = Math.max(0, Math.round((cap.end - trim) * 1000) / 1000); + } + timeline.video.startFromSeconds = + (timeline.video.startFromSeconds ?? 0) + trim; + timeline.durationSeconds = + Math.round((timeline.durationSeconds - trim) * 1000) / 1000; + if (narration) { + for (const clip of narration.clips) { + clip.startSeconds = Math.max( + 0, + Math.round((clip.startSeconds - trim) * 1000) / 1000, + ); + } + } +} + +writeFileSync(timelinePath, JSON.stringify(timeline, null, 2)); +if (narration) { + writeFileSync(narrationPath, JSON.stringify(narration, null, 2)); +} + +console.log( + `fit: trimmed ${trim.toFixed(2)}s, video ${timeline.durationSeconds.toFixed(2)}s` + + (narration + ? `, ${narration.clips.length} narration clips (paced at capture)` + : ', captions-only'), +); +for (const cap of timeline.captions) { + console.log( + ` line "${cap.text}" ${cap.start.toFixed(2)}-${cap.end.toFixed(2)}`, + ); +} diff --git a/packages/cloud-agents/tsconfig.json b/packages/cloud-agents/tsconfig.json index 62afcee4e..a58412869 100644 --- a/packages/cloud-agents/tsconfig.json +++ b/packages/cloud-agents/tsconfig.json @@ -4,5 +4,11 @@ "types": ["node", "vitest/globals"] }, "include": ["src", "evals"], - "exclude": ["node_modules"] + // The feature-demo skill bundles a standalone Remotion render project that + // is copied into a sandbox work dir at demo time; it is not part of this + // package's program (JSX, remotion deps). + "exclude": [ + "node_modules", + "src/server/workflows/skills/standard/feature-demo/render" + ] } diff --git a/packages/compute-providers/src/adapters/modal.test.ts b/packages/compute-providers/src/adapters/modal.test.ts index 76e88106a..157361fa2 100644 --- a/packages/compute-providers/src/adapters/modal.test.ts +++ b/packages/compute-providers/src/adapters/modal.test.ts @@ -927,9 +927,16 @@ describe('ModalClient', () => { expect(installBrowserAgentScript).toContain( 'AGENT_BROWSER_FORWARD_ARGS=()', ); + // Default headless preserved, with an explicit per-invocation headed + // opt-in (the feature-demo capture runner passes --headed so GPU-backed + // canvases present to the compositor). expect(installBrowserAgentScript).toContain( 'export AGENT_BROWSER_HEADED=false', ); + expect(installBrowserAgentScript).toContain( + 'export AGENT_BROWSER_HEADED=true', + ); + expect(installBrowserAgentScript).toContain('AGENT_BROWSER_WANT_HEADED=1'); expect(installBrowserAgentScript).toContain( 'exec "$AGENT_BROWSER_BIN" "${AGENT_BROWSER_FORWARD_ARGS[@]}"', ); @@ -962,6 +969,42 @@ describe('ModalClient', () => { ); }); + it('expects the Remotion chrome-headless-shell to be baked into the worker runtime', () => { + const dockerfile = fs.readFileSync( + new URL('../../../../apps/worker/Dockerfile', import.meta.url), + 'utf8', + ); + + expect(dockerfile).toContain('ARG REMOTION_VERSION='); + expect(dockerfile).toContain( + 'ENV REMOTION_HEADLESS_SHELL_PATH="/opt/remotion/headless-shell"', + ); + expect(dockerfile).toContain('"@remotion/cli@${REMOTION_VERSION}"'); + expect(dockerfile).toContain('"remotion@${REMOTION_VERSION}"'); + expect(dockerfile).toContain('npx remotion browser ensure'); + expect(dockerfile).toContain( + 'cp -R node_modules/.remotion/chrome-headless-shell /opt/remotion/chrome-headless-shell', + ); + expect(dockerfile).toContain( + 'ln -sf "$REMOTION_SHELL" "${REMOTION_HEADLESS_SHELL_PATH}"', + ); + expect(dockerfile).toContain('/opt/remotion/.installed'); + }); + + it('pre-installs the feature-demo render dependencies from the skill package.json', () => { + const dockerfile = fs.readFileSync( + new URL('../../../../apps/worker/Dockerfile', import.meta.url), + 'utf8', + ); + + // Keyed off the skill's own pinned package.json so the baked node_modules + // cannot drift from what the render project expects. + expect(dockerfile).toContain( + 'COPY packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json /opt/feature-demo/render/package.json', + ); + expect(dockerfile).toContain('/opt/feature-demo/render/.installed'); + }); + it('expects PM2 to be available in the worker runtime', () => { const dockerfile = fs.readFileSync( new URL('../../../../apps/worker/Dockerfile', import.meta.url), diff --git a/packages/env/src/__tests__/index.test.ts b/packages/env/src/__tests__/index.test.ts index 98a991ae6..d3de661ca 100644 --- a/packages/env/src/__tests__/index.test.ts +++ b/packages/env/src/__tests__/index.test.ts @@ -116,7 +116,7 @@ describe('Env', () => { expect(env.DOCKER_WORKER_CPU_LIMIT).toBe(2); expect(env.DOCKER_WORKER_MEMORY_LIMIT).toBe('4g'); expect(env.DOCKER_TASK_DAEMON_MEMORY_LIMIT).toBe('8g'); - expect(env.DOCKER_WORKER_PIDS_LIMIT).toBe(512); + expect(env.DOCKER_WORKER_PIDS_LIMIT).toBe(2048); expect(env.DOCKER_WORKER_DISK_LIMIT).toBe('20g'); expect(env.DOCKER_WORKER_ALLOW_UNBOUNDED_DISK).toBe(false); expect(env.DOCKER_WORKER_LOG_MAX_SIZE).toBe('10m'); diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts index a97d92081..aab3248e8 100644 --- a/packages/env/src/index.ts +++ b/packages/env/src/index.ts @@ -85,7 +85,11 @@ const serverSchema = { DOCKER_WORKER_CPU_LIMIT: z.coerce.number().positive().default(2), DOCKER_WORKER_MEMORY_LIMIT: dockerSize().default('4g'), DOCKER_TASK_DAEMON_MEMORY_LIMIT: dockerSize().default('8g'), - DOCKER_WORKER_PIDS_LIMIT: z.coerce.number().int().positive().default(512), + // Headroom for browser-driving tasks: a Chrome process tree plus the + // agent-browser daemon and worker toolchain runs well over the old 512 cap + // under load. `--init` reaps zombies (see docker-sandbox-security), but the + // live process count still needs room. Env-overridable. + DOCKER_WORKER_PIDS_LIMIT: z.coerce.number().int().positive().default(2048), DOCKER_WORKER_DISK_LIMIT: dockerSize().default('20g'), DOCKER_WORKER_ALLOW_UNBOUNDED_DISK: optInBoolean(), DOCKER_WORKER_LOG_MAX_SIZE: dockerSize().default('10m'), @@ -131,6 +135,12 @@ const serverSchema = { // private networks; a CIDR list rather than a boolean so opening one // internal host does not re-expose every adjacent service. R_CUSTOM_MCP_ALLOWED_PRIVATE_CIDRS: z.string().min(1).optional(), + // ElevenLabs credentials for the narration TTS endpoint. The key stays on + // the control plane; task sandboxes reach TTS only through /api/tts with + // their run-scoped token (see apps/api/src/handlers/tts). Unset means the + // feature is off and the endpoint 404s. + R_ELEVENLABS_API_KEY: z.string().min(1).optional(), + R_ELEVENLABS_VOICE_ID: z.string().min(1).optional(), R_INTERCOM_APP_ID: z.string().min(1).optional(), R_POSTHOG_PROJECT_KEY: z.string().min(1).optional(), R_POSTHOG_HOST: z.string().url().optional(), @@ -449,6 +459,8 @@ const OPTIONAL_NON_EMPTY_KEYS = new Set([ 'R_INSTANCE_ID', 'STATUSPAGE_INCIDENTS_ENABLED_INSTANCE_ID', 'R_CUSTOM_MCP_ALLOWED_PRIVATE_CIDRS', + 'R_ELEVENLABS_API_KEY', + 'R_ELEVENLABS_VOICE_ID', 'R_INTERCOM_APP_ID', 'R_POSTHOG_PROJECT_KEY', 'R_POSTHOG_HOST', diff --git a/packages/sdk/src/server/routers/mcp-connections.test.ts b/packages/sdk/src/server/routers/mcp-connections.test.ts index 5102f3f72..d8b58a63a 100644 --- a/packages/sdk/src/server/routers/mcp-connections.test.ts +++ b/packages/sdk/src/server/routers/mcp-connections.test.ts @@ -481,6 +481,31 @@ describe('mcpConnectionsRouter.getMcpServerConfigs', () => { expect(JSON.stringify(result)).not.toContain('enc:secret'); }); + it('never delivers a credential-only ElevenLabs connection to agents', async () => { + mockOrderBy.mockResolvedValue([ + buildJoinedConnectionRow({ + id: 'conn-elevenlabs', + userId: null, + mcpId: 'elevenlabs', + authConfig: { + type: 'elevenlabs', + encryptedApiKey: 'enc:secret', + voiceId: 'v1', + }, + }), + ]); + + const result = await createCaller( + 'https://api.preview.roomote.run/trpc/mcpConnections.getMcpServerConfigs', + ).getMcpServerConfigs(); + + // Credential-only: no MCP server, and the secret never leaves the + // control plane toward a task sandbox. + expect(result).toEqual({ servers: {} }); + expect(JSON.stringify(result)).not.toContain('enc:secret'); + expect(JSON.stringify(result)).not.toContain('v1'); + }); + it('returns Vercel proxy config without requesting OAuth tokens', async () => { mockOrderBy.mockResolvedValue([ buildJoinedConnectionRow({ diff --git a/packages/sdk/src/server/routers/mcp-connections.ts b/packages/sdk/src/server/routers/mcp-connections.ts index 431290478..bd4d0e08c 100644 --- a/packages/sdk/src/server/routers/mcp-connections.ts +++ b/packages/sdk/src/server/routers/mcp-connections.ts @@ -393,6 +393,18 @@ async function buildCuratedMcpServerConfigs(ctx: { continue; } + // Credential-only integrations (e.g. ElevenLabs narration) are consumed + // by control-plane features exclusively: no MCP server exists for them + // and their credentials must never be delivered toward a task sandbox. + if (integration.serverMode === 'credential_only') { + console.info('[getMcpServerConfigs] Skipping connection:', { + connectionId: connection.id, + mcpId: connection.mcpId, + reason: 'credential_only_integration', + }); + continue; + } + if (connection.mcpId === 'linear') { if (!INTEGRATION_PROXY_MCP_IDS.has(connection.mcpId)) { continue; diff --git a/packages/types/src/__tests__/mcp-oauth.test.ts b/packages/types/src/__tests__/mcp-oauth.test.ts index f4d1eba57..e9a046031 100644 --- a/packages/types/src/__tests__/mcp-oauth.test.ts +++ b/packages/types/src/__tests__/mcp-oauth.test.ts @@ -5,6 +5,7 @@ import { getMcpIntegrationDefaultDisabledTools, getMcpIntegrationOauthScopeMode, getMcpIntegrationOauthScopes, + isMcpConnectionElevenLabsConfig, LINEAR_APP_OAUTH_SCOPES, MONDAY_MCP_READ_ONLY_OAUTH_SCOPES, RESEND_DEFAULT_DISABLED_TOOL_NAMES, @@ -63,6 +64,43 @@ describe('Granola API key connection', () => { }); }); +describe('ElevenLabs credential-only integration', () => { + it('is a deployment-scoped credential_only entry with no MCP url', () => { + expect(getMcpIntegration('elevenlabs')).toMatchObject({ + name: 'ElevenLabs', + connectionScope: 'deployment', + connectionMode: 'admin_configured', + serverMode: 'credential_only', + }); + expect(getMcpIntegration('elevenlabs')?.url).toBeUndefined(); + expect(getMcpIntegrationDefaultDisabledTools('elevenlabs')).toEqual([]); + }); + + it('recognizes a valid stored ElevenLabs config and rejects others', () => { + expect( + isMcpConnectionElevenLabsConfig({ + type: 'elevenlabs', + encryptedApiKey: 'enc', + voiceId: 'v1', + }), + ).toBe(true); + // Missing voiceId, wrong type, and empty configs must not pass. + expect( + isMcpConnectionElevenLabsConfig({ + type: 'elevenlabs', + encryptedApiKey: 'enc', + } as never), + ).toBe(false); + expect( + isMcpConnectionElevenLabsConfig({ + type: 'granola', + encryptedApiKey: 'x', + }), + ).toBe(false); + expect(isMcpConnectionElevenLabsConfig({})).toBe(false); + }); +}); + describe('Resend OAuth', () => { it('uses a deployment-scoped hosted MCP with risky tools disabled initially', () => { expect(getMcpIntegration('resend')).toMatchObject({ diff --git a/packages/types/src/control-plane-env-vars.test.ts b/packages/types/src/control-plane-env-vars.test.ts index f9844a777..b88246fa7 100644 --- a/packages/types/src/control-plane-env-vars.test.ts +++ b/packages/types/src/control-plane-env-vars.test.ts @@ -27,6 +27,8 @@ describe('CONTROL_PLANE_ENV_VAR_NAMES', () => { 'DATABASE_URL', 'S3_SECRET_ACCESS_KEY', 'R_LICENSE_KEY', + 'R_ELEVENLABS_API_KEY', + 'R_ELEVENLABS_VOICE_ID', ]) { expect(CONTROL_PLANE_ENV_VAR_NAMES.has(name)).toBe(true); } diff --git a/packages/types/src/control-plane-env-vars.ts b/packages/types/src/control-plane-env-vars.ts index 65fd1b405..24a05ae98 100644 --- a/packages/types/src/control-plane-env-vars.ts +++ b/packages/types/src/control-plane-env-vars.ts @@ -99,6 +99,17 @@ export const INSTANCE_SECRET_ENV_VAR_NAMES: ReadonlySet = new Set([ 'R_LICENSE_KEY', ]); +/** + * Media-generation provider credentials. Task sandboxes reach these providers + * only through control-plane endpoints (e.g. /api/tts) that inject the key + * server-side, so the raw values must never enter a sandbox. The voice/model + * ids are not secrets, but a task never needs them either. + */ +export const MEDIA_PROVIDER_ENV_VAR_NAMES: ReadonlySet = new Set([ + 'R_ELEVENLABS_API_KEY', + 'R_ELEVENLABS_VOICE_ID', +]); + /** * Declarative environment provisioning inputs, managed through the deployment * environment. Not secrets per se, but they are control-plane configuration @@ -134,6 +145,7 @@ export const CONTROL_PLANE_ENV_VAR_NAMES: ReadonlySet = new Set( ...INTEGRATION_BOT_SECRET_ENV_VAR_NAMES, ...PROVIDER_IDENTIFIER_ENV_VAR_NAMES, ...INSTANCE_SECRET_ENV_VAR_NAMES, + ...MEDIA_PROVIDER_ENV_VAR_NAMES, ...DECLARATIVE_ENVIRONMENT_ENV_VAR_NAMES, ...DISABLED_MODEL_PROVIDER_ENV_VAR_NAMES, ], diff --git a/packages/types/src/mcp-oauth.ts b/packages/types/src/mcp-oauth.ts index a94a15827..e4ced8451 100644 --- a/packages/types/src/mcp-oauth.ts +++ b/packages/types/src/mcp-oauth.ts @@ -133,6 +133,21 @@ export interface McpConnectionGranolaConfig { encryptedApiKey: string; } +/** + * Deployment-scoped ElevenLabs connection config stored in + * mcpConnections.authConfig. + * + * Credential-only: consumed by the control-plane narration TTS endpoint + * (/api/tts) and deliberately excluded from agent MCP config delivery — the + * key never reaches a task sandbox. The API key is expected to be encrypted + * before persistence; the voice id is plain configuration. + */ +export interface McpConnectionElevenLabsConfig { + type: 'elevenlabs'; + encryptedApiKey: string; + voiceId: string; +} + /** * Organization-scoped Vercel connection config stored in mcpConnections.authConfig. * @@ -168,6 +183,7 @@ export type McpConnectionAuthConfig = | McpConnectionSnowflakeConfig | McpConnectionAsanaConfig | McpConnectionGranolaConfig + | McpConnectionElevenLabsConfig | McpConnectionVercelConfig | McpConnectionGrafanaConfig | Record; @@ -244,7 +260,18 @@ export type LinkedAccountSetup = { export type McpIntegrationOauthScopeMode = 'all' | 'read-only'; export type McpIntegrationConnectionMode = 'oauth' | 'admin_configured'; -export type McpIntegrationServerMode = 'upstream_proxy' | 'native'; +/** + * - `upstream_proxy`: tools come from a remote MCP server reached through the + * Roomote proxy. + * - `native`: tools come from a Roomote-implemented MCP handler. + * - `credential_only`: the integration stores admin-configured credentials + * consumed by control-plane features; it exposes no MCP tools to agents + * and is never delivered to task sandboxes. + */ +export type McpIntegrationServerMode = + | 'upstream_proxy' + | 'native' + | 'credential_only'; export type McpIntegrationOAuthClientEnv = { clientIdEnv: string; @@ -501,6 +528,15 @@ export const MCP_INTEGRATIONS: McpIntegration[] = [ instructions: 'Use Granola to browse and read meeting notes, transcripts, folders, decisions, and action items through the deployment API key. The built-in tools are read-only.', }, + { + id: 'elevenlabs', + name: 'ElevenLabs', + description: `Connect ElevenLabs so ${PRODUCT_NAME} can narrate feature-demo videos with your voice. The API key stays on the control plane; agents get no ElevenLabs tools and the key never enters a task sandbox`, + icon: 'elevenlabs', + connectionScope: 'deployment', + connectionMode: 'admin_configured', + serverMode: 'credential_only', + }, { id: 'supermemory', name: 'Supermemory', @@ -636,6 +672,26 @@ export function isNativeMcpIntegration( return integration?.serverMode === 'native'; } +/** + * Credential-only integrations store admin-configured credentials for + * control-plane features and expose no MCP tools to agents, so they get no + * proxy route and are never delivered to task sandboxes. + */ +export function isCredentialOnlyMcpIntegration( + integrationOrId: McpIntegration | string | undefined, +): boolean { + if (!integrationOrId) { + return false; + } + + const integration = + typeof integrationOrId === 'string' + ? getMcpIntegration(integrationOrId) + : integrationOrId; + + return integration?.serverMode === 'credential_only'; +} + export function getMcpIntegrationUpstreamUrl( integrationOrId: McpIntegration | string | undefined, ): string | null { @@ -822,6 +878,21 @@ export function isMcpConnectionGranolaConfig( ); } +export function isMcpConnectionElevenLabsConfig( + authConfig: McpConnectionAuthConfig | null | undefined, +): authConfig is McpConnectionElevenLabsConfig { + return Boolean( + authConfig && + typeof authConfig === 'object' && + 'type' in authConfig && + authConfig.type === 'elevenlabs' && + 'encryptedApiKey' in authConfig && + typeof authConfig.encryptedApiKey === 'string' && + 'voiceId' in authConfig && + typeof authConfig.voiceId === 'string', + ); +} + export function isMcpConnectionVercelConfig( authConfig: McpConnectionAuthConfig | null | undefined, ): authConfig is McpConnectionVercelConfig {