From 5684bd89e7a637c4728b172d3f4287632c6d170e Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:47:59 -0400 Subject: [PATCH 01/27] [Feat] Add feature-demo skill for polished, narrated demo videos A new standard skill that produces Screen Studio-style feature demos: the demo script drives real browser interactions (delegated to the proof-runner subagent via agent-browser) while logging cursor, click, and element-rect timelines on the same clock, then a bundled Remotion composition post-produces the recording with zoom-to-element moves, click ripples, captions, and optional voice-over narration. - packages/cloud-agents .../feature-demo: SKILL.md orchestration, capture runner, narration + timing-fit scripts, render project (excluded from tsc/knip/oxlint like agent-browser; copied verbatim into the sandbox skills dir at activation) - apps/api /api/tts/narration: control-plane TTS endpoint (task-token policy + rate limit). ElevenLabs credentials live only on the control plane; R_ELEVENLABS_* are control-plane env names, stripped from sandbox env injection. Unconfigured deployments 404 and the skill degrades to captions-only. - apps/worker/Dockerfile: bake Remotion's chrome-headless-shell at /opt/remotion/headless-shell (the bundled Chrome removed old headless); runtime fallback via remotion browser ensure for older snapshots. - docs: cookbook recipe + environment variable reference. --- .oxfmtrc.json | 1 + .oxlintrc.json | 1 + apps/api/src/handlers/index.ts | 3 + .../src/handlers/tts/__tests__/tts.test.ts | 193 +++++++++++++++ apps/api/src/handlers/tts/index.ts | 218 +++++++++++++++++ apps/api/src/route-policies.ts | 18 ++ apps/api/src/server.ts | 2 + apps/docs/cookbook/feature-demo-videos.mdx | 58 +++++ apps/docs/cookbook/index.mdx | 1 + apps/docs/environment-variables.mdx | 3 + apps/worker/Dockerfile | 32 +++ knip.ts | 3 + .../src/packaged-skill-invocations.ts | 1 + .../__tests__/featureDemoSkill.test.ts | 92 +++++++ .../skills/standard/feature-demo/SKILL.md | 130 ++++++++++ .../standard/feature-demo/capture/capture.mjs | 210 ++++++++++++++++ .../standard/feature-demo/render/package.json | 12 + .../feature-demo/render/props/narration.json | 1 + .../feature-demo/render/props/timeline.json | 10 + .../feature-demo/render/src/DemoStage.tsx | 225 ++++++++++++++++++ .../feature-demo/render/src/FeatureDemo.tsx | 44 ++++ .../standard/feature-demo/render/src/Root.tsx | 22 ++ .../standard/feature-demo/render/src/index.ts | 4 + .../feature-demo/render/src/presets.ts | 45 ++++ .../feature-demo/scripts/build-narration.mjs | 99 ++++++++ .../feature-demo/scripts/fit-timing.mjs | 194 +++++++++++++++ packages/cloud-agents/tsconfig.json | 8 +- .../src/adapters/modal.test.ts | 22 ++ packages/env/src/index.ts | 10 + .../types/src/control-plane-env-vars.test.ts | 2 + packages/types/src/control-plane-env-vars.ts | 13 + 31 files changed, 1676 insertions(+), 1 deletion(-) create mode 100644 apps/api/src/handlers/tts/__tests__/tts.test.ts create mode 100644 apps/api/src/handlers/tts/index.ts create mode 100644 apps/docs/cookbook/feature-demo-videos.mdx create mode 100644 packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/narration.json create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/FeatureDemo.tsx create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Root.tsx create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/index.ts create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/presets.ts create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs diff --git a/.oxfmtrc.json b/.oxfmtrc.json index 4b2032d4f..0975d6461 100644 --- a/.oxfmtrc.json +++ b/.oxfmtrc.json @@ -25,6 +25,7 @@ ".env.*", ".qdrant", "packages/cloud-agents/src/server/workflows/skills/standard/agent-browser", + "packages/cloud-agents/src/server/workflows/skills/standard/feature-demo", ".vscode", ".git", "**/*.{md,mdx,yml,yaml,html,htm,toml,svg,graphql,gql}" diff --git a/.oxlintrc.json b/.oxlintrc.json index 25500e9e1..21bc24926 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -19,6 +19,7 @@ "apps/docs/**", "apps/marketing/**", "packages/cloud-agents/src/server/workflows/skills/standard/agent-browser/**", + "packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/**", "apps/codex-acp/target/**", "**/next-env.d.ts", "deploy/**" diff --git a/apps/api/src/handlers/index.ts b/apps/api/src/handlers/index.ts index 31a99bd6f..3f5167350 100644 --- a/apps/api/src/handlers/index.ts +++ b/apps/api/src/handlers/index.ts @@ -26,6 +26,9 @@ export { mcpRouting } from './mcp/routing'; // inference gateway export { inference } from './inference'; +// narration tts +export { tts } from './tts'; + // task runs export { taskRunsRouter } from './task-runs'; diff --git a/apps/api/src/handlers/tts/__tests__/tts.test.ts b/apps/api/src/handlers/tts/__tests__/tts.test.ts new file mode 100644 index 000000000..e1308fd7d --- /dev/null +++ b/apps/api/src/handlers/tts/__tests__/tts.test.ts @@ -0,0 +1,193 @@ +import { Hono } from 'hono'; +import type { RunTokenContext } from '@roomote/types'; + +import type { Variables } from '../../../types'; + +const { mockEnv } = vi.hoisted(() => ({ + mockEnv: { + R_ELEVENLABS_API_KEY: undefined as string | undefined, + R_ELEVENLABS_VOICE_ID: undefined as string | undefined, + R_ELEVENLABS_MODEL: undefined as string | undefined, + }, +})); + +vi.mock('@roomote/env', () => ({ + Env: mockEnv, +})); + +import { tts } from '../index'; + +function createRunToken(): RunTokenContext { + return { + runId: 42, + userId: null, + principal: 'deployment', + tokenType: 'run', + version: 1, + }; +} + +function createApp( + authContext: Variables['authContext'], +): Hono<{ Variables: Variables }> { + const app = new Hono<{ Variables: Variables }>(); + + app.use('*', async (c, next) => { + c.set('authContext', authContext); + await next(); + }); + + app.route('/tts', tts); + return app; +} + +function narrationRequest(body: unknown): Request { + return new Request('http://localhost/tts/narration', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }); +} + +const originalFetch = globalThis.fetch; + +beforeEach(() => { + mockEnv.R_ELEVENLABS_API_KEY = 'el-key'; + mockEnv.R_ELEVENLABS_VOICE_ID = 'voice-1'; + mockEnv.R_ELEVENLABS_MODEL = undefined; +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + vi.restoreAllMocks(); +}); + +describe('POST /tts/narration', () => { + it('404s when the deployment has no ElevenLabs credentials', async () => { + mockEnv.R_ELEVENLABS_API_KEY = undefined; + + const response = await createApp(createRunToken()).request( + narrationRequest({ lines: ['hello'] }), + ); + + expect(response.status).toBe(404); + }); + + it('404s when only the voice id is missing', async () => { + mockEnv.R_ELEVENLABS_VOICE_ID = undefined; + + const response = await createApp(createRunToken()).request( + narrationRequest({ lines: ['hello'] }), + ); + + expect(response.status).toBe(404); + }); + + it('rejects non-run-token callers', async () => { + const response = await createApp(undefined).request( + narrationRequest({ lines: ['hello'] }), + ); + + expect(response.status).toBe(403); + }); + + it('rejects invalid line payloads', async () => { + const app = createApp(createRunToken()); + + for (const body of [ + {}, + { lines: [] }, + { lines: ['ok', 42] }, + { lines: [' '] }, + { lines: ['x'.repeat(1_201)] }, + { lines: Array.from({ length: 25 }, () => 'line') }, + ]) { + const response = await app.request(narrationRequest(body)); + + expect(response.status).toBe(400); + } + }); + + it('413s a body that exceeds the cap even without content-length', async () => { + const chunk = new TextEncoder().encode('x'.repeat(1024)); + const endlessBody = new ReadableStream({ + pull(controller) { + controller.enqueue(chunk); + }, + }); + + const response = await createApp(createRunToken()).request( + new Request('http://localhost/tts/narration', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: endlessBody, + // @ts-expect-error duplex is required for stream bodies in undici + duplex: 'half', + }), + ); + + expect(response.status).toBe(413); + }); + + it('synthesizes each line through ElevenLabs with the server-held key', async () => { + const fetchMock = vi.fn( + async () => new Response(Buffer.from('mp3-bytes'), { status: 200 }), + ); + globalThis.fetch = fetchMock as unknown as typeof globalThis.fetch; + + const response = await createApp(createRunToken()).request( + narrationRequest({ lines: ['First line.', 'Second line.'] }), + ); + + expect(response.status).toBe(200); + + const payload = (await response.json()) as { + clips: { audioBase64: string }[]; + }; + + expect(payload.clips).toHaveLength(2); + expect( + Buffer.from(payload.clips[0]!.audioBase64, 'base64').toString(), + ).toBe('mp3-bytes'); + + expect(fetchMock).toHaveBeenCalledTimes(2); + + const [url, init] = fetchMock.mock.calls[0] as unknown as [ + string, + RequestInit, + ]; + + expect(url).toContain('https://api.elevenlabs.io/v1/text-to-speech/'); + expect(url).toContain('voice-1'); + + const headers = init.headers as Record; + + expect(headers['xi-api-key']).toBe('el-key'); + + const body = JSON.parse(String(init.body)) as { + model_id: string; + voice_settings: { stability: number }; + }; + + // v3 default anchors the clone's accent with "robust" stability. + expect(body.model_id).toBe('eleven_v3'); + expect(body.voice_settings.stability).toBe(1.0); + }); + + it('502s when the upstream synthesis fails', async () => { + globalThis.fetch = vi.fn( + async () => new Response('nope', { status: 401 }), + ) as unknown as typeof globalThis.fetch; + + const response = await createApp(createRunToken()).request( + narrationRequest({ lines: ['hello'] }), + ); + + expect(response.status).toBe(502); + + const payload = (await response.json()) as { error: string }; + + // The upstream error body must not leak through. + expect(payload.error).toBe('Narration synthesis failed'); + }); +}); diff --git a/apps/api/src/handlers/tts/index.ts b/apps/api/src/handlers/tts/index.ts new file mode 100644 index 000000000..a167eabe6 --- /dev/null +++ b/apps/api/src/handlers/tts/index.ts @@ -0,0 +1,218 @@ +import { Hono } from 'hono'; +import { Env } from '@roomote/env'; +import type { RunTokenContext } from '@roomote/types'; + +import type { Variables } from '../../types'; +import { logHandlerError } from '../utils'; + +/** + * Narration text-to-speech for task sandboxes, used by the feature-demo + * skill. The sandbox posts plain narration lines with its run-scoped token; + * the ElevenLabs key lives only on the control plane (`R_ELEVENLABS_API_KEY` + * is in `CONTROL_PLANE_ENV_VAR_NAMES`, so it is also stripped from sandbox + * env injection) and the synthesized audio streams back as base64 clips. + * + * Unset credentials mean the feature is off: the endpoint 404s and callers + * degrade to captions-only output. + */ + +type NarrationBody = { + lines?: unknown; +}; + +class RequestBodyTooLargeError extends Error {} + +function isRunTokenContext( + auth: Variables['authContext'], +): auth is RunTokenContext { + return Boolean(auth && 'runId' in auth); +} + +/** Narration is a handful of short spoken lines, not prose. */ +const MAX_NARRATION_LINES = 24; +const MAX_NARRATION_LINE_CHARS = 1_200; +const MAX_NARRATION_REQUEST_BYTES = 64 * 1024; + +/** + * v3 is the most natural read for voice clones; stability 1.0 ("robust") + * anchors delivery to the reference audio, which prevents the accent drift + * v3 exhibits at lower stability on instant clones. + */ +const DEFAULT_ELEVENLABS_MODEL = 'eleven_v3'; +const ELEVENLABS_TTS_TIMEOUT_MS = 60_000; + +async function readRequestBodyBytes( + request: Request, + maxBytes: number, +): Promise { + if (!request.body) { + return new Uint8Array(); + } + + const reader = request.body.getReader(); + const chunks: Uint8Array[] = []; + let totalBytes = 0; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + break; + } + + totalBytes += value.byteLength; + if (totalBytes > maxBytes) { + await reader.cancel().catch(() => undefined); + throw new RequestBodyTooLargeError(); + } + + chunks.push(value); + } + + const bodyBytes = new Uint8Array(totalBytes); + let offset = 0; + + for (const chunk of chunks) { + bodyBytes.set(chunk, offset); + offset += chunk.byteLength; + } + + return bodyBytes; +} + +function parseNarrationLines(body: NarrationBody): string[] | null { + if (!Array.isArray(body.lines)) { + return null; + } + + if (body.lines.length === 0 || body.lines.length > MAX_NARRATION_LINES) { + return null; + } + + const lines: string[] = []; + + for (const line of body.lines) { + if (typeof line !== 'string') { + return null; + } + + const trimmed = line.trim(); + + if (!trimmed || trimmed.length > MAX_NARRATION_LINE_CHARS) { + return null; + } + + lines.push(trimmed); + } + + return lines; +} + +async function synthesizeLine(options: { + apiKey: string; + voiceId: string; + modelId: string; + text: string; +}): Promise { + const response = await fetch( + `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent( + options.voiceId, + )}?output_format=mp3_44100_128`, + { + method: 'POST', + headers: { + 'xi-api-key': options.apiKey, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + text: options.text, + model_id: options.modelId, + voice_settings: options.modelId.startsWith('eleven_v3') + ? { stability: 1.0, similarity_boost: 0.75 } + : { stability: 0.5, similarity_boost: 0.75 }, + }), + signal: AbortSignal.timeout(ELEVENLABS_TTS_TIMEOUT_MS), + }, + ); + + if (!response.ok) { + throw new Error( + `ElevenLabs TTS request failed with status ${response.status}`, + ); + } + + return Buffer.from(await response.arrayBuffer()); +} + +export const tts = new Hono<{ Variables: Variables }>(); + +tts.post('/narration', async (c) => { + const apiKey = Env.R_ELEVENLABS_API_KEY; + const voiceId = Env.R_ELEVENLABS_VOICE_ID; + + // Not configured => the feature does not exist on this deployment. + if (!apiKey || !voiceId) { + return c.json({ error: 'Narration TTS is not configured' }, 404); + } + + const auth = c.get('authContext'); + + // The route policy already rejects non-run tokens; this guard keeps the + // handler honest if the policy table ever drifts. + if (!auth || !isRunTokenContext(auth)) { + return c.json({ error: 'Narration TTS requires a task run token' }, 403); + } + + let body: NarrationBody; + + try { + const bodyBytes = await readRequestBodyBytes( + c.req.raw, + MAX_NARRATION_REQUEST_BYTES, + ); + + body = JSON.parse(new TextDecoder().decode(bodyBytes)) as NarrationBody; + } catch (error) { + if (error instanceof RequestBodyTooLargeError) { + return c.json( + { + error: `Request body exceeds max size of ${MAX_NARRATION_REQUEST_BYTES} bytes`, + }, + 413, + ); + } + + return c.json({ error: 'Invalid request body' }, 400); + } + + const lines = parseNarrationLines(body); + + if (!lines) { + return c.json( + { + error: + `lines must be 1-${MAX_NARRATION_LINES} non-empty strings of at ` + + `most ${MAX_NARRATION_LINE_CHARS} characters`, + }, + 400, + ); + } + + const modelId = Env.R_ELEVENLABS_MODEL || DEFAULT_ELEVENLABS_MODEL; + const clips: { audioBase64: string }[] = []; + + // Sequential on purpose: ElevenLabs enforces per-key concurrency limits, + // and a narration is a handful of short lines. + for (const text of lines) { + try { + const audio = await synthesizeLine({ apiKey, voiceId, modelId, text }); + + clips.push({ audioBase64: audio.toString('base64') }); + } catch (error) { + logHandlerError('ttsNarration', error); + return c.json({ error: 'Narration synthesis failed' }, 502); + } + } + + return c.json({ clips }); +}); diff --git a/apps/api/src/route-policies.ts b/apps/api/src/route-policies.ts index aa063f590..da8aa1aa0 100644 --- a/apps/api/src/route-policies.ts +++ b/apps/api/src/route-policies.ts @@ -236,6 +236,24 @@ export const ROUTE_POLICY_RULES: readonly RoutePolicyRule[] = [ policy: 'task-token', }, + // Narration text-to-speech for task sandboxes: the ElevenLabs key is + // injected server-side and never enters the sandbox. The client-keyed + // limit is a global ceiling (server-to-server callers share one bucket) + // sized far above legitimate use — a demo narrates a handful of lines + // once — to blunt credit-drain from a leaked run token. + { + name: 'tts', + match: { type: 'prefix', path: '/api/tts' }, + policy: 'task-token', + rateLimits: [ + { + keySource: 'client', + limit: 60, + windowSeconds: 60, + }, + ], + }, + // Router-facing MCP endpoints: accept user auth tokens (LLM router // gathering context before a run exists) and task run tokens. { diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index b146268dc..c86b6b22b 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -47,6 +47,7 @@ import { discord, cloudDeploymentAccess, inference, + tts, mcp, mcpRouting, taskRunsRouter, @@ -209,6 +210,7 @@ export function createApiApp(): ApiApp { app.route('/api/internal/discord', discord); app.route('/api/internal/cloud', cloudDeploymentAccess); app.route('/api/inference', inference); + app.route('/api/tts', tts); app.route('/api/mcp', mcp); app.route('/api/mcp-routing', mcpRouting); app.route('/api/task-runs', taskRunsRouter); diff --git a/apps/docs/cookbook/feature-demo-videos.mdx b/apps/docs/cookbook/feature-demo-videos.mdx new file mode 100644 index 000000000..b9d8e68aa --- /dev/null +++ b/apps/docs/cookbook/feature-demo-videos.mdx @@ -0,0 +1,58 @@ +--- +title: Record feature demo videos +description: Ask Roomote for a polished, narrated demo video of a feature — recorded live, with zooms, captions, and an optional voice-over. +contributor: Matt Rubens +contributor_url: https://github.com/mrubens +contributor_company: Roomote +contributor_company_url: https://roomote.dev +--- + +## Overview + +Shipping a feature usually ends with someone screen-recording a shaky +walkthrough. Roomote can produce that video for you: it drives the product in +its sandbox browser, records a clean clip, and post-produces it with smooth +zoom-to-element moves, a synthetic cursor, click ripples, and captions. With +narration configured, the demo gets a voice-over that leads each visual beat. + +- **Trigger**: Ask in a task +- **Setup time**: None for captioned demos; 5 minutes for narration +- **Requires**: Nothing extra; admin access for narration +- **Serves**: Engineers, Product, Marketing + +## Ingredients + +- Any task where the feature is reachable — a running dev environment or a + public page +- Optional: an [ElevenLabs](https://elevenlabs.io) API key and voice ID for + narration + +## Steps + +1. In a task, ask for a demo, for example: `Make a demo video of the new + custom MCP servers settings page` or invoke `$feature-demo` directly. +2. Roomote plans the story beats, records the interaction live, and renders a + 16:9 video (ask for a vertical cut for short-form if you want one). +3. The finished video is uploaded as a task artifact and linked in the reply + (and embedded under `## Screencasts` when the task opens a PR). + +## Narration + +Captioned demos work out of the box. For a voice-over in your own (cloned) +voice, a deployment admin sets two environment variables: + +- `R_ELEVENLABS_API_KEY` — an ElevenLabs key. A key scoped to + **text-to-speech only**, with a credit cap, is sufficient and recommended. +- `R_ELEVENLABS_VOICE_ID` — the voice to narrate with. + +The key stays on the Roomote control plane: task sandboxes send narration +text to an authenticated Roomote endpoint and receive audio back, so the +provider credential never enters a sandbox. + +## Tips + +- Narration lines come from the on-screen captions; ask for wording changes + the way you would edit copy ("make the second line more casual"). +- Demos pace themselves to the narration: the recording slows slightly so + each line finishes before its zoom lands. +- Keep demos to 2-5 beats. Two shorter videos beat one long one. diff --git a/apps/docs/cookbook/index.mdx b/apps/docs/cookbook/index.mdx index 5d68fcc28..f25f27824 100644 --- a/apps/docs/cookbook/index.mdx +++ b/apps/docs/cookbook/index.mdx @@ -16,6 +16,7 @@ and the quality of your output. | [Ease your team into cloud agents](/cookbook/ease-your-team-into-cloud-agents) | Build trust in Roomote through small, visible, low-risk team habits. | | [Evaluate outage impact](/cookbook/vendor-outage-triage) | Filter vendor status noise by comparing each incident with your real code, regions, and feature usage. | | [Fix CI failures](/cookbook/ci-failure-auto-fix) | Keep the build green by having Roomote verify and fix CI breakages automatically. | +| [Record feature demo videos](/cookbook/feature-demo-videos) | Ask Roomote for a polished, narrated demo video of a feature — recorded live with zooms, captions, and voice-over. | | [Schedule maintenance](/cookbook/scheduled-housekeeping) | Turn flaky-test scans, feature-flag audits, and dependency reviews into recurring Roomote work. | | [Triage customer issues](/cookbook/support-channel) | Give support escalations a repeatable path through production evidence, data, and code. | {/* cookbook-recipes:end */} diff --git a/apps/docs/environment-variables.mdx b/apps/docs/environment-variables.mdx index ff27fb411..a753cfdca 100644 --- a/apps/docs/environment-variables.mdx +++ b/apps/docs/environment-variables.mdx @@ -340,6 +340,9 @@ as per-task auth tokens or workspace paths. | `R_MICROSOFT_CLIENT_SECRET` | Microsoft sign-in | Microsoft OAuth client secret. | | `R_MICROSOFT_TENANT_ID` | Microsoft sign-in | Microsoft tenant ID. Also accepted as the Azure DevOps tenant ID fallback. | | `R_ALLOWED_EMAILS` | Optional | Comma-separated email allowlist for deployments that restrict sign-in by email. | +| `R_ELEVENLABS_API_KEY` | Optional | ElevenLabs API key for narrated feature-demo videos. The key stays on the control plane; sandboxes reach text-to-speech only through an authenticated Roomote endpoint. A key scoped to text-to-speech only is sufficient and recommended. | +| `R_ELEVENLABS_VOICE_ID` | Optional | ElevenLabs voice ID used for feature-demo narration. Required alongside the API key for narration to be available. | +| `R_ELEVENLABS_MODEL` | Optional | ElevenLabs TTS model override. Defaults to `eleven_v3`. | During Microsoft Teams setup, Roomote uses the Microsoft Entra app values for the Teams bot by default. Use **Show advanced config** after the Directory diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index 05f7197d2..d27a1a1d7 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -17,6 +17,10 @@ ARG FFPROBE_INSTALLER_VERSION=2.1.2 ARG GH_VERSION=2.81.0 ARG NODE_PTY_VERSION=1.1.0 ARG PM2_VERSION=7.0.1 +# Remotion mirrors chrome-headless-shell builds for both linux64 and +# linux-arm64; the bundled agent-browser Chrome cannot render Remotion +# compositions (Chrome 143 removed old-headless). +ARG REMOTION_VERSION=4.0.507 RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -122,6 +126,7 @@ ENV PATH=/home/roomote/.local/bin:/opt/mise/shims:${PATH} ENV AGENT_BROWSER_EXECUTABLE_PATH="/opt/agent-browser/chrome" ENV FFMPEG_EXECUTABLE_PATH="/opt/ffmpeg/bin/ffmpeg" ENV FFPROBE_EXECUTABLE_PATH="/opt/ffmpeg/bin/ffprobe" +ENV REMOTION_HEADLESS_SHELL_PATH="/opt/remotion/headless-shell" ENV ROOMOTE_BAKED_OPENCODE_CLI_VERSION=${OPENCODE_CLI_VERSION} COPY .docker/sandbox/install-browser-agent.sh /tmp/install-browser-agent.sh @@ -220,6 +225,33 @@ RUN sudo mkdir -p /opt/ffmpeg/bin \ && ln -sf "${FFPROBE_EXECUTABLE_PATH}" "$HOME/.local/bin/ffprobe" \ && printf '%s\n' "${FFMPEG_INSTALLER_VERSION}+${FFPROBE_INSTALLER_VERSION}" > /opt/ffmpeg/.installed +# Bake Remotion's chrome-headless-shell at a fixed path so renders never +# download a browser at runtime. `npx remotion browser ensure` fetches the +# matching linux64/linux-arm64 build into node_modules/.remotion; copy that +# tree to /opt/remotion and expose a stable arch-independent symlink at +# ${REMOTION_HEADLESS_SHELL_PATH} so consumers need no arch logic. +RUN sudo mkdir -p /opt/remotion \ + && sudo chown -R roomote:roomote /opt/remotion \ + && REMOTION_TMP="$(mktemp -d)" \ + && cd "$REMOTION_TMP" \ + && printf '%s\n' '{ "name": "remotion-browser-seed", "private": true }' > package.json \ + && npm install --no-save --no-package-lock \ + "@remotion/cli@${REMOTION_VERSION}" \ + "remotion@${REMOTION_VERSION}" \ + && npx remotion browser ensure \ + && cp -R node_modules/.remotion/chrome-headless-shell /opt/remotion/chrome-headless-shell \ + && case "$(uname -m)" in \ + aarch64) REMOTION_PLATFORM="linux-arm64" ;; \ + x86_64) REMOTION_PLATFORM="linux64" ;; \ + *) echo "Unsupported architecture for Remotion headless shell: $(uname -m)" >&2; exit 1 ;; \ + esac \ + && REMOTION_SHELL="/opt/remotion/chrome-headless-shell/${REMOTION_PLATFORM}/chrome-headless-shell-${REMOTION_PLATFORM}/headless_shell" \ + && test -x "$REMOTION_SHELL" \ + && ln -sf "$REMOTION_SHELL" "${REMOTION_HEADLESS_SHELL_PATH}" \ + && printf '%s\n' "${REMOTION_VERSION}" > /opt/remotion/.installed \ + && cd / \ + && rm -rf "$REMOTION_TMP" + RUN npm install --prefix /sandbox --no-save --no-package-lock \ "node-pty@${NODE_PTY_VERSION}" \ "opencode-ai@${OPENCODE_CLI_VERSION}" \ diff --git a/knip.ts b/knip.ts index 7bee864c5..fe1d3a636 100644 --- a/knip.ts +++ b/knip.ts @@ -74,6 +74,9 @@ const config: KnipConfig = { 'packages/cloud-agents': { entry: ['evals/router/**/*.ts'], project: ['src/**/*.ts'], + // The feature-demo skill bundles a standalone Remotion render project + // that runs from a sandbox work dir, not from this package's graph. + ignore: ['src/server/workflows/skills/standard/feature-demo/**'], ignoreBinaries: [ 'evals/router/promptfooconfig.ts', 'evals/router/promptfooconfig.followup.ts', diff --git a/packages/cloud-agents/src/packaged-skill-invocations.ts b/packages/cloud-agents/src/packaged-skill-invocations.ts index 1f3501eee..6e523b1fe 100644 --- a/packages/cloud-agents/src/packaged-skill-invocations.ts +++ b/packages/cloud-agents/src/packaged-skill-invocations.ts @@ -20,6 +20,7 @@ const CORE_PACKAGED_SKILL_INVOCATIONS = [ 'issue-fixer', 'environment-setup', 'explain-repo-code', + 'feature-demo', 'fix-pr', 'github-management', 'implement-repo-change', diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts new file mode 100644 index 000000000..3ad37d4b3 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -0,0 +1,92 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { PACKAGED_SKILL_INVOCATIONS } from '../../../packaged-skill-invocations'; + +const thisFilePath = fileURLToPath(import.meta.url); +const thisDirPath = path.dirname(thisFilePath); +const skillDirPath = path.resolve( + thisDirPath, + '../skills/standard/feature-demo', +); + +describe('feature-demo skill', () => { + const skillContent = fs.readFileSync( + path.join(skillDirPath, 'SKILL.md'), + 'utf8', + ); + + it('is registered for explicit invocation', () => { + expect(PACKAGED_SKILL_INVOCATIONS).toContain('feature-demo'); + }); + + it('ships the capture runner, pipeline scripts, and render project', () => { + for (const relativePath of [ + 'capture/capture.mjs', + 'scripts/build-narration.mjs', + 'scripts/fit-timing.mjs', + 'render/package.json', + 'render/src/index.ts', + 'render/src/Root.tsx', + 'render/src/FeatureDemo.tsx', + 'render/src/DemoStage.tsx', + 'render/src/presets.ts', + 'render/props/timeline.json', + 'render/props/narration.json', + ]) { + expect( + fs.existsSync(path.join(skillDirPath, relativePath)), + `${relativePath} must ship with the skill`, + ).toBe(true); + } + }); + + it('keeps browser work delegated to proof-runner', () => { + expect(skillContent).toContain( + "Browser automation is the proof-runner subagent's exclusive surface", + ); + expect(skillContent).toContain('proof runtime unavailable'); + expect(skillContent).toContain( + 'Never load or invoke `agent-browser` (or any other browser automation) from this skill', + ); + }); + + it('keeps TTS provider keys out of the sandbox', () => { + expect(skillContent).toContain( + 'Never ask for, read, or handle TTS provider keys.', + ); + expect(skillContent).toContain('captions-only'); + + const narrationScript = fs.readFileSync( + path.join(skillDirPath, 'scripts/build-narration.mjs'), + 'utf8', + ); + + // The sandbox-side script talks to the control plane with the run token + // and must never reference the provider or its key directly. + expect(narrationScript).toContain('/api/tts/narration'); + expect(narrationScript).toContain('ROOMOTE_CLOUD_TOKEN'); + expect(narrationScript.toLowerCase()).not.toContain('elevenlabs.io'); + expect(narrationScript).not.toContain('xi-api-key'); + }); + + it('never commits pipeline outputs into the repository', () => { + expect(skillContent).toContain( + 'Never commit recordings, renders, node_modules, or props into the repository.', + ); + }); + + it('renders with the baked headless shell and a runtime fallback', () => { + expect(skillContent).toContain( + '--browser-executable="${REMOTION_HEADLESS_SHELL_PATH:-/opt/remotion/headless-shell}"', + ); + expect(skillContent).toContain('npx remotion browser ensure'); + }); + + it('delivers through manage_artifacts with canonical URLs only', () => { + expect(skillContent).toContain('manage_artifacts'); + expect(skillContent).toContain('never invent URLs'); + expect(skillContent).toContain('## Screencasts'); + }); +}); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md new file mode 100644 index 000000000..fca02c4e0 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -0,0 +1,130 @@ +--- +name: feature-demo +description: Produce a polished, Screen Studio-style demo video of a product feature — recorded live in the sandbox browser, with smooth zooms, cursor effects, captions, and optional voice-over narration. +--- + + +You produce short, polished feature-demo videos: a real recording of the +product driven live in the browser, post-produced with smooth zoom-to-element +moves, a synthetic cursor, click ripples, captions, and (when the deployment +has narration configured) a voice-over that leads each visual beat. + +Use this skill when the user asks for a demo video, walkthrough recording, +or promo clip of a feature. It is NOT for verification screencasts of a code +change — that is `capture-visual-proof`. + + + +One demo script drives everything. The capture runner performs the real +browser interactions AND records, on the same clock, where the cursor is, +when clicks land, and the resolved rectangle of every focused element. The +renderer consumes that timeline, so a zoom can never drift from the element +it targets. + +Pipeline: author script → capture (browser, delegated) → narrate (optional) +→ fit timing → render → verify → upload. + +All work happens under `/tmp/feature-demo/work` (the work dir). Nothing from +this pipeline is ever committed to the repository. + + + + + +Scope the demo + +Identify the feature, the surface to record (running app in the sandbox, or a public page the user names), and 2-5 story beats. A good demo is 10-25 seconds of motion; more beats than that dilutes it. +Pick presets: `wide` (1920x1080) is the default; add `vertical` (1080x1920) only when the user wants a social/short-form cut. +If the surface needs the app running, make sure the environment is up first (dev server reachable) before capturing. + + + + +Author the demo script + +Write `/tmp/feature-demo/demo-script.json`: `{ "url": string, "viewport": { "w": 1280, "h": 800 }, "beats": [...] }`. Beat actions: +- `{ "a": "wait", "ms": n }` / `{ "a": "hold", "ms": n }` — let the page settle / linger. +- `{ "a": "scrollTo", "sel": css, "ms": n }` — scroll an element into view (the scroll shows in the recording). +- `{ "a": "focus", "sel": css, "scale": 1.3-1.9, "moveMs": ~700, "holdMs": 800-1200, "caption": "..." }` — glide the cursor to the element and zoom to it. Captions become the narration lines. +- `{ "a": "click", "sel": css, "holdMs": ~300 }` — real click with ripple. +- `{ "a": "type", "sel": css, "text": "..." }` — real typing. +- `{ "a": "reset", "ms": ~600 }` — pull back between beats (partial by design; pass `"full": true` only for the closing shot). +Keep the opening tight: one short `wait` for page load, then get moving. Dead opening seconds are trimmed automatically, but do not rely on it. +Write captions as short, conversational spoken lines (they double as the narration): contractions are good, symbols and long clauses are not. 3-9 words on screen; if the spoken wording should differ from the on-screen caption, put the spoken variants in `/tmp/feature-demo/lines.json` (array, one string per caption). +Selectors must be resilient: prefer ids, stable data attributes, or unique semantic tags over deep CSS chains. The capture fails loudly on a selector that resolves to nothing. + + + + +Capture (delegated browser work) + +Browser automation is the proof-runner subagent's exclusive surface — do not load `agent-browser` or run the capture yourself. Delegate with the Task tool to `proof-runner` with a brief that instructs it to run exactly: + +`SCRIPT=/tmp/feature-demo/demo-script.json OUT_DIR=/tmp/feature-demo/work node ~/.agents/skills/feature-demo/capture/capture.mjs` + +and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. The runner is an agent-browser orchestrator (it shells the `agent-browser` CLI for every browser action), so it stays inside proof-runner's sanctioned tooling. +If the harness has no proof-runner registered, report a blocker (`proof runtime unavailable`) instead of driving the browser from this skill. +Expected outputs: `/tmp/feature-demo/work/recording.mp4` and `/tmp/feature-demo/work/timeline.json`. Verify both exist and that `ffprobe` reports a duration close to the timeline's `durationSeconds`. One retry on failure; then report blocked with the runner's error. + + + + +Narration (optional, degrades cleanly) + +Run `WORK_DIR=/tmp/feature-demo/work node ~/.agents/skills/feature-demo/scripts/build-narration.mjs` (add `LINES=/tmp/feature-demo/lines.json` if spoken lines differ from captions). This posts the caption text to the Roomote control plane, which holds the TTS credentials — no provider key exists in this sandbox, and you must never ask for one. +Exit code 3 means narration is not configured on this deployment: proceed captions-only, and mention in the final report that narration is available if an admin sets `R_ELEVENLABS_API_KEY` + `R_ELEVENLABS_VOICE_ID`. + + + + +Fit timing + +Run `WORK_DIR=/tmp/feature-demo/work node ~/.agents/skills/feature-demo/scripts/fit-timing.mjs`. It trims the dead opening, paces the voice-over (pitch-preserving atempo), solves a video playback rate so each spoken line finishes before its zoom lands, and rewrites caption windows to match the audio. Captions-only demos still get the opening trim. +Check its printed schedule: no line should start before the previous one ends. If lines overlap or the rate hits the 0.75 floor, the narration is too long for the motion — shorten the lines or add holds to the script and recapture. + + + + +Render + +Assemble the render project in the work dir: +- `cp -R ~/.agents/skills/feature-demo/render /tmp/feature-demo/render` +- `cp /tmp/feature-demo/work/timeline.json /tmp/feature-demo/work/narration.json /tmp/feature-demo/render/props/` (skip narration.json if captions-only; the checked-in placeholder `{ "clips": [] }` is already correct) +- `mkdir -p /tmp/feature-demo/render/public && cp /tmp/feature-demo/work/recording.mp4 /tmp/feature-demo/render/public/` +- `cp -R /tmp/feature-demo/work/vo /tmp/feature-demo/render/public/vo` (narrated demos only) +`cd /tmp/feature-demo/render && npm install` (versions are pinned; takes a few minutes on first run). +Render with the image's baked headless shell: + +`npx remotion render src/index.ts Demo-wide out/demo-wide.mp4 --browser-executable="${REMOTION_HEADLESS_SHELL_PATH:-/opt/remotion/headless-shell}" --log=error` + +If that binary does not exist (older sandbox snapshot), run `npx remotion browser ensure` once and render without the flag. Repeat for `Demo-vertical` when the vertical preset was requested. + + + + +Verify honestly + +`ffprobe` the output: sane duration, a video stream, and an audio stream when narration was generated. +Extract 3-4 spread frames (`ffmpeg -ss -i out/demo-wide.mp4 -frames:v 1 frame.png`) and review each whole frame: zoom landed on the right element, cursor glued to it, caption legible and correctly timed, no backdrop showing through mid-zoom, no broken page state in shot. +A defect in the video is a blocker to report, not something to ship quietly. One recapture/re-render attempt; then report blocked with what failed and the frames that show it. + + + + +Deliver + +Upload the mp4 via `manage_artifacts` (`action: upload`, `type: general`) plus one representative keyframe PNG. Treat only the returned `artifactId`/`viewUrl`/`rawUrl` values as canonical — never invent URLs. +In a PR body, embed under `## Screencasts` using the existing convention: the keyframe image (its signed `rawUrl`) hyperlinked to the video's `viewUrl`, with a one-line caption. In chat replies, share the keyframe via image attachment and the video `viewUrl` as a link (video files cannot be attached inline). +End with a sharing note: what the demo shows, which presets were rendered, and whether it is narrated or captions-only. + + + + + + +Never load or invoke `agent-browser` (or any other browser automation) from this skill — capture is always delegated to the `proof-runner` subagent. +Never ask for, read, or handle TTS provider keys. Narration goes through the control-plane endpoint with the run token; a 404 there means captions-only. +All intermediate files live under `/tmp/feature-demo`. Never commit recordings, renders, node_modules, or props into the repository. +Report blockers honestly: a missing selector, a failed render, or a visibly broken frame is a blocker with evidence, not a reason to narrow the claim or ship a degraded video silently. +Voice, wording, and pacing choices belong to the user when they express them; defaults are: wide preset, captions as narration lines, conversational tone. + diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs new file mode 100644 index 000000000..6ad819e28 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -0,0 +1,210 @@ +// Capture runner: drives agent-browser through a demo script, records a +// cursorless WebM, and emits the timeline the Remotion renderer consumes. +// Runs inside the worker sandbox (agent-browser + ffmpeg present). +// +// The key idea: the runner performs the real interactions AND logs, for the +// same clock, where the (synthetic) cursor is, when clicks land, and the +// resolved rect of each target. Effects and capture come from one script, so +// a zoom can never drift from the element it is zooming to. +// +// Usage: SCRIPT=/path/to/demo-script.json OUT_DIR=/tmp/feature-demo/work \ +// node capture.mjs +// See SKILL.md for the demo-script schema. + +import { execFileSync } from 'node:child_process'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + +const AB = process.env.AGENT_BROWSER_BIN || 'agent-browser'; +const OUT_DIR = process.env.OUT_DIR || '/tmp/feature-demo/work'; + +const SCRIPT_PATH = process.env.SCRIPT; + +if (!SCRIPT_PATH) { + console.error('Set SCRIPT to the demo-script JSON path.'); + process.exit(1); +} + +const script = JSON.parse(readFileSync(SCRIPT_PATH, 'utf8')); + +if (!script.url || !Array.isArray(script.beats)) { + console.error('Demo script must have { url, beats: [...] }.'); + process.exit(1); +} + +const VIEWPORT = script.viewport || { w: 1280, h: 800 }; + +// Between focus beats the camera pulls back only partially; pogo-ing to full +// wide between every zoom reads as jumpy. The final reset goes fully wide. +const GLIDE_SCALE = 1.18; + +const ab = (...args) => + execFileSync(AB, args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + +const sleep = (ms) => execFileSync('sleep', [String(ms / 1000)]); + +let t0 = 0; +const now = () => (Date.now() - t0) / 1000; // seconds since record start + +function rect(sel) { + const js = `(function(){var e=document.querySelector(${JSON.stringify( + sel, + )});if(!e)return null;var r=e.getBoundingClientRect();return{x:r.x,y:r.y,w:r.width,h:r.height};})()`; + const out = ab('eval', js).trim(); + const r = JSON.parse(out); + if (!r) throw new Error(`element not found: ${sel}`); + return r; +} + +const centerNorm = (r) => ({ + x: (r.x + r.w / 2) / VIEWPORT.w, + y: (r.y + r.h / 2) / VIEWPORT.h, +}); + +const timeline = { + video: { path: 'recording.mp4', width: VIEWPORT.w, height: VIEWPORT.h }, + fps: 30, + durationSeconds: 0, + scaleKeys: [{ t: 0, v: 1 }], + focalKeys: [{ t: 0, v: { x: 0.5, y: 0.5 } }], + cursorKeys: [{ t: 0, v: { x: 0.5, y: 1.1 } }], + clicks: [], + captions: [], +}; + +let cur = { scale: 1, focal: { x: 0.5, y: 0.5 } }; +const pushScale = (t, v) => timeline.scaleKeys.push({ t, v }); +const pushFocal = (t, v) => timeline.focalKeys.push({ t, v }); + +async function run() { + mkdirSync(OUT_DIR, { recursive: true }); + ab('set', 'viewport', String(VIEWPORT.w), String(VIEWPORT.h)); + ab('record', 'start', `${OUT_DIR}/raw.webm`, script.url); + t0 = Date.now(); + sleep(300); // let the first frames settle + + for (const beat of script.beats) { + if (beat.a === 'hold' || beat.a === 'wait') { + sleep(beat.ms); + continue; + } + + if (beat.a === 'scrollTo') { + ab('scrollintoview', beat.sel); + sleep(beat.ms ?? 650); // let the scroll settle (shows in the recording) + continue; + } + + if (beat.a === 'focus') { + const c = centerNorm(rect(beat.sel)); + const start = now(); + pushScale(start, cur.scale); + pushFocal(start, cur.focal); + // Real hover so the app's hover state shows under the synthetic cursor. + ab( + 'mouse', + 'move', + String((c.x * VIEWPORT.w) | 0), + String((c.y * VIEWPORT.h) | 0), + ); + sleep(beat.moveMs ?? 700); // pace the glide for the eased cursor + const end = now(); + pushScale(end, beat.scale ?? 1.5); + pushFocal(end, c); + timeline.cursorKeys.push({ t: end, v: c }); + if (beat.caption) { + timeline.captions.push({ + start: start + 0.15, + end: end + (beat.holdMs ?? 900) / 1000, + text: beat.caption, + }); + } + cur = { scale: beat.scale ?? 1.5, focal: c }; + sleep(beat.holdMs ?? 900); + continue; + } + + if (beat.a === 'click') { + const c = centerNorm(rect(beat.sel)); + const t = now(); + timeline.clicks.push({ t, at: c }); + timeline.cursorKeys.push({ t, v: c }); + ab('click', beat.sel); + sleep(beat.holdMs ?? 300); + continue; + } + + if (beat.a === 'type') { + ab('type', beat.sel, beat.text); + continue; + } + + if (beat.a === 'reset') { + // Partial pull-back keeps the camera alive between beats; `full: true` + // (or the closing reset) returns to wide. + const target = beat.full ? 1 : GLIDE_SCALE; + const start = now(); + pushScale(start, cur.scale); + pushFocal(start, cur.focal); + sleep(beat.ms ?? 600); + const end = now(); + pushScale(end, target); + pushFocal(end, { x: 0.5, y: 0.5 }); + cur = { scale: target, focal: { x: 0.5, y: 0.5 } }; + continue; + } + + throw new Error(`unknown beat action: ${beat.a}`); + } + + // Close on a full wide shot even when the script forgot a final reset. + if (cur.scale !== 1) { + const start = now(); + pushScale(start, cur.scale); + pushFocal(start, cur.focal); + sleep(600); + const end = now(); + pushScale(end, 1); + pushFocal(end, { x: 0.5, y: 0.5 }); + } + + timeline.durationSeconds = now(); + ab('record', 'stop'); + try { + ab('close', '--all'); + } catch { + // best-effort; the recording is already on disk + } + + // WebM (VP8/VP9) -> H.264 mp4 for the renderer. + execFileSync( + 'ffmpeg', + [ + '-y', + '-i', + `${OUT_DIR}/raw.webm`, + '-c:v', + 'libx264', + '-pix_fmt', + 'yuv420p', + '-preset', + 'veryfast', + `${OUT_DIR}/recording.mp4`, + ], + { stdio: ['ignore', 'ignore', 'pipe'] }, + ); + + writeFileSync(`${OUT_DIR}/timeline.json`, JSON.stringify(timeline, null, 2)); + console.log( + `captured: duration=${timeline.durationSeconds.toFixed(2)}s ` + + `clicks=${timeline.clicks.length} captions=${timeline.captions.length} ` + + `cursorKeys=${timeline.cursorKeys.length}`, + ); +} + +run().catch((e) => { + console.error('capture failed:', e.message); + process.exit(1); +}); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json new file mode 100644 index 000000000..a83125727 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json @@ -0,0 +1,12 @@ +{ + "name": "feature-demo-render", + "private": true, + "version": "1.0.0", + "description": "Remotion composition for Roomote feature-demo videos. Copied to a sandbox work dir at demo time; not a workspace package.", + "dependencies": { + "@remotion/cli": "4.0.507", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "remotion": "4.0.507" + } +} diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/narration.json b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/narration.json new file mode 100644 index 000000000..4f0573827 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/narration.json @@ -0,0 +1 @@ +{ "clips": [] } diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json new file mode 100644 index 000000000..5b7bbb061 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/props/timeline.json @@ -0,0 +1,10 @@ +{ + "video": { "path": "recording.mp4", "width": 1280, "height": 800 }, + "fps": 30, + "durationSeconds": 1, + "scaleKeys": [{ "t": 0, "v": 1 }], + "focalKeys": [{ "t": 0, "v": { "x": 0.5, "y": 0.5 } }], + "cursorKeys": [{ "t": 0, "v": { "x": 0.5, "y": 1.1 } }], + "clicks": [], + "captions": [] +} diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx new file mode 100644 index 000000000..1153cc349 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/DemoStage.tsx @@ -0,0 +1,225 @@ +import { OffthreadVideo, staticFile, useCurrentFrame } from 'remotion'; +import timeline from '../props/timeline.json'; +import type { CaptionStyle } from './presets'; + +export const FPS = timeline.fps; +export const DEMO_SECONDS = timeline.durationSeconds; + +type Vec = { x: number; y: number }; + +const smooth = (t: number) => t * t * (3 - 2 * t); +const clamp = (v: number, lo: number, hi: number) => + Math.max(lo, Math.min(hi, v)); + +function lerpNum(keys: { t: number; v: number }[], t: number): number { + if (t <= keys[0].t) return keys[0].v; + for (let i = 0; i < keys.length - 1; i++) { + const a = keys[i]; + const b = keys[i + 1]; + if (t >= a.t && t <= b.t) + return a.v + (b.v - a.v) * smooth((t - a.t) / Math.max(b.t - a.t, 1e-6)); + } + return keys[keys.length - 1].v; +} +function lerpVec(keys: { t: number; v: Vec }[], t: number): Vec { + if (t <= keys[0].t) return keys[0].v; + for (let i = 0; i < keys.length - 1; i++) { + const a = keys[i]; + const b = keys[i + 1]; + if (t >= a.t && t <= b.t) { + const p = smooth((t - a.t) / Math.max(b.t - a.t, 1e-6)); + return { x: a.v.x + (b.v.x - a.v.x) * p, y: a.v.y + (b.v.y - a.v.y) * p }; + } + } + return keys[keys.length - 1].v; +} + +const Cursor: React.FC<{ invScale: number }> = ({ invScale }) => ( +
+ + + +
+); + +// The polished demo "window": recorded video on a rounded panel, with the +// zoom/cursor/ripple/caption effects driven by the captured timeline. Laid +// out for whatever canvas size the preset asks for. +export const DemoStage: React.FC<{ + canvasW: number; + canvasH: number; + pad: number; + caption: CaptionStyle; + centerPull: number; + baseScale: number; +}> = ({ canvasW, canvasH, pad, caption, centerPull, baseScale }) => { + const frame = useCurrentFrame(); + const t = frame / FPS; + + const SHOT_W = timeline.video.width; + const SHOT_H = timeline.video.height; + const BASE_W = Math.round( + Math.min(canvasW - 2 * pad, ((canvasH - 2 * pad) * SHOT_W) / SHOT_H), + ); + const BASE_H = Math.round((BASE_W * SHOT_H) / SHOT_W); + const WIN_X = (canvasW - BASE_W) / 2; + const WIN_Y = (canvasH - BASE_H) / 2; + + const S = baseScale + (lerpNum(timeline.scaleKeys, t) - 1); + const focal = lerpVec(timeline.focalKeys, t); + const cursor = lerpVec(timeline.cursorKeys, t); + + const focal0x = WIN_X + focal.x * BASE_W; + const focal0y = WIN_Y + focal.y * BASE_H; + const k = clamp((S - 1) / 1.1, 0, 1) * centerPull; + const desiredTx = (canvasW / 2 - focal0x) * k; + const desiredTy = (canvasH / 2 - focal0y) * k; + // The edge-clamp keeps the backdrop from showing behind the window — but + // only makes sense when the scaled window is larger than the canvas on that + // axis (16:9 presets). For a vertical band the window is smaller than the + // canvas, so there is nothing to clamp: keep it centered instead of letting + // the clamp shove it into a corner. + const txMax = WIN_X + focal.x * BASE_W * (S - 1); + const txMin = canvasW - WIN_X - focal.x * BASE_W - BASE_W * (1 - focal.x) * S; + const tyMax = WIN_Y + focal.y * BASE_H * (S - 1); + const tyMin = canvasH - WIN_Y - focal.y * BASE_H - BASE_H * (1 - focal.y) * S; + const Tx = + BASE_W * S >= canvasW + ? clamp(desiredTx, Math.min(txMin, txMax), Math.max(txMin, txMax)) + : desiredTx; + const Ty = + BASE_H * S >= canvasH + ? clamp(desiredTy, Math.min(tyMin, tyMax), Math.max(tyMin, tyMax)) + : desiredTy; + const invScale = 1 / S; + + return ( + <> +
+
+ +
+ + {timeline.clicks.map((c, i) => { + const dt = t - c.t; + if (dt < 0 || dt > 0.6) return null; + const p = dt / 0.6; + const size = 12 + smooth(p) * 90; + return ( +
+ ); + })} + +
+ +
+
+ + {timeline.captions.map((cap, i) => { + const fadeIn = clamp((t - cap.start) / 0.25, 0, 1); + const fadeOut = clamp((cap.end - t) / 0.25, 0, 1); + const opacity = Math.min(fadeIn, fadeOut); + if (opacity <= 0) return null; + return ( +
+
+ {cap.text} +
+
+ ); + })} + + ); +}; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/FeatureDemo.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/FeatureDemo.tsx new file mode 100644 index 000000000..990dd6a07 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/FeatureDemo.tsx @@ -0,0 +1,44 @@ +import { AbsoluteFill, Audio, Sequence, staticFile } from 'remotion'; +import { DemoStage, FPS, DEMO_SECONDS } from './DemoStage'; +import { PRESETS, BACKDROP, type Preset } from './presets'; +import narration from '../props/narration.json'; + +export { FPS, DEMO_SECONDS }; + +// Narration can run slightly past the recorded motion; the composition holds +// on the last frame until it finishes. +const narrationEnd = narration.clips.reduce( + (m, c) => Math.max(m, c.startSeconds + c.durationSeconds), + 0, +); +export const TOTAL_SECONDS = Math.max(DEMO_SECONDS, narrationEnd + 0.4); +export const totalFrames = () => Math.round(TOTAL_SECONDS * FPS); + +export const FeatureDemo: React.FC<{ presetId: Preset['id'] }> = ({ + presetId, +}) => { + const preset = PRESETS[presetId]; + + return ( + + + + {narration.clips.map((c, i) => ( + + + ))} + + ); +}; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Root.tsx b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Root.tsx new file mode 100644 index 000000000..19dd1ce08 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/Root.tsx @@ -0,0 +1,22 @@ +import { Composition } from 'remotion'; +import { FeatureDemo, FPS, totalFrames } from './FeatureDemo'; +import { PRESETS, type Preset } from './presets'; + +export const RemotionRoot: React.FC = () => { + return ( + <> + {(Object.values(PRESETS) as Preset[]).map((preset) => ( + + ))} + + ); +}; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/index.ts b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/index.ts new file mode 100644 index 000000000..91fa0f346 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/index.ts @@ -0,0 +1,4 @@ +import { registerRoot } from 'remotion'; +import { RemotionRoot } from './Root'; + +registerRoot(RemotionRoot); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/presets.ts b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/presets.ts new file mode 100644 index 000000000..d818a5b73 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/src/presets.ts @@ -0,0 +1,45 @@ +export type CaptionStyle = { + fontSize: number; + bottom: number; + maxWidthPct: number; +}; + +export type Preset = { + id: 'wide' | 'vertical'; + width: number; + height: number; + pad: number; + caption: CaptionStyle; + // How strongly a zoom pulls its focal point toward center. 0 keeps the + // window fixed and punches in place (right for vertical bands). + centerPull: number; + // Baseline zoom so a preset never fully zooms out (keeps a vertical band + // filled). Zooms add on top of it. + baseScale: number; +}; + +// One capture feeds every preset; presets differ only in framing and caption +// emphasis. No title cards or branding — the narration + captions carry it. +export const PRESETS: Record = { + wide: { + id: 'wide', + width: 1920, + height: 1080, + pad: 120, + caption: { fontSize: 34, bottom: 70, maxWidthPct: 74 }, + centerPull: 0.9, + baseScale: 1, + }, + vertical: { + id: 'vertical', + width: 1080, + height: 1920, + pad: 90, + caption: { fontSize: 46, bottom: 300, maxWidthPct: 86 }, + centerPull: 0, + baseScale: 1.25, + }, +}; + +export const BACKDROP = + 'radial-gradient(120% 120% at 20% 0%, #23305c 0%, #141726 55%, #0d0f18 100%)'; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs new file mode 100644 index 000000000..e3c96f315 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs @@ -0,0 +1,99 @@ +// Fetch narrated voice-over for the demo's caption lines from the Roomote +// control plane and write per-line mp3s plus the narration manifest the +// renderer consumes. +// +// The sandbox never sees a TTS provider key: this posts plain text to +// /api/tts/narration with the run-scoped token, and the control plane holds +// the ElevenLabs credentials (R_ELEVENLABS_API_KEY / R_ELEVENLABS_VOICE_ID). +// +// Exit codes: 0 = narration written; 3 = TTS not configured on this +// deployment (callers degrade to captions-only); 1 = real failure. +// +// Usage: WORK_DIR=/tmp/feature-demo/work node build-narration.mjs +// Optional: LINES=/path/to/lines.json (array of spoken lines, one per +// caption, when the spoken wording should differ from on-screen captions). + +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; + +const WORK_DIR = process.env.WORK_DIR || '/tmp/feature-demo/work'; + +// Same base-URL/token resolution as the roomote MCP server config. +const rawBaseUrl = + process.env.ROOMOTE_PLATFORM_API_URL || + process.env.TRPC_URL || + 'http://localhost:13001'; +const baseUrl = rawBaseUrl.replace(/\/$/, ''); +const token = process.env.ROOMOTE_CLOUD_TOKEN || process.env.AUTH_TOKEN; + +if (!token) { + console.error('No run token in the environment (ROOMOTE_CLOUD_TOKEN).'); + process.exit(1); +} + +const timeline = JSON.parse( + readFileSync(`${WORK_DIR}/timeline.json`, 'utf8'), +); +const lines = process.env.LINES + ? JSON.parse(readFileSync(process.env.LINES, 'utf8')) + : timeline.captions.map((c) => c.text); + +if (!Array.isArray(lines) || lines.length === 0) { + console.error('No narration lines (timeline has no captions?).'); + process.exit(1); +} + +const run = async () => { + const response = await fetch(`${baseUrl}/api/tts/narration`, { + method: 'POST', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ lines }), + }); + + if (response.status === 404) { + console.error( + 'Narration TTS is not configured on this deployment; ' + + 'produce a captions-only demo.', + ); + process.exit(3); + } + + if (!response.ok) { + throw new Error(`TTS request failed with status ${response.status}`); + } + + const { clips } = await response.json(); + + if (!Array.isArray(clips) || clips.length !== lines.length) { + throw new Error('TTS response did not include one clip per line'); + } + + mkdirSync(`${WORK_DIR}/vo`, { recursive: true }); + + const manifest = []; + + for (let i = 0; i < clips.length; i++) { + const file = `vo/${i}.mp3`; + + writeFileSync( + `${WORK_DIR}/${file}`, + Buffer.from(clips[i].audioBase64, 'base64'), + ); + // startSeconds/durationSeconds are placeholders here; fit-timing.mjs + // measures the real durations and lays the clips out against the beats. + manifest.push({ file, startSeconds: 0, durationSeconds: 0 }); + } + + writeFileSync( + `${WORK_DIR}/narration.json`, + JSON.stringify({ clips: manifest }, null, 2), + ); + console.log(`narration: wrote ${manifest.length} clips to ${WORK_DIR}/vo`); +}; + +run().catch((e) => { + console.error(e.message); + process.exit(1); +}); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs new file mode 100644 index 000000000..f1fbc9ce3 --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs @@ -0,0 +1,194 @@ +// Post-capture timing fit. Takes the captured timeline plus the narration +// clips and produces the final pacing: +// +// 1. Trims the dead opening hold so the demo starts immediately. +// 2. Speeds the voice-over slightly (pitch-preserving atempo) into a +// conversational pace band. +// 3. Solves a video playback rate (<= 1) so every narration line finishes +// before the next visual beat: the voice leads each zoom instead of the +// video freezing at the end waiting for the audio. +// 4. Rewrites caption windows to match the spoken lines. +// +// Captions-only mode (no narration.json clips) still gets the opening trim. +// +// Usage: WORK_DIR=/tmp/feature-demo/work node fit-timing.mjs [atempo] + +import { execFileSync } from 'node:child_process'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; + +const WORK_DIR = process.env.WORK_DIR || '/tmp/feature-demo/work'; +const ATEMPO = Number(process.argv[2] || '1.12'); + +const INTRO_SECONDS = 1.2; // what survives of the opening hold +const FIRST_LINE_AT = 0.7; // narration start after the video opens +const LINE_GAP = 0.35; // breathing room between spoken lines +const BEAT_LEAD = 0.4; // how far the voice leads its zoom +const MIN_RATE = 0.75; // never slow the recording below this + +function ffprobeDuration(file) { + const out = execFileSync('ffprobe', [ + '-v', + 'error', + '-show_entries', + 'format=duration', + '-of', + 'default=nw=1:nk=1', + file, + ]); + return parseFloat(out.toString().trim()); +} + +const timelinePath = `${WORK_DIR}/timeline.json`; +const narrationPath = `${WORK_DIR}/narration.json`; +const timeline = JSON.parse(readFileSync(timelinePath, 'utf8')); +const narration = existsSync(narrationPath) + ? JSON.parse(readFileSync(narrationPath, 'utf8')) + : { clips: [] }; + +// --- 1. Trim the dead opening (all times still in source-video seconds) --- + +const firstBeat = timeline.captions[0]?.start; +const trim = + firstBeat && firstBeat > INTRO_SECONDS + 0.5 ? firstBeat - INTRO_SECONDS : 0; + +function shiftKeys(keys) { + const out = []; + for (const key of keys) { + const t = key.t - trim; + if (t < 0) { + // keep at most one pre-zero key as the t=0 anchor + const anchored = { ...key, t: 0 }; + if (out.length > 0 && out[out.length - 1].t === 0) { + out[out.length - 1] = anchored; + } else { + out.push(anchored); + } + } else { + out.push({ ...key, t: Math.round(t * 1000) / 1000 }); + } + } + return out; +} + +if (trim > 0) { + timeline.scaleKeys = shiftKeys(timeline.scaleKeys); + timeline.focalKeys = shiftKeys(timeline.focalKeys); + timeline.cursorKeys = shiftKeys(timeline.cursorKeys); + timeline.clicks = timeline.clicks + .filter((c) => c.t >= trim) + .map((c) => ({ ...c, t: Math.round((c.t - trim) * 1000) / 1000 })); + for (const cap of timeline.captions) { + cap.start = Math.max(0, cap.start - trim); + cap.end = Math.max(0, cap.end - trim); + } + timeline.video.startFromSeconds = + (timeline.video.startFromSeconds ?? 0) + trim; + timeline.durationSeconds -= trim; +} + +// --- captions-only: trim is all there is to do --- + +if (narration.clips.length === 0) { + writeFileSync(timelinePath, JSON.stringify(timeline, null, 2)); + console.log( + `fit: captions-only, trimmed ${trim.toFixed(2)}s, ` + + `video ${timeline.durationSeconds.toFixed(2)}s`, + ); + process.exit(0); +} + +// --- 2. atempo the voice-over into pace, measure real durations --- + +const durations = []; + +for (let i = 0; i < narration.clips.length; i++) { + const file = `${WORK_DIR}/${narration.clips[i].file}`; + if (ATEMPO !== 1.0) { + execFileSync( + 'ffmpeg', + ['-y', '-v', 'error', '-i', file, '-filter:a', `atempo=${ATEMPO}`, `${file}.f.mp3`], + { stdio: ['ignore', 'ignore', 'pipe'] }, + ); + execFileSync('mv', [`${file}.f.mp3`, file]); + } + durations.push(ffprobeDuration(file)); +} + +// --- 3. solve the playback rate so the voice leads every beat --- + +// Beat anchors are the (trimmed) caption starts in source-video seconds; the +// first line is anchored to the video open instead of its beat. +const beats = timeline.captions.map((c) => c.start); +let rate = 1; + +for (let i = 1; i < beats.length && i < durations.length; i++) { + let needed = FIRST_LINE_AT; + for (let j = 0; j < i; j++) { + needed += durations[j] + LINE_GAP; + } + // Need beats[i]/rate - BEAT_LEAD >= needed => rate <= beats[i]/(needed+LEAD) + const maxRate = beats[i] / (needed + BEAT_LEAD); + rate = Math.min(rate, maxRate); +} + +rate = Math.max(MIN_RATE, Math.min(1, Math.round(rate * 100) / 100)); + +if (rate !== 1) { + const stretch = 1 / rate; + for (const keys of [ + timeline.scaleKeys, + timeline.focalKeys, + timeline.cursorKeys, + ]) { + for (const key of keys) { + key.t = Math.round(key.t * stretch * 1000) / 1000; + } + } + for (const click of timeline.clicks) { + click.t = Math.round(click.t * stretch * 1000) / 1000; + } + timeline.durationSeconds = + Math.round(timeline.durationSeconds * stretch * 1000) / 1000; +} + +timeline.video.playbackRate = rate; + +// --- 4. schedule lines (voice leads each beat) + matching captions --- + +let prevEnd = 0; +const starts = []; + +for (let i = 0; i < durations.length; i++) { + const beatAt = i === 0 ? FIRST_LINE_AT : beats[i] / rate - BEAT_LEAD; + const start = + Math.round(Math.max(i === 0 ? 0 : prevEnd + LINE_GAP, beatAt) * 100) / 100; + starts.push(start); + prevEnd = start + durations[i]; +} + +for (let i = 0; i < durations.length; i++) { + narration.clips[i].startSeconds = starts[i]; + narration.clips[i].durationSeconds = Math.round(durations[i] * 1000) / 1000; + if (timeline.captions[i]) { + timeline.captions[i].start = starts[i]; + timeline.captions[i].end = + Math.round((starts[i] + durations[i] + 0.25) * 1000) / 1000; + } +} + +writeFileSync(timelinePath, JSON.stringify(timeline, null, 2)); +writeFileSync(narrationPath, JSON.stringify(narration, null, 2)); + +const narrationEnd = starts[starts.length - 1] + durations[durations.length - 1]; + +console.log( + `fit: trimmed ${trim.toFixed(2)}s, rate ${rate}, ` + + `video ${timeline.durationSeconds.toFixed(2)}s, ` + + `narration ends ${narrationEnd.toFixed(2)}s`, +); +for (let i = 0; i < starts.length; i++) { + console.log( + ` line${i} ${starts[i].toFixed(2)}+${durations[i].toFixed(2)}` + + `=${(starts[i] + durations[i]).toFixed(2)}`, + ); +} diff --git a/packages/cloud-agents/tsconfig.json b/packages/cloud-agents/tsconfig.json index 62afcee4e..a58412869 100644 --- a/packages/cloud-agents/tsconfig.json +++ b/packages/cloud-agents/tsconfig.json @@ -4,5 +4,11 @@ "types": ["node", "vitest/globals"] }, "include": ["src", "evals"], - "exclude": ["node_modules"] + // The feature-demo skill bundles a standalone Remotion render project that + // is copied into a sandbox work dir at demo time; it is not part of this + // package's program (JSX, remotion deps). + "exclude": [ + "node_modules", + "src/server/workflows/skills/standard/feature-demo/render" + ] } diff --git a/packages/compute-providers/src/adapters/modal.test.ts b/packages/compute-providers/src/adapters/modal.test.ts index 76e88106a..7d81ee597 100644 --- a/packages/compute-providers/src/adapters/modal.test.ts +++ b/packages/compute-providers/src/adapters/modal.test.ts @@ -962,6 +962,28 @@ describe('ModalClient', () => { ); }); + it('expects the Remotion chrome-headless-shell to be baked into the worker runtime', () => { + const dockerfile = fs.readFileSync( + new URL('../../../../apps/worker/Dockerfile', import.meta.url), + 'utf8', + ); + + expect(dockerfile).toContain('ARG REMOTION_VERSION='); + expect(dockerfile).toContain( + 'ENV REMOTION_HEADLESS_SHELL_PATH="/opt/remotion/headless-shell"', + ); + expect(dockerfile).toContain('"@remotion/cli@${REMOTION_VERSION}"'); + expect(dockerfile).toContain('"remotion@${REMOTION_VERSION}"'); + expect(dockerfile).toContain('npx remotion browser ensure'); + expect(dockerfile).toContain( + 'cp -R node_modules/.remotion/chrome-headless-shell /opt/remotion/chrome-headless-shell', + ); + expect(dockerfile).toContain( + 'ln -sf "$REMOTION_SHELL" "${REMOTION_HEADLESS_SHELL_PATH}"', + ); + expect(dockerfile).toContain('/opt/remotion/.installed'); + }); + it('expects PM2 to be available in the worker runtime', () => { const dockerfile = fs.readFileSync( new URL('../../../../apps/worker/Dockerfile', import.meta.url), diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts index a97d92081..8acba1abc 100644 --- a/packages/env/src/index.ts +++ b/packages/env/src/index.ts @@ -131,6 +131,13 @@ const serverSchema = { // private networks; a CIDR list rather than a boolean so opening one // internal host does not re-expose every adjacent service. R_CUSTOM_MCP_ALLOWED_PRIVATE_CIDRS: z.string().min(1).optional(), + // ElevenLabs credentials for the narration TTS endpoint. The key stays on + // the control plane; task sandboxes reach TTS only through /api/tts with + // their run-scoped token (see apps/api/src/handlers/tts). Unset means the + // feature is off and the endpoint 404s. + R_ELEVENLABS_API_KEY: z.string().min(1).optional(), + R_ELEVENLABS_VOICE_ID: z.string().min(1).optional(), + R_ELEVENLABS_MODEL: z.string().min(1).optional(), R_INTERCOM_APP_ID: z.string().min(1).optional(), R_POSTHOG_PROJECT_KEY: z.string().min(1).optional(), R_POSTHOG_HOST: z.string().url().optional(), @@ -449,6 +456,9 @@ const OPTIONAL_NON_EMPTY_KEYS = new Set([ 'R_INSTANCE_ID', 'STATUSPAGE_INCIDENTS_ENABLED_INSTANCE_ID', 'R_CUSTOM_MCP_ALLOWED_PRIVATE_CIDRS', + 'R_ELEVENLABS_API_KEY', + 'R_ELEVENLABS_VOICE_ID', + 'R_ELEVENLABS_MODEL', 'R_INTERCOM_APP_ID', 'R_POSTHOG_PROJECT_KEY', 'R_POSTHOG_HOST', diff --git a/packages/types/src/control-plane-env-vars.test.ts b/packages/types/src/control-plane-env-vars.test.ts index f9844a777..b88246fa7 100644 --- a/packages/types/src/control-plane-env-vars.test.ts +++ b/packages/types/src/control-plane-env-vars.test.ts @@ -27,6 +27,8 @@ describe('CONTROL_PLANE_ENV_VAR_NAMES', () => { 'DATABASE_URL', 'S3_SECRET_ACCESS_KEY', 'R_LICENSE_KEY', + 'R_ELEVENLABS_API_KEY', + 'R_ELEVENLABS_VOICE_ID', ]) { expect(CONTROL_PLANE_ENV_VAR_NAMES.has(name)).toBe(true); } diff --git a/packages/types/src/control-plane-env-vars.ts b/packages/types/src/control-plane-env-vars.ts index 65fd1b405..f2ebdb687 100644 --- a/packages/types/src/control-plane-env-vars.ts +++ b/packages/types/src/control-plane-env-vars.ts @@ -99,6 +99,18 @@ export const INSTANCE_SECRET_ENV_VAR_NAMES: ReadonlySet = new Set([ 'R_LICENSE_KEY', ]); +/** + * Media-generation provider credentials. Task sandboxes reach these providers + * only through control-plane endpoints (e.g. /api/tts) that inject the key + * server-side, so the raw values must never enter a sandbox. The voice/model + * ids are not secrets, but a task never needs them either. + */ +export const MEDIA_PROVIDER_ENV_VAR_NAMES: ReadonlySet = new Set([ + 'R_ELEVENLABS_API_KEY', + 'R_ELEVENLABS_VOICE_ID', + 'R_ELEVENLABS_MODEL', +]); + /** * Declarative environment provisioning inputs, managed through the deployment * environment. Not secrets per se, but they are control-plane configuration @@ -134,6 +146,7 @@ export const CONTROL_PLANE_ENV_VAR_NAMES: ReadonlySet = new Set( ...INTEGRATION_BOT_SECRET_ENV_VAR_NAMES, ...PROVIDER_IDENTIFIER_ENV_VAR_NAMES, ...INSTANCE_SECRET_ENV_VAR_NAMES, + ...MEDIA_PROVIDER_ENV_VAR_NAMES, ...DECLARATIVE_ENVIRONMENT_ENV_VAR_NAMES, ...DISABLED_MODEL_PROVIDER_ENV_VAR_NAMES, ], From d93c834fe7b999a2584a08759f99629ff8681dec Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:55:09 -0400 Subject: [PATCH 02/27] [Fix] Report the TTS model so timing fit picks the right pace eleven_v3 reads slowly and needs the 1.12 atempo speed-up; v2-family models pace naturally and should not be sped up. The narration endpoint now returns its modelId, build-narration records it in the manifest, and fit-timing derives the atempo default from it (argv still wins). --- apps/api/src/handlers/tts/index.ts | 4 +++- .../standard/feature-demo/scripts/build-narration.mjs | 9 ++++++--- .../skills/standard/feature-demo/scripts/fit-timing.mjs | 9 ++++++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/apps/api/src/handlers/tts/index.ts b/apps/api/src/handlers/tts/index.ts index a167eabe6..275145c51 100644 --- a/apps/api/src/handlers/tts/index.ts +++ b/apps/api/src/handlers/tts/index.ts @@ -214,5 +214,7 @@ tts.post('/narration', async (c) => { } } - return c.json({ clips }); + // The model id lets callers adapt post-processing (e.g. v3 reads slowly + // and benefits from a pitch-preserving speed-up; v2 paces naturally). + return c.json({ clips, modelId }); }); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs index e3c96f315..b8602d9cb 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs @@ -64,7 +64,7 @@ const run = async () => { throw new Error(`TTS request failed with status ${response.status}`); } - const { clips } = await response.json(); + const { clips, modelId } = await response.json(); if (!Array.isArray(clips) || clips.length !== lines.length) { throw new Error('TTS response did not include one clip per line'); @@ -88,9 +88,12 @@ const run = async () => { writeFileSync( `${WORK_DIR}/narration.json`, - JSON.stringify({ clips: manifest }, null, 2), + JSON.stringify({ clips: manifest, modelId: modelId ?? null }, null, 2), + ); + console.log( + `narration: wrote ${manifest.length} clips to ${WORK_DIR}/vo` + + (modelId ? ` (model ${modelId})` : ''), ); - console.log(`narration: wrote ${manifest.length} clips to ${WORK_DIR}/vo`); }; run().catch((e) => { diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs index f1fbc9ce3..3927f6888 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs @@ -17,7 +17,6 @@ import { execFileSync } from 'node:child_process'; import { existsSync, readFileSync, writeFileSync } from 'node:fs'; const WORK_DIR = process.env.WORK_DIR || '/tmp/feature-demo/work'; -const ATEMPO = Number(process.argv[2] || '1.12'); const INTRO_SECONDS = 1.2; // what survives of the opening hold const FIRST_LINE_AT = 0.7; // narration start after the video opens @@ -45,6 +44,14 @@ const narration = existsSync(narrationPath) ? JSON.parse(readFileSync(narrationPath, 'utf8')) : { clips: [] }; +// v3 reads noticeably slower than a conversational pace and benefits from a +// pitch-preserving speed-up; v2-family models pace naturally. An explicit +// argv override wins either way. +const defaultAtempo = (narration.modelId ?? '').startsWith('eleven_v3') + ? 1.12 + : 1.0; +const ATEMPO = Number(process.argv[2] || defaultAtempo); + // --- 1. Trim the dead opening (all times still in source-video seconds) --- const firstBeat = timeline.captions[0]?.start; From e809dc85ed9656b0232f94f40b29f2f7d33873c0 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:58:57 -0400 Subject: [PATCH 03/27] [Refactor] Frame the render project as an adaptable reference template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review direction: the bundled Remotion project is a starting point the agent copies and adapts (branding, captions, layout, presets), not a fixed pipeline stage. The timeline JSON stays the stable contract between capture and render. When adapting, the skill installs Remotion's official agent skills (remotion-dev/skills — their sanctioned successor to the deprecated Remotion MCP) for current API guidance. A render/ README documents the bug classes worth preserving (counter-scaled cursor, edge-clamp guard, timeline-driven interpolation). --- .../__tests__/featureDemoSkill.test.ts | 8 ++++++ .../skills/standard/feature-demo/SKILL.md | 8 ++++++ .../standard/feature-demo/render/README.md | 26 +++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index 3ad37d4b3..885d6e3bf 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -77,6 +77,14 @@ describe('feature-demo skill', () => { ); }); + it('frames the render project as an adaptable reference template', () => { + expect(skillContent).toContain('**reference template**'); + expect(skillContent).toContain('npx -y skills add remotion-dev/skills'); + expect(skillContent).toContain( + 'The timeline JSON schema is the stable contract', + ); + }); + it('renders with the baked headless shell and a runtime fallback', () => { expect(skillContent).toContain( '--browser-executable="${REMOTION_HEADLESS_SHELL_PATH:-/opt/remotion/headless-shell}"', diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index fca02c4e0..67cc0663f 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -24,6 +24,12 @@ it targets. Pipeline: author script → capture (browser, delegated) → narrate (optional) → fit timing → render → verify → upload. +The bundled `render/` project is a **reference template**, not a fixed +pipeline stage: copy it to the work dir and adapt the copy freely (branding, +caption styling, layout, extra presets) when the demo calls for it. The +timeline JSON is the stable contract between capture and render — keep the +copy consuming it and any adaptation stays in sync with the recording. + All work happens under `/tmp/feature-demo/work` (the work dir). Nothing from this pipeline is ever committed to the repository. @@ -92,6 +98,7 @@ and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. - `cp /tmp/feature-demo/work/timeline.json /tmp/feature-demo/work/narration.json /tmp/feature-demo/render/props/` (skip narration.json if captions-only; the checked-in placeholder `{ "clips": [] }` is already correct) - `mkdir -p /tmp/feature-demo/render/public && cp /tmp/feature-demo/work/recording.mp4 /tmp/feature-demo/render/public/` - `cp -R /tmp/feature-demo/work/vo /tmp/feature-demo/render/public/vo` (narrated demos only) +Adapt the copied composition when the default look does not fit the request — different backdrop, caption treatment, brand colors, an extra preset, a layout change. Before writing Remotion code, install Remotion's official agent skills into the render copy and read them for current API guidance: `cd /tmp/feature-demo/render && npx -y skills add remotion-dev/skills`. Preserve the pieces that encode hard-won correctness unless you have a reason not to: the zoom transform with its counter-scaled cursor, the edge-clamp guard (only clamp an axis when the scaled window exceeds the canvas), and the timeline-driven keyframe interpolation. `cd /tmp/feature-demo/render && npm install` (versions are pinned; takes a few minutes on first run). Render with the image's baked headless shell: @@ -125,6 +132,7 @@ If that binary does not exist (older sandbox snapshot), run `npx remotion browse Never load or invoke `agent-browser` (or any other browser automation) from this skill — capture is always delegated to the `proof-runner` subagent. Never ask for, read, or handle TTS provider keys. Narration goes through the control-plane endpoint with the run token; a 404 there means captions-only. All intermediate files live under `/tmp/feature-demo`. Never commit recordings, renders, node_modules, or props into the repository. +Adapt the work-dir copy of the render template, never the installed skill sources under `~/.agents/skills/feature-demo`. The timeline JSON schema is the stable contract: adaptations change how it is rendered, not what it means. Report blockers honestly: a missing selector, a failed render, or a visibly broken frame is a blocker with evidence, not a reason to narrow the claim or ship a degraded video silently. Voice, wording, and pacing choices belong to the user when they express them; defaults are: wide preset, captions as narration lines, conversational tone. diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md new file mode 100644 index 000000000..c9d1e58ac --- /dev/null +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/README.md @@ -0,0 +1,26 @@ +# feature-demo render template + +A reference Remotion composition for feature-demo videos. At demo time the +agent copies this directory into the sandbox work dir and may adapt the copy +(branding, caption styling, layout, presets) — see `../SKILL.md`. + +The stable contract is `props/timeline.json` (emitted by the capture runner) +and `props/narration.json` (emitted by the narration scripts): compositions +render those; adaptations change how they look, not what they mean. + +Worth preserving when adapting — each encodes a bug class found the hard way: + +- **Counter-scaled cursor/ripple** (`DemoStage.tsx`): cursor and ripple are + children of the zoom-transform container with a `1/S` counter-scale, so + they stay glued to page coordinates at constant size through any zoom. +- **Edge-clamp guard** (`DemoStage.tsx`): the translate that keeps backdrop + from showing behind the window must only clamp an axis when the scaled + window exceeds the canvas on that axis; clamping a smaller-than-canvas + window (vertical presets) shoves it into a corner. +- **Timeline-driven interpolation**: all motion eases between capture-emitted + keys. Do not invent motion that is not in the timeline — it will drift + from the recording. + +This directory is excluded from the repo's typecheck/lint/knip on purpose: +it is not part of the package graph and runs only from a work-dir copy with +its own pinned `npm install`. From b35d8c78577d1f5c9262e66a08a19202c07cdf0e Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:02:15 -0400 Subject: [PATCH 04/27] [Refactor] Hardcode the narration model to eleven_multilingual_v2 Simplify per review: one model, one settings profile, no R_ELEVENLABS_MODEL knob and no model-aware atempo plumbing. multilingual_v2 reads voice clones naturally at a natural pace; the timing fit's atempo stays available as an explicit argv override. --- .../src/handlers/tts/__tests__/tts.test.ts | 7 ++--- apps/api/src/handlers/tts/index.ts | 26 +++++++++---------- apps/docs/environment-variables.mdx | 1 - .../feature-demo/scripts/build-narration.mjs | 9 +++---- .../feature-demo/scripts/fit-timing.mjs | 14 ++++------ packages/env/src/index.ts | 2 -- packages/types/src/control-plane-env-vars.ts | 1 - 7 files changed, 22 insertions(+), 38 deletions(-) diff --git a/apps/api/src/handlers/tts/__tests__/tts.test.ts b/apps/api/src/handlers/tts/__tests__/tts.test.ts index e1308fd7d..65dcf5372 100644 --- a/apps/api/src/handlers/tts/__tests__/tts.test.ts +++ b/apps/api/src/handlers/tts/__tests__/tts.test.ts @@ -7,7 +7,6 @@ const { mockEnv } = vi.hoisted(() => ({ mockEnv: { R_ELEVENLABS_API_KEY: undefined as string | undefined, R_ELEVENLABS_VOICE_ID: undefined as string | undefined, - R_ELEVENLABS_MODEL: undefined as string | undefined, }, })); @@ -54,7 +53,6 @@ const originalFetch = globalThis.fetch; beforeEach(() => { mockEnv.R_ELEVENLABS_API_KEY = 'el-key'; mockEnv.R_ELEVENLABS_VOICE_ID = 'voice-1'; - mockEnv.R_ELEVENLABS_MODEL = undefined; }); afterEach(() => { @@ -169,9 +167,8 @@ describe('POST /tts/narration', () => { voice_settings: { stability: number }; }; - // v3 default anchors the clone's accent with "robust" stability. - expect(body.model_id).toBe('eleven_v3'); - expect(body.voice_settings.stability).toBe(1.0); + expect(body.model_id).toBe('eleven_multilingual_v2'); + expect(body.voice_settings.stability).toBe(0.35); }); it('502s when the upstream synthesis fails', async () => { diff --git a/apps/api/src/handlers/tts/index.ts b/apps/api/src/handlers/tts/index.ts index 275145c51..82155e088 100644 --- a/apps/api/src/handlers/tts/index.ts +++ b/apps/api/src/handlers/tts/index.ts @@ -34,11 +34,11 @@ const MAX_NARRATION_LINE_CHARS = 1_200; const MAX_NARRATION_REQUEST_BYTES = 64 * 1024; /** - * v3 is the most natural read for voice clones; stability 1.0 ("robust") - * anchors delivery to the reference audio, which prevents the accent drift - * v3 exhibits at lower stability on instant clones. + * multilingual_v2 gives voice clones a natural, conversational read at a + * natural pace (the more "expressive" models lean announcer-voice on + * professional clones and read too slowly). */ -const DEFAULT_ELEVENLABS_MODEL = 'eleven_v3'; +const ELEVENLABS_MODEL_ID = 'eleven_multilingual_v2'; const ELEVENLABS_TTS_TIMEOUT_MS = 60_000; async function readRequestBodyBytes( @@ -111,7 +111,6 @@ function parseNarrationLines(body: NarrationBody): string[] | null { async function synthesizeLine(options: { apiKey: string; voiceId: string; - modelId: string; text: string; }): Promise { const response = await fetch( @@ -126,10 +125,12 @@ async function synthesizeLine(options: { }, body: JSON.stringify({ text: options.text, - model_id: options.modelId, - voice_settings: options.modelId.startsWith('eleven_v3') - ? { stability: 1.0, similarity_boost: 0.75 } - : { stability: 0.5, similarity_boost: 0.75 }, + model_id: ELEVENLABS_MODEL_ID, + voice_settings: { + stability: 0.35, + similarity_boost: 0.75, + style: 0.45, + }, }), signal: AbortSignal.timeout(ELEVENLABS_TTS_TIMEOUT_MS), }, @@ -198,14 +199,13 @@ tts.post('/narration', async (c) => { ); } - const modelId = Env.R_ELEVENLABS_MODEL || DEFAULT_ELEVENLABS_MODEL; const clips: { audioBase64: string }[] = []; // Sequential on purpose: ElevenLabs enforces per-key concurrency limits, // and a narration is a handful of short lines. for (const text of lines) { try { - const audio = await synthesizeLine({ apiKey, voiceId, modelId, text }); + const audio = await synthesizeLine({ apiKey, voiceId, text }); clips.push({ audioBase64: audio.toString('base64') }); } catch (error) { @@ -214,7 +214,5 @@ tts.post('/narration', async (c) => { } } - // The model id lets callers adapt post-processing (e.g. v3 reads slowly - // and benefits from a pitch-preserving speed-up; v2 paces naturally). - return c.json({ clips, modelId }); + return c.json({ clips }); }); diff --git a/apps/docs/environment-variables.mdx b/apps/docs/environment-variables.mdx index a753cfdca..f7b000260 100644 --- a/apps/docs/environment-variables.mdx +++ b/apps/docs/environment-variables.mdx @@ -342,7 +342,6 @@ as per-task auth tokens or workspace paths. | `R_ALLOWED_EMAILS` | Optional | Comma-separated email allowlist for deployments that restrict sign-in by email. | | `R_ELEVENLABS_API_KEY` | Optional | ElevenLabs API key for narrated feature-demo videos. The key stays on the control plane; sandboxes reach text-to-speech only through an authenticated Roomote endpoint. A key scoped to text-to-speech only is sufficient and recommended. | | `R_ELEVENLABS_VOICE_ID` | Optional | ElevenLabs voice ID used for feature-demo narration. Required alongside the API key for narration to be available. | -| `R_ELEVENLABS_MODEL` | Optional | ElevenLabs TTS model override. Defaults to `eleven_v3`. | During Microsoft Teams setup, Roomote uses the Microsoft Entra app values for the Teams bot by default. Use **Show advanced config** after the Directory diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs index b8602d9cb..e3c96f315 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/build-narration.mjs @@ -64,7 +64,7 @@ const run = async () => { throw new Error(`TTS request failed with status ${response.status}`); } - const { clips, modelId } = await response.json(); + const { clips } = await response.json(); if (!Array.isArray(clips) || clips.length !== lines.length) { throw new Error('TTS response did not include one clip per line'); @@ -88,12 +88,9 @@ const run = async () => { writeFileSync( `${WORK_DIR}/narration.json`, - JSON.stringify({ clips: manifest, modelId: modelId ?? null }, null, 2), - ); - console.log( - `narration: wrote ${manifest.length} clips to ${WORK_DIR}/vo` + - (modelId ? ` (model ${modelId})` : ''), + JSON.stringify({ clips: manifest }, null, 2), ); + console.log(`narration: wrote ${manifest.length} clips to ${WORK_DIR}/vo`); }; run().catch((e) => { diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs index 3927f6888..c77cd3985 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs @@ -2,8 +2,8 @@ // clips and produces the final pacing: // // 1. Trims the dead opening hold so the demo starts immediately. -// 2. Speeds the voice-over slightly (pitch-preserving atempo) into a -// conversational pace band. +// 2. Optionally retimes the voice-over (pitch-preserving atempo, off by +// default). // 3. Solves a video playback rate (<= 1) so every narration line finishes // before the next visual beat: the voice leads each zoom instead of the // video freezing at the end waiting for the audio. @@ -44,13 +44,9 @@ const narration = existsSync(narrationPath) ? JSON.parse(readFileSync(narrationPath, 'utf8')) : { clips: [] }; -// v3 reads noticeably slower than a conversational pace and benefits from a -// pitch-preserving speed-up; v2-family models pace naturally. An explicit -// argv override wins either way. -const defaultAtempo = (narration.modelId ?? '').startsWith('eleven_v3') - ? 1.12 - : 1.0; -const ATEMPO = Number(process.argv[2] || defaultAtempo); +// The narration model paces naturally, so no speed-up by default; pass an +// atempo argv (e.g. 1.1) to tighten a read that came back slow. +const ATEMPO = Number(process.argv[2] || '1.0'); // --- 1. Trim the dead opening (all times still in source-video seconds) --- diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts index 8acba1abc..4d55332ea 100644 --- a/packages/env/src/index.ts +++ b/packages/env/src/index.ts @@ -137,7 +137,6 @@ const serverSchema = { // feature is off and the endpoint 404s. R_ELEVENLABS_API_KEY: z.string().min(1).optional(), R_ELEVENLABS_VOICE_ID: z.string().min(1).optional(), - R_ELEVENLABS_MODEL: z.string().min(1).optional(), R_INTERCOM_APP_ID: z.string().min(1).optional(), R_POSTHOG_PROJECT_KEY: z.string().min(1).optional(), R_POSTHOG_HOST: z.string().url().optional(), @@ -458,7 +457,6 @@ const OPTIONAL_NON_EMPTY_KEYS = new Set([ 'R_CUSTOM_MCP_ALLOWED_PRIVATE_CIDRS', 'R_ELEVENLABS_API_KEY', 'R_ELEVENLABS_VOICE_ID', - 'R_ELEVENLABS_MODEL', 'R_INTERCOM_APP_ID', 'R_POSTHOG_PROJECT_KEY', 'R_POSTHOG_HOST', diff --git a/packages/types/src/control-plane-env-vars.ts b/packages/types/src/control-plane-env-vars.ts index f2ebdb687..24a05ae98 100644 --- a/packages/types/src/control-plane-env-vars.ts +++ b/packages/types/src/control-plane-env-vars.ts @@ -108,7 +108,6 @@ export const INSTANCE_SECRET_ENV_VAR_NAMES: ReadonlySet = new Set([ export const MEDIA_PROVIDER_ENV_VAR_NAMES: ReadonlySet = new Set([ 'R_ELEVENLABS_API_KEY', 'R_ELEVENLABS_VOICE_ID', - 'R_ELEVENLABS_MODEL', ]); /** From b995257bdb20459fedd72ecd1353d9c1b3fcfac9 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:09:06 -0400 Subject: [PATCH 05/27] [Fix] Address review feedback on capture and timing fit - capture: bracket cursor motion with a hold key at motion start so the synthetic cursor no longer drifts toward its next target through waits, holds, and scrolls (and clicks on a fresh element get a short bracketed hop instead of a slow glide). - capture: pass selectors into the page-eval as alphabet-checked base64 so no selector bytes are interpolated into code (CodeQL). - fit-timing: solve the playback rate by simulating the schedule instead of a closed-form cumulative estimate; a line that waits for a late anchor ends later than the estimate, which could push later lines past their anchors. Lines intentionally START just before their zoom and play over its hold; the docs now say so explicitly. --- .../standard/feature-demo/capture/capture.mjs | 32 ++++++++-- .../feature-demo/scripts/fit-timing.mjs | 63 ++++++++++++------- 2 files changed, 67 insertions(+), 28 deletions(-) diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs index 6ad819e28..b993abf0d 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -49,9 +49,17 @@ let t0 = 0; const now = () => (Date.now() - t0) / 1000; // seconds since record start function rect(sel) { - const js = `(function(){var e=document.querySelector(${JSON.stringify( - sel, - )});if(!e)return null;var r=e.getBoundingClientRect();return{x:r.x,y:r.y,w:r.width,h:r.height};})()`; + // The selector travels into the page-eval as base64 so no selector bytes + // are ever interpolated into code; the alphabet check makes that a hard + // guarantee rather than an encoding assumption. + const selB64 = Buffer.from(String(sel), 'utf8').toString('base64'); + if (!/^[A-Za-z0-9+/=]*$/.test(selB64)) { + throw new Error(`unencodable selector: ${sel}`); + } + const js = + `(function(){var e=document.querySelector(atob("${selB64}"));` + + `if(!e)return null;var r=e.getBoundingClientRect();` + + `return{x:r.x,y:r.y,w:r.width,h:r.height};})()`; const out = ab('eval', js).trim(); const r = JSON.parse(out); if (!r) throw new Error(`element not found: ${sel}`); @@ -75,8 +83,18 @@ const timeline = { }; let cur = { scale: 1, focal: { x: 0.5, y: 0.5 } }; +// Track the cursor so motion can be bracketed by a hold key: the renderer +// eases between consecutive keys, so without a hold at motion start the +// synthetic cursor would drift toward the next target through every wait, +// hold, and scroll in between while the real mouse is stationary. +let curCursor = { x: 0.5, y: 1.1 }; const pushScale = (t, v) => timeline.scaleKeys.push({ t, v }); const pushFocal = (t, v) => timeline.focalKeys.push({ t, v }); +const pushCursorMove = (startT, endT, target) => { + timeline.cursorKeys.push({ t: startT, v: curCursor }); + timeline.cursorKeys.push({ t: endT, v: target }); + curCursor = target; +}; async function run() { mkdirSync(OUT_DIR, { recursive: true }); @@ -113,7 +131,7 @@ async function run() { const end = now(); pushScale(end, beat.scale ?? 1.5); pushFocal(end, c); - timeline.cursorKeys.push({ t: end, v: c }); + pushCursorMove(start, end, c); if (beat.caption) { timeline.captions.push({ start: start + 0.15, @@ -130,7 +148,11 @@ async function run() { const c = centerNorm(rect(beat.sel)); const t = now(); timeline.clicks.push({ t, at: c }); - timeline.cursorKeys.push({ t, v: c }); + // Usually the cursor is already here from a preceding focus; when it + // is not, give it a short bracketed hop instead of a slow drift. + if (curCursor.x !== c.x || curCursor.y !== c.y) { + pushCursorMove(Math.max(0, t - 0.25), t, c); + } ab('click', beat.sel); sleep(beat.holdMs ?? 300); continue; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs index c77cd3985..afb988a54 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs @@ -4,9 +4,10 @@ // 1. Trims the dead opening hold so the demo starts immediately. // 2. Optionally retimes the voice-over (pitch-preserving atempo, off by // default). -// 3. Solves a video playback rate (<= 1) so every narration line finishes -// before the next visual beat: the voice leads each zoom instead of the -// video freezing at the end waiting for the audio. +// 3. Solves a video playback rate (<= 1) so every narration line STARTS +// just before its zoom lands and then plays over the zoom's hold — the +// voice leads each beat, and no line is pushed past its anchor by an +// earlier line still speaking. // 4. Rewrites caption windows to match the spoken lines. // // Captions-only mode (no narration.json clips) still gets the opening trim. @@ -120,21 +121,46 @@ for (let i = 0; i < narration.clips.length; i++) { // --- 3. solve the playback rate so the voice leads every beat --- // Beat anchors are the (trimmed) caption starts in source-video seconds; the -// first line is anchored to the video open instead of its beat. +// first line is anchored to the video open instead of its beat. A line is +// meant to START at its anchor (just before the zoom lands) and play over +// the zoom's hold; the schedule fails only when a line cannot start on time +// because an earlier line is still speaking. +// +// Solved by simulation rather than closed form: a line that waits for a +// late anchor ends later than any cumulative-duration estimate, so the only +// reliable check is to lay the schedule out at a candidate rate and look +// for a line pushed past its anchor. const beats = timeline.captions.map((c) => c.start); -let rate = 1; -for (let i = 1; i < beats.length && i < durations.length; i++) { - let needed = FIRST_LINE_AT; - for (let j = 0; j < i; j++) { - needed += durations[j] + LINE_GAP; +function scheduleAt(candidateRate) { + let prevEnd = 0; + const lineStarts = []; + let pushedPastAnchor = false; + + for (let i = 0; i < durations.length; i++) { + // A line without a matching caption beat just runs sequentially. + const anchor = + i === 0 + ? FIRST_LINE_AT + : beats[i] !== undefined + ? beats[i] / candidateRate - BEAT_LEAD + : prevEnd + LINE_GAP; + const start = Math.max(i === 0 ? 0 : prevEnd + LINE_GAP, anchor); + if (start > anchor + 0.01) { + pushedPastAnchor = true; + } + lineStarts.push(Math.round(start * 100) / 100); + prevEnd = start + durations[i]; } - // Need beats[i]/rate - BEAT_LEAD >= needed => rate <= beats[i]/(needed+LEAD) - const maxRate = beats[i] / (needed + BEAT_LEAD); - rate = Math.min(rate, maxRate); + + return { lineStarts, pushedPastAnchor }; } -rate = Math.max(MIN_RATE, Math.min(1, Math.round(rate * 100) / 100)); +let rate = 1; +while (rate > MIN_RATE && scheduleAt(rate).pushedPastAnchor) { + rate = Math.round((rate - 0.01) * 100) / 100; +} +rate = Math.max(MIN_RATE, rate); if (rate !== 1) { const stretch = 1 / rate; @@ -158,16 +184,7 @@ timeline.video.playbackRate = rate; // --- 4. schedule lines (voice leads each beat) + matching captions --- -let prevEnd = 0; -const starts = []; - -for (let i = 0; i < durations.length; i++) { - const beatAt = i === 0 ? FIRST_LINE_AT : beats[i] / rate - BEAT_LEAD; - const start = - Math.round(Math.max(i === 0 ? 0 : prevEnd + LINE_GAP, beatAt) * 100) / 100; - starts.push(start); - prevEnd = start + durations[i]; -} +const { lineStarts: starts } = scheduleAt(rate); for (let i = 0; i < durations.length; i++) { narration.clips[i].startSeconds = starts[i]; From 3e7f7bcd0a773dd6e5f24ce4507579fe17ff06ad Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:10:42 -0400 Subject: [PATCH 06/27] [Docs] Regenerate the cookbook index and clean up the recipe copy The cookbook index is generated from recipe frontmatter; run the generator instead of hand-editing between the markers. --- apps/docs/cookbook/feature-demo-videos.mdx | 8 ++++---- apps/docs/cookbook/index.mdx | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/docs/cookbook/feature-demo-videos.mdx b/apps/docs/cookbook/feature-demo-videos.mdx index b9d8e68aa..be9e9b2e8 100644 --- a/apps/docs/cookbook/feature-demo-videos.mdx +++ b/apps/docs/cookbook/feature-demo-videos.mdx @@ -1,6 +1,6 @@ --- title: Record feature demo videos -description: Ask Roomote for a polished, narrated demo video of a feature — recorded live, with zooms, captions, and an optional voice-over. +description: Ask Roomote for a polished, narrated demo video of a feature, recorded live with zooms, captions, and an optional voice-over. contributor: Matt Rubens contributor_url: https://github.com/mrubens contributor_company: Roomote @@ -22,7 +22,7 @@ narration configured, the demo gets a voice-over that leads each visual beat. ## Ingredients -- Any task where the feature is reachable — a running dev environment or a +- Any task where the feature is reachable: a running dev environment or a public page - Optional: an [ElevenLabs](https://elevenlabs.io) API key and voice ID for narration @@ -41,9 +41,9 @@ narration configured, the demo gets a voice-over that leads each visual beat. Captioned demos work out of the box. For a voice-over in your own (cloned) voice, a deployment admin sets two environment variables: -- `R_ELEVENLABS_API_KEY` — an ElevenLabs key. A key scoped to +- `R_ELEVENLABS_API_KEY`: an ElevenLabs key. A key scoped to **text-to-speech only**, with a credit cap, is sufficient and recommended. -- `R_ELEVENLABS_VOICE_ID` — the voice to narrate with. +- `R_ELEVENLABS_VOICE_ID`: the voice to narrate with. The key stays on the Roomote control plane: task sandboxes send narration text to an authenticated Roomote endpoint and receive audio back, so the diff --git a/apps/docs/cookbook/index.mdx b/apps/docs/cookbook/index.mdx index f25f27824..10e059573 100644 --- a/apps/docs/cookbook/index.mdx +++ b/apps/docs/cookbook/index.mdx @@ -16,7 +16,7 @@ and the quality of your output. | [Ease your team into cloud agents](/cookbook/ease-your-team-into-cloud-agents) | Build trust in Roomote through small, visible, low-risk team habits. | | [Evaluate outage impact](/cookbook/vendor-outage-triage) | Filter vendor status noise by comparing each incident with your real code, regions, and feature usage. | | [Fix CI failures](/cookbook/ci-failure-auto-fix) | Keep the build green by having Roomote verify and fix CI breakages automatically. | -| [Record feature demo videos](/cookbook/feature-demo-videos) | Ask Roomote for a polished, narrated demo video of a feature — recorded live with zooms, captions, and voice-over. | +| [Record feature demo videos](/cookbook/feature-demo-videos) | Ask Roomote for a polished, narrated demo video of a feature, recorded live with zooms, captions, and an optional voice-over. | | [Schedule maintenance](/cookbook/scheduled-housekeeping) | Turn flaky-test scans, feature-flag audits, and dependency reviews into recurring Roomote work. | | [Triage customer issues](/cookbook/support-channel) | Give support escalations a repeatable path through production evidence, data, and code. | {/* cookbook-recipes:end */} From 169aa52b7492f11adefddcd790e8183fed163d4f Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:15:26 -0400 Subject: [PATCH 07/27] [Fix] Resolve the Remotion headless-shell binary by search The binary name differs per platform: headless_shell on linux-arm64, chrome-headless-shell on linux64, so the hardcoded name failed the amd64 image build. Resolve it with a find over both names (validated under amd64 emulation and in the arm64 sandbox image). --- apps/worker/Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index d27a1a1d7..451f2f9aa 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -240,12 +240,11 @@ RUN sudo mkdir -p /opt/remotion \ "remotion@${REMOTION_VERSION}" \ && npx remotion browser ensure \ && cp -R node_modules/.remotion/chrome-headless-shell /opt/remotion/chrome-headless-shell \ - && case "$(uname -m)" in \ - aarch64) REMOTION_PLATFORM="linux-arm64" ;; \ - x86_64) REMOTION_PLATFORM="linux64" ;; \ - *) echo "Unsupported architecture for Remotion headless shell: $(uname -m)" >&2; exit 1 ;; \ - esac \ - && REMOTION_SHELL="/opt/remotion/chrome-headless-shell/${REMOTION_PLATFORM}/chrome-headless-shell-${REMOTION_PLATFORM}/headless_shell" \ + # The binary name differs per platform (headless_shell on linux-arm64, + # chrome-headless-shell on linux64), so resolve it by search instead of + # assuming either name. + && REMOTION_SHELL="$(find /opt/remotion/chrome-headless-shell -type f \( -name headless_shell -o -name chrome-headless-shell \) | head -n 1)" \ + && test -n "$REMOTION_SHELL" \ && test -x "$REMOTION_SHELL" \ && ln -sf "$REMOTION_SHELL" "${REMOTION_HEADLESS_SHELL_PATH}" \ && printf '%s\n' "${REMOTION_VERSION}" > /opt/remotion/.installed \ From 88a7f37ea0f22f7e424a418f3aeba9947b91fe44 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:23:21 -0400 Subject: [PATCH 08/27] [Fix] Anchor narration to when each zoom lands, not the caption fade-in Capture emits an explicit per-caption anchor (the zoom's landing time); the timing fit schedules against it. caption.start begins with the cursor glide ~moveMs earlier, so anchoring there started narration before the zoom even began moving. Also hoist the Dockerfile block comment above the RUN chain. --- apps/worker/Dockerfile | 8 ++++---- .../skills/standard/feature-demo/capture/capture.mjs | 3 +++ .../skills/standard/feature-demo/scripts/fit-timing.mjs | 8 +++++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index 451f2f9aa..4ca8f80cc 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -229,7 +229,10 @@ RUN sudo mkdir -p /opt/ffmpeg/bin \ # download a browser at runtime. `npx remotion browser ensure` fetches the # matching linux64/linux-arm64 build into node_modules/.remotion; copy that # tree to /opt/remotion and expose a stable arch-independent symlink at -# ${REMOTION_HEADLESS_SHELL_PATH} so consumers need no arch logic. +# ${REMOTION_HEADLESS_SHELL_PATH} so consumers need no arch logic. The +# binary name differs per platform (headless_shell on linux-arm64, +# chrome-headless-shell on linux64), so it is resolved by search instead of +# assuming either name. RUN sudo mkdir -p /opt/remotion \ && sudo chown -R roomote:roomote /opt/remotion \ && REMOTION_TMP="$(mktemp -d)" \ @@ -240,9 +243,6 @@ RUN sudo mkdir -p /opt/remotion \ "remotion@${REMOTION_VERSION}" \ && npx remotion browser ensure \ && cp -R node_modules/.remotion/chrome-headless-shell /opt/remotion/chrome-headless-shell \ - # The binary name differs per platform (headless_shell on linux-arm64, - # chrome-headless-shell on linux64), so resolve it by search instead of - # assuming either name. && REMOTION_SHELL="$(find /opt/remotion/chrome-headless-shell -type f \( -name headless_shell -o -name chrome-headless-shell \) | head -n 1)" \ && test -n "$REMOTION_SHELL" \ && test -x "$REMOTION_SHELL" \ diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs index b993abf0d..d8bd15ed4 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -137,6 +137,9 @@ async function run() { start: start + 0.15, end: end + (beat.holdMs ?? 900) / 1000, text: beat.caption, + // When the zoom LANDS — narration schedules against this, not the + // caption fade-in (which begins with the glide, ~moveMs earlier). + anchor: end, }); } cur = { scale: beat.scale ?? 1.5, focal: c }; diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs index afb988a54..fed91f054 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/scripts/fit-timing.mjs @@ -84,6 +84,9 @@ if (trim > 0) { for (const cap of timeline.captions) { cap.start = Math.max(0, cap.start - trim); cap.end = Math.max(0, cap.end - trim); + if (cap.anchor !== undefined) { + cap.anchor = Math.max(0, cap.anchor - trim); + } } timeline.video.startFromSeconds = (timeline.video.startFromSeconds ?? 0) + trim; @@ -130,7 +133,10 @@ for (let i = 0; i < narration.clips.length; i++) { // late anchor ends later than any cumulative-duration estimate, so the only // reliable check is to lay the schedule out at a candidate rate and look // for a line pushed past its anchor. -const beats = timeline.captions.map((c) => c.start); +// Anchor to when each zoom LANDS (capture emits it as `anchor`); the caption +// `start` begins with the glide, ~moveMs before the zoom arrives, and using +// it would start narration before the zoom even begins moving. +const beats = timeline.captions.map((c) => c.anchor ?? c.start); function scheduleAt(candidateRate) { let prevEnd = 0; From 8dc12b236eeb71399e18894ba44b6455de1b684f Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:39:04 -0400 Subject: [PATCH 09/27] [Fix] Keep screencast frames flowing on statically-rendered pages Field failure from a real game demo: CDP screencast only emits frames on visual damage and stamps them without wall-clock gaps, so a 52s interaction over a static-rendering surface collapsed into a sub-second video. The capture runner now injects an imperceptible 2px rAF ticker after record start so compositor damage flows at wall-clock rate (static-page regression: 7.93s timeline -> 8.0s recording), closes any pre-existing pages before recording (record start creates a fresh browser context; a page left over from inspection would split beats and recording across contexts), and fails loudly when the recording comes back much shorter than the interaction instead of letting a garbage video flow downstream. SKILL.md routes that failure to a 'stale sandbox runtime' blocker (old snapshots carry a 2018 legacy ffmpeg that crashes record stop under real frame volume). --- .../skills/standard/feature-demo/SKILL.md | 3 +- .../standard/feature-demo/capture/capture.mjs | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index 67cc0663f..66d75521f 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -70,7 +70,8 @@ this pipeline is ever committed to the repository. and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. The runner is an agent-browser orchestrator (it shells the `agent-browser` CLI for every browser action), so it stays inside proof-runner's sanctioned tooling. If the harness has no proof-runner registered, report a blocker (`proof runtime unavailable`) instead of driving the browser from this skill. -Expected outputs: `/tmp/feature-demo/work/recording.mp4` and `/tmp/feature-demo/work/timeline.json`. Verify both exist and that `ffprobe` reports a duration close to the timeline's `durationSeconds`. One retry on failure; then report blocked with the runner's error. +Expected outputs: `/tmp/feature-demo/work/recording.mp4` and `/tmp/feature-demo/work/timeline.json`. Verify both exist and that `ffprobe` reports a duration close to the timeline's `durationSeconds` (the runner itself fails loudly when the recording is much shorter than the interaction). One retry on failure; then report blocked with the runner's error. +If the runner's error mentions sparse screencast frames or `record stop` reports an ffmpeg failure, the sandbox is likely running a stale snapshot with an outdated runtime ffmpeg — report the blocker as `stale sandbox runtime` so an operator refreshes the snapshot pool; do not retry more than once. diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs index d8bd15ed4..83e46a937 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -96,11 +96,34 @@ const pushCursorMove = (startT, endT, target) => { curCursor = target; }; +// The recorder captures via CDP screencast, which only emits frames when the +// page visually changes — and stamps them without wall-clock gaps. On a +// statically-rendered surface (games between state changes, idle dashboards) +// a long session collapses into a sub-second video. Injecting an +// imperceptible 2px corner dot that re-paints every animation frame keeps +// compositor damage — and therefore frames — flowing at wall-clock rate. +const TICKER_JS = + '(function(){var d=document.createElement("div");' + + 'd.style.cssText="position:fixed;left:0;bottom:0;width:2px;height:2px;' + + 'z-index:2147483647;pointer-events:none;background:#000;opacity:0.01";' + + 'document.body.appendChild(d);var f=0;' + + '(function t(){d.style.opacity=(f++%2)?"0.02":"0.01";' + + 'requestAnimationFrame(t)})()})()'; + async function run() { mkdirSync(OUT_DIR, { recursive: true }); + // `record start` creates a fresh browser context; a pre-existing page + // (e.g. from an earlier inspection step) would leave the beats driving one + // page while the recorder watches another. Start from a clean slate. + try { + ab('close', '--all'); + } catch { + // no active session is fine + } ab('set', 'viewport', String(VIEWPORT.w), String(VIEWPORT.h)); ab('record', 'start', `${OUT_DIR}/raw.webm`, script.url); t0 = Date.now(); + ab('eval', TICKER_JS); sleep(300); // let the first frames settle for (const beat of script.beats) { @@ -203,6 +226,36 @@ async function run() { // best-effort; the recording is already on disk } + // Honest-state gate: a recording much shorter than the interaction means + // the screencast stalled (or frames stayed sparse despite the ticker) and + // the demo would be garbage. Fail loudly instead of shipping it. + const recordedSeconds = parseFloat( + execFileSync('ffprobe', [ + '-v', + 'error', + '-show_entries', + 'format=duration', + '-of', + 'default=nw=1:nk=1', + `${OUT_DIR}/raw.webm`, + ]) + .toString() + .trim(), + ); + + if ( + !Number.isFinite(recordedSeconds) || + recordedSeconds < timeline.durationSeconds * 0.75 + ) { + throw new Error( + `recording is ${recordedSeconds}s but the interaction ran ` + + `${timeline.durationSeconds.toFixed(2)}s — screencast frames were ` + + `sparse or the recorder stalled. If \`record stop\` also reported an ` + + `ffmpeg error, the sandbox runtime is likely a stale snapshot with ` + + `an outdated ffmpeg. Report this as a blocker; do not render.`, + ); + } + // WebM (VP8/VP9) -> H.264 mp4 for the renderer. execFileSync( 'ffmpeg', From b17b6c462926aa5ec5e910e8232f49118e7b7e17 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 01:47:39 -0400 Subject: [PATCH 10/27] [Fix] Sanction the staged feature-demo runner in the proof-runner prompt Second dogfood run failed at the delegation seam in two ways: the brief's home-relative runner path did not resolve in the delegated runtime, and the proof-runner refused the capture script as a disallowed browser path (its instructions allow only the agent-browser CLI, and a per-brief assurance is not reliable authority). - The skill now stages the runner at /tmp/feature-demo/capture.mjs before delegating (no home-relative paths across the boundary) and parent-side paths use $HOME explicitly. - The proof-runner prompt carries a narrow standing exception: the staged runner is an agent-browser orchestrator and running it as the brief specifies is compliant; the exception names that one path and no other script. --- .../run-task/__tests__/proof-runner-prompt.test.ts | 14 ++++++++++++++ apps/worker/src/run-task/proof-runner-prompt.ts | 1 + .../workflows/__tests__/featureDemoSkill.test.ts | 9 +++++++++ .../skills/standard/feature-demo/SKILL.md | 13 ++++++++----- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts index 4d9e03d58..f774dbdde 100644 --- a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts +++ b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts @@ -25,6 +25,20 @@ describe('createProofRunnerAgentPrompt', () => { ); }); + it('sanctions exactly the staged feature-demo capture runner', () => { + const prompt = createProofRunnerAgentPrompt('http://127.0.0.1:3000'); + + expect(prompt).toContain( + 'the feature-demo capture runner at `/tmp/feature-demo/capture.mjs`', + ); + expect(prompt).toContain( + 'running it via `node` with the environment variables the brief specifies is compliant browser work', + ); + expect(prompt).toContain( + 'This exception covers exactly that staged runner and no other script.', + ); + }); + it('returns artifact IDs and explicit parent sharing guidance', () => { const prompt = createProofRunnerAgentPrompt('http://127.0.0.1:3000'); diff --git a/apps/worker/src/run-task/proof-runner-prompt.ts b/apps/worker/src/run-task/proof-runner-prompt.ts index 0c4786ef6..b2285b739 100644 --- a/apps/worker/src/run-task/proof-runner-prompt.ts +++ b/apps/worker/src/run-task/proof-runner-prompt.ts @@ -27,6 +27,7 @@ export function createProofRunnerAgentPrompt(browserTarget: string): string { '- Before the first browser command, explicitly load the `agent-browser` skill or CLI-served guidance exactly once. If the OpenCode Skill tool is available, invoke the `agent-browser` skill. If it is not available, run `agent-browser skills get core --full` in the shell and treat that output as the browser usage guide.', '- `agent-browser` is a command-line executable, not an OpenCode tool or MCP tool. Invoke it with shell commands such as `agent-browser get url`; do not look for an internal tool named `agent-browser`.', '- `agent-browser` is the only allowed browser automation CLI. Do not use Playwright, browser DevTools, curl-only screenshot substitutes, or any other browser automation path.', + '- One sanctioned exception: the feature-demo capture runner at `/tmp/feature-demo/capture.mjs`, staged there by the parent. It is an `agent-browser` orchestrator — every browser action it performs goes through the `agent-browser` CLI — and running it via `node` with the environment variables the brief specifies is compliant browser work, not a disallowed automation path. Its outputs land where the brief says (typically `/tmp/feature-demo/work/`), not under `/tmp/capture-visual-proof/`. This exception covers exactly that staged runner and no other script.', '- If the `agent-browser` skill/guidance cannot be loaded, or if the `agent-browser` executable is unavailable, report blocked with blocker type `proof runtime unavailable` and describe the missing skill guidance or CLI explicitly.', '- Treat the browser target above as the primary proof surface. Preserve its hostname exactly; do not rewrite `localhost` to `127.0.0.1` or the reverse, and do not substitute another surface.', '- Before deep diagnostics, inspect the live state with `agent-browser get url`, `agent-browser get title`, or `agent-browser snapshot -i`.', diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index 885d6e3bf..5020719fb 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -52,6 +52,15 @@ describe('feature-demo skill', () => { ); }); + it('stages the capture runner at the sanctioned /tmp path for delegation', () => { + // Home-directory paths do not survive the delegation boundary, and the + // proof-runner prompt sanctions exactly this staged path. + expect(skillContent).toContain( + 'cp "$HOME/.agents/skills/feature-demo/capture/capture.mjs" /tmp/feature-demo/capture.mjs', + ); + expect(skillContent).toContain('node /tmp/feature-demo/capture.mjs'); + }); + it('keeps TTS provider keys out of the sandbox', () => { expect(skillContent).toContain( 'Never ask for, read, or handle TTS provider keys.', diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index 66d75521f..153a01c29 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -64,11 +64,14 @@ this pipeline is ever committed to the repository. Capture (delegated browser work) +Stage the capture runner where the delegated runtime can see it — home-directory paths do not survive the delegation boundary, so always copy first: + +`mkdir -p /tmp/feature-demo && cp "$HOME/.agents/skills/feature-demo/capture/capture.mjs" /tmp/feature-demo/capture.mjs` Browser automation is the proof-runner subagent's exclusive surface — do not load `agent-browser` or run the capture yourself. Delegate with the Task tool to `proof-runner` with a brief that instructs it to run exactly: -`SCRIPT=/tmp/feature-demo/demo-script.json OUT_DIR=/tmp/feature-demo/work node ~/.agents/skills/feature-demo/capture/capture.mjs` +`SCRIPT=/tmp/feature-demo/demo-script.json OUT_DIR=/tmp/feature-demo/work node /tmp/feature-demo/capture.mjs` -and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. The runner is an agent-browser orchestrator (it shells the `agent-browser` CLI for every browser action), so it stays inside proof-runner's sanctioned tooling. +and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. The staged runner is proof-runner's one sanctioned script exception (its own instructions say so): an agent-browser orchestrator that shells the `agent-browser` CLI for every browser action. If the harness has no proof-runner registered, report a blocker (`proof runtime unavailable`) instead of driving the browser from this skill. Expected outputs: `/tmp/feature-demo/work/recording.mp4` and `/tmp/feature-demo/work/timeline.json`. Verify both exist and that `ffprobe` reports a duration close to the timeline's `durationSeconds` (the runner itself fails loudly when the recording is much shorter than the interaction). One retry on failure; then report blocked with the runner's error. If the runner's error mentions sparse screencast frames or `record stop` reports an ffmpeg failure, the sandbox is likely running a stale snapshot with an outdated runtime ffmpeg — report the blocker as `stale sandbox runtime` so an operator refreshes the snapshot pool; do not retry more than once. @@ -78,7 +81,7 @@ and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. Narration (optional, degrades cleanly) -Run `WORK_DIR=/tmp/feature-demo/work node ~/.agents/skills/feature-demo/scripts/build-narration.mjs` (add `LINES=/tmp/feature-demo/lines.json` if spoken lines differ from captions). This posts the caption text to the Roomote control plane, which holds the TTS credentials — no provider key exists in this sandbox, and you must never ask for one. +Run `WORK_DIR=/tmp/feature-demo/work node "$HOME/.agents/skills/feature-demo/scripts/build-narration.mjs"` (add `LINES=/tmp/feature-demo/lines.json` if spoken lines differ from captions). This posts the caption text to the Roomote control plane, which holds the TTS credentials — no provider key exists in this sandbox, and you must never ask for one. Exit code 3 means narration is not configured on this deployment: proceed captions-only, and mention in the final report that narration is available if an admin sets `R_ELEVENLABS_API_KEY` + `R_ELEVENLABS_VOICE_ID`. @@ -86,7 +89,7 @@ and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. Fit timing -Run `WORK_DIR=/tmp/feature-demo/work node ~/.agents/skills/feature-demo/scripts/fit-timing.mjs`. It trims the dead opening, paces the voice-over (pitch-preserving atempo), solves a video playback rate so each spoken line finishes before its zoom lands, and rewrites caption windows to match the audio. Captions-only demos still get the opening trim. +Run `WORK_DIR=/tmp/feature-demo/work node "$HOME/.agents/skills/feature-demo/scripts/fit-timing.mjs"`. It trims the dead opening, optionally paces the voice-over (pitch-preserving atempo), solves a video playback rate so each spoken line starts just before its zoom lands, and rewrites caption windows to match the audio. Captions-only demos still get the opening trim. Check its printed schedule: no line should start before the previous one ends. If lines overlap or the rate hits the 0.75 floor, the narration is too long for the motion — shorten the lines or add holds to the script and recapture. @@ -95,7 +98,7 @@ and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. Render Assemble the render project in the work dir: -- `cp -R ~/.agents/skills/feature-demo/render /tmp/feature-demo/render` +- `cp -R "$HOME/.agents/skills/feature-demo/render" /tmp/feature-demo/render` - `cp /tmp/feature-demo/work/timeline.json /tmp/feature-demo/work/narration.json /tmp/feature-demo/render/props/` (skip narration.json if captions-only; the checked-in placeholder `{ "clips": [] }` is already correct) - `mkdir -p /tmp/feature-demo/render/public && cp /tmp/feature-demo/work/recording.mp4 /tmp/feature-demo/render/public/` - `cp -R /tmp/feature-demo/work/vo /tmp/feature-demo/render/public/vo` (narrated demos only) From b3175e1cd5bbcdecd514034580f4e87e033f5990 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 02:24:57 -0400 Subject: [PATCH 11/27] [Docs] Route WebGL-surface recording stalls to a specific blocker Live-sandbox controls showed the sparse-frames failure is page-specific: a WebGL game stalls headless frame production for the whole page while plain pages on the same sandbox record at full wall-clock rate. SKILL.md now names that blocker (webgl surface stalls headless recording) so agents diagnose by target instead of burning retries; the stale-runtime ffmpeg hint stays scoped to actual record-stop ffmpeg errors. --- .../src/server/workflows/skills/standard/feature-demo/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index 153a01c29..feafb5101 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -74,7 +74,7 @@ this pipeline is ever committed to the repository. and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. The staged runner is proof-runner's one sanctioned script exception (its own instructions say so): an agent-browser orchestrator that shells the `agent-browser` CLI for every browser action. If the harness has no proof-runner registered, report a blocker (`proof runtime unavailable`) instead of driving the browser from this skill. Expected outputs: `/tmp/feature-demo/work/recording.mp4` and `/tmp/feature-demo/work/timeline.json`. Verify both exist and that `ffprobe` reports a duration close to the timeline's `durationSeconds` (the runner itself fails loudly when the recording is much shorter than the interaction). One retry on failure; then report blocked with the runner's error. -If the runner's error mentions sparse screencast frames or `record stop` reports an ffmpeg failure, the sandbox is likely running a stale snapshot with an outdated runtime ffmpeg — report the blocker as `stale sandbox runtime` so an operator refreshes the snapshot pool; do not retry more than once. +If the runner fails with a recording much shorter than the interaction, diagnose by target: WebGL/canvas-heavy surfaces (games, 3D visualizations) can stall headless frame production for the whole page, so ordinary web UIs on the same sandbox record fine while that one surface collapses. Report the blocker as `webgl surface stalls headless recording`, naming the surface — do not burn retries. Separately, if `record stop` itself reports an ffmpeg error, the sandbox is likely a stale snapshot with an outdated runtime ffmpeg (`stale sandbox runtime`). From d293811b09d915ee27836a96ca98a8ee0772e83f Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:10:16 -0400 Subject: [PATCH 12/27] [Fix] Bind the proof-runner capture sanction to a content digest Review caught that the sanction trusted a parent-writable /tmp path: anything staged there would inherit the narrow agent-browser allowance. The worker now computes the sha256 of the activated feature-demo capture runner at config-generation time and pins it into the proof-runner prompt; the runner verifies the staged file's digest before executing and reports 'capture runner integrity mismatch' otherwise. Without a digest (skill absent) the sanction is omitted entirely. --- .../__tests__/proof-runner-prompt.test.ts | 21 ++++++++-- apps/worker/src/run-task/agent-home.ts | 40 ++++++++++++++++++- .../src/run-task/proof-runner-prompt.ts | 17 +++++++- 3 files changed, 71 insertions(+), 7 deletions(-) diff --git a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts index f774dbdde..d611e6b1b 100644 --- a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts +++ b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts @@ -25,8 +25,12 @@ describe('createProofRunnerAgentPrompt', () => { ); }); - it('sanctions exactly the staged feature-demo capture runner', () => { - const prompt = createProofRunnerAgentPrompt('http://127.0.0.1:3000'); + it('sanctions only the digest-verified feature-demo capture runner', () => { + const digest = 'a'.repeat(64); + const prompt = createProofRunnerAgentPrompt( + 'http://127.0.0.1:3000', + digest, + ); expect(prompt).toContain( 'the feature-demo capture runner at `/tmp/feature-demo/capture.mjs`', @@ -34,11 +38,22 @@ describe('createProofRunnerAgentPrompt', () => { expect(prompt).toContain( 'running it via `node` with the environment variables the brief specifies is compliant browser work', ); + // The staging path is parent-writable, so the sanction binds to content. + expect(prompt).toContain('sha256sum /tmp/feature-demo/capture.mjs'); + expect(prompt).toContain(`\`${digest}\``); + expect(prompt).toContain('capture runner integrity mismatch'); expect(prompt).toContain( - 'This exception covers exactly that staged runner and no other script.', + 'This exception covers exactly that digest-verified file and no other script.', ); }); + it('omits the capture-runner sanction entirely without a digest', () => { + const prompt = createProofRunnerAgentPrompt('http://127.0.0.1:3000'); + + expect(prompt).not.toContain('/tmp/feature-demo/capture.mjs'); + expect(prompt).not.toContain('sanctioned exception'); + }); + it('returns artifact IDs and explicit parent sharing guidance', () => { const prompt = createProofRunnerAgentPrompt('http://127.0.0.1:3000'); diff --git a/apps/worker/src/run-task/agent-home.ts b/apps/worker/src/run-task/agent-home.ts index 2f2c92265..1f6021915 100644 --- a/apps/worker/src/run-task/agent-home.ts +++ b/apps/worker/src/run-task/agent-home.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import * as fs from 'node:fs'; import * as path from 'node:path'; @@ -1304,15 +1305,47 @@ function createArchitectAgentConfig(options: { }; } +/** + * Digest of the activated feature-demo capture runner, pinned into the + * proof-runner prompt so its sanctioned /tmp staging path (writable by any + * parent flow) cannot be used to smuggle a different script into the + * runner's browser sanction. Undefined (skill absent/unreadable) omits the + * sanction entirely. + */ +function resolveFeatureDemoCaptureRunnerDigest( + homeDir: string, +): string | undefined { + const runnerPath = path.join( + homeDir, + '.agents', + 'skills', + 'feature-demo', + 'capture', + 'capture.mjs', + ); + + try { + return createHash('sha256') + .update(fs.readFileSync(runnerPath)) + .digest('hex'); + } catch { + return undefined; + } +} + function createProofRunnerAgentConfig( browserTarget: string, + featureDemoCaptureRunnerSha256: string | undefined, ): Record { return { description: 'Delegated browser proof runner that captures and uploads screenshot and screencast proof from the sandbox-local browser surface.', mode: 'subagent', hidden: true, - prompt: createProofRunnerAgentPrompt(browserTarget), + prompt: createProofRunnerAgentPrompt( + browserTarget, + featureDemoCaptureRunnerSha256, + ), permission: { read: 'allow', list: 'allow', @@ -1960,7 +1993,10 @@ export function generateOpenCodeConfig({ if (proofBrowserTarget) { operatorAgent[ROOMOTE_OPENCODE_PROOF_RUNNER_AGENT_NAME] = - createProofRunnerAgentConfig(proofBrowserTarget); + createProofRunnerAgentConfig( + proofBrowserTarget, + resolveFeatureDemoCaptureRunnerDigest(homeDir), + ); const proofRunnerInstructionsPath = path.join( openCodeConfigDir, diff --git a/apps/worker/src/run-task/proof-runner-prompt.ts b/apps/worker/src/run-task/proof-runner-prompt.ts index b2285b739..9c7d31c63 100644 --- a/apps/worker/src/run-task/proof-runner-prompt.ts +++ b/apps/worker/src/run-task/proof-runner-prompt.ts @@ -4,8 +4,16 @@ export const ROOMOTE_OPENCODE_PROOF_RUNNER_AGENT_NAME = 'proof-runner'; * Builds the hidden proof-runner subagent prompt. The browser target is baked * in at config-generation time so the runner never depends on a staged * runtime handoff file for its capture configuration. + * + * `featureDemoCaptureRunnerSha256` is the digest of the activated + * feature-demo capture runner. The runner's staging path (/tmp) is writable + * by any parent flow, so the sanction binds to verified content, not the + * pathname; without a digest the sanction is omitted entirely. */ -export function createProofRunnerAgentPrompt(browserTarget: string): string { +export function createProofRunnerAgentPrompt( + browserTarget: string, + featureDemoCaptureRunnerSha256?: string, +): string { return [ 'You are the single delegated proof runner for this task.', '', @@ -27,7 +35,12 @@ export function createProofRunnerAgentPrompt(browserTarget: string): string { '- Before the first browser command, explicitly load the `agent-browser` skill or CLI-served guidance exactly once. If the OpenCode Skill tool is available, invoke the `agent-browser` skill. If it is not available, run `agent-browser skills get core --full` in the shell and treat that output as the browser usage guide.', '- `agent-browser` is a command-line executable, not an OpenCode tool or MCP tool. Invoke it with shell commands such as `agent-browser get url`; do not look for an internal tool named `agent-browser`.', '- `agent-browser` is the only allowed browser automation CLI. Do not use Playwright, browser DevTools, curl-only screenshot substitutes, or any other browser automation path.', - '- One sanctioned exception: the feature-demo capture runner at `/tmp/feature-demo/capture.mjs`, staged there by the parent. It is an `agent-browser` orchestrator — every browser action it performs goes through the `agent-browser` CLI — and running it via `node` with the environment variables the brief specifies is compliant browser work, not a disallowed automation path. Its outputs land where the brief says (typically `/tmp/feature-demo/work/`), not under `/tmp/capture-visual-proof/`. This exception covers exactly that staged runner and no other script.', + ...(featureDemoCaptureRunnerSha256 + ? [ + '- One sanctioned exception: the feature-demo capture runner at `/tmp/feature-demo/capture.mjs`, staged there by the parent. It is an `agent-browser` orchestrator — every browser action it performs goes through the `agent-browser` CLI — and running it via `node` with the environment variables the brief specifies is compliant browser work, not a disallowed automation path. The staging path is parent-writable, so verify content before executing: run `sha256sum /tmp/feature-demo/capture.mjs` and confirm the digest is exactly ' + + `\`${featureDemoCaptureRunnerSha256}\`. On any mismatch or missing file, report blocked with blocker type \`capture runner integrity mismatch\` and do not execute it. Its outputs land where the brief says (typically \`/tmp/feature-demo/work/\`), not under \`/tmp/capture-visual-proof/\`. This exception covers exactly that digest-verified file and no other script.`, + ] + : []), '- If the `agent-browser` skill/guidance cannot be loaded, or if the `agent-browser` executable is unavailable, report blocked with blocker type `proof runtime unavailable` and describe the missing skill guidance or CLI explicitly.', '- Treat the browser target above as the primary proof surface. Preserve its hostname exactly; do not rewrite `localhost` to `127.0.0.1` or the reverse, and do not substitute another surface.', '- Before deep diagnostics, inspect the live state with `agent-browser get url`, `agent-browser get title`, or `agent-browser snapshot -i`.', From 915793605b4da9350e2306c4a5893f51ed48db94 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:42:06 -0400 Subject: [PATCH 13/27] [Feat] Add ElevenLabs as a built-in credential-only integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Admins configure narration TTS under Settings → Integrations → ElevenLabs (API key + voice ID) instead of deployment env vars. Encrypted at rest, deployment-scoped, admin-only — same storage as Granola. New 'credential_only' serverMode: the integration exposes no MCP tools to agents and is explicitly excluded from getMcpServerConfigs, so the key is consumed only by the control-plane /api/tts endpoint and never travels toward a task sandbox (proven by an inverse delivery test). The TTS endpoint now resolves credentials from the connection first, falling back to R_ELEVENLABS_* env vars for self-hosters. UI, tRPC commands/procedures, hooks, icon, self-setup catalog metadata, and docs updated; env-var mocks in the TTS test replaced with a resolver mock. --- .../src/handlers/tts/__tests__/tts.test.ts | 29 +- apps/api/src/handlers/tts/connection.ts | 67 ++++ apps/api/src/handlers/tts/index.ts | 16 +- apps/docs/cookbook/feature-demo-videos.mdx | 22 +- .../components/settings/Integrations.test.tsx | 14 + .../src/components/settings/Integrations.tsx | 306 +++++++++++++++++- .../system/custom/logos/brand-icon.tsx | 2 + apps/web/src/hooks/mcp-connections/index.ts | 2 + .../useElevenLabsConnection.ts | 14 + .../useSaveElevenLabsConnection.ts | 26 ++ .../trpc/commands/mcp-connections/index.ts | 111 +++++++ apps/web/src/trpc/routers/_app.ts | 13 + apps/web/src/types/mcp-connections.ts | 12 + .../src/server/mcp-self-setup/catalog.ts | 7 + .../server/routers/mcp-connections.test.ts | 25 ++ .../sdk/src/server/routers/mcp-connections.ts | 12 + .../types/src/__tests__/mcp-oauth.test.ts | 38 +++ packages/types/src/mcp-oauth.ts | 53 ++- 18 files changed, 741 insertions(+), 28 deletions(-) create mode 100644 apps/api/src/handlers/tts/connection.ts create mode 100644 apps/web/src/hooks/mcp-connections/useElevenLabsConnection.ts create mode 100644 apps/web/src/hooks/mcp-connections/useSaveElevenLabsConnection.ts diff --git a/apps/api/src/handlers/tts/__tests__/tts.test.ts b/apps/api/src/handlers/tts/__tests__/tts.test.ts index 65dcf5372..bdb09e050 100644 --- a/apps/api/src/handlers/tts/__tests__/tts.test.ts +++ b/apps/api/src/handlers/tts/__tests__/tts.test.ts @@ -3,15 +3,12 @@ import type { RunTokenContext } from '@roomote/types'; import type { Variables } from '../../../types'; -const { mockEnv } = vi.hoisted(() => ({ - mockEnv: { - R_ELEVENLABS_API_KEY: undefined as string | undefined, - R_ELEVENLABS_VOICE_ID: undefined as string | undefined, - }, +const { mockResolveCredentials } = vi.hoisted(() => ({ + mockResolveCredentials: vi.fn(), })); -vi.mock('@roomote/env', () => ({ - Env: mockEnv, +vi.mock('../connection', () => ({ + resolveElevenLabsCredentials: mockResolveCredentials, })); import { tts } from '../index'; @@ -51,8 +48,10 @@ function narrationRequest(body: unknown): Request { const originalFetch = globalThis.fetch; beforeEach(() => { - mockEnv.R_ELEVENLABS_API_KEY = 'el-key'; - mockEnv.R_ELEVENLABS_VOICE_ID = 'voice-1'; + mockResolveCredentials.mockResolvedValue({ + apiKey: 'el-key', + voiceId: 'voice-1', + }); }); afterEach(() => { @@ -62,7 +61,7 @@ afterEach(() => { describe('POST /tts/narration', () => { it('404s when the deployment has no ElevenLabs credentials', async () => { - mockEnv.R_ELEVENLABS_API_KEY = undefined; + mockResolveCredentials.mockResolvedValue(undefined); const response = await createApp(createRunToken()).request( narrationRequest({ lines: ['hello'] }), @@ -71,14 +70,18 @@ describe('POST /tts/narration', () => { expect(response.status).toBe(404); }); - it('404s when only the voice id is missing', async () => { - mockEnv.R_ELEVENLABS_VOICE_ID = undefined; + it('500s without leaking details when credential resolution throws', async () => { + mockResolveCredentials.mockRejectedValue(new Error('db exploded')); const response = await createApp(createRunToken()).request( narrationRequest({ lines: ['hello'] }), ); - expect(response.status).toBe(404); + expect(response.status).toBe(500); + + const payload = (await response.json()) as { error: string }; + + expect(payload.error).not.toContain('db exploded'); }); it('rejects non-run-token callers', async () => { diff --git a/apps/api/src/handlers/tts/connection.ts b/apps/api/src/handlers/tts/connection.ts new file mode 100644 index 000000000..fa3d0e06a --- /dev/null +++ b/apps/api/src/handlers/tts/connection.ts @@ -0,0 +1,67 @@ +import { areCuratedIntegrationsDisabled, Env } from '@roomote/env'; +import { + and, + db, + deploymentMcpEnablements, + eq, + isNull, + mcpConnections, +} from '@roomote/db/server'; +import { decrypt } from '@roomote/db/encryption'; +import { isMcpConnectionElevenLabsConfig } from '@roomote/types'; + +type ElevenLabsCredentials = { + apiKey: string; + voiceId: string; +}; + +/** + * Resolves narration TTS credentials for the deployment. + * + * The admin-configured ElevenLabs integration (Settings → Integrations) is + * the primary source; the `R_ELEVENLABS_*` environment variables remain an + * operator fallback. Returns undefined when neither is configured — the + * feature is off and the endpoint 404s. + * + * The integration is credential-only: this control-plane resolver is its + * single consumer, and the connection is deliberately excluded from agent + * MCP config delivery, so the key never travels toward a task sandbox. + */ +export async function resolveElevenLabsCredentials(): Promise< + ElevenLabsCredentials | undefined +> { + if (!areCuratedIntegrationsDisabled(Env.R_CURATED_INTEGRATIONS_DISABLED)) { + const connection = await db.query.mcpConnections.findFirst({ + where: and( + eq(mcpConnections.mcpId, 'elevenlabs'), + isNull(mcpConnections.userId), + eq(mcpConnections.enabled, true), + eq(mcpConnections.authStatus, 'authenticated'), + ), + }); + + if (connection && isMcpConnectionElevenLabsConfig(connection.authConfig)) { + const enablement = await db.query.deploymentMcpEnablements.findFirst({ + where: eq(deploymentMcpEnablements.mcpId, 'elevenlabs'), + }); + + if (enablement?.enabled !== false) { + const apiKey = decrypt(connection.authConfig.encryptedApiKey).trim(); + const voiceId = connection.authConfig.voiceId.trim(); + + if (apiKey && voiceId) { + return { apiKey, voiceId }; + } + } + } + } + + if (Env.R_ELEVENLABS_API_KEY && Env.R_ELEVENLABS_VOICE_ID) { + return { + apiKey: Env.R_ELEVENLABS_API_KEY, + voiceId: Env.R_ELEVENLABS_VOICE_ID, + }; + } + + return undefined; +} diff --git a/apps/api/src/handlers/tts/index.ts b/apps/api/src/handlers/tts/index.ts index 82155e088..2e6916d9c 100644 --- a/apps/api/src/handlers/tts/index.ts +++ b/apps/api/src/handlers/tts/index.ts @@ -1,9 +1,9 @@ import { Hono } from 'hono'; -import { Env } from '@roomote/env'; import type { RunTokenContext } from '@roomote/types'; import type { Variables } from '../../types'; import { logHandlerError } from '../utils'; +import { resolveElevenLabsCredentials } from './connection'; /** * Narration text-to-speech for task sandboxes, used by the feature-demo @@ -148,14 +148,22 @@ async function synthesizeLine(options: { export const tts = new Hono<{ Variables: Variables }>(); tts.post('/narration', async (c) => { - const apiKey = Env.R_ELEVENLABS_API_KEY; - const voiceId = Env.R_ELEVENLABS_VOICE_ID; + let credentials; + + try { + credentials = await resolveElevenLabsCredentials(); + } catch (error) { + logHandlerError('ttsNarration', error); + return c.json({ error: 'Narration synthesis failed' }, 500); + } // Not configured => the feature does not exist on this deployment. - if (!apiKey || !voiceId) { + if (!credentials) { return c.json({ error: 'Narration TTS is not configured' }, 404); } + const { apiKey, voiceId } = credentials; + const auth = c.get('authContext'); // The route policy already rejects non-run tokens; this guard keeps the diff --git a/apps/docs/cookbook/feature-demo-videos.mdx b/apps/docs/cookbook/feature-demo-videos.mdx index be9e9b2e8..f9d176c6f 100644 --- a/apps/docs/cookbook/feature-demo-videos.mdx +++ b/apps/docs/cookbook/feature-demo-videos.mdx @@ -39,15 +39,19 @@ narration configured, the demo gets a voice-over that leads each visual beat. ## Narration Captioned demos work out of the box. For a voice-over in your own (cloned) -voice, a deployment admin sets two environment variables: - -- `R_ELEVENLABS_API_KEY`: an ElevenLabs key. A key scoped to - **text-to-speech only**, with a credit cap, is sufficient and recommended. -- `R_ELEVENLABS_VOICE_ID`: the voice to narrate with. - -The key stays on the Roomote control plane: task sandboxes send narration -text to an authenticated Roomote endpoint and receive audio back, so the -provider credential never enters a sandbox. +voice, a deployment admin connects ElevenLabs under **Settings → +Integrations → ElevenLabs** with: + +- an ElevenLabs API key, scoped to **text-to-speech only** with a credit cap + (sufficient and recommended), and +- the voice ID to narrate with. + +The key is encrypted at rest and stays on the Roomote control plane: task +sandboxes send narration text to an authenticated Roomote endpoint and +receive audio back, so the provider credential never enters a sandbox and +agents get no ElevenLabs tools. (Operators self-hosting can alternatively set +the `R_ELEVENLABS_API_KEY` and `R_ELEVENLABS_VOICE_ID` environment variables; +the integration takes precedence when both are present.) ## Tips diff --git a/apps/web/src/components/settings/Integrations.test.tsx b/apps/web/src/components/settings/Integrations.test.tsx index 252e0c026..7d7ed0b70 100644 --- a/apps/web/src/components/settings/Integrations.test.tsx +++ b/apps/web/src/components/settings/Integrations.test.tsx @@ -38,6 +38,10 @@ const state = vi.hoisted(() => ({ granolaConnection: null as null | { authStatus?: string | null; }, + elevenLabsConnection: null as null | { + authStatus?: string | null; + voiceId?: string; + }, grafanaConnection: null as null | { authStatus?: string | null; baseUrl: string; @@ -95,6 +99,7 @@ const { mutations, selectMock } = vi.hoisted(() => ({ setDisabledTools: vi.fn(), saveAsanaConnection: vi.fn(), saveGranolaConnection: vi.fn(), + saveElevenLabsConnection: vi.fn(), saveGrafanaConnection: vi.fn(), saveSnowflakeConnection: vi.fn(), saveVercelConnection: vi.fn(), @@ -254,6 +259,14 @@ vi.mock('@/hooks/mcp-connections', () => ({ data: state.granolaConnection, isPending: false, }), + useSaveElevenLabsConnection: () => ({ + isPending: false, + mutate: mutations.saveElevenLabsConnection, + }), + useElevenLabsConnection: () => ({ + data: state.elevenLabsConnection, + isPending: false, + }), useSaveGrafanaConnection: () => ({ isPending: false, mutate: mutations.saveGrafanaConnection, @@ -821,6 +834,7 @@ describe('Integrations settings', () => { 'Asana', 'Better Stack', 'Braintrust', + 'ElevenLabs', 'Grafana', 'Granola', 'Jira', diff --git a/apps/web/src/components/settings/Integrations.tsx b/apps/web/src/components/settings/Integrations.tsx index cdec85843..ce6cf2cb0 100644 --- a/apps/web/src/components/settings/Integrations.tsx +++ b/apps/web/src/components/settings/Integrations.tsx @@ -25,11 +25,13 @@ import { useDisconnectMcp, useGrafanaConnection, useGranolaConnection, + useElevenLabsConnection, useDeploymentMcpEnablements, useMcpOauthReadiness, useSaveAsanaConnection, useSaveGrafanaConnection, useSaveGranolaConnection, + useSaveElevenLabsConnection, useSaveSnowflakeConnection, useSaveVercelConnection, useSetDeploymentMcpEnabled, @@ -48,6 +50,7 @@ import { saveAsanaConnectionSchema, saveGrafanaConnectionSchema, saveGranolaConnectionSchema, + saveElevenLabsConnectionSchema, saveSnowflakeConnectionSchema, saveVercelConnectionSchema, } from '@/types'; @@ -89,6 +92,8 @@ const DEEP_LINK_ENABLE_DESCRIPTIONS: Record = { 'Roomote will be able to inspect dashboards, alert rules, live alert state, annotations, and data sources.', granola: 'Roomote will use one deployment-wide Granola connection to browse meeting notes, transcripts, decisions, and action items.', + elevenlabs: + 'Roomote will use one deployment-wide ElevenLabs connection to narrate feature-demo videos. The key stays on the control plane; agents get no ElevenLabs tools.', github: 'Roomote will be able to inspect PRs, issues, and repository context.', jira: 'Roomote will be able to inspect Jira issues, workflows, and JQL search results.', @@ -173,6 +178,16 @@ type GranolaFormState = { apiKey: string; }; +type ElevenLabsFormState = { + apiKey: string; + voiceId: string; +}; + +type ElevenLabsConnectionData = { + authStatus?: 'pending' | 'authenticated' | 'error' | null; + voiceId?: string; +}; + type GrafanaFormState = { baseUrl: string; serviceAccountToken: string; @@ -215,6 +230,26 @@ function buildEmptyGranolaForm(): GranolaFormState { }; } +function buildEmptyElevenLabsForm(): ElevenLabsFormState { + return { + apiKey: '', + voiceId: '', + }; +} + +function buildElevenLabsForm( + connection: ElevenLabsConnectionData | null | undefined, +): ElevenLabsFormState { + if (!connection) { + return buildEmptyElevenLabsForm(); + } + + return { + apiKey: '', + voiceId: connection.voiceId ?? '', + }; +} + function buildEmptyGrafanaForm(): GrafanaFormState { return { baseUrl: '', @@ -291,6 +326,21 @@ function getGranolaFieldErrors( }; } +function getElevenLabsFieldErrors( + result: ReturnType, +): Partial> { + if (result.success) { + return {}; + } + + const fieldErrors = result.error.flatten().fieldErrors; + + return { + apiKey: fieldErrors.apiKey, + voiceId: fieldErrors.voiceId, + }; +} + function buildGrafanaForm( connection: GrafanaConnectionData | null | undefined, ): GrafanaFormState { @@ -411,7 +461,10 @@ function buildAdminConfiguredIntegrationItem({ } : undefined, secondaryAction: - canManageTools && enabled && integration.serverMode !== 'native' + canManageTools && + enabled && + integration.serverMode !== 'native' && + integration.serverMode !== 'credential_only' ? { label: 'Manage tools', ariaLabel: `Manage ${integration.name} tools`, @@ -808,6 +861,82 @@ function GranolaConnectionFields({ ); } +function ElevenLabsConnectionFields({ + form, + fieldErrors, + formError, + allowBlankApiKey, + onFieldChange, +}: { + form: ElevenLabsFormState; + fieldErrors: Partial>; + formError: string | null; + allowBlankApiKey: boolean; + onFieldChange: (field: keyof ElevenLabsFormState, value: string) => void; +}) { + const fieldClassName = + 'mt-2 w-full border-border/70 bg-background data-[invalid=true]:border-destructive'; + + return ( + <> +
+ + onFieldChange('apiKey', event.target.value)} + data-invalid={fieldErrors.apiKey ? 'true' : undefined} + className={fieldClassName} + autoCapitalize="off" + autoCorrect="off" + spellCheck={false} + data-1p-ignore + /> +

+ We recommend a key scoped to text-to-speech only, with a credit limit. + The key is used exclusively by this deployment's control plane to + narrate feature-demo videos; it is never sent to agents or task + sandboxes. +

+ {allowBlankApiKey ? ( +

+ Leave blank to keep the existing API key. +

+ ) : null} + {fieldErrors.apiKey ? ( +

{fieldErrors.apiKey[0]}

+ ) : null} +
+
+ + onFieldChange('voiceId', event.target.value)} + data-invalid={fieldErrors.voiceId ? 'true' : undefined} + className={fieldClassName} + autoCapitalize="off" + autoCorrect="off" + spellCheck={false} + /> +

+ The ElevenLabs voice used for narration. Find voice IDs in the + ElevenLabs voice library. +

+ {fieldErrors.voiceId ? ( +

{fieldErrors.voiceId[0]}

+ ) : null} +
+ {formError ? ( +

{formError}

+ ) : null} + + ); +} + function GrafanaConnectionFields({ form, fieldErrors, @@ -1007,6 +1136,16 @@ export function Integrations() { const [granolaForm, setGranolaForm] = useState( buildEmptyGranolaForm(), ); + const [isElevenLabsDialogOpen, setIsElevenLabsDialogOpen] = useState(false); + const [elevenLabsForm, setElevenLabsForm] = useState( + buildEmptyElevenLabsForm(), + ); + const [elevenLabsFieldErrors, setElevenLabsFieldErrors] = useState< + Partial> + >({}); + const [elevenLabsFormError, setElevenLabsFormError] = useState( + null, + ); const [granolaFieldErrors, setGranolaFieldErrors] = useState< Partial> >({}); @@ -1065,6 +1204,7 @@ export function Integrations() { const saveAsanaConnection = useSaveAsanaConnection(); const saveGrafanaConnection = useSaveGrafanaConnection(); const saveGranolaConnection = useSaveGranolaConnection(); + const saveElevenLabsConnection = useSaveElevenLabsConnection(); const saveSnowflakeConnection = useSaveSnowflakeConnection(); const saveVercelConnection = useSaveVercelConnection(); const asanaConnectionSummary = useMemo(() => { @@ -1091,6 +1231,18 @@ export function Integrations() { const granolaConnection = useGranolaConnection( isAdmin && (isGranolaConnected || isGranolaDialogOpen), ); + const elevenLabsConnectionSummary = useMemo(() => { + const connection = (userMcpConnections.data ?? []).find( + (entry) => entry.mcpId === 'elevenlabs', + ); + + return connection; + }, [userMcpConnections.data]); + const isElevenLabsConnected = + elevenLabsConnectionSummary?.authStatus === 'authenticated'; + const elevenLabsConnection = useElevenLabsConnection( + isAdmin && (isElevenLabsConnected || isElevenLabsDialogOpen), + ); const grafanaConnectionSummary = useMemo(() => { const connection = (userMcpConnections.data ?? []).find( (entry) => entry.mcpId === 'grafana', @@ -1158,6 +1310,44 @@ export function Integrations() { setGranolaForm(buildEmptyGranolaForm()); }, [granolaConnection.isPending, isGranolaConnected, isGranolaDialogOpen]); + useEffect(() => { + if (!isElevenLabsDialogOpen) { + return; + } + + if (elevenLabsConnection.isPending && isElevenLabsConnected) { + return; + } + + setElevenLabsFieldErrors({}); + setElevenLabsFormError(null); + setElevenLabsForm(buildElevenLabsForm(elevenLabsConnection.data)); + }, [ + elevenLabsConnection.data, + elevenLabsConnection.isPending, + isElevenLabsConnected, + isElevenLabsDialogOpen, + ]); + + useEffect(() => { + if (!isElevenLabsDialogOpen) { + return; + } + + if (elevenLabsConnection.isPending && isElevenLabsConnected) { + return; + } + + setElevenLabsFieldErrors({}); + setElevenLabsFormError(null); + setElevenLabsForm(buildElevenLabsForm(elevenLabsConnection.data)); + }, [ + elevenLabsConnection.data, + elevenLabsConnection.isPending, + isElevenLabsConnected, + isElevenLabsDialogOpen, + ]); + useEffect(() => { if (!isSnowflakeDialogOpen) { return; @@ -1392,6 +1582,27 @@ export function Integrations() { }); } + if (integration.id === 'elevenlabs') { + return buildAdminConfiguredIntegrationItem({ + integration, + connection: userConnectionMap.get(integration.id), + orgEnabled: orgEnablementMap.get(integration.id) ?? false, + highlightedIntegrationId, + savePending: saveElevenLabsConnection.isPending, + disconnectPending: disconnectMcp.isPending, + disconnectingMcpId: disconnectMcp.variables?.mcpId, + dialogOpen: isElevenLabsDialogOpen, + connectionPending: elevenLabsConnection.isPending, + canConfigure: isAdmin, + // Credential-only: no agent tools to manage. + canManageTools: false, + openDialog: () => setIsElevenLabsDialogOpen(true), + openToolDialog: () => openMcpToolDialog(integration), + disconnectIntegration: () => + disconnectAdminConfiguredIntegration(integration), + }); + } + if (integration.id === 'snowflake') { return buildAdminConfiguredIntegrationItem({ integration, @@ -1497,6 +1708,7 @@ export function Integrations() { isAdmin && enabled && integration.serverMode !== 'native' && + integration.serverMode !== 'credential_only' && (!isDeploymentScoped || isConnected) ? { label: 'Manage tools', @@ -1592,6 +1804,7 @@ export function Integrations() { disconnectMcp, grafanaConnection.isPending, granolaConnection.isPending, + elevenLabsConnection.isPending, linearInstallation.data, linearInstallation.isPending, linearOauthSetup.isPending, @@ -1601,10 +1814,12 @@ export function Integrations() { isAdmin, isGrafanaDialogOpen, isGranolaDialogOpen, + isElevenLabsDialogOpen, isLinearOauthSetupOpen, saveAsanaConnection.isPending, saveGrafanaConnection.isPending, saveGranolaConnection.isPending, + saveElevenLabsConnection.isPending, saveVercelConnection.isPending, deploymentEnablements.data, pathname, @@ -1721,6 +1936,21 @@ export function Integrations() { setGranolaFormError(null); }; + const handleElevenLabsFieldChange = ( + field: keyof ElevenLabsFormState, + value: string, + ) => { + setElevenLabsForm((current) => ({ ...current, [field]: value })); + setElevenLabsFieldErrors((current) => { + if (!current[field]) { + return current; + } + + return { ...current, [field]: undefined }; + }); + setElevenLabsFormError(null); + }; + const handleGrafanaFieldChange = ( field: keyof GrafanaFormState, value: string, @@ -1777,6 +2007,19 @@ export function Integrations() { setGranolaForm(buildEmptyGranolaForm()); }; + const handleElevenLabsDialogOpenChange = (open: boolean) => { + setIsElevenLabsDialogOpen(open); + + setElevenLabsFieldErrors({}); + setElevenLabsFormError(null); + + if (!open) { + return; + } + + setElevenLabsForm(buildElevenLabsForm(elevenLabsConnection.data)); + }; + const handleSnowflakeDialogOpenChange = (open: boolean) => { setIsSnowflakeDialogOpen(open); @@ -1894,6 +2137,43 @@ export function Integrations() { }); }; + const handleElevenLabsSubmit = (event: FormEvent) => { + event.preventDefault(); + + const parsed = saveElevenLabsConnectionSchema.safeParse({ + apiKey: elevenLabsForm.apiKey, + voiceId: elevenLabsForm.voiceId, + }); + if (!parsed.success) { + setElevenLabsFieldErrors(getElevenLabsFieldErrors(parsed)); + return; + } + + if (!isElevenLabsConnected && parsed.data.apiKey.length === 0) { + setElevenLabsFieldErrors({ + apiKey: ['API key is required'], + }); + return; + } + + setElevenLabsFieldErrors({}); + setElevenLabsFormError(null); + + saveElevenLabsConnection.mutate(parsed.data, { + onSuccess: () => { + toast.success( + isElevenLabsConnected + ? 'ElevenLabs connection updated for this deployment.' + : 'ElevenLabs connected for this deployment.', + ); + handleElevenLabsDialogOpenChange(false); + }, + onError: (error) => { + setElevenLabsFormError(error.message); + }, + }); + }; + const handleSnowflakeSubmit = (event: FormEvent) => { event.preventDefault(); @@ -2106,6 +2386,30 @@ export function Integrations() { onFieldChange={handleGranolaFieldChange} /> + + Store an ElevenLabs API key and voice ID for this deployment. The + key stays encrypted server-side and is used only by the control + plane to narrate feature-demo videos. + + } + onSubmit={handleElevenLabsSubmit} + > + + = { asana: siAsana, betterstack: siBetterstack, braintrust: siBraintrust, + elevenlabs: siElevenlabs, dependabot: siDependabot, datadog: siDatadog, discord: siDiscord, diff --git a/apps/web/src/hooks/mcp-connections/index.ts b/apps/web/src/hooks/mcp-connections/index.ts index dc07469c5..42902ab64 100644 --- a/apps/web/src/hooks/mcp-connections/index.ts +++ b/apps/web/src/hooks/mcp-connections/index.ts @@ -13,6 +13,8 @@ export { useAsanaConnection } from './useAsanaConnection'; export { useSaveAsanaConnection } from './useSaveAsanaConnection'; export { useGranolaConnection } from './useGranolaConnection'; export { useSaveGranolaConnection } from './useSaveGranolaConnection'; +export { useElevenLabsConnection } from './useElevenLabsConnection'; +export { useSaveElevenLabsConnection } from './useSaveElevenLabsConnection'; export { useSaveGrafanaConnection } from './useSaveGrafanaConnection'; export { useSaveSnowflakeConnection } from './useSaveSnowflakeConnection'; export { useSnowflakeConnection } from './useSnowflakeConnection'; diff --git a/apps/web/src/hooks/mcp-connections/useElevenLabsConnection.ts b/apps/web/src/hooks/mcp-connections/useElevenLabsConnection.ts new file mode 100644 index 000000000..1314897f4 --- /dev/null +++ b/apps/web/src/hooks/mcp-connections/useElevenLabsConnection.ts @@ -0,0 +1,14 @@ +'use client'; + +import { useQuery } from '@tanstack/react-query'; + +import { useTRPC } from '@/trpc/client'; + +export function useElevenLabsConnection(enabled = true) { + const trpc = useTRPC(); + + return useQuery({ + ...trpc.mcpConnections.elevenLabsConnection.queryOptions(), + enabled, + }); +} diff --git a/apps/web/src/hooks/mcp-connections/useSaveElevenLabsConnection.ts b/apps/web/src/hooks/mcp-connections/useSaveElevenLabsConnection.ts new file mode 100644 index 000000000..ea7fa5515 --- /dev/null +++ b/apps/web/src/hooks/mcp-connections/useSaveElevenLabsConnection.ts @@ -0,0 +1,26 @@ +'use client'; + +import { useMutation, useQueryClient } from '@tanstack/react-query'; + +import { useTRPC } from '@/trpc/client'; + +export function useSaveElevenLabsConnection() { + const trpc = useTRPC(); + const queryClient = useQueryClient(); + + return useMutation( + trpc.mcpConnections.saveElevenLabsConnection.mutationOptions({ + onSuccess: () => { + queryClient.invalidateQueries({ + queryKey: trpc.mcpConnections.deploymentEnablements.queryKey(), + }); + queryClient.invalidateQueries({ + queryKey: trpc.mcpConnections.userConnections.queryKey(), + }); + queryClient.invalidateQueries({ + queryKey: trpc.mcpConnections.elevenLabsConnection.queryKey(), + }); + }, + }), + ); +} diff --git a/apps/web/src/trpc/commands/mcp-connections/index.ts b/apps/web/src/trpc/commands/mcp-connections/index.ts index 12b3b73bc..8dc911f2f 100644 --- a/apps/web/src/trpc/commands/mcp-connections/index.ts +++ b/apps/web/src/trpc/commands/mcp-connections/index.ts @@ -18,6 +18,7 @@ import { type McpConnectionRole, isMcpConnectionAsanaConfig, isMcpConnectionGranolaConfig, + isMcpConnectionElevenLabsConfig, isMcpConnectionGrafanaConfig, isMcpConnectionSnowflakeConfig, isMcpConnectionVercelConfig, @@ -42,6 +43,7 @@ import { MCP_TOOL_CATALOG_REQUIRES_PERSONAL_CONNECTION } from '@/lib/mcp-tool-er import type { SaveAsanaConnectionInput, SaveGranolaConnectionInput, + SaveElevenLabsConnectionInput, SaveGrafanaConnectionInput, SaveSnowflakeConnectionInput, SaveVercelConnectionInput, @@ -776,6 +778,30 @@ export async function getGranolaConnectionCommand(auth: UserAuthSuccess) { }; } +export async function getElevenLabsConnectionCommand(auth: UserAuthSuccess) { + assertAdmin(auth); + + const connection = await db.query.mcpConnections.findFirst({ + where: and( + eq(mcpConnections.mcpId, 'elevenlabs'), + isNull(mcpConnections.userId), + ), + columns: { + authConfig: true, + authStatus: true, + }, + }); + + if (!connection || !isMcpConnectionElevenLabsConfig(connection.authConfig)) { + return null; + } + + return { + authStatus: connection.authStatus, + voiceId: connection.authConfig.voiceId, + }; +} + export async function getVercelConnectionCommand( auth: UserAuthSuccess, ): Promise { @@ -1146,6 +1172,91 @@ export async function saveGranolaConnectionCommand( }; } +export async function saveElevenLabsConnectionCommand( + auth: UserAuthSuccess, + input: SaveElevenLabsConnectionInput, +) { + assertAdmin(auth); + assertCuratedIntegrationsEnabled(); + + const existingConnection = await db.query.mcpConnections.findFirst({ + where: and( + eq(mcpConnections.mcpId, 'elevenlabs'), + isNull(mcpConnections.userId), + ), + columns: { + authConfig: true, + }, + }); + + const existingConfig = isMcpConnectionElevenLabsConfig( + existingConnection?.authConfig, + ) + ? existingConnection.authConfig + : null; + const nextEncryptedApiKey = + input.apiKey.length > 0 + ? encrypt(input.apiKey) + : existingConfig?.encryptedApiKey; + + if (!nextEncryptedApiKey) { + throw new Error( + 'ElevenLabs API key is required when no ElevenLabs key is already stored.', + ); + } + + const authConfig = { + type: 'elevenlabs' as const, + encryptedApiKey: nextEncryptedApiKey, + voiceId: input.voiceId, + }; + + await db + .insert(mcpConnections) + .values({ + userId: null, + mcpId: 'elevenlabs', + connectionRole: 'default', + authConfig, + enabled: true, + authStatus: 'authenticated', + }) + .onConflictDoUpdate({ + target: [ + mcpConnections.userId, + mcpConnections.mcpId, + mcpConnections.connectionRole, + ], + set: { + connectionRole: 'default', + authConfig, + enabled: true, + authStatus: 'authenticated', + updatedAt: new Date(), + }, + }); + + await db + .insert(deploymentMcpEnablements) + .values({ + mcpId: 'elevenlabs', + enabled: true, + enabledByUserId: auth.userId, + }) + .onConflictDoUpdate({ + target: [deploymentMcpEnablements.mcpId], + set: { + enabled: true, + enabledByUserId: auth.userId, + updatedAt: new Date(), + }, + }); + + return { + authStatus: 'authenticated' as const, + }; +} + export async function saveVercelConnectionCommand( auth: UserAuthSuccess, input: SaveVercelConnectionInput, diff --git a/apps/web/src/trpc/routers/_app.ts b/apps/web/src/trpc/routers/_app.ts index 2c61968fc..8c0f68e36 100644 --- a/apps/web/src/trpc/routers/_app.ts +++ b/apps/web/src/trpc/routers/_app.ts @@ -39,6 +39,7 @@ import { filterSchema, saveAsanaConnectionSchema, saveGranolaConnectionSchema, + saveElevenLabsConnectionSchema, saveGrafanaConnectionSchema, saveSnowflakeConnectionSchema, saveVercelConnectionSchema, @@ -199,12 +200,14 @@ import { getUserMcpConnectionsCommand, getAsanaConnectionCommand, getGranolaConnectionCommand, + getElevenLabsConnectionCommand, getGrafanaConnectionCommand, getSnowflakeConnectionCommand, getVercelConnectionCommand, listDeploymentMcpIntegrationToolsCommand, saveAsanaConnectionCommand, saveGranolaConnectionCommand, + saveElevenLabsConnectionCommand, saveGrafanaConnectionCommand, saveSnowflakeConnectionCommand, saveVercelConnectionCommand, @@ -1693,6 +1696,10 @@ export const appRouter = createRouter({ getGranolaConnectionCommand(auth), ), + elevenLabsConnection: protectedProcedure.query(({ ctx: { auth } }) => + getElevenLabsConnectionCommand(auth), + ), + grafanaConnection: protectedProcedure.query(({ ctx: { auth } }) => getGrafanaConnectionCommand(auth), ), @@ -1761,6 +1768,12 @@ export const appRouter = createRouter({ saveGranolaConnectionCommand(auth, input), ), + saveElevenLabsConnection: protectedProcedure + .input(saveElevenLabsConnectionSchema) + .mutation(({ ctx: { auth }, input }) => + saveElevenLabsConnectionCommand(auth, input), + ), + saveGrafanaConnection: protectedProcedure .input(saveGrafanaConnectionSchema) .mutation(({ ctx: { auth }, input }) => diff --git a/apps/web/src/types/mcp-connections.ts b/apps/web/src/types/mcp-connections.ts index b7fdc5514..b55620eab 100644 --- a/apps/web/src/types/mcp-connections.ts +++ b/apps/web/src/types/mcp-connections.ts @@ -44,6 +44,18 @@ export type SaveGranolaConnectionInput = z.infer< typeof saveGranolaConnectionSchema >; +export const saveElevenLabsConnectionSchema = z.object({ + apiKey: z.string().transform((value) => value.trim()), + voiceId: z + .string() + .transform((value) => value.trim()) + .pipe(z.string().min(1, 'Voice ID is required')), +}); + +export type SaveElevenLabsConnectionInput = z.infer< + typeof saveElevenLabsConnectionSchema +>; + export const saveVercelConnectionSchema = z.object({ accessToken: z.string().transform((value) => value.trim()), defaultTeamIdOrSlug: z diff --git a/packages/cloud-agents/src/server/mcp-self-setup/catalog.ts b/packages/cloud-agents/src/server/mcp-self-setup/catalog.ts index 3958d9a1e..b3be68b56 100644 --- a/packages/cloud-agents/src/server/mcp-self-setup/catalog.ts +++ b/packages/cloud-agents/src/server/mcp-self-setup/catalog.ts @@ -165,6 +165,13 @@ export const MCP_SETUP_INTEGRATION_METADATA: Record< 'Keep access read-only and limited by the Granola key configuration', ], }, + elevenlabs: { + capabilities: [ + 'Narrate feature-demo videos in a configured ElevenLabs voice', + 'Keep the API key on the control plane, never exposed to agents', + 'Configure once per deployment with a text-to-speech-scoped key', + ], + }, supermemory: { capabilities: [ 'Save important decisions and context as shared memories during tasks', diff --git a/packages/sdk/src/server/routers/mcp-connections.test.ts b/packages/sdk/src/server/routers/mcp-connections.test.ts index 5102f3f72..d8b58a63a 100644 --- a/packages/sdk/src/server/routers/mcp-connections.test.ts +++ b/packages/sdk/src/server/routers/mcp-connections.test.ts @@ -481,6 +481,31 @@ describe('mcpConnectionsRouter.getMcpServerConfigs', () => { expect(JSON.stringify(result)).not.toContain('enc:secret'); }); + it('never delivers a credential-only ElevenLabs connection to agents', async () => { + mockOrderBy.mockResolvedValue([ + buildJoinedConnectionRow({ + id: 'conn-elevenlabs', + userId: null, + mcpId: 'elevenlabs', + authConfig: { + type: 'elevenlabs', + encryptedApiKey: 'enc:secret', + voiceId: 'v1', + }, + }), + ]); + + const result = await createCaller( + 'https://api.preview.roomote.run/trpc/mcpConnections.getMcpServerConfigs', + ).getMcpServerConfigs(); + + // Credential-only: no MCP server, and the secret never leaves the + // control plane toward a task sandbox. + expect(result).toEqual({ servers: {} }); + expect(JSON.stringify(result)).not.toContain('enc:secret'); + expect(JSON.stringify(result)).not.toContain('v1'); + }); + it('returns Vercel proxy config without requesting OAuth tokens', async () => { mockOrderBy.mockResolvedValue([ buildJoinedConnectionRow({ diff --git a/packages/sdk/src/server/routers/mcp-connections.ts b/packages/sdk/src/server/routers/mcp-connections.ts index 431290478..bd4d0e08c 100644 --- a/packages/sdk/src/server/routers/mcp-connections.ts +++ b/packages/sdk/src/server/routers/mcp-connections.ts @@ -393,6 +393,18 @@ async function buildCuratedMcpServerConfigs(ctx: { continue; } + // Credential-only integrations (e.g. ElevenLabs narration) are consumed + // by control-plane features exclusively: no MCP server exists for them + // and their credentials must never be delivered toward a task sandbox. + if (integration.serverMode === 'credential_only') { + console.info('[getMcpServerConfigs] Skipping connection:', { + connectionId: connection.id, + mcpId: connection.mcpId, + reason: 'credential_only_integration', + }); + continue; + } + if (connection.mcpId === 'linear') { if (!INTEGRATION_PROXY_MCP_IDS.has(connection.mcpId)) { continue; diff --git a/packages/types/src/__tests__/mcp-oauth.test.ts b/packages/types/src/__tests__/mcp-oauth.test.ts index f4d1eba57..e9a046031 100644 --- a/packages/types/src/__tests__/mcp-oauth.test.ts +++ b/packages/types/src/__tests__/mcp-oauth.test.ts @@ -5,6 +5,7 @@ import { getMcpIntegrationDefaultDisabledTools, getMcpIntegrationOauthScopeMode, getMcpIntegrationOauthScopes, + isMcpConnectionElevenLabsConfig, LINEAR_APP_OAUTH_SCOPES, MONDAY_MCP_READ_ONLY_OAUTH_SCOPES, RESEND_DEFAULT_DISABLED_TOOL_NAMES, @@ -63,6 +64,43 @@ describe('Granola API key connection', () => { }); }); +describe('ElevenLabs credential-only integration', () => { + it('is a deployment-scoped credential_only entry with no MCP url', () => { + expect(getMcpIntegration('elevenlabs')).toMatchObject({ + name: 'ElevenLabs', + connectionScope: 'deployment', + connectionMode: 'admin_configured', + serverMode: 'credential_only', + }); + expect(getMcpIntegration('elevenlabs')?.url).toBeUndefined(); + expect(getMcpIntegrationDefaultDisabledTools('elevenlabs')).toEqual([]); + }); + + it('recognizes a valid stored ElevenLabs config and rejects others', () => { + expect( + isMcpConnectionElevenLabsConfig({ + type: 'elevenlabs', + encryptedApiKey: 'enc', + voiceId: 'v1', + }), + ).toBe(true); + // Missing voiceId, wrong type, and empty configs must not pass. + expect( + isMcpConnectionElevenLabsConfig({ + type: 'elevenlabs', + encryptedApiKey: 'enc', + } as never), + ).toBe(false); + expect( + isMcpConnectionElevenLabsConfig({ + type: 'granola', + encryptedApiKey: 'x', + }), + ).toBe(false); + expect(isMcpConnectionElevenLabsConfig({})).toBe(false); + }); +}); + describe('Resend OAuth', () => { it('uses a deployment-scoped hosted MCP with risky tools disabled initially', () => { expect(getMcpIntegration('resend')).toMatchObject({ diff --git a/packages/types/src/mcp-oauth.ts b/packages/types/src/mcp-oauth.ts index a94a15827..2a07ed14b 100644 --- a/packages/types/src/mcp-oauth.ts +++ b/packages/types/src/mcp-oauth.ts @@ -133,6 +133,21 @@ export interface McpConnectionGranolaConfig { encryptedApiKey: string; } +/** + * Deployment-scoped ElevenLabs connection config stored in + * mcpConnections.authConfig. + * + * Credential-only: consumed by the control-plane narration TTS endpoint + * (/api/tts) and deliberately excluded from agent MCP config delivery — the + * key never reaches a task sandbox. The API key is expected to be encrypted + * before persistence; the voice id is plain configuration. + */ +export interface McpConnectionElevenLabsConfig { + type: 'elevenlabs'; + encryptedApiKey: string; + voiceId: string; +} + /** * Organization-scoped Vercel connection config stored in mcpConnections.authConfig. * @@ -168,6 +183,7 @@ export type McpConnectionAuthConfig = | McpConnectionSnowflakeConfig | McpConnectionAsanaConfig | McpConnectionGranolaConfig + | McpConnectionElevenLabsConfig | McpConnectionVercelConfig | McpConnectionGrafanaConfig | Record; @@ -244,7 +260,18 @@ export type LinkedAccountSetup = { export type McpIntegrationOauthScopeMode = 'all' | 'read-only'; export type McpIntegrationConnectionMode = 'oauth' | 'admin_configured'; -export type McpIntegrationServerMode = 'upstream_proxy' | 'native'; +/** + * - `upstream_proxy`: tools come from a remote MCP server reached through the + * Roomote proxy. + * - `native`: tools come from a Roomote-implemented MCP handler. + * - `credential_only`: the integration stores admin-configured credentials + * consumed by control-plane features; it exposes no MCP tools to agents + * and is never delivered to task sandboxes. + */ +export type McpIntegrationServerMode = + | 'upstream_proxy' + | 'native' + | 'credential_only'; export type McpIntegrationOAuthClientEnv = { clientIdEnv: string; @@ -501,6 +528,15 @@ export const MCP_INTEGRATIONS: McpIntegration[] = [ instructions: 'Use Granola to browse and read meeting notes, transcripts, folders, decisions, and action items through the deployment API key. The built-in tools are read-only.', }, + { + id: 'elevenlabs', + name: 'ElevenLabs', + description: `Connect ElevenLabs so ${PRODUCT_NAME} can narrate feature-demo videos with your voice. The API key stays on the control plane; agents get no ElevenLabs tools and the key never enters a task sandbox`, + icon: 'elevenlabs', + connectionScope: 'deployment', + connectionMode: 'admin_configured', + serverMode: 'credential_only', + }, { id: 'supermemory', name: 'Supermemory', @@ -822,6 +858,21 @@ export function isMcpConnectionGranolaConfig( ); } +export function isMcpConnectionElevenLabsConfig( + authConfig: McpConnectionAuthConfig | null | undefined, +): authConfig is McpConnectionElevenLabsConfig { + return Boolean( + authConfig && + typeof authConfig === 'object' && + 'type' in authConfig && + authConfig.type === 'elevenlabs' && + 'encryptedApiKey' in authConfig && + typeof authConfig.encryptedApiKey === 'string' && + 'voiceId' in authConfig && + typeof authConfig.voiceId === 'string', + ); +} + export function isMcpConnectionVercelConfig( authConfig: McpConnectionAuthConfig | null | undefined, ): authConfig is McpConnectionVercelConfig { From bbcbb92747cbf5f6b0bbd5e644309e48b763aa5d Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:51:25 -0400 Subject: [PATCH 14/27] [Fix] Exclude credential-only integrations from MCP proxy registration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The API server registers a proxy route for every non-native catalog entry, and route construction throws for an integration without an upstream URL — so adding credential-only ElevenLabs broke server construction (caught by the live-server integration suite). Add isCredentialOnlyMcpIntegration and skip those entries in the proxy registration loop, alongside native and Linear. --- apps/api/src/handlers/mcp/index.ts | 10 ++++++++-- packages/types/src/mcp-oauth.ts | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/apps/api/src/handlers/mcp/index.ts b/apps/api/src/handlers/mcp/index.ts index fb41599d1..b4382948c 100644 --- a/apps/api/src/handlers/mcp/index.ts +++ b/apps/api/src/handlers/mcp/index.ts @@ -4,7 +4,11 @@ import { areCuratedIntegrationsDisabled, isCustomMcpDisabled, } from '@roomote/env'; -import { isNativeMcpIntegration, MCP_INTEGRATIONS } from '@roomote/types'; +import { + isCredentialOnlyMcpIntegration, + isNativeMcpIntegration, + MCP_INTEGRATIONS, +} from '@roomote/types'; import type { Variables } from '../../types'; @@ -66,7 +70,9 @@ mcp.route('/vercel', vercelMcp); for (const integration of MCP_INTEGRATIONS.filter( (candidate) => - !isNativeMcpIntegration(candidate) && candidate.id !== 'linear', + !isNativeMcpIntegration(candidate) && + !isCredentialOnlyMcpIntegration(candidate) && + candidate.id !== 'linear', )) { mcp.route( `/${integration.id}`, diff --git a/packages/types/src/mcp-oauth.ts b/packages/types/src/mcp-oauth.ts index 2a07ed14b..e4ced8451 100644 --- a/packages/types/src/mcp-oauth.ts +++ b/packages/types/src/mcp-oauth.ts @@ -672,6 +672,26 @@ export function isNativeMcpIntegration( return integration?.serverMode === 'native'; } +/** + * Credential-only integrations store admin-configured credentials for + * control-plane features and expose no MCP tools to agents, so they get no + * proxy route and are never delivered to task sandboxes. + */ +export function isCredentialOnlyMcpIntegration( + integrationOrId: McpIntegration | string | undefined, +): boolean { + if (!integrationOrId) { + return false; + } + + const integration = + typeof integrationOrId === 'string' + ? getMcpIntegration(integrationOrId) + : integrationOrId; + + return integration?.serverMode === 'credential_only'; +} + export function getMcpIntegrationUpstreamUrl( integrationOrId: McpIntegration | string | undefined, ): string | null { From c86f1b8264c611d989d312bf282c3bb252ba2645 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 11:55:23 -0400 Subject: [PATCH 15/27] [Fix] Drop credential-only integrations from the worker proxy map too Completes the review point on bbcbb9274: the worker's curated proxy-path map no longer carries a dead entry for integrations that are never delivered to sandboxes. --- apps/worker/src/commands/setup/setup-mcps.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/worker/src/commands/setup/setup-mcps.ts b/apps/worker/src/commands/setup/setup-mcps.ts index 51dcfc1cf..9ce1fcfbe 100644 --- a/apps/worker/src/commands/setup/setup-mcps.ts +++ b/apps/worker/src/commands/setup/setup-mcps.ts @@ -3,6 +3,7 @@ import * as path from 'node:path'; import { CUSTOM_MCP_PROXY_PATH_PREFIX, getMcpIntegrationUpstreamUrl, + isCredentialOnlyMcpIntegration, isReservedRuntimeMcpEnvVarName, isRoomoteNamespacedEnvVarName, MCP_INTEGRATIONS, @@ -170,7 +171,11 @@ function resolveConfigValues( function buildIntegrationProxyMap(): Map { const integrationConfigs: IntegrationProxyConfig[] = []; - for (const integration of MCP_INTEGRATIONS) { + // Credential-only integrations have no MCP server and are never delivered + // to sandboxes, so they get no proxy-path entry. + for (const integration of MCP_INTEGRATIONS.filter( + (candidate) => !isCredentialOnlyMcpIntegration(candidate), + )) { const upstreamUrl = getMcpIntegrationUpstreamUrl(integration); const upstream = upstreamUrl ? parseUrl(upstreamUrl) : null; From 6b206ef3b77f0ae28fdb4268ab144cc9ec12af6d Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:00:08 -0400 Subject: [PATCH 16/27] [Fix] Close the verify/execute race on the staged capture runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review caught a TOCTOU in the digest sanction: sha256sum then node as separate commands over the parent-writable staging path leaves a swap window. The proof-runner now copies the staged file to a fresh private path, digests the copy, and executes exactly that verified copy — checked bytes are executed bytes, and the staging path is never re-read after verification. --- .../run-task/__tests__/proof-runner-prompt.test.ts | 12 +++++++++--- apps/worker/src/run-task/proof-runner-prompt.ts | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts index d611e6b1b..e3c940f0f 100644 --- a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts +++ b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts @@ -38,12 +38,18 @@ describe('createProofRunnerAgentPrompt', () => { expect(prompt).toContain( 'running it via `node` with the environment variables the brief specifies is compliant browser work', ); - // The staging path is parent-writable, so the sanction binds to content. - expect(prompt).toContain('sha256sum /tmp/feature-demo/capture.mjs'); + // The staging path is parent-writable, so the sanction binds to content — + // and to close the verify/execute race, the runner digests and executes a + // private copy, never the swappable staging path. + expect(prompt).toContain('copy the file to a fresh private path first'); + expect(prompt).toContain('sha256sum "$RUNNER"'); + expect(prompt).toContain( + 'execute that verified copy — never the original path', + ); expect(prompt).toContain(`\`${digest}\``); expect(prompt).toContain('capture runner integrity mismatch'); expect(prompt).toContain( - 'This exception covers exactly that digest-verified file and no other script.', + 'This exception covers exactly that digest-verified copy and no other script.', ); }); diff --git a/apps/worker/src/run-task/proof-runner-prompt.ts b/apps/worker/src/run-task/proof-runner-prompt.ts index 9c7d31c63..010ce1c86 100644 --- a/apps/worker/src/run-task/proof-runner-prompt.ts +++ b/apps/worker/src/run-task/proof-runner-prompt.ts @@ -37,8 +37,8 @@ export function createProofRunnerAgentPrompt( '- `agent-browser` is the only allowed browser automation CLI. Do not use Playwright, browser DevTools, curl-only screenshot substitutes, or any other browser automation path.', ...(featureDemoCaptureRunnerSha256 ? [ - '- One sanctioned exception: the feature-demo capture runner at `/tmp/feature-demo/capture.mjs`, staged there by the parent. It is an `agent-browser` orchestrator — every browser action it performs goes through the `agent-browser` CLI — and running it via `node` with the environment variables the brief specifies is compliant browser work, not a disallowed automation path. The staging path is parent-writable, so verify content before executing: run `sha256sum /tmp/feature-demo/capture.mjs` and confirm the digest is exactly ' + - `\`${featureDemoCaptureRunnerSha256}\`. On any mismatch or missing file, report blocked with blocker type \`capture runner integrity mismatch\` and do not execute it. Its outputs land where the brief says (typically \`/tmp/feature-demo/work/\`), not under \`/tmp/capture-visual-proof/\`. This exception covers exactly that digest-verified file and no other script.`, + '- One sanctioned exception: the feature-demo capture runner staged by the parent at `/tmp/feature-demo/capture.mjs`. It is an `agent-browser` orchestrator — every browser action it performs goes through the `agent-browser` CLI — and running it via `node` with the environment variables the brief specifies is compliant browser work, not a disallowed automation path. The staging path is parent-writable, so the checked bytes must be the executed bytes: copy the file to a fresh private path first (for example `RUNNER="$(mktemp /tmp/fd-runner-XXXXXX.mjs)" && cp /tmp/feature-demo/capture.mjs "$RUNNER"`), verify the COPY with `sha256sum "$RUNNER"` against exactly ' + + `\`${featureDemoCaptureRunnerSha256}\`, and execute that verified copy — never the original path, and never re-read the staging path after verification. On any mismatch or missing file, report blocked with blocker type \`capture runner integrity mismatch\` and do not execute anything. Its outputs land where the brief says (typically \`/tmp/feature-demo/work/\`), not under \`/tmp/capture-visual-proof/\`. This exception covers exactly that digest-verified copy and no other script.`, ] : []), '- If the `agent-browser` skill/guidance cannot be loaded, or if the `agent-browser` executable is unavailable, report blocked with blocker type `proof runtime unavailable` and describe the missing skill guidance or CLI explicitly.', From 48921e6471d56ebcf223bef59890ca06dbe0c5cb Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:01:23 -0400 Subject: [PATCH 17/27] [Fix] Update proof-runner prompt test for the reworded sanction Follow-up to 6b206ef3b: the first assertion still matched the old 'runner at' phrasing; the sanction now reads 'runner staged by the parent at'. --- apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts index e3c940f0f..b8958fbff 100644 --- a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts +++ b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts @@ -33,7 +33,7 @@ describe('createProofRunnerAgentPrompt', () => { ); expect(prompt).toContain( - 'the feature-demo capture runner at `/tmp/feature-demo/capture.mjs`', + 'the feature-demo capture runner staged by the parent at `/tmp/feature-demo/capture.mjs`', ); expect(prompt).toContain( 'running it via `node` with the environment variables the brief specifies is compliant browser work', From 51950ca4e3d48ddfff5c26706e9f03bae42e9ae1 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:13:55 -0400 Subject: [PATCH 18/27] [Fix] Atomically hash-and-execute the staged capture runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior mktemp copy still lived in shared /tmp, so a same-uid parent could swap it between sha256sum and node opening it — copying does not close a TOCTOU. The proof-runner now runs a single node process that reads capture.mjs exactly once, hashes those bytes, and executes the same bytes via a data: import only on an exact match. The file is opened once, so there is no post-hash window; the digest is still baked in at config-generation time. SKILL.md hands off paths and lets the proof-runner own the verified invocation. --- .../__tests__/proof-runner-prompt.test.ts | 24 ++++++++++++------- .../src/run-task/proof-runner-prompt.ts | 9 +++++-- .../__tests__/featureDemoSkill.test.ts | 7 +++++- .../skills/standard/feature-demo/SKILL.md | 6 +---- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts index b8958fbff..7117d94ff 100644 --- a/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts +++ b/apps/worker/src/run-task/__tests__/proof-runner-prompt.test.ts @@ -35,21 +35,27 @@ describe('createProofRunnerAgentPrompt', () => { expect(prompt).toContain( 'the feature-demo capture runner staged by the parent at `/tmp/feature-demo/capture.mjs`', ); + + // The staging path is shared/parent-writable, so the sanction closes the + // check/use race by reading the file exactly once and executing the same + // bytes it hashed (a single node process; no separate hash-then-run step, + // no re-open of the swappable path). + expect(prompt).toContain('reading the file exactly once'); + expect(prompt).toContain( + 'const b=readFileSync("/tmp/feature-demo/capture.mjs")', + ); expect(prompt).toContain( - 'running it via `node` with the environment variables the brief specifies is compliant browser work', + `createHash("sha256").update(b).digest("hex")!=="${digest}"`, ); - // The staging path is parent-writable, so the sanction binds to content — - // and to close the verify/execute race, the runner digests and executes a - // private copy, never the swappable staging path. - expect(prompt).toContain('copy the file to a fresh private path first'); - expect(prompt).toContain('sha256sum "$RUNNER"'); expect(prompt).toContain( - 'execute that verified copy — never the original path', + 'await import("data:text/javascript;base64,"+b.toString("base64"))', ); - expect(prompt).toContain(`\`${digest}\``); expect(prompt).toContain('capture runner integrity mismatch'); expect(prompt).toContain( - 'This exception covers exactly that digest-verified copy and no other script.', + 'never fall back to `node /tmp/feature-demo/capture.mjs`', + ); + expect(prompt).toContain( + 'This exception covers exactly this verified-execution command and no other script.', ); }); diff --git a/apps/worker/src/run-task/proof-runner-prompt.ts b/apps/worker/src/run-task/proof-runner-prompt.ts index 010ce1c86..89b256c6a 100644 --- a/apps/worker/src/run-task/proof-runner-prompt.ts +++ b/apps/worker/src/run-task/proof-runner-prompt.ts @@ -37,8 +37,13 @@ export function createProofRunnerAgentPrompt( '- `agent-browser` is the only allowed browser automation CLI. Do not use Playwright, browser DevTools, curl-only screenshot substitutes, or any other browser automation path.', ...(featureDemoCaptureRunnerSha256 ? [ - '- One sanctioned exception: the feature-demo capture runner staged by the parent at `/tmp/feature-demo/capture.mjs`. It is an `agent-browser` orchestrator — every browser action it performs goes through the `agent-browser` CLI — and running it via `node` with the environment variables the brief specifies is compliant browser work, not a disallowed automation path. The staging path is parent-writable, so the checked bytes must be the executed bytes: copy the file to a fresh private path first (for example `RUNNER="$(mktemp /tmp/fd-runner-XXXXXX.mjs)" && cp /tmp/feature-demo/capture.mjs "$RUNNER"`), verify the COPY with `sha256sum "$RUNNER"` against exactly ' + - `\`${featureDemoCaptureRunnerSha256}\`, and execute that verified copy — never the original path, and never re-read the staging path after verification. On any mismatch or missing file, report blocked with blocker type \`capture runner integrity mismatch\` and do not execute anything. Its outputs land where the brief says (typically \`/tmp/feature-demo/work/\`), not under \`/tmp/capture-visual-proof/\`. This exception covers exactly that digest-verified copy and no other script.`, + '- One sanctioned exception: the feature-demo capture runner staged by the parent at `/tmp/feature-demo/capture.mjs`. It is an `agent-browser` orchestrator — every browser action it performs goes through the `agent-browser` CLI — so running it (with the environment variables the brief specifies) is compliant browser work, not a disallowed automation path. The staging path is in shared, parent-writable `/tmp`, so a separate hash step then a `node ` step would leave the file swappable in between. Close that window by reading the file exactly once and executing the same bytes you hashed — run precisely this command (it hashes the loaded bytes and, only on an exact match, executes those same bytes, never re-opening the file):', + '', + ' ```', + ` SCRIPT=/tmp/feature-demo/demo-script.json OUT_DIR=/tmp/feature-demo/work node --input-type=module -e 'import{readFileSync}from"node:fs";import{createHash}from"node:crypto";const b=readFileSync("/tmp/feature-demo/capture.mjs");if(createHash("sha256").update(b).digest("hex")!=="${featureDemoCaptureRunnerSha256}"){console.error("capture runner integrity mismatch");process.exit(1)}await import("data:text/javascript;base64,"+b.toString("base64"))'`, + ' ```', + '', + ' On the `capture runner integrity mismatch` exit, or if the file is missing, report blocked with blocker type `capture runner integrity mismatch` and do not run any other command in its place. Adjust only the `SCRIPT`/`OUT_DIR` values per the brief; never change the read-hash-execute body, and never fall back to `node /tmp/feature-demo/capture.mjs`. This exception covers exactly this verified-execution command and no other script.', ] : []), '- If the `agent-browser` skill/guidance cannot be loaded, or if the `agent-browser` executable is unavailable, report blocked with blocker type `proof runtime unavailable` and describe the missing skill guidance or CLI explicitly.', diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index 5020719fb..9ec950723 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -58,7 +58,12 @@ describe('feature-demo skill', () => { expect(skillContent).toContain( 'cp "$HOME/.agents/skills/feature-demo/capture/capture.mjs" /tmp/feature-demo/capture.mjs', ); - expect(skillContent).toContain('node /tmp/feature-demo/capture.mjs'); + expect(skillContent).toContain('/tmp/feature-demo/capture.mjs'); + // The parent hands off the paths; the proof-runner owns the integrity- + // verified node invocation, so the skill must not dictate a raw node run. + expect(skillContent).toContain( + 'Do not dictate the `node` invocation yourself; the proof-runner owns that.', + ); }); it('keeps TTS provider keys out of the sandbox', () => { diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index feafb5101..acdeef0cc 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -67,11 +67,7 @@ this pipeline is ever committed to the repository. Stage the capture runner where the delegated runtime can see it — home-directory paths do not survive the delegation boundary, so always copy first: `mkdir -p /tmp/feature-demo && cp "$HOME/.agents/skills/feature-demo/capture/capture.mjs" /tmp/feature-demo/capture.mjs` -Browser automation is the proof-runner subagent's exclusive surface — do not load `agent-browser` or run the capture yourself. Delegate with the Task tool to `proof-runner` with a brief that instructs it to run exactly: - -`SCRIPT=/tmp/feature-demo/demo-script.json OUT_DIR=/tmp/feature-demo/work node /tmp/feature-demo/capture.mjs` - -and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. The staged runner is proof-runner's one sanctioned script exception (its own instructions say so): an agent-browser orchestrator that shells the `agent-browser` CLI for every browser action. +Browser automation is the proof-runner subagent's exclusive surface — do not load `agent-browser` or run the capture yourself. Delegate with the Task tool to `proof-runner`, telling it the script path (`/tmp/feature-demo/demo-script.json`), the output dir (`/tmp/feature-demo/work`), and the staged runner path (`/tmp/feature-demo/capture.mjs`), and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. The staged runner is proof-runner's one sanctioned script exception — an agent-browser orchestrator that shells the `agent-browser` CLI for every browser action — and its own instructions define the exact integrity-verified command it must use to execute it. Do not dictate the `node` invocation yourself; the proof-runner owns that. If the harness has no proof-runner registered, report a blocker (`proof runtime unavailable`) instead of driving the browser from this skill. Expected outputs: `/tmp/feature-demo/work/recording.mp4` and `/tmp/feature-demo/work/timeline.json`. Verify both exist and that `ffprobe` reports a duration close to the timeline's `durationSeconds` (the runner itself fails loudly when the recording is much shorter than the interaction). One retry on failure; then report blocked with the runner's error. If the runner fails with a recording much shorter than the interaction, diagnose by target: WebGL/canvas-heavy surfaces (games, 3D visualizations) can stall headless frame production for the whole page, so ordinary web UIs on the same sandbox record fine while that one surface collapses. Report the blocker as `webgl surface stalls headless recording`, naming the surface — do not burn retries. Separately, if `record stop` itself reports an ffmpeg error, the sandbox is likely a stale snapshot with an outdated runtime ffmpeg (`stale sandbox runtime`). From 7c3998a81644417b8de098411d9c1e73d64e1d18 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:33:53 -0400 Subject: [PATCH 19/27] [Feat] Record feature demos headed so GPU canvases don't stall MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live dogfood run collapsed a 47s game interaction into a 0.7s clip: GPU-backed canvases (games, 3D, WebGL/WebGPU) never present to the headless compositor, starving the screencast. agent-browser's own docs prescribe `--headed` for exactly this, and on displayless Linux it auto-starts a private Xvfb (present in the worker image). The capture runner now prefixes --headed on every agent-browser command by default (HEADED=0 opts out); harmless on ordinary pages. Verified end-to-end in roomote-worker:arm64-real against an animated WebGL canvas: full pipeline, real-duration recording, auto-Xvfb, exit 0. SKILL.md updated; this supersedes the earlier 'file an upstream issue' plan — the flag already exists. --- .../workflows/__tests__/featureDemoSkill.test.ts | 14 ++++++++++++++ .../skills/standard/feature-demo/SKILL.md | 2 +- .../standard/feature-demo/capture/capture.mjs | 10 +++++++++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index 9ec950723..29419cc5b 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -91,6 +91,20 @@ describe('feature-demo skill', () => { ); }); + it('records headed so GPU-backed canvases do not stall the screencast', () => { + const captureRunner = fs.readFileSync( + path.join(skillDirPath, 'capture/capture.mjs'), + 'utf8', + ); + + // Default headed with an explicit opt-out; the flag rides every command. + expect(captureRunner).toContain( + "const HEADED = process.env.HEADED !== '0'", + ); + expect(captureRunner).toContain("HEADED ? ['--headed', ...args] : args"); + expect(skillContent).toContain('records headed by default'); + }); + it('frames the render project as an adaptable reference template', () => { expect(skillContent).toContain('**reference template**'); expect(skillContent).toContain('npx -y skills add remotion-dev/skills'); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index acdeef0cc..57636b34d 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -70,7 +70,7 @@ this pipeline is ever committed to the repository. Browser automation is the proof-runner subagent's exclusive surface — do not load `agent-browser` or run the capture yourself. Delegate with the Task tool to `proof-runner`, telling it the script path (`/tmp/feature-demo/demo-script.json`), the output dir (`/tmp/feature-demo/work`), and the staged runner path (`/tmp/feature-demo/capture.mjs`), and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. The staged runner is proof-runner's one sanctioned script exception — an agent-browser orchestrator that shells the `agent-browser` CLI for every browser action — and its own instructions define the exact integrity-verified command it must use to execute it. Do not dictate the `node` invocation yourself; the proof-runner owns that. If the harness has no proof-runner registered, report a blocker (`proof runtime unavailable`) instead of driving the browser from this skill. Expected outputs: `/tmp/feature-demo/work/recording.mp4` and `/tmp/feature-demo/work/timeline.json`. Verify both exist and that `ffprobe` reports a duration close to the timeline's `durationSeconds` (the runner itself fails loudly when the recording is much shorter than the interaction). One retry on failure; then report blocked with the runner's error. -If the runner fails with a recording much shorter than the interaction, diagnose by target: WebGL/canvas-heavy surfaces (games, 3D visualizations) can stall headless frame production for the whole page, so ordinary web UIs on the same sandbox record fine while that one surface collapses. Report the blocker as `webgl surface stalls headless recording`, naming the surface — do not burn retries. Separately, if `record stop` itself reports an ffmpeg error, the sandbox is likely a stale snapshot with an outdated runtime ffmpeg (`stale sandbox runtime`). +The runner records headed by default (`agent-browser --headed`, auto-Xvfb in the sandbox) so GPU-backed canvases (games, 3D, WebGL/WebGPU) present frames instead of stalling the screencast. If a recording still comes back much shorter than the interaction, the runner fails loudly: for a WebGL/WebGPU-heavy surface, first retry is warranted, and if it persists report `webgl surface stalls headless recording` naming the surface (some WebGPU capture paths remain unsupported upstream even headed). If `record stop` itself reports an ffmpeg error, the sandbox is likely a stale snapshot with an outdated runtime ffmpeg (`stale sandbox runtime`). diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs index 83e46a937..5599e3ba8 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -37,8 +37,16 @@ const VIEWPORT = script.viewport || { w: 1280, h: 800 }; // wide between every zoom reads as jumpy. The final reset goes fully wide. const GLIDE_SCALE = 1.18; +// Record headed by default. GPU-backed canvases (games, 3D, WebGL/WebGPU) +// never present to the headless compositor, which starves the screencast and +// collapses a long interaction into a sub-second clip — agent-browser's own +// docs prescribe `--headed` for exactly this, and on displayless Linux it +// auto-starts a private Xvfb (present in the worker image). Harmless on +// ordinary pages. Set HEADED=0 to force headless. +const HEADED = process.env.HEADED !== '0'; + const ab = (...args) => - execFileSync(AB, args, { + execFileSync(AB, HEADED ? ['--headed', ...args] : args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], }); From 2fa47ef504dea6c86e9f6e8e9b212b1fe8bcf901 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 12:45:58 -0400 Subject: [PATCH 20/27] [Fix] Let the agent-browser wrapper honor an explicit --headed opt-in The generated wrapper stripped --headed and force-exported AGENT_BROWSER_HEADED=false, so the capture runner's --headed was a no-op (review catch). The wrapper now forwards --headed and exports AGENT_BROWSER_HEADED=true only when a caller explicitly passes it; default behavior is byte-identical (headless), preview-cookie seeding stays headless, and --headed false/=false explicitly opts out. Verified the arg handling in isolation across the command set. --- .docker/sandbox/install-browser-agent.sh | 23 ++++++++++++++++++- .../src/adapters/modal.test.ts | 7 ++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.docker/sandbox/install-browser-agent.sh b/.docker/sandbox/install-browser-agent.sh index abdfb156a..e474470e5 100644 --- a/.docker/sandbox/install-browser-agent.sh +++ b/.docker/sandbox/install-browser-agent.sh @@ -374,21 +374,35 @@ collect_cli_browser_args() { local i=0 AGENT_BROWSER_EXEC_ARGS=() AGENT_BROWSER_FORWARD_ARGS=() + # Default headless; a caller may opt a single invocation into headed mode + # (e.g. the feature-demo capture runner, so GPU-backed canvases present to + # the compositor). On displayless Linux agent-browser auto-starts Xvfb. + AGENT_BROWSER_WANT_HEADED=0 while [ "$i" -lt "${#args[@]}" ]; do local arg="${args[$i]}" case "$arg" in --headed) + AGENT_BROWSER_WANT_HEADED=1 + AGENT_BROWSER_FORWARD_ARGS+=("$arg") if [ "$i" -lt $(( ${#args[@]} - 1 )) ]; then case "${args[$((i + 1))]}" in true|false) i=$((i + 1)) + [ "${args[$i]}" = false ] && AGENT_BROWSER_WANT_HEADED=0 + AGENT_BROWSER_FORWARD_ARGS+=("${args[$i]}") ;; esac fi ;; + --headed=false) + AGENT_BROWSER_WANT_HEADED=0 + AGENT_BROWSER_FORWARD_ARGS+=("$arg") + ;; --headed=*) + AGENT_BROWSER_WANT_HEADED=1 + AGENT_BROWSER_FORWARD_ARGS+=("$arg") ;; --args) AGENT_BROWSER_FORWARD_ARGS+=("$arg") @@ -512,7 +526,14 @@ if should_clear_seed_cache; then fi resolve_cli_paths -export AGENT_BROWSER_HEADED=false +# Preview-cookie seeding above always runs headless (inline env). The task +# command itself honors an explicit --headed opt-in; everything else stays +# headless as before. +if [ "${AGENT_BROWSER_WANT_HEADED:-0}" = 1 ]; then + export AGENT_BROWSER_HEADED=true +else + export AGENT_BROWSER_HEADED=false +fi exec "$AGENT_BROWSER_BIN" "${AGENT_BROWSER_FORWARD_ARGS[@]}" EOF_WRAPPER } diff --git a/packages/compute-providers/src/adapters/modal.test.ts b/packages/compute-providers/src/adapters/modal.test.ts index 7d81ee597..80d766d32 100644 --- a/packages/compute-providers/src/adapters/modal.test.ts +++ b/packages/compute-providers/src/adapters/modal.test.ts @@ -927,9 +927,16 @@ describe('ModalClient', () => { expect(installBrowserAgentScript).toContain( 'AGENT_BROWSER_FORWARD_ARGS=()', ); + // Default headless preserved, with an explicit per-invocation headed + // opt-in (the feature-demo capture runner passes --headed so GPU-backed + // canvases present to the compositor). expect(installBrowserAgentScript).toContain( 'export AGENT_BROWSER_HEADED=false', ); + expect(installBrowserAgentScript).toContain( + 'export AGENT_BROWSER_HEADED=true', + ); + expect(installBrowserAgentScript).toContain('AGENT_BROWSER_WANT_HEADED=1'); expect(installBrowserAgentScript).toContain( 'exec "$AGENT_BROWSER_BIN" "${AGENT_BROWSER_FORWARD_ARGS[@]}"', ); From d41b387e6058a9be674e84350d3b4832970839a2 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:27:06 -0400 Subject: [PATCH 21/27] [Fix] Reap zombie browser processes and force fresh daemon for headed capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Forensics on a stalled demo capture (live worker container) traced the real root cause past the misleading agent narrative: worker containers run with no init process and a 512 pids-limit, so agent-browser daemon restarts leak Chrome process trees whose defunct children accumulate (observed 174 defunct, pids.current 494/512) until fresh Chrome dies with pthread_create: Resource temporarily unavailable — surfaced as Chrome exiting early and misread as a display problem. - Run docker workers and task daemons with --init (tini) so orphaned children are reaped. - Raise DOCKER_WORKER_PIDS_LIMIT default 512 -> 2048 (still overridable). - feature-demo capture: stop any existing agent-browser daemon (agent-browser close, not just close --all) before recording so --headed actually applies, and hard-fail when record start reports '--headed ignored: daemon already running' instead of silently recording a headless (stalling) session. Diagnosis from the Roomote-Evals forensic session; verified the code gaps directly. Capture path re-smoked end-to-end in the worker image including the daemon-already-running trap. --- .../src/__tests__/RoomoteController.test.ts | 6 +-- .../__tests__/docker-sandbox-security.test.ts | 14 ++++++ .../docker-sandbox-security.ts | 8 ++++ .../skills/standard/feature-demo/SKILL.md | 3 +- .../standard/feature-demo/capture/capture.mjs | 44 ++++++++++++++++--- packages/env/src/__tests__/index.test.ts | 2 +- packages/env/src/index.ts | 6 ++- 7 files changed, 70 insertions(+), 13 deletions(-) diff --git a/apps/controller/src/__tests__/RoomoteController.test.ts b/apps/controller/src/__tests__/RoomoteController.test.ts index 2ce64159e..67db6e1d6 100644 --- a/apps/controller/src/__tests__/RoomoteController.test.ts +++ b/apps/controller/src/__tests__/RoomoteController.test.ts @@ -30,7 +30,7 @@ const { DOCKER_WORKER_CPU_LIMIT: 2, DOCKER_WORKER_MEMORY_LIMIT: '4g', DOCKER_TASK_DAEMON_MEMORY_LIMIT: '8g', - DOCKER_WORKER_PIDS_LIMIT: 512, + DOCKER_WORKER_PIDS_LIMIT: 2048, DOCKER_WORKER_DISK_LIMIT: '20g', DOCKER_WORKER_ALLOW_UNBOUNDED_DISK: false, DOCKER_WORKER_LOG_MAX_SIZE: '10m', @@ -132,7 +132,7 @@ describe('RoomoteController', () => { mockEnv.DOCKER_WORKER_CPU_LIMIT = 2; mockEnv.DOCKER_WORKER_MEMORY_LIMIT = '4g'; mockEnv.DOCKER_TASK_DAEMON_MEMORY_LIMIT = '8g'; - mockEnv.DOCKER_WORKER_PIDS_LIMIT = 512; + mockEnv.DOCKER_WORKER_PIDS_LIMIT = 2048; mockEnv.DOCKER_WORKER_DISK_LIMIT = '20g'; mockEnv.DOCKER_WORKER_ALLOW_UNBOUNDED_DISK = false; mockEnv.DOCKER_WORKER_LOG_MAX_SIZE = '10m'; @@ -195,7 +195,7 @@ describe('RoomoteController', () => { cpuLimit: 2, memoryLimit: '4g', taskDaemonMemoryLimit: '8g', - pidsLimit: 512, + pidsLimit: 2048, diskLimit: '20g', allowUnboundedDisk: false, logMaxSize: '10m', diff --git a/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts b/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts index 7c7c23352..ef201034d 100644 --- a/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts +++ b/apps/controller/src/compute-providers/__tests__/docker-sandbox-security.test.ts @@ -37,6 +37,7 @@ describe('buildDockerWorkerResourceArgs', () => { logMaxFiles: 3, }), ).toEqual([ + '--init', '--cpus', '2', '--memory', @@ -60,6 +61,18 @@ describe('buildDockerWorkerResourceArgs', () => { ]); }); + it('runs an init process to reap orphaned browser child processes', () => { + expect( + buildDockerWorkerResourceArgs({ + cpuLimit: 2, + memoryLimit: '4g', + pidsLimit: 512, + logMaxSize: '10m', + logMaxFiles: 3, + }), + ).toContain('--init'); + }); + it('omits the storage option when the driver cannot enforce a disk limit', () => { expect( buildDockerWorkerResourceArgs({ @@ -85,6 +98,7 @@ describe('buildDockerTaskDaemonResourceArgs', () => { }); expect(args).toContain('--pids-limit'); + expect(args).toContain('--init'); expect(args).toEqual( expect.arrayContaining(['--memory', '8g', '--memory-swap', '8g']), ); diff --git a/apps/controller/src/compute-providers/docker-sandbox-security.ts b/apps/controller/src/compute-providers/docker-sandbox-security.ts index 343c71bb8..4776f18ab 100644 --- a/apps/controller/src/compute-providers/docker-sandbox-security.ts +++ b/apps/controller/src/compute-providers/docker-sandbox-security.ts @@ -367,6 +367,12 @@ export function buildDockerWorkerResourceArgs( limits: DockerWorkerResourceLimits, ): string[] { return [ + // Run an init process (tini) as PID 1 so orphaned child processes are + // reaped. Browser-driving tasks (agent-browser daemon restarts) leak + // Chrome process trees whose defunct children otherwise accumulate until + // they exhaust --pids-limit, after which fresh Chrome launches fail with + // `pthread_create: Resource temporarily unavailable`. + '--init', '--cpus', String(limits.cpuLimit), '--memory', @@ -393,6 +399,8 @@ export function buildDockerTaskDaemonResourceArgs( limits: DockerWorkerResourceLimits, ): string[] { return [ + // Reap orphaned children (see buildDockerWorkerResourceArgs). + '--init', '--cpus', String(limits.cpuLimit), '--memory', diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index 57636b34d..e55415d35 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -70,7 +70,8 @@ this pipeline is ever committed to the repository. Browser automation is the proof-runner subagent's exclusive surface — do not load `agent-browser` or run the capture yourself. Delegate with the Task tool to `proof-runner`, telling it the script path (`/tmp/feature-demo/demo-script.json`), the output dir (`/tmp/feature-demo/work`), and the staged runner path (`/tmp/feature-demo/capture.mjs`), and to report the runner's printed summary plus `ls -la /tmp/feature-demo/work`. The staged runner is proof-runner's one sanctioned script exception — an agent-browser orchestrator that shells the `agent-browser` CLI for every browser action — and its own instructions define the exact integrity-verified command it must use to execute it. Do not dictate the `node` invocation yourself; the proof-runner owns that. If the harness has no proof-runner registered, report a blocker (`proof runtime unavailable`) instead of driving the browser from this skill. Expected outputs: `/tmp/feature-demo/work/recording.mp4` and `/tmp/feature-demo/work/timeline.json`. Verify both exist and that `ffprobe` reports a duration close to the timeline's `durationSeconds` (the runner itself fails loudly when the recording is much shorter than the interaction). One retry on failure; then report blocked with the runner's error. -The runner records headed by default (`agent-browser --headed`, auto-Xvfb in the sandbox) so GPU-backed canvases (games, 3D, WebGL/WebGPU) present frames instead of stalling the screencast. If a recording still comes back much shorter than the interaction, the runner fails loudly: for a WebGL/WebGPU-heavy surface, first retry is warranted, and if it persists report `webgl surface stalls headless recording` naming the surface (some WebGPU capture paths remain unsupported upstream even headed). If `record stop` itself reports an ffmpeg error, the sandbox is likely a stale snapshot with an outdated runtime ffmpeg (`stale sandbox runtime`). +The runner records headed by default (`agent-browser --headed`, auto-Xvfb in the sandbox) so GPU-backed canvases (games, 3D, WebGL/WebGPU) present frames instead of stalling the screencast. `--headed` only applies when agent-browser launches a fresh daemon, so the runner stops any existing daemon (`agent-browser close`) before recording and hard-fails if `record start` reports `--headed ignored: daemon already running` — do not let an earlier browser step (inspection, etc.) leave a headless daemon up. If that error fires, ensure nothing else is driving agent-browser and retry. +If a recording still comes back much shorter than the interaction, the runner fails loudly: for a WebGL/WebGPU-heavy surface, a first retry is warranted, and if it persists report `webgl surface stalls headless recording` naming the surface (some WebGPU capture paths remain unsupported upstream even headed). If `record stop` itself reports an ffmpeg error, the sandbox is likely a stale snapshot with an outdated runtime ffmpeg (`stale sandbox runtime`). diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs index 5599e3ba8..709e7a8d8 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/capture/capture.mjs @@ -11,7 +11,7 @@ // node capture.mjs // See SKILL.md for the demo-script schema. -import { execFileSync } from 'node:child_process'; +import { execFileSync, spawnSync } from 'node:child_process'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; const AB = process.env.AGENT_BROWSER_BIN || 'agent-browser'; @@ -51,6 +51,21 @@ const ab = (...args) => stdio: ['ignore', 'pipe', 'pipe'], }); +// Same invocation, but returns combined stdout+stderr so warnings that +// agent-browser writes to stderr (e.g. the "--headed ignored" notice) can be +// inspected. Throws on a non-zero exit like execFileSync would. +const abCapture = (...args) => { + const res = spawnSync(AB, HEADED ? ['--headed', ...args] : args, { + encoding: 'utf8', + }); + if (res.error) throw res.error; + const out = `${res.stdout ?? ''}${res.stderr ?? ''}`; + if (res.status !== 0) { + throw new Error(`agent-browser ${args.join(' ')} failed: ${out}`); + } + return out; +}; + const sleep = (ms) => execFileSync('sleep', [String(ms / 1000)]); let t0 = 0; @@ -120,16 +135,31 @@ const TICKER_JS = async function run() { mkdirSync(OUT_DIR, { recursive: true }); - // `record start` creates a fresh browser context; a pre-existing page - // (e.g. from an earlier inspection step) would leave the beats driving one - // page while the recorder watches another. Start from a clean slate. + // Fully stop any existing agent-browser daemon before recording. `--headed` + // (and other launch options) only take effect when the daemon launches the + // browser; against an already-running daemon agent-browser prints + // "--headed ignored: daemon already running" and records headless. `close` + // (alias quit/exit) tears the daemon down; `close --all` alone only clears + // sessions. This also gives `record start` a clean slate so the beats and + // the recorder share one page. try { - ab('close', '--all'); + ab('close'); } catch { - // no active session is fine + // no active daemon is fine } ab('set', 'viewport', String(VIEWPORT.w), String(VIEWPORT.h)); - ab('record', 'start', `${OUT_DIR}/raw.webm`, script.url); + + const startOut = abCapture('record', 'start', `${OUT_DIR}/raw.webm`, script.url); + // If the daemon survived the close and swallowed our launch options, the + // recording would silently be headless — fail loudly instead of producing + // a stalled clip on a GPU-backed surface. + if (HEADED && /--headed ignored|daemon already running/i.test(startOut)) { + throw new Error( + 'agent-browser ignored --headed because a daemon was already running; ' + + 'the recording would be headless. Ensure the daemon is stopped ' + + '(`agent-browser close`) before capture and retry.', + ); + } t0 = Date.now(); ab('eval', TICKER_JS); sleep(300); // let the first frames settle diff --git a/packages/env/src/__tests__/index.test.ts b/packages/env/src/__tests__/index.test.ts index 98a991ae6..d3de661ca 100644 --- a/packages/env/src/__tests__/index.test.ts +++ b/packages/env/src/__tests__/index.test.ts @@ -116,7 +116,7 @@ describe('Env', () => { expect(env.DOCKER_WORKER_CPU_LIMIT).toBe(2); expect(env.DOCKER_WORKER_MEMORY_LIMIT).toBe('4g'); expect(env.DOCKER_TASK_DAEMON_MEMORY_LIMIT).toBe('8g'); - expect(env.DOCKER_WORKER_PIDS_LIMIT).toBe(512); + expect(env.DOCKER_WORKER_PIDS_LIMIT).toBe(2048); expect(env.DOCKER_WORKER_DISK_LIMIT).toBe('20g'); expect(env.DOCKER_WORKER_ALLOW_UNBOUNDED_DISK).toBe(false); expect(env.DOCKER_WORKER_LOG_MAX_SIZE).toBe('10m'); diff --git a/packages/env/src/index.ts b/packages/env/src/index.ts index 4d55332ea..aab3248e8 100644 --- a/packages/env/src/index.ts +++ b/packages/env/src/index.ts @@ -85,7 +85,11 @@ const serverSchema = { DOCKER_WORKER_CPU_LIMIT: z.coerce.number().positive().default(2), DOCKER_WORKER_MEMORY_LIMIT: dockerSize().default('4g'), DOCKER_TASK_DAEMON_MEMORY_LIMIT: dockerSize().default('8g'), - DOCKER_WORKER_PIDS_LIMIT: z.coerce.number().int().positive().default(512), + // Headroom for browser-driving tasks: a Chrome process tree plus the + // agent-browser daemon and worker toolchain runs well over the old 512 cap + // under load. `--init` reaps zombies (see docker-sandbox-security), but the + // live process count still needs room. Env-overridable. + DOCKER_WORKER_PIDS_LIMIT: z.coerce.number().int().positive().default(2048), DOCKER_WORKER_DISK_LIMIT: dockerSize().default('20g'), DOCKER_WORKER_ALLOW_UNBOUNDED_DISK: optInBoolean(), DOCKER_WORKER_LOG_MAX_SIZE: dockerSize().default('10m'), From 0abbb3a3672d96c3787a8bfbb4be5668e2b74b7f Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 13:53:22 -0400 Subject: [PATCH 22/27] [Fix] Bump agent-browser 0.27.0 -> 0.33.2 for headed auto-Xvfb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.27 does not auto-start a virtual display for --headed, so headed recording died with 'Missing X server or $DISPLAY' (and, against an already-running daemon, '--headed ignored'). 0.33.2 auto-starts Xvfb on displayless Linux, which is what the feature-demo capture path needs. Verified in the worker image: headed recording of an ordinary page now succeeds end-to-end where 0.27 failed. (WebGL-heavy game surfaces can still hang the headed browser — a separate, known limitation the skill reports as a blocker.) --- .docker/sandbox/install-browser-agent.sh | 2 +- apps/worker/Dockerfile | 2 +- apps/worker/src/commands/setup/legacy-runtime-tools.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.docker/sandbox/install-browser-agent.sh b/.docker/sandbox/install-browser-agent.sh index e474470e5..c5cf9ad91 100644 --- a/.docker/sandbox/install-browser-agent.sh +++ b/.docker/sandbox/install-browser-agent.sh @@ -9,7 +9,7 @@ set -euo pipefail # 1. Worker image builds (Dockerfiles) to prebake browser automation. # 2. Worker setup compatibility fallbacks on older Linux images. -AGENT_BROWSER_VERSION="${AGENT_BROWSER_VERSION:-0.27.0}" +AGENT_BROWSER_VERSION="${AGENT_BROWSER_VERSION:-0.33.2}" AGENT_BROWSER_INSTALL_ROOT="${AGENT_BROWSER_INSTALL_ROOT:-/opt/agent-browser}" AGENT_BROWSER_EXECUTABLE_PATH="${AGENT_BROWSER_EXECUTABLE_PATH:-${AGENT_BROWSER_INSTALL_ROOT}/chrome}" AGENT_BROWSER_SAVED_CLI_PATH="${AGENT_BROWSER_SAVED_CLI_PATH:-${AGENT_BROWSER_INSTALL_ROOT}/.cli-path}" diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index 4ca8f80cc..60ea48c90 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -8,7 +8,7 @@ FROM ubuntu:24.04 AS runtime # to their source. LABEL org.opencontainers.image.source=https://github.com/RooCodeInc/Roomote -ARG AGENT_BROWSER_VERSION=0.27.0 +ARG AGENT_BROWSER_VERSION=0.33.2 ARG OPENCODE_CLI_VERSION=1.18.10 ARG FFMPEG_INSTALLER_VERSION=1.1.0 # @ffprobe-installer (not ffprobe-static, which ships no linux/arm64 binary diff --git a/apps/worker/src/commands/setup/legacy-runtime-tools.ts b/apps/worker/src/commands/setup/legacy-runtime-tools.ts index 5c5b1557c..b770ed7a7 100644 --- a/apps/worker/src/commands/setup/legacy-runtime-tools.ts +++ b/apps/worker/src/commands/setup/legacy-runtime-tools.ts @@ -10,7 +10,7 @@ import type { StartupLogger } from '../../logging'; import { isCommandAvailable } from './command-availability'; import { withAptLock } from './package-manager'; -const AGENT_BROWSER_VERSION = '0.27.0'; +const AGENT_BROWSER_VERSION = '0.33.2'; const AGENT_BROWSER_INSTALL_ROOT = '/opt/agent-browser'; From 58e37aa2333585dbc7f5d6048dda3a916d027686 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 17:33:44 -0400 Subject: [PATCH 23/27] [Feat] Plan feature demos through the advisor subagent The demo's story beats, captions, and selector candidates are creative planning work, so the skill now delegates that step to the existing advisor subagent (R_PLANNING_MODEL when configured, else the coding model at the advisor reasoning level) with a complete brief. The parent keeps ownership of everything the advisor cannot do: verifying proposed selectors against real page source, writing demo-script.json, and the capture/render pipeline. Follows the harness advisor contract: internal guidance only, one retry on empty output, then plan directly. --- .../workflows/__tests__/featureDemoSkill.test.ts | 13 +++++++++++++ .../workflows/skills/standard/feature-demo/SKILL.md | 7 ++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index 29419cc5b..9c9a32fc3 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -42,6 +42,19 @@ describe('feature-demo skill', () => { } }); + it('delegates demo planning to the advisor with parent-owned verification', () => { + expect(skillContent).toContain( + 'Delegate the creative plan to the `advisor` subagent with the Task tool', + ); + // The advisor cannot see the live page; the parent must verify selectors. + expect(skillContent).toContain( + "Treat the advisor's plan as internal guidance, not finished work", + ); + expect(skillContent).toContain( + 'Verify each selector the advisor proposed against the actual page source', + ); + }); + it('keeps browser work delegated to proof-runner', () => { expect(skillContent).toContain( "Browser automation is the proof-runner subagent's exclusive surface", diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index e55415d35..9f9afe912 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -37,9 +37,10 @@ this pipeline is ever committed to the repository. -Scope the demo +Scope and plan the demo (advisor) -Identify the feature, the surface to record (running app in the sandbox, or a public page the user names), and 2-5 story beats. A good demo is 10-25 seconds of motion; more beats than that dilutes it. +Delegate the creative plan to the `advisor` subagent with the Task tool — it runs the planning model at deeper reasoning and can read the repository to study the feature. Give it a complete brief: the user's request, the surface to record (URL), the relevant source paths for the feature's UI, the beat vocabulary and pacing constraints from step 2 verbatim, and the caption style rules. Ask it to return, as short final text: 2-5 story beats in order (each with the on-screen element to focus, a proposed resilient CSS selector from the source, a 3-9 word caption that doubles as the spoken narration line, and any click/type interactions the story needs), plus which preset(s) fit. A good demo is 10-25 seconds of motion; more beats than that dilutes it. +Treat the advisor's plan as internal guidance, not finished work: the advisor cannot run commands or see the live page, so its selectors are educated guesses from source. You own verification (step 2) and every artifact. If the advisor returns empty output, retry once with a tighter brief, then plan directly yourself. Pick presets: `wide` (1920x1080) is the default; add `vertical` (1080x1920) only when the user wants a social/short-form cut. If the surface needs the app running, make sure the environment is up first (dev server reachable) before capturing. @@ -57,7 +58,7 @@ this pipeline is ever committed to the repository. - `{ "a": "reset", "ms": ~600 }` — pull back between beats (partial by design; pass `"full": true` only for the closing shot). Keep the opening tight: one short `wait` for page load, then get moving. Dead opening seconds are trimmed automatically, but do not rely on it. Write captions as short, conversational spoken lines (they double as the narration): contractions are good, symbols and long clauses are not. 3-9 words on screen; if the spoken wording should differ from the on-screen caption, put the spoken variants in `/tmp/feature-demo/lines.json` (array, one string per caption). -Selectors must be resilient: prefer ids, stable data attributes, or unique semantic tags over deep CSS chains. The capture fails loudly on a selector that resolves to nothing. +Selectors must be resilient: prefer ids, stable data attributes, or unique semantic tags over deep CSS chains. Verify each selector the advisor proposed against the actual page source (grep the component/template files) before writing the script — the capture fails loudly on a selector that resolves to nothing, and a wrong selector costs a full capture round. From fe37ea3d089e8d5020afa100a2e593d576668abb Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 17:41:00 -0400 Subject: [PATCH 24/27] [Fix] Give external-page demos a proof-runner selector-verification path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Source-file selector verification is impossible for a public page the user names: no local source to grep, and the parent must not drive the browser. Split verification by surface — repository-backed surfaces grep source; external pages delegate a read-only resolve-only pre-flight to the proof-runner (open URL, snapshot/get box, confirm or correct each selector) before authoring the script. --- .../server/workflows/__tests__/featureDemoSkill.test.ts | 7 ++++++- .../server/workflows/skills/standard/feature-demo/SKILL.md | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index 9c9a32fc3..ed8ac8f56 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -50,8 +50,13 @@ describe('feature-demo skill', () => { expect(skillContent).toContain( "Treat the advisor's plan as internal guidance, not finished work", ); + // Repository-backed surfaces verify against source; external public + // pages have no local source, so selectors are validated through a + // read-only proof-runner pre-flight instead. + expect(skillContent).toContain('Repository-backed surface'); + expect(skillContent).toContain('External public page named by the user'); expect(skillContent).toContain( - 'Verify each selector the advisor proposed against the actual page source', + 'delegate a lightweight resolve-only brief to the `proof-runner`', ); }); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index 9f9afe912..3d87e2972 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -58,7 +58,9 @@ this pipeline is ever committed to the repository. - `{ "a": "reset", "ms": ~600 }` — pull back between beats (partial by design; pass `"full": true` only for the closing shot). Keep the opening tight: one short `wait` for page load, then get moving. Dead opening seconds are trimmed automatically, but do not rely on it. Write captions as short, conversational spoken lines (they double as the narration): contractions are good, symbols and long clauses are not. 3-9 words on screen; if the spoken wording should differ from the on-screen caption, put the spoken variants in `/tmp/feature-demo/lines.json` (array, one string per caption). -Selectors must be resilient: prefer ids, stable data attributes, or unique semantic tags over deep CSS chains. Verify each selector the advisor proposed against the actual page source (grep the component/template files) before writing the script — the capture fails loudly on a selector that resolves to nothing, and a wrong selector costs a full capture round. +Selectors must be resilient: prefer ids, stable data attributes, or unique semantic tags over deep CSS chains. Verify them before a full capture round (the capture fails loudly on a selector that resolves to nothing), by whichever path fits the surface: +- Repository-backed surface (the app's own UI): grep the component/template source for each proposed selector and confirm it exists. +- External public page named by the user (no local source, and the parent must not drive the browser directly): you cannot grep it. Instead, first delegate a lightweight resolve-only brief to the `proof-runner` — ask it to open the URL and report, via `agent-browser snapshot`/`get box`, whether each proposed selector resolves and its box, returning corrected selectors where they do not. Author the script from what it confirms. (This is the same delegated browser surface capture uses, just a read-only pre-flight.) From 2be7dcb8c4df058405cca8fac47166afd16dde30 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 17:48:36 -0400 Subject: [PATCH 25/27] [Perf] Pre-install feature-demo render deps in the worker image Baking the chrome-headless-shell but not the render project's deps left every demo paying a multi-minute Remotion + React npm install. Bake node_modules at /opt/feature-demo/render from the skill's own pinned package.json (single source of truth, no drift), and have the render step reuse them (cp the baked node_modules into the work-dir copy); npm install is now only the fallback for an adapted composition that adds deps or an older snapshot. Validated in the worker image: rendered fully offline (npx --offline) reusing the baked modules, no install. --- apps/worker/Dockerfile | 13 +++++++++++++ .../workflows/__tests__/featureDemoSkill.test.ts | 11 +++++++++++ .../skills/standard/feature-demo/SKILL.md | 2 +- .../compute-providers/src/adapters/modal.test.ts | 14 ++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/apps/worker/Dockerfile b/apps/worker/Dockerfile index 60ea48c90..f1a2a8efa 100644 --- a/apps/worker/Dockerfile +++ b/apps/worker/Dockerfile @@ -251,6 +251,19 @@ RUN sudo mkdir -p /opt/remotion \ && cd / \ && rm -rf "$REMOTION_TMP" +# Pre-install the feature-demo render project's dependencies (Remotion + +# React) so demos reuse them instead of running a multi-minute npm install on +# every render. Keyed off the skill's own pinned package.json (the single +# source of truth), so the baked modules cannot drift from what the render +# project expects. The skill reuses /opt/feature-demo/render/node_modules and +# falls back to npm install only when it is absent or an adapted composition +# adds dependencies. +COPY packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json /opt/feature-demo/render/package.json +RUN sudo chown -R roomote:roomote /opt/feature-demo \ + && cd /opt/feature-demo/render \ + && npm install --no-audit --no-fund \ + && printf '%s\n' "ready" > /opt/feature-demo/render/.installed + RUN npm install --prefix /sandbox --no-save --no-package-lock \ "node-pty@${NODE_PTY_VERSION}" \ "opencode-ai@${OPENCODE_CLI_VERSION}" \ diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index ed8ac8f56..977879824 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -138,6 +138,17 @@ describe('feature-demo skill', () => { expect(skillContent).toContain('npx remotion browser ensure'); }); + it('reuses the image-baked render dependencies instead of installing', () => { + // Fast path copies the baked node_modules; npm install is only the + // adapted-deps / old-snapshot fallback. + expect(skillContent).toContain( + 'cp -R /opt/feature-demo/render/node_modules /tmp/feature-demo/render/node_modules', + ); + expect(skillContent).toContain( + 'only if you adapted the composition to add dependencies', + ); + }); + it('delivers through manage_artifacts with canonical URLs only', () => { expect(skillContent).toContain('manage_artifacts'); expect(skillContent).toContain('never invent URLs'); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index 3d87e2972..44196ba93 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -103,7 +103,7 @@ this pipeline is ever committed to the repository. - `mkdir -p /tmp/feature-demo/render/public && cp /tmp/feature-demo/work/recording.mp4 /tmp/feature-demo/render/public/` - `cp -R /tmp/feature-demo/work/vo /tmp/feature-demo/render/public/vo` (narrated demos only) Adapt the copied composition when the default look does not fit the request — different backdrop, caption treatment, brand colors, an extra preset, a layout change. Before writing Remotion code, install Remotion's official agent skills into the render copy and read them for current API guidance: `cd /tmp/feature-demo/render && npx -y skills add remotion-dev/skills`. Preserve the pieces that encode hard-won correctness unless you have a reason not to: the zoom transform with its counter-scaled cursor, the edge-clamp guard (only clamp an axis when the scaled window exceeds the canvas), and the timeline-driven keyframe interpolation. -`cd /tmp/feature-demo/render && npm install` (versions are pinned; takes a few minutes on first run). +Provide the dependencies. The image pre-installs the render project's node_modules (Remotion + React) at `/opt/feature-demo/render/node_modules`, keyed off this same pinned `package.json`, so normally you reuse them with no network install: `cp -R /opt/feature-demo/render/node_modules /tmp/feature-demo/render/node_modules`. Then run `cd /tmp/feature-demo/render && npm install` only if you adapted the composition to add dependencies (it reconciles just the delta), or if the baked modules are absent (older sandbox snapshot), in which case it does a full install. Render with the image's baked headless shell: `npx remotion render src/index.ts Demo-wide out/demo-wide.mp4 --browser-executable="${REMOTION_HEADLESS_SHELL_PATH:-/opt/remotion/headless-shell}" --log=error` diff --git a/packages/compute-providers/src/adapters/modal.test.ts b/packages/compute-providers/src/adapters/modal.test.ts index 80d766d32..157361fa2 100644 --- a/packages/compute-providers/src/adapters/modal.test.ts +++ b/packages/compute-providers/src/adapters/modal.test.ts @@ -991,6 +991,20 @@ describe('ModalClient', () => { expect(dockerfile).toContain('/opt/remotion/.installed'); }); + it('pre-installs the feature-demo render dependencies from the skill package.json', () => { + const dockerfile = fs.readFileSync( + new URL('../../../../apps/worker/Dockerfile', import.meta.url), + 'utf8', + ); + + // Keyed off the skill's own pinned package.json so the baked node_modules + // cannot drift from what the render project expects. + expect(dockerfile).toContain( + 'COPY packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/render/package.json /opt/feature-demo/render/package.json', + ); + expect(dockerfile).toContain('/opt/feature-demo/render/.installed'); + }); + it('expects PM2 to be available in the worker runtime', () => { const dockerfile = fs.readFileSync( new URL('../../../../apps/worker/Dockerfile', import.meta.url), From 86e50995f095100fcb4520a68f8953e774452fed Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 17:49:25 -0400 Subject: [PATCH 26/27] [Fix] External-page selectors validate via capture's own loud failure The previous pre-flight conflicted with the proof-runner contract (fixed browser target, one brief per task), so it was not actually usable. For an external public page the browser is reachable only through the single capture delegation, and the capture runner already resolves every selector live and fails loudly naming any that miss. Author best-effort resilient selectors and correct them from that named failure within the allowed retry, rather than inventing a second browser trip. --- .../server/workflows/__tests__/featureDemoSkill.test.ts | 7 ++++--- .../server/workflows/skills/standard/feature-demo/SKILL.md | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index 977879824..46c6f966c 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -51,12 +51,13 @@ describe('feature-demo skill', () => { "Treat the advisor's plan as internal guidance, not finished work", ); // Repository-backed surfaces verify against source; external public - // pages have no local source, so selectors are validated through a - // read-only proof-runner pre-flight instead. + // pages have no local source and no separate browser pre-flight (the + // proof-runner takes one brief per task), so they rely on capture's own + // loud per-selector failure within the allowed retry. expect(skillContent).toContain('Repository-backed surface'); expect(skillContent).toContain('External public page named by the user'); expect(skillContent).toContain( - 'delegate a lightweight resolve-only brief to the `proof-runner`', + 'the runner resolves every selector live and fails loudly naming any that do not resolve', ); }); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index 44196ba93..38a6eec6e 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -58,9 +58,9 @@ this pipeline is ever committed to the repository. - `{ "a": "reset", "ms": ~600 }` — pull back between beats (partial by design; pass `"full": true` only for the closing shot). Keep the opening tight: one short `wait` for page load, then get moving. Dead opening seconds are trimmed automatically, but do not rely on it. Write captions as short, conversational spoken lines (they double as the narration): contractions are good, symbols and long clauses are not. 3-9 words on screen; if the spoken wording should differ from the on-screen caption, put the spoken variants in `/tmp/feature-demo/lines.json` (array, one string per caption). -Selectors must be resilient: prefer ids, stable data attributes, or unique semantic tags over deep CSS chains. Verify them before a full capture round (the capture fails loudly on a selector that resolves to nothing), by whichever path fits the surface: -- Repository-backed surface (the app's own UI): grep the component/template source for each proposed selector and confirm it exists. -- External public page named by the user (no local source, and the parent must not drive the browser directly): you cannot grep it. Instead, first delegate a lightweight resolve-only brief to the `proof-runner` — ask it to open the URL and report, via `agent-browser snapshot`/`get box`, whether each proposed selector resolves and its box, returning corrected selectors where they do not. Author the script from what it confirms. (This is the same delegated browser surface capture uses, just a read-only pre-flight.) +Selectors must be resilient: prefer ids, stable data attributes, or unique semantic tags over deep CSS chains. How you gain confidence before capture depends on the surface: +- Repository-backed surface (the app's own UI): grep the component/template source for each proposed selector and confirm it exists — cheap and worth doing every time. +- External public page named by the user: there is no local source to grep, and the browser is reachable only through the single capture delegation (the `proof-runner` takes one brief per task and keeps its configured surface, so a separate pre-flight is not available). Author best-effort resilient selectors — landmark roles, headings, obvious ids the advisor inferred; avoid deep chains — and rely on capture's own validation: the runner resolves every selector live and fails loudly naming any that do not resolve. Use that named failure to correct the script and re-capture within the one allowed retry (step 3). From b27d504f086a4e917d9d150f7794997045c6723c Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Sat, 8 Aug 2026 18:01:19 -0400 Subject: [PATCH 27/27] [Refactor] Install only the three relevant Remotion skills for adaptation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adapting an existing composition needs markup, render, and doc-lookup guidance — not the full remotion-dev/skills bundle (which also pulls create/maps/saas/upgrade/interactivity). Pin the install to --skill remotion-markup --skill remotion-render --skill remotion-docs --yes. Verified the exact CLI syntax in the worker image: installs exactly those three, non-interactively, project scope, no warnings. --- .../src/server/workflows/__tests__/featureDemoSkill.test.ts | 6 +++++- .../server/workflows/skills/standard/feature-demo/SKILL.md | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts index 46c6f966c..27add1ac1 100644 --- a/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts +++ b/packages/cloud-agents/src/server/workflows/__tests__/featureDemoSkill.test.ts @@ -126,7 +126,11 @@ describe('feature-demo skill', () => { it('frames the render project as an adaptable reference template', () => { expect(skillContent).toContain('**reference template**'); - expect(skillContent).toContain('npx -y skills add remotion-dev/skills'); + // Pin to the three skills relevant to editing a composition, not the + // full bundle. + expect(skillContent).toContain( + 'npx -y skills add remotion-dev/skills --skill remotion-markup --skill remotion-render --skill remotion-docs --yes', + ); expect(skillContent).toContain( 'The timeline JSON schema is the stable contract', ); diff --git a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md index 38a6eec6e..732d994ba 100644 --- a/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md +++ b/packages/cloud-agents/src/server/workflows/skills/standard/feature-demo/SKILL.md @@ -102,7 +102,7 @@ this pipeline is ever committed to the repository. - `cp /tmp/feature-demo/work/timeline.json /tmp/feature-demo/work/narration.json /tmp/feature-demo/render/props/` (skip narration.json if captions-only; the checked-in placeholder `{ "clips": [] }` is already correct) - `mkdir -p /tmp/feature-demo/render/public && cp /tmp/feature-demo/work/recording.mp4 /tmp/feature-demo/render/public/` - `cp -R /tmp/feature-demo/work/vo /tmp/feature-demo/render/public/vo` (narrated demos only) -Adapt the copied composition when the default look does not fit the request — different backdrop, caption treatment, brand colors, an extra preset, a layout change. Before writing Remotion code, install Remotion's official agent skills into the render copy and read them for current API guidance: `cd /tmp/feature-demo/render && npx -y skills add remotion-dev/skills`. Preserve the pieces that encode hard-won correctness unless you have a reason not to: the zoom transform with its counter-scaled cursor, the edge-clamp guard (only clamp an axis when the scaled window exceeds the canvas), and the timeline-driven keyframe interpolation. +Adapt the copied composition when the default look does not fit the request — different backdrop, caption treatment, brand colors, an extra preset, a layout change. Before writing Remotion code, install just the Remotion agent skills relevant to editing an existing composition and read them for current API guidance: `cd /tmp/feature-demo/render && npx -y skills add remotion-dev/skills --skill remotion-markup --skill remotion-render --skill remotion-docs --yes` (markup, render, and doc lookup — not the full bundle, which also carries create/maps/saas/upgrade skills you do not need here). Preserve the pieces that encode hard-won correctness unless you have a reason not to: the zoom transform with its counter-scaled cursor, the edge-clamp guard (only clamp an axis when the scaled window exceeds the canvas), and the timeline-driven keyframe interpolation. Provide the dependencies. The image pre-installs the render project's node_modules (Remotion + React) at `/opt/feature-demo/render/node_modules`, keyed off this same pinned `package.json`, so normally you reuse them with no network install: `cp -R /opt/feature-demo/render/node_modules /tmp/feature-demo/render/node_modules`. Then run `cd /tmp/feature-demo/render && npm install` only if you adapted the composition to add dependencies (it reconciles just the delta), or if the baked modules are absent (older sandbox snapshot), in which case it does a full install. Render with the image's baked headless shell: