From 08c8749b0c1ee6eb32a0c2029bd36654becbf913 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 19 Aug 2026 12:41:53 -0700 Subject: [PATCH 1/7] fix(web): make expandable section titles keyboard accessible (#42310) --- .../trace-viewer/src/ui/attachmentsTab.tsx | 10 ++--- .../src/ui/networkResourceDetails.tsx | 10 ++--- packages/web/src/common.css | 6 +++ packages/web/src/components/expandable.css | 21 +++++++++ .../web/src/components/expandable.spec.ts | 30 ++++++++++++- .../web/src/components/expandable.story.tsx | 10 ++++- packages/web/src/components/expandable.tsx | 45 +++++++++++-------- .../ui-mode-test-network-tab.spec.ts | 29 ++++++++++++ 8 files changed, 129 insertions(+), 32 deletions(-) diff --git a/packages/trace-viewer/src/ui/attachmentsTab.tsx b/packages/trace-viewer/src/ui/attachmentsTab.tsx index ede74609510c3..e69a9db980af9 100644 --- a/packages/trace-viewer/src/ui/attachmentsTab.tsx +++ b/packages/trace-viewer/src/ui/attachmentsTab.tsx @@ -67,16 +67,14 @@ const ExpandableAttachment: React.FunctionComponent = return Math.min(Math.max(5, lineCount), 20) * lineHeight; }, [attachmentText]); - const title = - {linkifyText(attachment.name)} - {hasContent && download} - ; + const title = {linkifyText(attachment.name)}; + const downloadLink = hasContent && download; if (!isTextAttachment || !hasContent) - return
{title}
; + return
{title}{downloadLink}
; return
- + {placeholder && {placeholder}} {expanded && attachmentText !== null &&
diff --git a/packages/trace-viewer/src/ui/networkResourceDetails.tsx b/packages/trace-viewer/src/ui/networkResourceDetails.tsx index 9a50112d885ad..392aa0a0ad3ac 100644 --- a/packages/trace-viewer/src/ui/networkResourceDetails.tsx +++ b/packages/trace-viewer/src/ui/networkResourceDetails.tsx @@ -140,13 +140,11 @@ const ExpandableSection: React.FC<{ setExpanded={setExpanded} expandOnTitleClick title={ - <> - {title} - {showCount && × {data?.length ?? 0}} - - { titleChildren } - + {title} + {showCount && × {data?.length ?? 0}} + } + titleSuffix={titleChildren} className={className} > {data && diff --git a/packages/web/src/common.css b/packages/web/src/common.css index 19f7cdc5749bf..f9ff41c41d1f9 100644 --- a/packages/web/src/common.css +++ b/packages/web/src/common.css @@ -55,6 +55,12 @@ a { color: var(--vscode-textLink-foreground); } +a:focus-visible { + outline: 1px solid var(--vscode-focusBorder); + outline-offset: 1px; + border-radius: 2px; +} + dialog { border: none; padding: 0; diff --git a/packages/web/src/components/expandable.css b/packages/web/src/components/expandable.css index 5127c0df7dc94..ba69850fadfa7 100644 --- a/packages/web/src/components/expandable.css +++ b/packages/web/src/components/expandable.css @@ -27,9 +27,30 @@ align-items: center; white-space: nowrap; user-select: none; +} + +.expandable-title-button { + all: unset; + display: flex; + align-items: center; + padding-right: 8px; + border-radius: 4px; cursor: pointer; } +.expandable-title > a { + padding: 0 4px; + border-radius: 4px; +} + +/* Links are flex items here, as tall as the row; the inset ring keeps the + content below from covering it. */ +.expandable-title-button:focus-visible, +.expandable-title > a:focus-visible { + outline: 1px solid var(--vscode-focusBorder); + outline-offset: -1px; +} + .expandable-content { margin-left: 25px; } diff --git a/packages/web/src/components/expandable.spec.ts b/packages/web/src/components/expandable.spec.ts index 660a97162f0ec..59a42358d26b6 100644 --- a/packages/web/src/components/expandable.spec.ts +++ b/packages/web/src/components/expandable.spec.ts @@ -16,7 +16,7 @@ import { expect, test } from '@playwright/test'; -import type { Collapsed, Expanded, Stateful, StatefulTitleClick } from './expandable.story'; +import type { Collapsed, Expanded, Stateful, StatefulTitleClick, StatefulTitleSuffix } from './expandable.story'; test.use({ viewport: { width: 500, height: 500 } }); @@ -53,3 +53,31 @@ test('title click should not expand by default', async ({ mount }) => { await expect(component.locator('.codicon-chevron-right')).toBeVisible(); await expect(component.getByTestId('expanded')).toHaveValue('false'); }); + +test('keyboard should toggle when title click is enabled', async ({ mount }) => { + const component = await mount('components/expandable/StatefulTitleClick'); + const button = component.getByRole('button', { name: 'Title' }); + await button.focus(); + await expect(button).toBeFocused(); + await button.press('Enter'); + await expect(component.getByTestId('expanded')).toHaveValue('true'); + await button.press(' '); + await expect(component.getByTestId('expanded')).toHaveValue('false'); +}); + +test('keyboard should toggle via chevron button by default', async ({ mount }) => { + const component = await mount('components/expandable/Stateful'); + const button = component.getByRole('button', { name: 'Expand' }); + await button.focus(); + await button.press('Enter'); + await expect(component.getByTestId('expanded')).toHaveValue('true'); + await expect(component.getByRole('button', { name: 'Collapse' })).toBeVisible(); +}); + +test('title suffix should render outside the toggle button', async ({ mount }) => { + const component = await mount('components/expandable/StatefulTitleSuffix'); + await component.getByRole('link', { name: 'download' }).click(); + await expect(component.getByTestId('expanded')).toHaveValue('false'); + await component.getByRole('button', { name: 'Title' }).click(); + await expect(component.getByTestId('expanded')).toHaveValue('true'); +}); diff --git a/packages/web/src/components/expandable.story.tsx b/packages/web/src/components/expandable.story.tsx index 99796b8135bd6..72aac5e0a2b32 100644 --- a/packages/web/src/components/expandable.story.tsx +++ b/packages/web/src/components/expandable.story.tsx @@ -26,7 +26,7 @@ export const Expanded = () => export const Stateful = () => { const [expanded, setExpanded] = React.useState(false); return <> - Details text + Title}>Details text ; }; @@ -38,3 +38,11 @@ export const StatefulTitleClick = () => { ; }; + +export const StatefulTitleSuffix = () => { + const [expanded, setExpanded] = React.useState(false); + return <> + download}>Details text + + ; +}; diff --git a/packages/web/src/components/expandable.tsx b/packages/web/src/components/expandable.tsx index 7bf551bc319fb..83cc35833aa7f 100644 --- a/packages/web/src/components/expandable.tsx +++ b/packages/web/src/components/expandable.tsx @@ -23,8 +23,9 @@ export const Expandable: React.FunctionComponent void, expanded: boolean, expandOnTitleClick?: boolean, + titleSuffix?: React.ReactNode, className?: string; -}>> = ({ title, children, setExpanded, expanded, expandOnTitleClick, className }) => { +}>> = ({ title, children, setExpanded, expanded, expandOnTitleClick, titleSuffix, className }) => { const titleId = React.useId(); const regionId = React.useId(); @@ -32,25 +33,33 @@ export const Expandable: React.FunctionComponent; + style={{ color: 'var(--vscode-foreground)', marginLeft: '5px' }} />; return
- {expandOnTitleClick ? -
- {chevron} - {title} -
: -
- {chevron} - {title} -
} +
+ {expandOnTitleClick ? + : + <> + + {title} + } + {titleSuffix} +
{expanded &&
{children}
}
; }; diff --git a/tests/playwright-test/ui-mode-test-network-tab.spec.ts b/tests/playwright-test/ui-mode-test-network-tab.spec.ts index 9de7f184242c6..da3369f843b53 100644 --- a/tests/playwright-test/ui-mode-test-network-tab.spec.ts +++ b/tests/playwright-test/ui-mode-test-network-tab.spec.ts @@ -429,6 +429,35 @@ test('should toggle sections inside network details', async ({ runUITest, server await expect(headersPanel.getByRole('region', { name: 'General' })).toContainText(/Start.+Duration\d+ms/); }); +test('should toggle sections inside network details with keyboard', { + annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/42263' }, +}, async ({ runUITest, server }) => { + const { page } = await runUITest({ + 'network-tab.test.ts': ` + import { test, expect } from '@playwright/test'; + test('network tab test', async ({ page }) => { + await page.goto('${server.PREFIX}/network-tab/network.html'); + await page.evaluate(() => (window as any).donePromise); + }); + `, + }); + + await page.getByRole('treeitem', { name: 'network tab test' }).dblclick(); + await expect(page.getByTestId('workbench-run-status')).toContainText('Passed'); + + await page.getByRole('tab', { name: 'Network' }).click(); + await page.getByRole('option').filter({ hasText: 'post-data-1' }).click(); + const headersPanel = page.getByRole('tabpanel', { name: 'Headers' }); + + const header = headersPanel.getByRole('button', { name: 'Request Headers × 16' }); + await header.focus(); + await expect(header).toBeFocused(); + await header.press('Enter'); + await expect(headersPanel.getByRole('region', { name: 'Request Headers × 16' })).toBeHidden(); + await header.press(' '); + await expect(headersPanel.getByRole('region', { name: 'Request Headers × 16' })).toBeVisible(); +}); + test('should copy network request', async ({ runUITest, server }) => { const { page } = await runUITest({ 'network-tab.test.ts': ` From 28b4fea796d7d78810d0e8dafb1172543a7e7b4d Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 19 Aug 2026 12:43:23 -0700 Subject: [PATCH 2/7] fix(web): make image diff mode switcher keyboard accessible (#42311) --- packages/web/src/shared/imageDiffView.css | 30 +++++++++++++++++++ packages/web/src/shared/imageDiffView.spec.ts | 16 ++++++++++ packages/web/src/shared/imageDiffView.tsx | 25 ++++++++-------- tests/playwright-test/reporter-html.spec.ts | 2 +- 4 files changed, 60 insertions(+), 13 deletions(-) create mode 100644 packages/web/src/shared/imageDiffView.css diff --git a/packages/web/src/shared/imageDiffView.css b/packages/web/src/shared/imageDiffView.css new file mode 100644 index 0000000000000..6a37ecec9c728 --- /dev/null +++ b/packages/web/src/shared/imageDiffView.css @@ -0,0 +1,30 @@ +/* + Copyright (c) Microsoft Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +.image-diff-mode { + all: unset; + flex: none; + margin: 0 10px; + cursor: pointer; + user-select: none; + border-radius: 4px; +} + +/* The trace viewer defines the vscode theme variables, the html reporter the color ones. */ +.image-diff-mode:focus-visible { + outline: 1px solid var(--vscode-focusBorder, var(--color-accent-fg)); + outline-offset: 2px; +} diff --git a/packages/web/src/shared/imageDiffView.spec.ts b/packages/web/src/shared/imageDiffView.spec.ts index 590b0f050bbe7..73a5b8d4a7f5b 100644 --- a/packages/web/src/shared/imageDiffView.spec.ts +++ b/packages/web/src/shared/imageDiffView.spec.ts @@ -35,3 +35,19 @@ test('should show diff by default', async ({ mount }) => { const box = await image.boundingBox(); expect(box).toEqual(expect.objectContaining({ width: 48, height: 48 })); }); + +test('should switch mode with keyboard', async ({ mount }) => { + const component = await mount('shared/imageDiffView/Default'); + const sxs = component.getByRole('tab', { name: 'Side by side' }); + await sxs.focus(); + await expect(sxs).toBeFocused(); + await sxs.press('Enter'); + await expect(sxs).toHaveAttribute('aria-selected', 'true'); + await expect(component.locator('img')).toHaveCount(2); + + const diffTab = component.getByRole('tab', { name: 'Diff' }); + await diffTab.focus(); + await diffTab.press(' '); + await expect(diffTab).toHaveAttribute('aria-selected', 'true'); + await expect(component.locator('img')).toHaveCount(1); +}); diff --git a/packages/web/src/shared/imageDiffView.tsx b/packages/web/src/shared/imageDiffView.tsx index 70c937460d23f..b34393c69c85b 100644 --- a/packages/web/src/shared/imageDiffView.tsx +++ b/packages/web/src/shared/imageDiffView.tsx @@ -15,6 +15,7 @@ */ import * as React from 'react'; +import './imageDiffView.css'; import { useMeasure } from '../uiUtils'; import { ResizeView } from './resizeView'; @@ -90,20 +91,20 @@ export const ImageDiffView: React.FC<{ const fitWidth = imageWidth * scale; const fitHeight = imageHeight * scale; - const modeStyle: React.CSSProperties = { - flex: 'none', - margin: '0 10px', - cursor: 'pointer', - userSelect: 'none', - }; + const modeTab = (m: typeof mode, title: string) => ; return
{isLoaded && <> -
- {diff.diff &&
setMode('diff')}>Diff
} -
setMode('actual')}>Actual
-
setMode('expected')}>{expectedImageTitle}
-
setMode('sxs')}>Side by side
-
setMode('slider')}>Slider
+
+ {diff.diff && modeTab('diff', 'Diff')} + {modeTab('actual', 'Actual')} + {modeTab('expected', expectedImageTitle)} + {modeTab('sxs', 'Side by side')} + {modeTab('slider', 'Slider')}
{diff.diff && mode === 'diff' && } diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index fc14d4d7f80e0..96e4e1907bf80 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -182,7 +182,7 @@ for (const useIntermediateMergeReport of [true, false] as const) { await expect(page.locator('text=Image mismatch')).toBeVisible(); await expect(page.locator('text=Snapshot mismatch')).toHaveCount(0); - await expect(page.getByTestId('test-screenshot-error-view').getByTestId('test-result-image-mismatch-tabs').locator('div')).toHaveText([ + await expect(page.getByTestId('test-screenshot-error-view').getByTestId('test-result-image-mismatch-tabs').getByRole('tab')).toHaveText([ 'Diff', 'Actual', 'Expected', From 642dcfd46c244e9bb4b73c65dba8f10ea6e46bb1 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 19 Aug 2026 12:44:15 -0700 Subject: [PATCH 3/7] docs(python): document that task cancellation is not supported (#42312) --- docs/src/library-python.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/src/library-python.md b/docs/src/library-python.md index 9dcf0cba99e10..6ec0eeb1db998 100644 --- a/docs/src/library-python.md +++ b/docs/src/library-python.md @@ -199,3 +199,7 @@ On Windows Python 3.7, Playwright sets the default event loop to `ProactorEventL ### Threading Playwright's API is not thread-safe. If you are using Playwright in a multi-threaded environment, you should create a playwright instance per thread. See [threading issue](https://github.com/microsoft/playwright-python/issues/623) for more details. + +### Cancelling `asyncio` tasks + +Cancelling a task that is running a Playwright call is not supported and results in undefined behavior. If an operation has to outlive its caller, run it in a separate task and protect it with [`asyncio.shield()`](https://docs.python.org/3/library/asyncio-task.html#asyncio.shield). From 582db26a9e22ba3f9926960263cc458b773b47de Mon Sep 17 00:00:00 2001 From: "microsoft-playwright-automation[bot]" <203992400+microsoft-playwright-automation[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:57:15 -0700 Subject: [PATCH 4/7] feat(webkit): roll to r2354 (#42303) --- packages/playwright-core/browsers.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playwright-core/browsers.json b/packages/playwright-core/browsers.json index a4981d493dfdd..55dabd29676af 100644 --- a/packages/playwright-core/browsers.json +++ b/packages/playwright-core/browsers.json @@ -24,7 +24,7 @@ }, { "name": "webkit", - "revision": "2349", + "revision": "2354", "installByDefault": true, "revisionOverrides": { "mac14": "2251", From 1a79e3e7fb4d7c3e14c81547b07db3eaa5a90534 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 19 Aug 2026 16:06:57 -0700 Subject: [PATCH 5/7] test(credentials): httpCredentials should not override page's Authorization header (#42313) --- .../browsercontext-credentials.spec.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/library/browsercontext-credentials.spec.ts b/tests/library/browsercontext-credentials.spec.ts index 4713245ff90a3..e420cb467a568 100644 --- a/tests/library/browsercontext-credentials.spec.ts +++ b/tests/library/browsercontext-credentials.spec.ts @@ -227,3 +227,22 @@ it('should fail with correct credentials and mismatching port', async ({ browser expect(responseOrError.status()).toBe(401); await context.close(); }); + +it('should not override Authorization header set by the page', async ({ browser, server }) => { + server.setAuth('/empty.html', 'user', 'pass'); + server.setRoute('/echo-auth', (req, res) => { + res.end(req.headers['authorization'] || ''); + }); + const context = await browser.newContext({ + httpCredentials: { username: 'user', password: 'pass' } + }); + const page = await context.newPage(); + const response = await page.goto(server.EMPTY_PAGE); + expect(response!.status()).toBe(200); + const received = await page.evaluate(async () => { + const response = await fetch('/echo-auth', { headers: { 'Authorization': 'Bearer my-own-app-token' } }); + return await response.text(); + }); + expect(received).toBe('Bearer my-own-app-token'); + await context.close(); +}); From 644132a6326cb12b0549b0f3c57071143c22b2bb Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Wed, 19 Aug 2026 16:47:06 -0700 Subject: [PATCH 6/7] fix(cli): add .playwright-cli/ to .gitignore on workspace install (#42318) --- .../src/tools/cli-daemon/program.ts | 17 +++++++++++++++++ tests/mcp/cli-misc.spec.ts | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/packages/playwright-core/src/tools/cli-daemon/program.ts b/packages/playwright-core/src/tools/cli-daemon/program.ts index 07c216829a998..40b5f5337765a 100644 --- a/packages/playwright-core/src/tools/cli-daemon/program.ts +++ b/packages/playwright-core/src/tools/cli-daemon/program.ts @@ -93,6 +93,7 @@ export async function initWorkspace(initSkills: string | undefined, initSkillsGl const playwrightDir = path.join(cwd, '.playwright'); await fs.promises.mkdir(playwrightDir, { recursive: true }); console.log(`✅ Workspace initialized at \`${cwd}\`.`); + await patchGitIgnore(cwd); } const skills = initSkillsGlobal ?? initSkills; @@ -111,6 +112,22 @@ export async function initWorkspace(initSkills: string | undefined, initSkillsGl await ensureConfiguredBrowserInstalled(); } +async function patchGitIgnore(cwd: string) { + if (!fs.existsSync(path.join(cwd, '.git'))) + return; + try { + const gitIgnorePath = path.join(cwd, '.gitignore'); + const existing = await fs.promises.readFile(gitIgnorePath, 'utf8').catch(() => ''); + if (existing.split('\n').some(line => line.trim() === '.playwright-cli/')) + return; + const separator = existing && !existing.endsWith('\n') ? '\n' : ''; + await fs.promises.appendFile(gitIgnorePath, separator + '# Playwright CLI output (may contain credentials)\n.playwright-cli/\n'); + console.log('✅ Added `.playwright-cli/` to `.gitignore`.'); + } catch (error) { + console.log(`⚠️ Failed to update \`.gitignore\`: ${error instanceof Error ? error.message : error}`); + } +} + async function ensureConfiguredBrowserInstalled() { if (getAsBooleanFromENV('PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD')) return; diff --git a/tests/mcp/cli-misc.spec.ts b/tests/mcp/cli-misc.spec.ts index a3e8630d56634..c99f422704f1f 100644 --- a/tests/mcp/cli-misc.spec.ts +++ b/tests/mcp/cli-misc.spec.ts @@ -40,6 +40,23 @@ test('install workspace', async ({ cli }, testInfo) => { expect(fs.existsSync(playwrightDir)).toBe(true); }); +test('install adds .playwright-cli/ to .gitignore', async ({ cli }, testInfo) => { + const outsideGitRepo = await cli('install'); + expect(outsideGitRepo.output).not.toContain('.gitignore'); + expect(fs.existsSync(testInfo.outputPath('.gitignore'))).toBe(false); + + await fs.promises.mkdir(testInfo.outputPath('.git'), { recursive: true }); + await fs.promises.writeFile(testInfo.outputPath('.gitignore'), 'node_modules/'); + const insideGitRepo = await cli('install'); + expect(insideGitRepo.output).toContain('Added `.playwright-cli/` to `.gitignore`.'); + const expectedContent = 'node_modules/\n# Playwright CLI output (may contain credentials)\n.playwright-cli/\n'; + expect(await fs.promises.readFile(testInfo.outputPath('.gitignore'), 'utf8')).toBe(expectedContent); + + const secondRun = await cli('install'); + expect(secondRun.output).not.toContain('.gitignore'); + expect(await fs.promises.readFile(testInfo.outputPath('.gitignore'), 'utf8')).toBe(expectedContent); +}); + test('install workspace w/skills', async ({ cli }, testInfo) => { const { output } = await cli('install', '--skills'); expect(output).toContain(`Skill installed to \`.claude${path.sep}skills${path.sep}playwright-cli\`.`); From 0d86401890abdcc2f99d78baae5022b42cdddd06 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Wed, 19 Aug 2026 18:30:19 -0700 Subject: [PATCH 7/7] feat(trace-viewer): webm-based film strip (#42319) --- packages/isomorphic/trace/entries.ts | 1 + packages/isomorphic/trace/traceLoader.ts | 1 + packages/isomorphic/trace/traceModel.ts | 5 + packages/isomorphic/trace/traceModernizer.ts | 4 + packages/trace-viewer/src/ui/filmStrip.tsx | 88 ++++++++++-- .../trace-viewer/src/ui/videoThumbnails.ts | 127 ++++++++++++++++++ packages/trace/src/trace.ts | 9 ++ 7 files changed, 225 insertions(+), 10 deletions(-) create mode 100644 packages/trace-viewer/src/ui/videoThumbnails.ts diff --git a/packages/isomorphic/trace/entries.ts b/packages/isomorphic/trace/entries.ts index 09c71959f30b1..a762d285bdd54 100644 --- a/packages/isomorphic/trace/entries.ts +++ b/packages/isomorphic/trace/entries.ts @@ -39,6 +39,7 @@ export type ContextEntry = { actions: ActionEntry[]; screenshots: trace.ScreenshotTraceEvent[]; ariaSnapshots: trace.AriaSnapshotTraceEvent[]; + videos: trace.VideoTraceEvent[]; events: (trace.EventTraceEvent | trace.ConsoleMessageTraceEvent)[]; stdio: trace.StdioTraceEvent[]; errors: trace.ErrorTraceEvent[]; diff --git a/packages/isomorphic/trace/traceLoader.ts b/packages/isomorphic/trace/traceLoader.ts index 40c0b0e4ba010..9daa159473037 100644 --- a/packages/isomorphic/trace/traceLoader.ts +++ b/packages/isomorphic/trace/traceLoader.ts @@ -152,6 +152,7 @@ function createEmptyContext(): ContextEntry { actions: [], screenshots: [], ariaSnapshots: [], + videos: [], events: [], errors: [], stdio: [], diff --git a/packages/isomorphic/trace/traceModel.ts b/packages/isomorphic/trace/traceModel.ts index 27185f56e72a1..459088a7e8305 100644 --- a/packages/isomorphic/trace/traceModel.ts +++ b/packages/isomorphic/trace/traceModel.ts @@ -66,6 +66,7 @@ export class TraceModel { readonly title?: string; readonly options: trace.BrowserContextEventOptions; readonly pages: PageEntry[]; + readonly videos: trace.VideoTraceEvent[]; readonly actions: ActionEntry[]; readonly attachments: Attachment[]; readonly visibleAttachments: Attachment[]; @@ -105,6 +106,7 @@ export class TraceModel { // Next call updates all timestamps for all events in library contexts, so it must be done first. this.actions = mergeActionsAndUpdateTiming(contexts); this.pages = ([] as PageEntry[]).concat(...contexts.map(c => c.pages)); + this.videos = []; this.wallTime = contexts.map(c => c.wallTime).reduce((prev, cur) => Math.min(prev || Number.MAX_VALUE, cur!), Number.MAX_VALUE); this.startTime = contexts.map(c => c.startTime).reduce((prev, cur) => Math.min(prev, cur), Number.MAX_VALUE); this.endTime = contexts.map(c => c.endTime).reduce((prev, cur) => Math.max(prev, cur), Number.MIN_VALUE); @@ -126,6 +128,7 @@ export class TraceModel { this._screenshots.set(`${event.callId}/${event.phase}`, event); for (const event of context.ariaSnapshots || []) this._ariaSnapshots.set(`${event.callId}/${event.phase}`, event); + this.videos.push(...(context.videos || [])); } this.attachments = this.actions.flatMap(action => action.attachments?.map(attachment => ({ ...attachment, callId: action.callId, traceUri })) ?? []); this.visibleAttachments = this.attachments.filter(attachment => !attachment.name.startsWith('_')); @@ -344,6 +347,8 @@ function adjustMonotonicTime(context: ContextEntry, monotonicTimeDelta: number) for (const frame of page.screencastFrames) frame.timestamp += monotonicTimeDelta; } + for (const video of context.videos || []) + video.timestampOrigin += monotonicTimeDelta; for (const resource of context.resources) { if (resource._monotonicTime) resource._monotonicTime += monotonicTimeDelta; diff --git a/packages/isomorphic/trace/traceModernizer.ts b/packages/isomorphic/trace/traceModernizer.ts index 48012f9b0d57f..f883aaa37f635 100644 --- a/packages/isomorphic/trace/traceModernizer.ts +++ b/packages/isomorphic/trace/traceModernizer.ts @@ -110,6 +110,10 @@ export class TraceModernizer { contextEntry.screenshots.push(event); break; } + case 'video': { + contextEntry.videos.push(event); + break; + } case 'aria-snapshot': { contextEntry.ariaSnapshots.push(event); break; diff --git a/packages/trace-viewer/src/ui/filmStrip.tsx b/packages/trace-viewer/src/ui/filmStrip.tsx index a264b28d4c64a..37a704ad61c34 100644 --- a/packages/trace-viewer/src/ui/filmStrip.tsx +++ b/packages/trace-viewer/src/ui/filmStrip.tsx @@ -20,6 +20,8 @@ import * as React from 'react'; import { useMeasure, upperBound } from '@web/uiUtils'; import type { PageEntry } from '@isomorphic/trace/entries'; import { useTraceModel } from './traceModelContext'; +import { useVideoThumbnails } from './videoThumbnails'; +import type { VideoThumbnail } from './videoThumbnails'; export type FilmStripPreviewPoint = { x: number; @@ -38,18 +40,29 @@ export const FilmStrip: React.FunctionComponent<{ const [measure, ref] = useMeasure(); const lanesRef = React.useRef(null); - let pageIndex = 0; + const video = model?.videos?.[0]; + const videoThumbnails = useVideoThumbnails(video, video && model ? model.createRelativeUrl(`file/${video.file}`) : undefined); + + let laneIndex = 0; if (lanesRef.current && previewPoint) { const bounds = lanesRef.current.getBoundingClientRect(); - pageIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0; + laneIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0; } - const screencastFrames = model?.pages?.[pageIndex]?.screencastFrames; + const pageLanes = (model?.pages ?? []).filter(page => page.screencastFrames.length); + const videoLanes: VideoThumbnail[][] = videoThumbnails.length ? [videoThumbnails] : []; + + let previewFrames: { timestamp: number, width: number, height: number, url: string }[] | undefined; + if (laneIndex < pageLanes.length) + previewFrames = model ? pageLanes[laneIndex]?.screencastFrames.map(frame => ({ ...frame, url: model.createRelativeUrl(`file/${frame.file}`) })) : undefined; + else + previewFrames = videoLanes[laneIndex - pageLanes.length]; + let previewImage = undefined; let previewSize = undefined; - if (previewPoint !== undefined && screencastFrames && screencastFrames.length) { + if (previewPoint !== undefined && previewFrames && previewFrames.length) { const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewPoint.x / measure.width; - previewImage = screencastFrames[upperBound(screencastFrames, previewTime, timeComparator) - 1]; + previewImage = previewFrames[upperBound(previewFrames, previewTime, timeComparator) - 1]; const fitInto = { width: Math.min(800, (window.innerWidth / 2) | 0), height: Math.min(800, (window.innerHeight / 2) | 0), @@ -58,14 +71,20 @@ export const FilmStrip: React.FunctionComponent<{ } return
-
{ - model?.pages.map((page, index) => page.screencastFrames.length ? + {pageLanes.map((page, index) => : null) - }
+ />)} + {videoLanes.map((thumbnails, index) => )} +
{model && previewPoint && previewImage && previewSize &&
- +
}
; }; +const VideoFilmStripLane: React.FunctionComponent<{ + boundaries: Boundaries, + thumbnails: VideoThumbnail[], + width: number, +}> = ({ boundaries, thumbnails, width }) => { + const viewportSize = { width: 0, height: 0 }; + for (const thumbnail of thumbnails) { + viewportSize.width = Math.max(viewportSize.width, thumbnail.width); + viewportSize.height = Math.max(viewportSize.height, thumbnail.height); + } + const frameSize = inscribe(viewportSize, tileSize); + const startTime = thumbnails[0].timestamp; + const endTime = thumbnails[thumbnails.length - 1].timestamp; + + const boundariesDuration = boundaries.maximum - boundaries.minimum; + const gapLeft = (startTime - boundaries.minimum) / boundariesDuration * width; + const gapRight = (boundaries.maximum - endTime) / boundariesDuration * width; + const effectiveWidth = (endTime - startTime) / boundariesDuration * width; + const frameCount = (effectiveWidth / (frameSize.width + 2 * frameMargin)) | 0; + const frameDuration = (endTime - startTime) / frameCount; + + const frames: React.JSX.Element[] = []; + for (let i = 0; startTime && frameDuration && i < frameCount; ++i) { + const time = startTime + frameDuration * i; + const index = upperBound(thumbnails, time, timeComparator) - 1; + frames.push(
); + } + frames.push(
); + + return
{frames}
; +}; + const FilmStripLane: React.FunctionComponent<{ boundaries: Boundaries, page: PageEntry, diff --git a/packages/trace-viewer/src/ui/videoThumbnails.ts b/packages/trace-viewer/src/ui/videoThumbnails.ts new file mode 100644 index 0000000000000..ebaac09bf6b20 --- /dev/null +++ b/packages/trace-viewer/src/ui/videoThumbnails.ts @@ -0,0 +1,127 @@ +/* + Copyright (c) Microsoft Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import * as React from 'react'; +import type * as trace from '@trace/trace'; + +export type VideoThumbnail = { + timestamp: number; + url: string; + width: number; + height: number; +}; + +type CacheEntry = { + thumbnails: VideoThumbnail[]; + listeners: Set<() => void>; + started: boolean; +}; + +const cache = new Map(); + +const maxThumbnails = 120; +const thumbnailsPerSecond = 2; + +export function useVideoThumbnails(video: trace.VideoTraceEvent | undefined, videoUrl: string | undefined): VideoThumbnail[] { + const [, setVersion] = React.useState(0); + const entry = video && videoUrl ? ensureEntry(video, videoUrl) : undefined; + React.useEffect(() => { + if (!entry) + return; + const listener = () => setVersion(version => version + 1); + entry.listeners.add(listener); + return () => { + entry.listeners.delete(listener); + }; + }, [entry]); + return entry?.thumbnails ?? []; +} + +function ensureEntry(video: trace.VideoTraceEvent, videoUrl: string): CacheEntry { + let entry = cache.get(videoUrl); + if (!entry) { + entry = { thumbnails: [], listeners: new Set(), started: false }; + cache.set(videoUrl, entry); + } + if (!entry.started) { + entry.started = true; + void generateThumbnails(entry, video, videoUrl).catch(() => {}); + } + return entry; +} + +async function generateThumbnails(entry: CacheEntry, video: trace.VideoTraceEvent, videoUrl: string): Promise { + const response = await fetch(videoUrl); + const blob = await response.blob(); + const objectUrl = URL.createObjectURL(blob); + const element = document.createElement('video'); + element.muted = true; + element.preload = 'auto'; + element.src = objectUrl; + try { + await new Promise((resolve, reject) => { + element.addEventListener('loadedmetadata', () => resolve(), { once: true }); + element.addEventListener('error', () => reject(new Error('video failed to load')), { once: true }); + }); + let duration = element.duration; + if (!isFinite(duration) || duration <= 0) { + element.currentTime = Number.MAX_SAFE_INTEGER; + await new Promise(resolve => element.addEventListener('seeked', () => resolve(), { once: true })); + duration = element.duration; + if (!isFinite(duration) || duration <= 0) + return; + } + const width = element.videoWidth || video.width; + const height = element.videoHeight || video.height; + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const context = canvas.getContext('2d'); + if (!context) + return; + const count = Math.max(1, Math.min(maxThumbnails, Math.ceil(duration * thumbnailsPerSecond))); + const step = duration / count; + for (let i = 0; i <= count; ++i) { + const time = Math.min(i * step, Math.max(0, duration - 0.001)); + await seek(element, time); + context.drawImage(element, 0, 0, width, height); + const frame = await new Promise(resolve => canvas.toBlob(resolve, 'image/jpeg', 0.8)); + if (!frame) + continue; + entry.thumbnails.push({ + timestamp: video.timestampOrigin + time * 1000, + url: URL.createObjectURL(frame), + width, + height, + }); + for (const listener of entry.listeners) + listener(); + } + } finally { + element.removeAttribute('src'); + element.load(); + URL.revokeObjectURL(objectUrl); + } +} + +async function seek(element: HTMLVideoElement, time: number): Promise { + if (Math.abs(element.currentTime - time) < 0.001 && element.readyState >= 2) + return; + await new Promise(resolve => { + element.addEventListener('seeked', () => resolve(), { once: true }); + element.currentTime = time; + }); +} diff --git a/packages/trace/src/trace.ts b/packages/trace/src/trace.ts index 46aa729f63b93..e5317b8b0825b 100644 --- a/packages/trace/src/trace.ts +++ b/packages/trace/src/trace.ts @@ -106,6 +106,14 @@ export type ScreencastFrameTraceEvent = { frameSwapWallTime?: number, }; +export type VideoTraceEvent = { + type: 'video', + file: string, + width: number, + height: number, + timestampOrigin: number, +}; + export type ActionPhase = 'before' | 'action' | 'after'; export type ScreenshotTraceEvent = { @@ -233,6 +241,7 @@ export type ErrorTraceEvent = { export type TraceEvent = ContextCreatedTraceEvent | ScreencastFrameTraceEvent | + VideoTraceEvent | ScreenshotTraceEvent | AriaSnapshotTraceEvent | ActionTraceEvent |