From 4e3176d1b29e0c16f2fc1eb8fb605cd18b8ffeb2 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 21 Aug 2026 16:05:07 -0700 Subject: [PATCH 1/3] feat(reporter): add chrome://tracing reporter (#42354) --- docs/src/test-api/class-fullconfig.md | 2 +- docs/src/test-api/class-testconfig.md | 2 +- docs/src/test-reporters-js.md | 26 ++ packages/playwright/src/common/config.ts | 2 +- .../playwright/src/reporters/chromeTrace.ts | 299 ++++++++++++++++++ packages/playwright/src/runner/reporters.ts | 20 +- packages/playwright/types/test.d.ts | 4 +- .../reporter-chrome-trace.spec.ts | 249 +++++++++++++++ utils/generate_types/overrides-test.d.ts | 4 +- 9 files changed, 594 insertions(+), 14 deletions(-) create mode 100644 packages/playwright/src/reporters/chromeTrace.ts create mode 100644 tests/playwright-test/reporter-chrome-trace.spec.ts diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index e3097cb16fa8b..8b52f0ba961c2 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -101,7 +101,7 @@ See [`property: TestConfig.quiet`]. ## property: FullConfig.reporter * since: v1.10 -- type: <[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html">> +- type: <[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html"|"chrome-trace">> - `0` <[string]> Reporter name or module or file path - `1` <[Object]> An object with reporter options if any diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 848882fe2ec20..68c1de594d0be 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -448,7 +448,7 @@ export default defineConfig({ ## property: TestConfig.reporter * since: v1.10 -- type: ?<[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html">> +- type: ?<[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html"|"chrome-trace">> - `0` <[string]> Reporter name or module or file path - `1` <[Object]> An object with reporter options if any diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 26bfa4803d384..0e049b8c6f84d 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -431,6 +431,32 @@ JUnit report supports following configuration options and environment variables: | `PLAYWRIGHT_JUNIT_SUITE_ID` | | Value of the `id` attribute on the root `` report entry. | Empty string. | `PLAYWRIGHT_JUNIT_SUITE_NAME` | | Value of the `name` attribute on the root `` report entry. | Empty string. +### Chrome tracing reporter + +Chrome tracing reporter produces a [Trace Event Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview) json file that can be opened in [`chrome://tracing`](chrome://tracing) or in the [Perfetto UI](https://ui.perfetto.dev). It renders the test run as a timeline with a lane per worker, where every test is a slice containing its before/after hooks, fixtures and steps as nested slices. Every slice carries the details of the test or step in its trace event arguments, including source locations, tags, annotations, errors, stdio and paths to the attachment files. + +```bash +npx playwright test --reporter=chrome-trace +``` + +By default the report is written to `test-results/chrome-trace.json`. + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: [['chrome-trace', { outputFile: 'chrome-trace.json' }]], +}); +``` + +Chrome tracing report supports following configuration options and environment variables: + +| Environment Variable Name | Reporter Config Option| Description | Default +|---|---|---|---| +| `PLAYWRIGHT_CHROME_TRACE_OUTPUT_DIR` | | Directory to save the output file. Ignored if output file is specified. | `test-results` +| `PLAYWRIGHT_CHROME_TRACE_OUTPUT_NAME` | | Base file name for the output, relative to the output dir. | `chrome-trace.json` +| `PLAYWRIGHT_CHROME_TRACE_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `PLAYWRIGHT_CHROME_TRACE_OUTPUT_DIR` and `PLAYWRIGHT_CHROME_TRACE_OUTPUT_NAME` will be ignored. | `undefined` + ### GitHub Actions annotations You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions. diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index e813edd7f0f38..6ebde8c68bac8 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -289,7 +289,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] | return reporters; } -export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob'] as const; +export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'chrome-trace'] as const; export type BuiltInReporter = typeof builtInReporters[number]; export type ContextReuseMode = 'none' | 'when-possible'; diff --git a/packages/playwright/src/reporters/chromeTrace.ts b/packages/playwright/src/reporters/chromeTrace.ts new file mode 100644 index 0000000000000..de268ea5aac10 --- /dev/null +++ b/packages/playwright/src/reporters/chromeTrace.ts @@ -0,0 +1,299 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; + +import { toPosixPath } from '@utils/fileUtils'; +import { getPlaywrightVersion } from 'playwright-core/lib/coreBundle'; + +import { formatError, nonTerminalScreen, resolveOutputFile, CommonReporterOptions } from './base'; +import { stripAnsiEscapes } from '../util'; + +import type { ReporterV2 } from './reporterV2'; +import type { ChromeTraceReporterOptions } from '../../types/test'; +import type { FullConfig, FullResult, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; + +type TraceEvent = { + name: string; + cat: string; + ph: 'X' | 'M' | 'i'; + ts: number; + dur?: number; + pid: number; + tid: number; + s?: 'g'; + cname?: string; + args?: Record; +}; + +type Attachment = { name: string, contentType: string, path?: string }; +type Annotation = { type: string, description?: string }; + +// Palette names understood by chrome://tracing. +const kStatusColors: Record = { + passed: 'good', + failed: 'bad', + timedOut: 'terrible', + interrupted: 'yellow', + skipped: 'grey', +}; + +const kProcessId = 1; +const kRunThreadId = 0; + +class ChromeTraceReporter implements ReporterV2 { + private _config!: FullConfig; + private _suite!: Suite; + private _resolvedOutputFile: string; + private _events: TraceEvent[] = []; + private _laneEndTime = new Map(); + private _globalErrors: { error: TestError, timestamp: number }[] = []; + + constructor(options: ChromeTraceReporterOptions & CommonReporterOptions) { + this._resolvedOutputFile = resolveOutputFile('CHROME_TRACE', { + ...options, + default: { + fileName: 'chrome-trace.json', + outputDir: 'test-results', + }, + })!.outputFile; + } + + version(): 'v2' { + return 'v2'; + } + + printsToStdio() { + return false; + } + + onConfigure(config: FullConfig) { + this._config = config; + } + + onBegin(suite: Suite) { + this._suite = suite; + } + + onError(error: TestError) { + this._globalErrors.push({ error, timestamp: Date.now() }); + } + + async onEnd(result: FullResult) { + const entries: { test: TestCase, result: TestResult }[] = []; + for (const test of this._suite?.allTests() ?? []) { + for (const testResult of test.results) + entries.push({ test, result: testResult }); + } + entries.sort((a, b) => a.result.startTime.getTime() - b.result.startTime.getTime()); + for (const entry of entries) + this._appendTestResult(entry.test, entry.result); + for (const { error, timestamp } of this._globalErrors) { + this._events.push({ + name: 'error', + cat: 'error', + ph: 'i', + s: 'g', + ts: timestamp, + pid: kProcessId, + tid: kRunThreadId, + cname: 'bad', + args: { error: this._formatError(error) }, + }); + } + await this._writeReport(result); + } + + private _appendTestResult(test: TestCase, result: TestResult) { + const startMs = result.startTime.getTime(); + let endMs = startMs + Math.max(0, result.duration); + for (const step of result.steps) + endMs = Math.max(endMs, stepEndTime(step)); + + const lane = this._allocateLane(result.parallelIndex, startMs); + this._laneEndTime.set(lane, endMs); + + const tid = lane + 1; + this._events.push({ + name: test.title, + cat: 'test', + ph: 'X', + ts: startMs, + dur: endMs - startMs, + pid: kProcessId, + tid, + cname: kStatusColors[result.status], + args: this._testArgs(test, result), + }); + for (const step of result.steps) + this._appendStep(step, tid, startMs, endMs); + } + + private _allocateLane(preferredLane: number, startMs: number): number { + const preferred = Math.max(0, preferredLane); + if (!(this._laneEndTime.get(preferred)! > startMs)) + return preferred; + for (let lane = 0; ; ++lane) { + if (!(this._laneEndTime.get(lane)! > startMs)) + return lane; + } + } + + private _appendStep(step: TestStep, tid: number, parentStart: number, parentEnd: number) { + const stepStart = step.startTime.getTime(); + const startMs = clamp(stepStart, parentStart, parentEnd); + const endMs = clamp(step.duration >= 0 ? stepStart + step.duration : parentEnd, startMs, parentEnd); + this._events.push({ + name: step.title, + cat: step.category, + ph: 'X', + ts: startMs, + dur: endMs - startMs, + pid: kProcessId, + tid, + cname: step.error ? 'bad' : undefined, + args: this._stepArgs(step), + }); + for (const child of step.steps) + this._appendStep(child, tid, startMs, endMs); + } + + private _testArgs(test: TestCase, result: TestResult): Record { + // root, project, file, ...describes, test + const [, projectName, , ...titles] = test.titlePath(); + const args: Record = { + status: result.status, + expectedStatus: test.expectedStatus, + testId: test.id, + workerIndex: result.workerIndex, + parallelIndex: result.parallelIndex, + timeout: test.timeout, + location: this._formatLocation(test.location), + }; + if (titles.length > 1) + args.title = titles.join(' › '); + if (projectName) + args.project = projectName; + if (result.retry) + args.retry = result.retry; + if (test.tags.length) + args.tags = test.tags.join(' '); + if (result.annotations.length) + args.annotations = result.annotations.map(annotation => formatAnnotation(annotation)); + if (result.attachments.length) + args.attachments = result.attachments.map(attachment => this._formatAttachment(attachment)); + if (result.errors.length) + args.errors = result.errors.map(error => this._formatError(error)); + const stdout = concatChunks(result.stdout); + if (stdout) + args.stdout = stdout; + const stderr = concatChunks(result.stderr); + if (stderr) + args.stderr = stderr; + return args; + } + + private _stepArgs(step: TestStep): Record | undefined { + const args: Record = {}; + if (step.location) + args.location = this._formatLocation(step.location); + if (step.annotations.length) + args.annotations = step.annotations.map(annotation => formatAnnotation(annotation)); + if (step.attachments.length) + args.attachments = step.attachments.map(attachment => this._formatAttachment(attachment)); + if (step.error) + args.error = this._formatError(step.error); + return Object.keys(args).length ? args : undefined; + } + + private _formatAttachment(attachment: Attachment): string { + return attachment.path ? this._relativePath(attachment.path) : attachment.name; + } + + private _formatLocation(location: Location | undefined): string | undefined { + if (!location) + return undefined; + return `${this._relativePath(location.file)}:${location.line}:${location.column}`; + } + + private _formatError(error: TestError): string { + return stripAnsiEscapes(formatError(nonTerminalScreen, error).message); + } + + private _relativePath(file: string): string { + return toPosixPath(path.relative(this._config.rootDir, file)); + } + + private async _writeReport(result: FullResult) { + let timeOrigin = result.startTime.getTime(); + for (const event of this._events) + timeOrigin = Math.min(timeOrigin, event.ts); + + const traceEvents: TraceEvent[] = [ + { name: 'process_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: kRunThreadId, args: { name: 'Playwright Test' } }, + { name: 'process_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: kRunThreadId, args: { sort_index: 0 } }, + ]; + for (const lane of [...this._laneEndTime.keys()].sort((a, b) => a - b)) { + traceEvents.push({ name: 'thread_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { name: `Worker ${lane}` } }); + traceEvents.push({ name: 'thread_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { sort_index: lane + 1 } }); + } + + // Sort is stable, so parent slices stay ahead of their children on ties. + this._events.sort((a, b) => a.ts - b.ts); + for (const event of this._events) { + event.ts = Math.round((event.ts - timeOrigin) * 1000); + if (event.dur !== undefined) + event.dur = Math.round(event.dur * 1000); + traceEvents.push(event); + } + + const report = { + traceEvents, + displayTimeUnit: 'ms', + metadata: { + 'playwright-version': getPlaywrightVersion(), + 'start-time': result.startTime.toISOString(), + 'duration': result.duration, + 'status': result.status, + }, + }; + + await fs.promises.mkdir(path.dirname(this._resolvedOutputFile), { recursive: true }); + await fs.promises.writeFile(this._resolvedOutputFile, JSON.stringify(report)); + } +} + +function stepEndTime(step: TestStep): number { + let end = step.startTime.getTime() + Math.max(0, step.duration); + for (const child of step.steps) + end = Math.max(end, stepEndTime(child)); + return end; +} + +function formatAnnotation(annotation: Annotation): string { + return annotation.description ? `${annotation.type}: ${annotation.description}` : annotation.type; +} + +function clamp(value: number, min: number, max: number): number { + return Math.min(Math.max(value, min), max); +} + +function concatChunks(chunks: (string | Buffer)[]): string { + return chunks.map(chunk => typeof chunk === 'string' ? chunk : chunk.toString('utf8')).join(''); +} + +export default ChromeTraceReporter; diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index ef3d45a79f086..1e760d4f91e36 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -19,6 +19,7 @@ import { calculateSha1 } from '@utils/crypto'; import { loadReporter } from './loadUtils'; import { formatError } from '../reporters/base'; import { BlobReporter } from '../reporters/blob'; +import ChromeTraceReporter from '../reporters/chromeTrace'; import DotReporter from '../reporters/dot'; import EmptyReporter from '../reporters/empty'; import GitHubReporter from '../reporters/github'; @@ -39,15 +40,16 @@ import type { TestRunOptions } from './tasks'; export async function createReporters(config: FullConfigInternal, mode: 'list' | 'test' | 'merge', descriptions?: ReporterDescription[], runOptions?: TestRunOptions): Promise { const defaultReporters: { [key in commonConfig.BuiltInReporter]: new(arg: any) => ReporterV2 } = { - blob: BlobReporter, - dot: mode === 'list' ? ListModeReporter : DotReporter, - line: mode === 'list' ? ListModeReporter : LineReporter, - list: mode === 'list' ? ListModeReporter : ListReporter, - github: GitHubReporter, - json: JSONReporter, - junit: JUnitReporter, - null: EmptyReporter, - html: HtmlReporter, + 'blob': BlobReporter, + 'chrome-trace': ChromeTraceReporter, + 'dot': mode === 'list' ? ListModeReporter : DotReporter, + 'line': mode === 'list' ? ListModeReporter : LineReporter, + 'list': mode === 'list' ? ListModeReporter : ListReporter, + 'github': GitHubReporter, + 'json': JSONReporter, + 'junit': JUnitReporter, + 'null': EmptyReporter, + 'html': HtmlReporter, }; const reporters: ReporterV2[] = []; descriptions ??= config.config.reporter; diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index f89fd4c330a8f..19e5f1a8e9cc7 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -25,6 +25,7 @@ export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: export type GitHubReporterOptions = { omitTags?: boolean }; export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean }; export type JsonReporterOptions = { outputFile?: string }; +export type ChromeTraceReporterOptions = { outputFile?: string }; export type HtmlReporterOptions = { outputFolder?: string; open?: 'always' | 'never' | 'on-failure'; @@ -46,6 +47,7 @@ export type ReporterDescription = Readonly< ['github'] | ['github', GitHubReporterOptions] | ['junit'] | ['junit', JUnitReporterOptions] | ['json'] | ['json', JsonReporterOptions] | + ['chrome-trace'] | ['chrome-trace', ChromeTraceReporterOptions] | ['html'] | ['html', HtmlReporterOptions] | ['null'] | [string] | [string, any] @@ -917,7 +919,7 @@ interface TestConfig { * ``` * */ - reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html', string> | ReporterDescription[]; + reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'chrome-trace', string> | ReporterDescription[]; /** * Global options for all tests, for example * [testOptions.browserName](https://playwright.dev/docs/api/class-testoptions#test-options-browser-name). Learn more diff --git a/tests/playwright-test/reporter-chrome-trace.spec.ts b/tests/playwright-test/reporter-chrome-trace.spec.ts new file mode 100644 index 0000000000000..77f3ca83f8520 --- /dev/null +++ b/tests/playwright-test/reporter-chrome-trace.spec.ts @@ -0,0 +1,249 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { test, expect } from './playwright-test-fixtures'; + +type TraceEvent = { + name: string; + cat: string; + ph: 'X' | 'M' | 'i'; + ts: number; + dur?: number; + pid: number; + tid: number; + cname?: string; + args?: any; +}; + +function readTrace(baseDir: string, fileName: string = 'test-results/chrome-trace.json') { + return JSON.parse(fs.readFileSync(path.join(baseDir, fileName), 'utf8')) as { + traceEvents: TraceEvent[], + displayTimeUnit: string, + metadata: any, + }; +} + +function slices(events: TraceEvent[]) { + return events.filter(e => e.ph === 'X'); +} + +function threadNames(events: TraceEvent[]) { + return events.filter(e => e.ph === 'M' && e.name === 'thread_name').map(e => e.args.name); +} + +function findSlice(events: TraceEvent[], name: string) { + return slices(events).find(e => e.name === name); +} + +const testFiles = { + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test.beforeAll(async () => {}); + test.beforeEach(async () => {}); + test.describe('suite', () => { + test('passing @smoke', { annotation: { type: 'issue', description: 'flaky' } }, async ({}) => { + console.log('hello from the test'); + await test.step('outer', async () => { + await test.step('inner', async () => { + expect(1).toBe(1); + }); + }); + }); + }); + test('failing', async ({}) => { + expect(1).toBe(2); + }); + `, +}; + +test('should write a chrome tracing report', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest(testFiles, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(1); + + const report = readTrace(testInfo.outputPath()); + expect(report.displayTimeUnit).toBe('ms'); + expect(report.metadata.status).toBe('failed'); + + const events = report.traceEvents; + expect(events.filter(e => e.ph === 'M' && e.name === 'process_name')[0].args.name).toBe('Playwright Test'); + expect(threadNames(events)).toEqual(['Worker 0']); + + const passing = findSlice(events, 'passing @smoke')!; + expect(passing.cat).toBe('test'); + expect(passing.cname).toBe('good'); + expect(passing.dur).toBeGreaterThan(0); + expect(passing.args).toEqual(expect.objectContaining({ + status: 'passed', + expectedStatus: 'passed', + title: 'suite › passing @smoke', + workerIndex: 0, + parallelIndex: 0, + timeout: 30000, + tags: '@smoke', + annotations: ['issue: flaky'], + stdout: 'hello from the test\n', + })); + expect(passing.args.location).toContain('a.test.ts:'); + + const failing = findSlice(events, 'failing')!; + expect(failing.cname).toBe('bad'); + expect(failing.args.status).toBe('failed'); + expect(failing.args.errors[0]).toContain('expect(received).toBe(expected)'); + + // Hooks, fixtures and steps are all rendered as slices. + expect(findSlice(events, 'Before Hooks')!.cat).toBe('hook'); + expect(findSlice(events, 'beforeAll hook')!.cat).toBe('hook'); + expect(findSlice(events, 'beforeEach hook')!.cat).toBe('hook'); + expect(findSlice(events, 'After Hooks')!.cat).toBe('hook'); + expect(findSlice(events, 'Expect "toBe"')!.cat).toBe('expect'); + + const outer = findSlice(events, 'outer')!; + expect(outer.cat).toBe('test.step'); + expect(outer.args.location).toContain('a.test.ts:'); +}); + +test('should nest steps within the test slice', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest(testFiles, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(1); + + const events = slices(readTrace(testInfo.outputPath()).traceEvents); + // Complete events on the same thread must form a proper stack. + const byThread = new Map(); + for (const event of events) { + let list = byThread.get(event.tid); + if (!list) { + list = []; + byThread.set(event.tid, list); + } + list.push(event); + } + for (const list of byThread.values()) { + list.sort((a, b) => a.ts - b.ts || b.dur! - a.dur!); + const stack: TraceEvent[] = []; + for (const event of list) { + while (stack.length && stack[stack.length - 1].ts + stack[stack.length - 1].dur! <= event.ts) + stack.pop(); + if (stack.length) { + const parent = stack[stack.length - 1]; + expect(event.ts + event.dur!, `${event.name} inside ${parent.name}`).toBeLessThanOrEqual(parent.ts + parent.dur!); + } + stack.push(event); + } + } + + const outer = findSlice(events, 'outer')!; + const inner = findSlice(events, 'inner')!; + expect(inner.ts).toBeGreaterThanOrEqual(outer.ts); + expect(inner.ts + inner.dur!).toBeLessThanOrEqual(outer.ts + outer.dur!); +}); + +test('should use a lane per worker', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { await new Promise(f => setTimeout(f, 500)); }); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('two', async ({}) => { await new Promise(f => setTimeout(f, 500)); }); + `, + }, { reporter: 'chrome-trace', workers: 2 }); + expect(result.exitCode).toBe(0); + + const events = readTrace(testInfo.outputPath()).traceEvents; + expect(threadNames(events)).toEqual(['Worker 0', 'Worker 1']); + expect(findSlice(events, 'one')!.tid).not.toBe(findSlice(events, 'two')!.tid); +}); + +test('should report attachment files', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import fs from 'fs'; + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + const file = test.info().outputPath('note.txt'); + fs.writeFileSync(file, 'hello'); + await test.info().attach('inline', { body: 'body' }); + await test.info().attach('file', { path: file }); + }); + `, + }, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(0); + + const one = findSlice(readTrace(testInfo.outputPath()).traceEvents, 'one')!; + expect(one.args.attachments).toEqual(['inline', expect.stringMatching(/^test-results\/a-one\/attachments\/file-.*\.txt$/)]); +}); + +test('should respect outputFile option', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { reporter: [['chrome-trace', { outputFile: 'reports/my-trace.json' }]] }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => {}); + `, + }); + expect(result.exitCode).toBe(0); + expect(findSlice(readTrace(testInfo.outputPath(), 'reports/my-trace.json').traceEvents, 'one')).toBeTruthy(); +}); + +test('should respect PLAYWRIGHT_CHROME_TRACE_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => {}); + `, + }, { reporter: 'chrome-trace' }, { PLAYWRIGHT_CHROME_TRACE_OUTPUT_FILE: testInfo.outputPath('env-trace.json') }); + expect(result.exitCode).toBe(0); + expect(findSlice(readTrace(testInfo.outputPath(), 'env-trace.json').traceEvents, 'one')).toBeTruthy(); +}); + +test('should report retries as separate slices', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('flaky', async ({}, testInfo) => { + expect(testInfo.retry).toBe(1); + }); + `, + }, { reporter: 'chrome-trace', retries: 1 }); + expect(result.exitCode).toBe(0); + + const flaky = slices(readTrace(testInfo.outputPath()).traceEvents).filter(e => e.name === 'flaky'); + expect(flaky).toHaveLength(2); + expect(flaky.map(e => e.args.status)).toEqual(['failed', 'passed']); + expect(flaky[1].args.retry).toBe(1); +}); + +test('should report global errors as instant events', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => {}); + `, + 'b.test.ts': ` + throw new Error('Oh my!'); + `, + }, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(1); + + const errors = readTrace(testInfo.outputPath()).traceEvents.filter(e => e.ph === 'i'); + expect(errors).toHaveLength(1); + expect(errors[0].args.error).toContain('Oh my!'); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index ee28e42b068aa..92ccd06110ecb 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -24,6 +24,7 @@ export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: export type GitHubReporterOptions = { omitTags?: boolean }; export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean }; export type JsonReporterOptions = { outputFile?: string }; +export type ChromeTraceReporterOptions = { outputFile?: string }; export type HtmlReporterOptions = { outputFolder?: string; open?: 'always' | 'never' | 'on-failure'; @@ -45,6 +46,7 @@ export type ReporterDescription = Readonly< ['github'] | ['github', GitHubReporterOptions] | ['junit'] | ['junit', JUnitReporterOptions] | ['json'] | ['json', JsonReporterOptions] | + ['chrome-trace'] | ['chrome-trace', ChromeTraceReporterOptions] | ['html'] | ['html', HtmlReporterOptions] | ['null'] | [string] | [string, any] @@ -67,7 +69,7 @@ type LiteralUnion = T | (U & { zz_IGNORE_ME?: never }); interface TestConfig { projects?: Project[]; - reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html', string> | ReporterDescription[]; + reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'chrome-trace', string> | ReporterDescription[]; use?: UseOptions; webServer?: TestConfigWebServer | TestConfigWebServer[]; } From 0530a54680d284f602954d0a147b64676eea6c07 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 21 Aug 2026 16:05:49 -0700 Subject: [PATCH 2/3] feat(reporter): report step params to the reporters (#42357) --- docs/src/test-api/class-test.md | 12 +++++++ docs/src/test-reporter-api/class-teststep.md | 21 +++++++++++ packages/playwright/src/common/ipc.ts | 1 + packages/playwright/src/common/testType.ts | 4 +-- packages/playwright/src/index.ts | 29 +++++++++++---- .../playwright/src/isomorphic/teleReceiver.ts | 3 ++ packages/playwright/src/matchers/expect.ts | 5 ++- packages/playwright/src/matchers/matchers.ts | 5 +-- .../playwright/src/reporters/teleEmitter.ts | 1 + packages/playwright/src/runner/dispatcher.ts | 1 + packages/playwright/src/worker/testInfo.ts | 11 ++++++ packages/playwright/types/test.d.ts | 4 +-- packages/playwright/types/testReporter.d.ts | 22 ++++++++++++ .../playwright-test/playwright.trace.spec.ts | 19 ++++++++++ tests/playwright-test/reporter-blob.spec.ts | 34 ++++++++++++++++++ tests/playwright-test/test-step.spec.ts | 36 +++++++++++++++++++ utils/generate_types/overrides-test.d.ts | 4 +-- 17 files changed, 196 insertions(+), 16 deletions(-) diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 6342b409018ab..f60e63a5a8181 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -1844,6 +1844,12 @@ Whether to box the step in the report. Defaults to `false`. When the step is box Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown. +### option: Test.step.params +* since: v1.63 +- `params` <[Object]<[string], [any]>> + +Arbitrary serializable parameters describing the step. They are reported to the reporters as `testStep.params` and are shown in the trace viewer. + ## async method: Test.step.skip * since: v1.50 - returns: <[void]> @@ -1891,6 +1897,12 @@ Whether to box the step in the report. Defaults to `false`. When the step is box Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown. +### option: Test.step.skip.params +* since: v1.63 +- `params` <[Object]<[string], [any]>> + +Arbitrary serializable parameters describing the step. They are reported to the reporters as `testStep.params` and are shown in the trace viewer. + ### option: Test.step.skip.timeout * since: v1.50 - `timeout` <[float]> diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index 0c18e6f768ddf..f437bab4d7f23 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -41,6 +41,27 @@ Error thrown during the step execution, if any. Parent step, if any. +## property: TestStep.params +* since: v1.63 +- type: ?<[Object]<[string], [any]>> + +Step-dependent parameters, when available. For example, steps produced by the Playwright API calls contain the target +`locator` and the call arguments such as `url`, while [`method: Test.step`] steps contain the parameters passed by the +test author. + +```js +// { locator: 'getByRole(\'button\')' } +await page.getByRole('button').click(); + +// { url: 'https://example.com' } +await page.goto('https://example.com'); + +// { orderId: 42 } +await test.step('checkout', async () => { + // ... +}, { params: { orderId: 42 } }); +``` + ## property: TestStep.startTime * since: v1.10 - type: <[Date]> diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 3943730099dcf..04ca1ef854060 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -127,6 +127,7 @@ export type StepBeginPayload = { parentStepId: string | undefined; title: string; category: string; + params?: Record; wallTime: number; // milliseconds since unix epoch location?: { file: string, line: number, column: number }; }; diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts index 08adee0f6b99a..81eb262f6d5c7 100644 --- a/packages/playwright/src/common/testType.ts +++ b/packages/playwright/src/common/testType.ts @@ -274,12 +274,12 @@ export class TestTypeImpl { suite._use.push({ fixtures, location }); } - async _step(expectation: 'pass'|'skip', title: string, body: (step: TestStepInfo) => T | Promise, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise { + async _step(expectation: 'pass'|'skip', title: string, body: (step: TestStepInfo) => T | Promise, options: {box?: boolean, location?: Location, timeout?: number, params?: Record } = {}): Promise { const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`test.step() can only be called from a test`); await testInfo._onUserStepBegin?.(title); - const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box }); + const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box, params: options.params }); return await currentZone().with('stepZone', step).run(async () => { try { let result: Awaited>> | undefined = undefined; diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index 4933dbc88fd78..6e61a76a85b10 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -112,11 +112,12 @@ const utilityFixtures: Fixtures = { } // In the general case, create a step for each api call and connect them through the stepId. + const params = renderParams(channel.params); const step = testInfo._addStep({ location: data.frames[0], category: 'pw:api', - title: renderTitle(channel.type, channel.method, channel.params, data.title), - params: channel.params, + title: renderTitle(channel.type, channel.method, channel.params, data.title, params?.locator), + params, group: getActionGroup({ type: channel.type, method: channel.method }), }, tracingGroupSteps[tracingGroupSteps.length - 1]); data.stepId = step.stepId; @@ -919,12 +920,26 @@ function createTestOverlay(parts: string[], position: string, fontSize: number) `; } -function renderTitle(type: string, method: string, params: Record | undefined, title?: string) { +function renderTitle(type: string, method: string, params: Record | undefined, title: string | undefined, locator: string | undefined) { const prefix = renderTitleForCall({ title, type, method, params }); - let selector; - if (params?.['selector'] && typeof params.selector === 'string') - selector = asLocatorDescription('javascript', params.selector); - return prefix + (selector ? ` ${selector}` : ''); + return prefix + (locator ? ` ${locator}` : ''); +} + +const kIncludedParams = new Set(['url', 'selector']); + +function renderParams(params: Record | undefined): Record | undefined { + if (!params) + return undefined; + const result: Record = {}; + for (const [name, value] of Object.entries(params)) { + if (!kIncludedParams.has(name)) + continue; + if (name === 'selector' && typeof value === 'string') + result.locator = asLocatorDescription('javascript', value); + else if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') + result[name] = value; + } + return Object.keys(result).length ? result : undefined; } function tracing() { diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index b8a15723b8408..441c6d24ac9ae 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -109,6 +109,7 @@ export type JsonTestStepStart = { parentStepId?: string; title: string; category: string, + params?: Record; startTime: number; location?: reporterTypes.Location; }; @@ -724,6 +725,7 @@ export class TeleTestCase implements reporterTypes.TestCase { class TeleTestStep implements reporterTypes.TestStep { title: string; category: string; + params: Record | undefined; location: reporterTypes.Location | undefined; parent: reporterTypes.TestStep | undefined; duration: number = -1; @@ -738,6 +740,7 @@ class TeleTestStep implements reporterTypes.TestStep { constructor(payload: JsonTestStepStart, parentStep: reporterTypes.TestStep | undefined, location: reporterTypes.Location | undefined, result: TeleTestResult) { this.title = payload.title; this.category = payload.category; + this.params = payload.params; this.location = location; this.parent = parentStep; this._startTime = payload.startTime; diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.ts index 751e960ce7511..c3879cd76af9c 100644 --- a/packages/playwright/src/matchers/expect.ts +++ b/packages/playwright/src/matchers/expect.ts @@ -336,12 +336,15 @@ function callMatcherAsStep(matcherName: string, info: ExpectMetaInfo, actual: un // This looks like it is unnecessary, but it isn't - we need to filter // out all the frames that belong to the test runner from caught runtime errors. const stackFrames = expectConfig().filteredStackTrace(captureRawStack()); + const params: Record = { ...suffixes.params }; + if (args[0]) + params.expected = args[0]; const stepData = { category: 'expect' as const, title: longTitle, shortTitle, location: stackFrames[0], - params: args[0] ? { expected: args[0] } : undefined, + params: Object.keys(params).length ? params : undefined, }; const step = testInfo?._addStep(stepData); diff --git a/packages/playwright/src/matchers/matchers.ts b/packages/playwright/src/matchers/matchers.ts index 2cbe2705d2b5b..e67724117207a 100644 --- a/packages/playwright/src/matchers/matchers.ts +++ b/packages/playwright/src/matchers/matchers.ts @@ -529,14 +529,15 @@ export async function toPass( return { pass: !this.isNot, message: () => '' }; } -export function computeMatcherTitleSuffix(matcherName: string, receiver: any, args: any[]): { short?: string, long?: string } { +export function computeMatcherTitleSuffix(matcherName: string, receiver: any, args: any[]): { short?: string, long?: string, params?: Record } { if (matcherName === 'toHaveScreenshot') { const title = toHaveScreenshotStepTitle(...args); return { short: title ? `(${title})` : '' }; } if (receiver && typeof receiver === 'object' && (receiver as any)._apiName === 'Locator') { try { - return { long: ' ' + asLocatorDescription('javascript', (receiver as LocatorEx)._selector) }; + const locator = asLocatorDescription('javascript', (receiver as LocatorEx)._selector); + return { long: ' ' + locator, params: { locator } }; } catch { } } diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 808239bee5cfc..424a8c203bbdb 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -317,6 +317,7 @@ export class TeleReporterEmitter implements ReporterV2 { parentStepId: (step.parent as any)?.[this._idSymbol], title: step.title, category: step.category, + params: step.params, startTime: +step.startTime, location: this._relativeLocation(step.location), }; diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 0999900ac1c0a..ba997fbffab96 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -405,6 +405,7 @@ class JobDispatcher { }, parent: parentStep, category: params.category, + params: params.params, startTime: new Date(params.wallTime), duration: -1, steps: [], diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index d7d107cd844f0..0a043b2d61c33 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -377,6 +377,7 @@ export class TestInfoImpl implements TestInfo { parentStepId: parentStep ? parentStep.stepId : undefined, title: step.title, category: step.category, + params: toReportedParams(step.params), wallTime: Date.now(), location: step.location, }; @@ -729,3 +730,13 @@ export class StepSkipError extends Error { } const stepSymbol = Symbol('step'); + +function toReportedParams(params: Record | undefined): Record | undefined { + if (!params) + return undefined; + try { + return JSON.parse(JSON.stringify(params)); + } catch { + return undefined; + } +} diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 19e5f1a8e9cc7..fd7b78e2c26d0 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -6727,7 +6727,7 @@ export interface TestType { * @param body Step body. * @param options */ - (title: string, body: (step: TestStepInfo) => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + (title: string, body: (step: TestStepInfo) => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise; /** * Mark a test step as "skip" to temporarily disable its execution, useful for steps that are currently failing and * planned for a near-term fix. Playwright will not run the step. See also @@ -6755,7 +6755,7 @@ export interface TestType { * @param body Step body. * @param options */ - skip(title: string, body: (step: TestStepInfo) => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + skip(title: string, body: (step: TestStepInfo) => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise; } /** * `expect` function can be used to create test assertions. Read more about [test assertions](https://playwright.dev/docs/test-assertions). diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 550b27c86090a..a9634b136a82f 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -875,6 +875,28 @@ export interface TestStep { */ location?: Location; + /** + * Step-dependent parameters, when available. For example, steps produced by the Playwright API calls contain the + * target `locator` and the call arguments such as `url`, while + * [test.step(title, body[, options])](https://playwright.dev/docs/api/class-test#test-step) steps contain the + * parameters passed by the test author. + * + * ```js + * // { locator: 'getByRole(\'button\')' } + * await page.getByRole('button').click(); + * + * // { url: 'https://example.com' } + * await page.goto('https://example.com'); + * + * // { orderId: 42 } + * await test.step('checkout', async () => { + * // ... + * }, { params: { orderId: 42 } }); + * ``` + * + */ + params?: { [key: string]: any; }; + /** * Parent step, if any. */ diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts index b7847e7834f24..4ed1a3874a161 100644 --- a/tests/playwright-test/playwright.trace.spec.ts +++ b/tests/playwright-test/playwright.trace.spec.ts @@ -1540,3 +1540,22 @@ test('should record test annotations in trace', async ({ runInlineTest }, testIn { type: 'note3' }, ]); }); + +test('should record step params in trace', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({}) => { + expect(1).toBe(1); + await test.step('my step', async () => {}, { params: { foo: 'bar' } }); + }); + `, + }, { trace: 'on' }); + + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + const trace = await parseTrace(testInfo.outputPath('test-results', 'a-pass', 'trace.zip')); + const actionByTitle = (title: string) => trace.model.actions.find(a => a.title === title)!; + expect(actionByTitle('my step').params).toEqual({ foo: 'bar' }); + expect(actionByTitle('Expect "toBe"').params).toEqual({ expected: '1' }); +}); diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 0f3590b0e01f8..3a6912791a597 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -1256,6 +1256,40 @@ test('preserve steps in html report', async ({ runInlineTest, mergeReports, show await expect(page.getByText('Expect "toBe"')).toBeVisible(); }); +test('preserve step params', async ({ runInlineTest, mergeReports }) => { + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'params-reporter.js': ` + class ParamsReporter { + onStepEnd(test, result, step) { + if (step.category === 'test.step' || step.title.startsWith('Navigate')) + console.log('%%' + step.title + ' | ' + JSON.stringify(step.params)); + } + } + module.exports = ParamsReporter; + `, + 'playwright.config.ts': ` + module.exports = { + reporter: [['blob']] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('test 1', async ({ page }) => { + await page.goto('about:blank'); + await test.step('my step', async () => {}, { params: { foo: 'bar', count: 7 } }); + }); + `, + }; + await runInlineTest(files); + const { exitCode, outputLines } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', './params-reporter.js'] }); + expect(exitCode).toBe(0); + expect(outputLines).toEqual([ + `Navigate to "about:blank" | {"url":"about:blank"}`, + `my step | {"foo":"bar","count":7}`, + ]); +}); + test('support fileName option', async ({ runInlineTest, mergeReports }) => { const files = (fileSuffix: string) => ({ 'playwright.config.ts': ` diff --git a/tests/playwright-test/test-step.spec.ts b/tests/playwright-test/test-step.spec.ts index 0a994c6d8576f..15a8b68116836 100644 --- a/tests/playwright-test/test-step.spec.ts +++ b/tests/playwright-test/test-step.spec.ts @@ -1861,3 +1861,39 @@ fixture | Fixture "context" pw:api | Close context `); }); + +test('should report step params', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter'; + export default class MyReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + if (step.location?.file.endsWith('a.test.ts')) + console.log('%%' + step.category + ' | ' + step.title + ' | ' + JSON.stringify(step.params)); + } + } + `, + 'playwright.config.ts': ` + module.exports = { reporter: './reporter' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.goto('about:blank'); + await page.setContent(''); + await page.getByRole('button').click(); + await expect(page.getByRole('button')).toBeVisible(); + await test.step('my step', async () => {}, { params: { foo: 'bar', count: 7 } }); + }); + ` + }, { reporter: '' }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + `pw:api | Navigate to "about:blank" | {"url":"about:blank"}`, + `pw:api | Set content | undefined`, + `pw:api | Click getByRole('button') | {"locator":"getByRole('button')"}`, + `expect | Expect "toBeVisible" getByRole('button') | {"locator":"getByRole('button')"}`, + `test.step | my step | {"foo":"bar","count":7}`, + ]); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 92ccd06110ecb..f8d7829f7ee89 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -197,8 +197,8 @@ export interface TestType { afterAll(title: string, inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void; step: { - (title: string, body: (step: TestStepInfo) => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; - skip(title: string, body: (step: TestStepInfo) => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + (title: string, body: (step: TestStepInfo) => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise; + skip(title: string, body: (step: TestStepInfo) => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise; } expect: Expect<{}>; extend(fixtures: Fixtures): TestType; From d4e1023f6c03a8dced50eb3db88c2217e7c1a86a Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 21 Aug 2026 20:59:21 -0700 Subject: [PATCH 3/3] feat(reporter): report step params in the chrome://tracing report (#42358) --- docs/src/test-reporter-api/class-teststep.md | 7 ++ docs/src/test-reporters-js.md | 7 +- packages/playwright/src/index.ts | 117 ++++++++++++++++-- .../playwright/src/reporters/chromeTrace.ts | 71 ++++++++--- packages/playwright/types/testReporter.d.ts | 6 + .../reporter-chrome-trace.spec.ts | 45 ++++++- tests/playwright-test/test-step.spec.ts | 79 ++++++++++++ 7 files changed, 304 insertions(+), 28 deletions(-) diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index f437bab4d7f23..8ff66f4be55d1 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -56,12 +56,19 @@ await page.getByRole('button').click(); // { url: 'https://example.com' } await page.goto('https://example.com'); +// { locator: 'getByLabel(\'Password\')', value: 'secret' } +await page.getByLabel('Password').fill('secret'); + // { orderId: 42 } await test.step('checkout', async () => { // ... }, { params: { orderId: 42 } }); ``` +To keep the reports small, Playwright API calls only report a curated set of arguments per call, and long string values +are truncated. Unbounded arguments such as the page content, evaluated expressions or request bodies are never +reported. + ## property: TestStep.startTime * since: v1.10 - type: <[Date]> diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 0e049b8c6f84d..c80500f3b0c4f 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -433,19 +433,20 @@ JUnit report supports following configuration options and environment variables: ### Chrome tracing reporter -Chrome tracing reporter produces a [Trace Event Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview) json file that can be opened in [`chrome://tracing`](chrome://tracing) or in the [Perfetto UI](https://ui.perfetto.dev). It renders the test run as a timeline with a lane per worker, where every test is a slice containing its before/after hooks, fixtures and steps as nested slices. Every slice carries the details of the test or step in its trace event arguments, including source locations, tags, annotations, errors, stdio and paths to the attachment files. +Chrome tracing reporter produces a [Trace Event Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview) json file that can be opened in [`chrome://tracing`](chrome://tracing) or in the [Perfetto UI](https://ui.perfetto.dev). It renders the test run as a timeline with a lane per worker, where every test is a slice containing its before/after hooks, fixtures and steps as nested slices. Every slice carries the details of the test or step in its trace event arguments, including source locations, [`property: TestStep.params`], tags, annotations, errors, stdio and paths to the attachment files. ```bash npx playwright test --reporter=chrome-trace ``` -By default the report is written to `test-results/chrome-trace.json`. +By default the report is written to `test-results/chrome-trace.json`. When the output file name ends with `.gz`, the +report is gzipped on the fly, which both viewers accept and which is worth it for large test runs. ```js title="playwright.config.ts" import { defineConfig } from '@playwright/test'; export default defineConfig({ - reporter: [['chrome-trace', { outputFile: 'chrome-trace.json' }]], + reporter: [['chrome-trace', { outputFile: 'chrome-trace.json.gz' }]], }); ``` diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index 6e61a76a85b10..eda97b3fa4d40 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -112,7 +112,7 @@ const utilityFixtures: Fixtures = { } // In the general case, create a step for each api call and connect them through the stepId. - const params = renderParams(channel.params); + const params = renderParams(channel.type, channel.method, channel.params); const step = testInfo._addStep({ location: data.frames[0], category: 'pw:api', @@ -925,23 +925,120 @@ function renderTitle(type: string, method: string, params: Record): Record | undefined { + switch (`${type}.${method}`) { + case 'APIRequestContext.fetch': + return { url: params.url, method: params.method }; + case 'Frame.goto': + case 'Frame.addScriptTag': + case 'Frame.addStyleTag': + return { url: params.url }; + + case 'Frame.click': + case 'Frame.dblclick': + case 'ElementHandle.click': + case 'ElementHandle.dblclick': + return { button: params.button, clickCount: params.clickCount, modifiers: params.modifiers, position: params.position }; + case 'Frame.hover': + case 'Frame.tap': + case 'ElementHandle.hover': + case 'ElementHandle.tap': + return { modifiers: params.modifiers, position: params.position }; + case 'Frame.check': + case 'Frame.uncheck': + case 'ElementHandle.check': + case 'ElementHandle.uncheck': + return { position: params.position }; + case 'Frame.dragAndDrop': + return { source: renderLocator(params.source), target: renderLocator(params.target) }; + case 'Frame.fill': + case 'ElementHandle.fill': + return { value: params.value }; + case 'Frame.press': + case 'ElementHandle.press': + case 'Page.keyboardDown': + case 'Page.keyboardUp': + case 'Page.keyboardPress': + return { key: params.key }; + case 'Frame.type': + case 'ElementHandle.type': + case 'Page.keyboardType': + case 'Page.keyboardInsertText': + return { text: params.text }; + case 'Frame.dispatchEvent': + case 'ElementHandle.dispatchEvent': + return { type: params.type }; + case 'Frame.selectOption': + case 'ElementHandle.selectOption': + return { options: params.options }; + case 'Frame.setInputFiles': + case 'ElementHandle.setInputFiles': + return { files: params.localPaths }; + + case 'Page.mouseClick': + return { x: params.x, y: params.y, button: params.button, clickCount: params.clickCount }; + case 'Page.mouseMove': + case 'Page.touchscreenTap': + return { x: params.x, y: params.y }; + case 'Page.mouseDown': + case 'Page.mouseUp': + return { button: params.button, clickCount: params.clickCount }; + case 'Page.mouseWheel': + return { deltaX: params.deltaX, deltaY: params.deltaY }; + + case 'Frame.waitForSelector': + case 'ElementHandle.waitForSelector': + case 'ElementHandle.waitForElementState': + return { state: params.state }; + case 'Frame.waitForTimeout': + return { timeout: params.waitTimeout }; + + case 'Page.emulateMedia': + return { media: params.media, colorScheme: params.colorScheme, reducedMotion: params.reducedMotion, forcedColors: params.forcedColors, contrast: params.contrast }; + case 'Page.setViewportSize': + return params.viewportSize; + case 'Page.screenshot': + case 'ElementHandle.screenshot': + return { type: params.type, fullPage: params.fullPage }; + case 'BrowserContext.setOffline': + return { offline: params.offline }; + case 'Dialog.accept': + return { promptText: params.promptText }; + case 'Tracing.tracingGroup': + return { name: params.name }; + } +} -function renderParams(params: Record | undefined): Record | undefined { +function renderParams(type: string, method: string, params: Record | undefined): Record | undefined { if (!params) return undefined; const result: Record = {}; - for (const [name, value] of Object.entries(params)) { - if (!kIncludedParams.has(name)) - continue; - if (name === 'selector' && typeof value === 'string') - result.locator = asLocatorDescription('javascript', value); - else if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') - result[name] = value; + const locator = renderLocator(params.selector); + if (locator !== undefined) + result.locator = locator; + for (const [name, value] of Object.entries(renderCallParams(type, method, params) ?? {})) { + if (value !== undefined) + result[name] = typeof value === 'string' ? truncateParam(value) : value; } return Object.keys(result).length ? result : undefined; } +function renderLocator(selector: any): string | undefined { + if (typeof selector !== 'string') + return undefined; + return truncateParam(asLocatorDescription('javascript', selector)); +} + +function truncateParam(value: string): string { + return value.length > kMaxParamLength ? value.substring(0, kMaxParamLength) + '\u2026' : value; +} + function tracing() { return (test.info() as TestInfoImpl)._tracing; } diff --git a/packages/playwright/src/reporters/chromeTrace.ts b/packages/playwright/src/reporters/chromeTrace.ts index de268ea5aac10..b3def8c83f882 100644 --- a/packages/playwright/src/reporters/chromeTrace.ts +++ b/packages/playwright/src/reporters/chromeTrace.ts @@ -16,6 +16,7 @@ import fs from 'fs'; import path from 'path'; +import zlib from 'zlib'; import { toPosixPath } from '@utils/fileUtils'; import { getPlaywrightVersion } from 'playwright-core/lib/coreBundle'; @@ -24,6 +25,7 @@ import { formatError, nonTerminalScreen, resolveOutputFile, CommonReporterOption import { stripAnsiEscapes } from '../util'; import type { ReporterV2 } from './reporterV2'; +import type { Writable } from 'stream'; import type { ChromeTraceReporterOptions } from '../../types/test'; import type { FullConfig, FullResult, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; @@ -209,6 +211,8 @@ class ChromeTraceReporter implements ReporterV2 { private _stepArgs(step: TestStep): Record | undefined { const args: Record = {}; + if (step.params) + args.params = step.params; if (step.location) args.location = this._formatLocation(step.location); if (step.annotations.length) @@ -243,13 +247,13 @@ class ChromeTraceReporter implements ReporterV2 { for (const event of this._events) timeOrigin = Math.min(timeOrigin, event.ts); - const traceEvents: TraceEvent[] = [ + const metadataEvents: TraceEvent[] = [ { name: 'process_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: kRunThreadId, args: { name: 'Playwright Test' } }, { name: 'process_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: kRunThreadId, args: { sort_index: 0 } }, ]; for (const lane of [...this._laneEndTime.keys()].sort((a, b) => a - b)) { - traceEvents.push({ name: 'thread_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { name: `Worker ${lane}` } }); - traceEvents.push({ name: 'thread_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { sort_index: lane + 1 } }); + metadataEvents.push({ name: 'thread_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { name: `Worker ${lane}` } }); + metadataEvents.push({ name: 'thread_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { sort_index: lane + 1 } }); } // Sort is stable, so parent slices stay ahead of their children on ties. @@ -258,22 +262,61 @@ class ChromeTraceReporter implements ReporterV2 { event.ts = Math.round((event.ts - timeOrigin) * 1000); if (event.dur !== undefined) event.dur = Math.round(event.dur * 1000); - traceEvents.push(event); } - const report = { - traceEvents, - displayTimeUnit: 'ms', - metadata: { - 'playwright-version': getPlaywrightVersion(), - 'start-time': result.startTime.toISOString(), - 'duration': result.duration, - 'status': result.status, - }, + const metadata = { + 'playwright-version': getPlaywrightVersion(), + 'start-time': result.startTime.toISOString(), + 'duration': result.duration, + 'status': result.status, }; + // Serialize event by event, the whole report does not have to fit into memory twice. await fs.promises.mkdir(path.dirname(this._resolvedOutputFile), { recursive: true }); - await fs.promises.writeFile(this._resolvedOutputFile, JSON.stringify(report)); + const writer = new ChunkWriter(this._resolvedOutputFile); + await writer.write('{"traceEvents":['); + let separator = ''; + for (const events of [metadataEvents, this._events]) { + for (const event of events) { + await writer.write(separator + JSON.stringify(event)); + separator = ','; + } + } + await writer.write(`],"displayTimeUnit":"ms","metadata":${JSON.stringify(metadata)}}`); + await writer.close(); + } +} + +// Writes into a ".gz" file through a gzip stream, into a plain file otherwise. +class ChunkWriter { + private _stream: Writable; + private _closed: Promise; + private _error: Error | undefined; + + constructor(file: string) { + const fileStream = fs.createWriteStream(file); + const gzip = file.endsWith('.gz') ? zlib.createGzip() : undefined; + gzip?.pipe(fileStream); + this._stream = gzip ?? fileStream; + // The file is only complete once the destination closes, which is later than + // the gzip stream ending. + this._closed = new Promise(resolve => fileStream.on('close', resolve)); + for (const stream of new Set([this._stream, fileStream])) + stream.on('error', error => this._error ??= error); + } + + async write(chunk: string) { + if (this._error) + throw this._error; + if (!this._stream.write(chunk)) + await new Promise(resolve => this._stream.once('drain', () => resolve())); + } + + async close() { + this._stream.end(); + await this._closed; + if (this._error) + throw this._error; } } diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index a9634b136a82f..0b84a322939e1 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -888,12 +888,18 @@ export interface TestStep { * // { url: 'https://example.com' } * await page.goto('https://example.com'); * + * // { locator: 'getByLabel(\'Password\')', value: 'secret' } + * await page.getByLabel('Password').fill('secret'); + * * // { orderId: 42 } * await test.step('checkout', async () => { * // ... * }, { params: { orderId: 42 } }); * ``` * + * To keep the reports small, Playwright API calls only report a curated set of arguments per call, and long string + * values are truncated. Unbounded arguments such as the page content, evaluated expressions or request bodies are + * never reported. */ params?: { [key: string]: any; }; diff --git a/tests/playwright-test/reporter-chrome-trace.spec.ts b/tests/playwright-test/reporter-chrome-trace.spec.ts index 77f3ca83f8520..6a9193b6364f4 100644 --- a/tests/playwright-test/reporter-chrome-trace.spec.ts +++ b/tests/playwright-test/reporter-chrome-trace.spec.ts @@ -16,6 +16,7 @@ import * as fs from 'fs'; import * as path from 'path'; +import * as zlib from 'zlib'; import { test, expect } from './playwright-test-fixtures'; type TraceEvent = { @@ -31,7 +32,9 @@ type TraceEvent = { }; function readTrace(baseDir: string, fileName: string = 'test-results/chrome-trace.json') { - return JSON.parse(fs.readFileSync(path.join(baseDir, fileName), 'utf8')) as { + const file = path.join(baseDir, fileName); + const content = fileName.endsWith('.gz') ? zlib.gunzipSync(fs.readFileSync(file)).toString('utf8') : fs.readFileSync(file, 'utf8'); + return JSON.parse(content) as { traceEvents: TraceEvent[], displayTimeUnit: string, metadata: any, @@ -189,6 +192,29 @@ test('should report attachment files', async ({ runInlineTest }, testInfo) => { expect(one.args.attachments).toEqual(['inline', expect.stringMatching(/^test-results\/a-one\/attachments\/file-.*\.txt$/)]); }); +test('should report step params', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({ page }) => { + await page.goto('about:blank'); + await page.setContent(''); + await page.getByRole('button').click(); + await expect(page.getByRole('button')).toBeVisible(); + await test.step('my step', async () => {}, { params: { foo: 'bar', count: 7 } }); + }); + `, + }, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(0); + + const events = slices(readTrace(testInfo.outputPath()).traceEvents); + expect(findSlice(events, 'Navigate to "about:blank"')!.args.params).toEqual({ url: 'about:blank' }); + expect(findSlice(events, 'Set content')!.args.params).toBe(undefined); + expect(findSlice(events, `Click getByRole('button')`)!.args.params).toEqual({ locator: `getByRole('button')` }); + expect(findSlice(events, `Expect "toBeVisible" getByRole('button')`)!.args.params).toEqual({ locator: `getByRole('button')` }); + expect(findSlice(events, 'my step')!.args.params).toEqual({ foo: 'bar', count: 7 }); +}); + test('should respect outputFile option', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'playwright.config.ts': ` @@ -203,6 +229,23 @@ test('should respect outputFile option', async ({ runInlineTest }, testInfo) => expect(findSlice(readTrace(testInfo.outputPath(), 'reports/my-trace.json').traceEvents, 'one')).toBeTruthy(); }); +test('should gzip the report when output file ends with .gz', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { reporter: [['chrome-trace', { outputFile: 'chrome-trace.json.gz' }]] }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => {}); + `, + }); + expect(result.exitCode).toBe(0); + + const gzipped = fs.readFileSync(testInfo.outputPath('chrome-trace.json.gz')); + expect(gzipped.subarray(0, 2)).toEqual(Buffer.from([0x1f, 0x8b])); + expect(findSlice(readTrace(testInfo.outputPath(), 'chrome-trace.json.gz').traceEvents, 'one')).toBeTruthy(); +}); + test('should respect PLAYWRIGHT_CHROME_TRACE_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => { const result = await runInlineTest({ 'a.test.ts': ` diff --git a/tests/playwright-test/test-step.spec.ts b/tests/playwright-test/test-step.spec.ts index 15a8b68116836..da821b6f25109 100644 --- a/tests/playwright-test/test-step.spec.ts +++ b/tests/playwright-test/test-step.spec.ts @@ -1897,3 +1897,82 @@ test('should report step params', async ({ runInlineTest }) => { `test.step | my step | {"foo":"bar","count":7}`, ]); }); + +test('should report input step params', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter'; + export default class MyReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + if (step.location?.file.endsWith('a.test.ts')) + console.log('%%' + step.title + ' | ' + JSON.stringify(step.params)); + } + } + `, + 'playwright.config.ts': ` + module.exports = { reporter: './reporter' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.setContent(''); + await page.locator('#i').fill('value'); + await page.locator('#i').press('Enter'); + await page.keyboard.type('typed'); + await page.getByRole('button').click({ button: 'right', clickCount: 2, modifiers: ['Shift'], position: { x: 3, y: 4 } }); + await page.mouse.move(10, 20); + await page.mouse.wheel(0, 100); + await page.locator('select').selectOption('b'); + await page.dispatchEvent('#i', 'focus'); + await page.locator('#i').waitFor({ state: 'visible' }); + await page.setViewportSize({ width: 800, height: 600 }); + await page.dragAndDrop('#i', 'select'); + await page.evaluate(() => 1); + }); + ` + }, { reporter: '' }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + `Set content | undefined`, + `Fill "value" locator('#i') | {"locator":"locator('#i')","value":"value"}`, + `Press "Enter" locator('#i') | {"locator":"locator('#i')","key":"Enter"}`, + `Type "typed" | {"text":"typed"}`, + `Click getByRole('button') | {"locator":"getByRole('button')","button":"right","clickCount":2,"modifiers":["Shift"],"position":{"x":3,"y":4}}`, + `Mouse move | {"x":10,"y":20}`, + `Mouse wheel | {"deltaX":0,"deltaY":100}`, + `Select option locator('select') | {"locator":"locator('select')","options":[{"valueOrLabel":"b"}]}`, + `Dispatch "focus" locator('#i') | {"locator":"locator('#i')","type":"focus"}`, + `Wait for selector locator('#i') | {"locator":"locator('#i')","state":"visible"}`, + `Set viewport size | {"width":800,"height":600}`, + `Drag and drop | {"source":"locator('#i')","target":"locator('select')"}`, + `Evaluate | undefined`, + ]); +}); + +test('should truncate long step params', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter'; + export default class MyReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + if (step.params?.value) + console.log('%%' + step.params.value); + } + } + `, + 'playwright.config.ts': ` + module.exports = { reporter: './reporter' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.setContent(''); + await page.locator('#i').fill('x'.repeat(1000)); + }); + ` + }, { reporter: '' }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual(['x'.repeat(200) + '\u2026']); +});