diff --git a/docs/src/test-api/class-fullconfig.md b/docs/src/test-api/class-fullconfig.md index e3097cb16fa8b..8b52f0ba961c2 100644 --- a/docs/src/test-api/class-fullconfig.md +++ b/docs/src/test-api/class-fullconfig.md @@ -101,7 +101,7 @@ See [`property: TestConfig.quiet`]. ## property: FullConfig.reporter * since: v1.10 -- type: <[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html">> +- type: <[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html"|"chrome-trace">> - `0` <[string]> Reporter name or module or file path - `1` <[Object]> An object with reporter options if any diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 6342b409018ab..f60e63a5a8181 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -1844,6 +1844,12 @@ Whether to box the step in the report. Defaults to `false`. When the step is box Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown. +### option: Test.step.params +* since: v1.63 +- `params` <[Object]<[string], [any]>> + +Arbitrary serializable parameters describing the step. They are reported to the reporters as `testStep.params` and are shown in the trace viewer. + ## async method: Test.step.skip * since: v1.50 - returns: <[void]> @@ -1891,6 +1897,12 @@ Whether to box the step in the report. Defaults to `false`. When the step is box Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown. +### option: Test.step.skip.params +* since: v1.63 +- `params` <[Object]<[string], [any]>> + +Arbitrary serializable parameters describing the step. They are reported to the reporters as `testStep.params` and are shown in the trace viewer. + ### option: Test.step.skip.timeout * since: v1.50 - `timeout` <[float]> diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index 848882fe2ec20..68c1de594d0be 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -448,7 +448,7 @@ export default defineConfig({ ## property: TestConfig.reporter * since: v1.10 -- type: ?<[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html">> +- type: ?<[string]|[Array]<[Object]>|[BuiltInReporter]<"list"|"dot"|"line"|"github"|"json"|"junit"|"null"|"html"|"chrome-trace">> - `0` <[string]> Reporter name or module or file path - `1` <[Object]> An object with reporter options if any diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index 0c18e6f768ddf..8ff66f4be55d1 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -41,6 +41,34 @@ Error thrown during the step execution, if any. Parent step, if any. +## property: TestStep.params +* since: v1.63 +- type: ?<[Object]<[string], [any]>> + +Step-dependent parameters, when available. For example, steps produced by the Playwright API calls contain the target +`locator` and the call arguments such as `url`, while [`method: Test.step`] steps contain the parameters passed by the +test author. + +```js +// { locator: 'getByRole(\'button\')' } +await page.getByRole('button').click(); + +// { url: 'https://example.com' } +await page.goto('https://example.com'); + +// { locator: 'getByLabel(\'Password\')', value: 'secret' } +await page.getByLabel('Password').fill('secret'); + +// { orderId: 42 } +await test.step('checkout', async () => { + // ... +}, { params: { orderId: 42 } }); +``` + +To keep the reports small, Playwright API calls only report a curated set of arguments per call, and long string values +are truncated. Unbounded arguments such as the page content, evaluated expressions or request bodies are never +reported. + ## property: TestStep.startTime * since: v1.10 - type: <[Date]> diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 26bfa4803d384..c80500f3b0c4f 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -431,6 +431,33 @@ JUnit report supports following configuration options and environment variables: | `PLAYWRIGHT_JUNIT_SUITE_ID` | | Value of the `id` attribute on the root `` report entry. | Empty string. | `PLAYWRIGHT_JUNIT_SUITE_NAME` | | Value of the `name` attribute on the root `` report entry. | Empty string. +### Chrome tracing reporter + +Chrome tracing reporter produces a [Trace Event Format](https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview) json file that can be opened in [`chrome://tracing`](chrome://tracing) or in the [Perfetto UI](https://ui.perfetto.dev). It renders the test run as a timeline with a lane per worker, where every test is a slice containing its before/after hooks, fixtures and steps as nested slices. Every slice carries the details of the test or step in its trace event arguments, including source locations, [`property: TestStep.params`], tags, annotations, errors, stdio and paths to the attachment files. + +```bash +npx playwright test --reporter=chrome-trace +``` + +By default the report is written to `test-results/chrome-trace.json`. When the output file name ends with `.gz`, the +report is gzipped on the fly, which both viewers accept and which is worth it for large test runs. + +```js title="playwright.config.ts" +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + reporter: [['chrome-trace', { outputFile: 'chrome-trace.json.gz' }]], +}); +``` + +Chrome tracing report supports following configuration options and environment variables: + +| Environment Variable Name | Reporter Config Option| Description | Default +|---|---|---|---| +| `PLAYWRIGHT_CHROME_TRACE_OUTPUT_DIR` | | Directory to save the output file. Ignored if output file is specified. | `test-results` +| `PLAYWRIGHT_CHROME_TRACE_OUTPUT_NAME` | | Base file name for the output, relative to the output dir. | `chrome-trace.json` +| `PLAYWRIGHT_CHROME_TRACE_OUTPUT_FILE` | `outputFile` | Full path to the output file. If defined, `PLAYWRIGHT_CHROME_TRACE_OUTPUT_DIR` and `PLAYWRIGHT_CHROME_TRACE_OUTPUT_NAME` will be ignored. | `undefined` + ### GitHub Actions annotations You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions. diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index e813edd7f0f38..6ebde8c68bac8 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -289,7 +289,7 @@ export function toReporters(reporters: BuiltInReporter | ReporterDescription[] | return reporters; } -export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob'] as const; +export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github', 'html', 'blob', 'chrome-trace'] as const; export type BuiltInReporter = typeof builtInReporters[number]; export type ContextReuseMode = 'none' | 'when-possible'; diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 3943730099dcf..04ca1ef854060 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -127,6 +127,7 @@ export type StepBeginPayload = { parentStepId: string | undefined; title: string; category: string; + params?: Record; wallTime: number; // milliseconds since unix epoch location?: { file: string, line: number, column: number }; }; diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts index 08adee0f6b99a..81eb262f6d5c7 100644 --- a/packages/playwright/src/common/testType.ts +++ b/packages/playwright/src/common/testType.ts @@ -274,12 +274,12 @@ export class TestTypeImpl { suite._use.push({ fixtures, location }); } - async _step(expectation: 'pass'|'skip', title: string, body: (step: TestStepInfo) => T | Promise, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise { + async _step(expectation: 'pass'|'skip', title: string, body: (step: TestStepInfo) => T | Promise, options: {box?: boolean, location?: Location, timeout?: number, params?: Record } = {}): Promise { const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`test.step() can only be called from a test`); await testInfo._onUserStepBegin?.(title); - const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box }); + const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box, params: options.params }); return await currentZone().with('stepZone', step).run(async () => { try { let result: Awaited>> | undefined = undefined; diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts index 4933dbc88fd78..eda97b3fa4d40 100644 --- a/packages/playwright/src/index.ts +++ b/packages/playwright/src/index.ts @@ -112,11 +112,12 @@ const utilityFixtures: Fixtures = { } // In the general case, create a step for each api call and connect them through the stepId. + const params = renderParams(channel.type, channel.method, channel.params); const step = testInfo._addStep({ location: data.frames[0], category: 'pw:api', - title: renderTitle(channel.type, channel.method, channel.params, data.title), - params: channel.params, + title: renderTitle(channel.type, channel.method, channel.params, data.title, params?.locator), + params, group: getActionGroup({ type: channel.type, method: channel.method }), }, tracingGroupSteps[tracingGroupSteps.length - 1]); data.stepId = step.stepId; @@ -919,12 +920,123 @@ function createTestOverlay(parts: string[], position: string, fontSize: number) `; } -function renderTitle(type: string, method: string, params: Record | undefined, title?: string) { +function renderTitle(type: string, method: string, params: Record | undefined, title: string | undefined, locator: string | undefined) { const prefix = renderTitleForCall({ title, type, method, params }); - let selector; - if (params?.['selector'] && typeof params.selector === 'string') - selector = asLocatorDescription('javascript', params.selector); - return prefix + (selector ? ` ${selector}` : ''); + return prefix + (locator ? ` ${locator}` : ''); +} + +const kMaxParamLength = 200; + +// Curated per-call parameters, keyed the same way as the protocol metainfo. Only the +// arguments that say what the call actually did are reported, and only when they are +// bounded in size: page content, evaluated expressions, request bodies and the like are +// never reported, and neither are options that repeat their default on every call. +function renderCallParams(type: string, method: string, params: Record): Record | undefined { + switch (`${type}.${method}`) { + case 'APIRequestContext.fetch': + return { url: params.url, method: params.method }; + case 'Frame.goto': + case 'Frame.addScriptTag': + case 'Frame.addStyleTag': + return { url: params.url }; + + case 'Frame.click': + case 'Frame.dblclick': + case 'ElementHandle.click': + case 'ElementHandle.dblclick': + return { button: params.button, clickCount: params.clickCount, modifiers: params.modifiers, position: params.position }; + case 'Frame.hover': + case 'Frame.tap': + case 'ElementHandle.hover': + case 'ElementHandle.tap': + return { modifiers: params.modifiers, position: params.position }; + case 'Frame.check': + case 'Frame.uncheck': + case 'ElementHandle.check': + case 'ElementHandle.uncheck': + return { position: params.position }; + case 'Frame.dragAndDrop': + return { source: renderLocator(params.source), target: renderLocator(params.target) }; + case 'Frame.fill': + case 'ElementHandle.fill': + return { value: params.value }; + case 'Frame.press': + case 'ElementHandle.press': + case 'Page.keyboardDown': + case 'Page.keyboardUp': + case 'Page.keyboardPress': + return { key: params.key }; + case 'Frame.type': + case 'ElementHandle.type': + case 'Page.keyboardType': + case 'Page.keyboardInsertText': + return { text: params.text }; + case 'Frame.dispatchEvent': + case 'ElementHandle.dispatchEvent': + return { type: params.type }; + case 'Frame.selectOption': + case 'ElementHandle.selectOption': + return { options: params.options }; + case 'Frame.setInputFiles': + case 'ElementHandle.setInputFiles': + return { files: params.localPaths }; + + case 'Page.mouseClick': + return { x: params.x, y: params.y, button: params.button, clickCount: params.clickCount }; + case 'Page.mouseMove': + case 'Page.touchscreenTap': + return { x: params.x, y: params.y }; + case 'Page.mouseDown': + case 'Page.mouseUp': + return { button: params.button, clickCount: params.clickCount }; + case 'Page.mouseWheel': + return { deltaX: params.deltaX, deltaY: params.deltaY }; + + case 'Frame.waitForSelector': + case 'ElementHandle.waitForSelector': + case 'ElementHandle.waitForElementState': + return { state: params.state }; + case 'Frame.waitForTimeout': + return { timeout: params.waitTimeout }; + + case 'Page.emulateMedia': + return { media: params.media, colorScheme: params.colorScheme, reducedMotion: params.reducedMotion, forcedColors: params.forcedColors, contrast: params.contrast }; + case 'Page.setViewportSize': + return params.viewportSize; + case 'Page.screenshot': + case 'ElementHandle.screenshot': + return { type: params.type, fullPage: params.fullPage }; + case 'BrowserContext.setOffline': + return { offline: params.offline }; + case 'Dialog.accept': + return { promptText: params.promptText }; + case 'Tracing.tracingGroup': + return { name: params.name }; + } +} + +function renderParams(type: string, method: string, params: Record | undefined): Record | undefined { + if (!params) + return undefined; + const result: Record = {}; + const locator = renderLocator(params.selector); + if (locator !== undefined) + result.locator = locator; + for (const [name, value] of Object.entries(renderCallParams(type, method, params) ?? {})) { + if (value !== undefined) + result[name] = typeof value === 'string' ? truncateParam(value) : value; + } + return Object.keys(result).length ? result : undefined; +} + +function renderLocator(selector: any): string | undefined { + if (typeof selector !== 'string') + return undefined; + return truncateParam(asLocatorDescription('javascript', selector)); +} + +function truncateParam(value: string): string { + return value.length > kMaxParamLength ? value.substring(0, kMaxParamLength) + '\u2026' : value; } function tracing() { diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index b8a15723b8408..441c6d24ac9ae 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -109,6 +109,7 @@ export type JsonTestStepStart = { parentStepId?: string; title: string; category: string, + params?: Record; startTime: number; location?: reporterTypes.Location; }; @@ -724,6 +725,7 @@ export class TeleTestCase implements reporterTypes.TestCase { class TeleTestStep implements reporterTypes.TestStep { title: string; category: string; + params: Record | undefined; location: reporterTypes.Location | undefined; parent: reporterTypes.TestStep | undefined; duration: number = -1; @@ -738,6 +740,7 @@ class TeleTestStep implements reporterTypes.TestStep { constructor(payload: JsonTestStepStart, parentStep: reporterTypes.TestStep | undefined, location: reporterTypes.Location | undefined, result: TeleTestResult) { this.title = payload.title; this.category = payload.category; + this.params = payload.params; this.location = location; this.parent = parentStep; this._startTime = payload.startTime; diff --git a/packages/playwright/src/matchers/expect.ts b/packages/playwright/src/matchers/expect.ts index 751e960ce7511..c3879cd76af9c 100644 --- a/packages/playwright/src/matchers/expect.ts +++ b/packages/playwright/src/matchers/expect.ts @@ -336,12 +336,15 @@ function callMatcherAsStep(matcherName: string, info: ExpectMetaInfo, actual: un // This looks like it is unnecessary, but it isn't - we need to filter // out all the frames that belong to the test runner from caught runtime errors. const stackFrames = expectConfig().filteredStackTrace(captureRawStack()); + const params: Record = { ...suffixes.params }; + if (args[0]) + params.expected = args[0]; const stepData = { category: 'expect' as const, title: longTitle, shortTitle, location: stackFrames[0], - params: args[0] ? { expected: args[0] } : undefined, + params: Object.keys(params).length ? params : undefined, }; const step = testInfo?._addStep(stepData); diff --git a/packages/playwright/src/matchers/matchers.ts b/packages/playwright/src/matchers/matchers.ts index 2cbe2705d2b5b..e67724117207a 100644 --- a/packages/playwright/src/matchers/matchers.ts +++ b/packages/playwright/src/matchers/matchers.ts @@ -529,14 +529,15 @@ export async function toPass( return { pass: !this.isNot, message: () => '' }; } -export function computeMatcherTitleSuffix(matcherName: string, receiver: any, args: any[]): { short?: string, long?: string } { +export function computeMatcherTitleSuffix(matcherName: string, receiver: any, args: any[]): { short?: string, long?: string, params?: Record } { if (matcherName === 'toHaveScreenshot') { const title = toHaveScreenshotStepTitle(...args); return { short: title ? `(${title})` : '' }; } if (receiver && typeof receiver === 'object' && (receiver as any)._apiName === 'Locator') { try { - return { long: ' ' + asLocatorDescription('javascript', (receiver as LocatorEx)._selector) }; + const locator = asLocatorDescription('javascript', (receiver as LocatorEx)._selector); + return { long: ' ' + locator, params: { locator } }; } catch { } } diff --git a/packages/playwright/src/reporters/chromeTrace.ts b/packages/playwright/src/reporters/chromeTrace.ts new file mode 100644 index 0000000000000..b3def8c83f882 --- /dev/null +++ b/packages/playwright/src/reporters/chromeTrace.ts @@ -0,0 +1,342 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import path from 'path'; +import zlib from 'zlib'; + +import { toPosixPath } from '@utils/fileUtils'; +import { getPlaywrightVersion } from 'playwright-core/lib/coreBundle'; + +import { formatError, nonTerminalScreen, resolveOutputFile, CommonReporterOptions } from './base'; +import { stripAnsiEscapes } from '../util'; + +import type { ReporterV2 } from './reporterV2'; +import type { Writable } from 'stream'; +import type { ChromeTraceReporterOptions } from '../../types/test'; +import type { FullConfig, FullResult, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; + +type TraceEvent = { + name: string; + cat: string; + ph: 'X' | 'M' | 'i'; + ts: number; + dur?: number; + pid: number; + tid: number; + s?: 'g'; + cname?: string; + args?: Record; +}; + +type Attachment = { name: string, contentType: string, path?: string }; +type Annotation = { type: string, description?: string }; + +// Palette names understood by chrome://tracing. +const kStatusColors: Record = { + passed: 'good', + failed: 'bad', + timedOut: 'terrible', + interrupted: 'yellow', + skipped: 'grey', +}; + +const kProcessId = 1; +const kRunThreadId = 0; + +class ChromeTraceReporter implements ReporterV2 { + private _config!: FullConfig; + private _suite!: Suite; + private _resolvedOutputFile: string; + private _events: TraceEvent[] = []; + private _laneEndTime = new Map(); + private _globalErrors: { error: TestError, timestamp: number }[] = []; + + constructor(options: ChromeTraceReporterOptions & CommonReporterOptions) { + this._resolvedOutputFile = resolveOutputFile('CHROME_TRACE', { + ...options, + default: { + fileName: 'chrome-trace.json', + outputDir: 'test-results', + }, + })!.outputFile; + } + + version(): 'v2' { + return 'v2'; + } + + printsToStdio() { + return false; + } + + onConfigure(config: FullConfig) { + this._config = config; + } + + onBegin(suite: Suite) { + this._suite = suite; + } + + onError(error: TestError) { + this._globalErrors.push({ error, timestamp: Date.now() }); + } + + async onEnd(result: FullResult) { + const entries: { test: TestCase, result: TestResult }[] = []; + for (const test of this._suite?.allTests() ?? []) { + for (const testResult of test.results) + entries.push({ test, result: testResult }); + } + entries.sort((a, b) => a.result.startTime.getTime() - b.result.startTime.getTime()); + for (const entry of entries) + this._appendTestResult(entry.test, entry.result); + for (const { error, timestamp } of this._globalErrors) { + this._events.push({ + name: 'error', + cat: 'error', + ph: 'i', + s: 'g', + ts: timestamp, + pid: kProcessId, + tid: kRunThreadId, + cname: 'bad', + args: { error: this._formatError(error) }, + }); + } + await this._writeReport(result); + } + + private _appendTestResult(test: TestCase, result: TestResult) { + const startMs = result.startTime.getTime(); + let endMs = startMs + Math.max(0, result.duration); + for (const step of result.steps) + endMs = Math.max(endMs, stepEndTime(step)); + + const lane = this._allocateLane(result.parallelIndex, startMs); + this._laneEndTime.set(lane, endMs); + + const tid = lane + 1; + this._events.push({ + name: test.title, + cat: 'test', + ph: 'X', + ts: startMs, + dur: endMs - startMs, + pid: kProcessId, + tid, + cname: kStatusColors[result.status], + args: this._testArgs(test, result), + }); + for (const step of result.steps) + this._appendStep(step, tid, startMs, endMs); + } + + private _allocateLane(preferredLane: number, startMs: number): number { + const preferred = Math.max(0, preferredLane); + if (!(this._laneEndTime.get(preferred)! > startMs)) + return preferred; + for (let lane = 0; ; ++lane) { + if (!(this._laneEndTime.get(lane)! > startMs)) + return lane; + } + } + + private _appendStep(step: TestStep, tid: number, parentStart: number, parentEnd: number) { + const stepStart = step.startTime.getTime(); + const startMs = clamp(stepStart, parentStart, parentEnd); + const endMs = clamp(step.duration >= 0 ? stepStart + step.duration : parentEnd, startMs, parentEnd); + this._events.push({ + name: step.title, + cat: step.category, + ph: 'X', + ts: startMs, + dur: endMs - startMs, + pid: kProcessId, + tid, + cname: step.error ? 'bad' : undefined, + args: this._stepArgs(step), + }); + for (const child of step.steps) + this._appendStep(child, tid, startMs, endMs); + } + + private _testArgs(test: TestCase, result: TestResult): Record { + // root, project, file, ...describes, test + const [, projectName, , ...titles] = test.titlePath(); + const args: Record = { + status: result.status, + expectedStatus: test.expectedStatus, + testId: test.id, + workerIndex: result.workerIndex, + parallelIndex: result.parallelIndex, + timeout: test.timeout, + location: this._formatLocation(test.location), + }; + if (titles.length > 1) + args.title = titles.join(' › '); + if (projectName) + args.project = projectName; + if (result.retry) + args.retry = result.retry; + if (test.tags.length) + args.tags = test.tags.join(' '); + if (result.annotations.length) + args.annotations = result.annotations.map(annotation => formatAnnotation(annotation)); + if (result.attachments.length) + args.attachments = result.attachments.map(attachment => this._formatAttachment(attachment)); + if (result.errors.length) + args.errors = result.errors.map(error => this._formatError(error)); + const stdout = concatChunks(result.stdout); + if (stdout) + args.stdout = stdout; + const stderr = concatChunks(result.stderr); + if (stderr) + args.stderr = stderr; + return args; + } + + private _stepArgs(step: TestStep): Record | undefined { + const args: Record = {}; + if (step.params) + args.params = step.params; + if (step.location) + args.location = this._formatLocation(step.location); + if (step.annotations.length) + args.annotations = step.annotations.map(annotation => formatAnnotation(annotation)); + if (step.attachments.length) + args.attachments = step.attachments.map(attachment => this._formatAttachment(attachment)); + if (step.error) + args.error = this._formatError(step.error); + return Object.keys(args).length ? args : undefined; + } + + private _formatAttachment(attachment: Attachment): string { + return attachment.path ? this._relativePath(attachment.path) : attachment.name; + } + + private _formatLocation(location: Location | undefined): string | undefined { + if (!location) + return undefined; + return `${this._relativePath(location.file)}:${location.line}:${location.column}`; + } + + private _formatError(error: TestError): string { + return stripAnsiEscapes(formatError(nonTerminalScreen, error).message); + } + + private _relativePath(file: string): string { + return toPosixPath(path.relative(this._config.rootDir, file)); + } + + private async _writeReport(result: FullResult) { + let timeOrigin = result.startTime.getTime(); + for (const event of this._events) + timeOrigin = Math.min(timeOrigin, event.ts); + + const metadataEvents: TraceEvent[] = [ + { name: 'process_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: kRunThreadId, args: { name: 'Playwright Test' } }, + { name: 'process_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: kRunThreadId, args: { sort_index: 0 } }, + ]; + for (const lane of [...this._laneEndTime.keys()].sort((a, b) => a - b)) { + metadataEvents.push({ name: 'thread_name', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { name: `Worker ${lane}` } }); + metadataEvents.push({ name: 'thread_sort_index', cat: '__metadata', ph: 'M', ts: 0, pid: kProcessId, tid: lane + 1, args: { sort_index: lane + 1 } }); + } + + // Sort is stable, so parent slices stay ahead of their children on ties. + this._events.sort((a, b) => a.ts - b.ts); + for (const event of this._events) { + event.ts = Math.round((event.ts - timeOrigin) * 1000); + if (event.dur !== undefined) + event.dur = Math.round(event.dur * 1000); + } + + const metadata = { + 'playwright-version': getPlaywrightVersion(), + 'start-time': result.startTime.toISOString(), + 'duration': result.duration, + 'status': result.status, + }; + + // Serialize event by event, the whole report does not have to fit into memory twice. + await fs.promises.mkdir(path.dirname(this._resolvedOutputFile), { recursive: true }); + const writer = new ChunkWriter(this._resolvedOutputFile); + await writer.write('{"traceEvents":['); + let separator = ''; + for (const events of [metadataEvents, this._events]) { + for (const event of events) { + await writer.write(separator + JSON.stringify(event)); + separator = ','; + } + } + await writer.write(`],"displayTimeUnit":"ms","metadata":${JSON.stringify(metadata)}}`); + await writer.close(); + } +} + +// Writes into a ".gz" file through a gzip stream, into a plain file otherwise. +class ChunkWriter { + private _stream: Writable; + private _closed: Promise; + private _error: Error | undefined; + + constructor(file: string) { + const fileStream = fs.createWriteStream(file); + const gzip = file.endsWith('.gz') ? zlib.createGzip() : undefined; + gzip?.pipe(fileStream); + this._stream = gzip ?? fileStream; + // The file is only complete once the destination closes, which is later than + // the gzip stream ending. + this._closed = new Promise(resolve => fileStream.on('close', resolve)); + for (const stream of new Set([this._stream, fileStream])) + stream.on('error', error => this._error ??= error); + } + + async write(chunk: string) { + if (this._error) + throw this._error; + if (!this._stream.write(chunk)) + await new Promise(resolve => this._stream.once('drain', () => resolve())); + } + + async close() { + this._stream.end(); + await this._closed; + if (this._error) + throw this._error; + } +} + +function stepEndTime(step: TestStep): number { + let end = step.startTime.getTime() + Math.max(0, step.duration); + for (const child of step.steps) + end = Math.max(end, stepEndTime(child)); + return end; +} + +function formatAnnotation(annotation: Annotation): string { + return annotation.description ? `${annotation.type}: ${annotation.description}` : annotation.type; +} + +function clamp(value: number, min: number, max: number): number { + return Math.min(Math.max(value, min), max); +} + +function concatChunks(chunks: (string | Buffer)[]): string { + return chunks.map(chunk => typeof chunk === 'string' ? chunk : chunk.toString('utf8')).join(''); +} + +export default ChromeTraceReporter; diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 808239bee5cfc..424a8c203bbdb 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -317,6 +317,7 @@ export class TeleReporterEmitter implements ReporterV2 { parentStepId: (step.parent as any)?.[this._idSymbol], title: step.title, category: step.category, + params: step.params, startTime: +step.startTime, location: this._relativeLocation(step.location), }; diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 0999900ac1c0a..ba997fbffab96 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -405,6 +405,7 @@ class JobDispatcher { }, parent: parentStep, category: params.category, + params: params.params, startTime: new Date(params.wallTime), duration: -1, steps: [], diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index ef3d45a79f086..1e760d4f91e36 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -19,6 +19,7 @@ import { calculateSha1 } from '@utils/crypto'; import { loadReporter } from './loadUtils'; import { formatError } from '../reporters/base'; import { BlobReporter } from '../reporters/blob'; +import ChromeTraceReporter from '../reporters/chromeTrace'; import DotReporter from '../reporters/dot'; import EmptyReporter from '../reporters/empty'; import GitHubReporter from '../reporters/github'; @@ -39,15 +40,16 @@ import type { TestRunOptions } from './tasks'; export async function createReporters(config: FullConfigInternal, mode: 'list' | 'test' | 'merge', descriptions?: ReporterDescription[], runOptions?: TestRunOptions): Promise { const defaultReporters: { [key in commonConfig.BuiltInReporter]: new(arg: any) => ReporterV2 } = { - blob: BlobReporter, - dot: mode === 'list' ? ListModeReporter : DotReporter, - line: mode === 'list' ? ListModeReporter : LineReporter, - list: mode === 'list' ? ListModeReporter : ListReporter, - github: GitHubReporter, - json: JSONReporter, - junit: JUnitReporter, - null: EmptyReporter, - html: HtmlReporter, + 'blob': BlobReporter, + 'chrome-trace': ChromeTraceReporter, + 'dot': mode === 'list' ? ListModeReporter : DotReporter, + 'line': mode === 'list' ? ListModeReporter : LineReporter, + 'list': mode === 'list' ? ListModeReporter : ListReporter, + 'github': GitHubReporter, + 'json': JSONReporter, + 'junit': JUnitReporter, + 'null': EmptyReporter, + 'html': HtmlReporter, }; const reporters: ReporterV2[] = []; descriptions ??= config.config.reporter; diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index d7d107cd844f0..0a043b2d61c33 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -377,6 +377,7 @@ export class TestInfoImpl implements TestInfo { parentStepId: parentStep ? parentStep.stepId : undefined, title: step.title, category: step.category, + params: toReportedParams(step.params), wallTime: Date.now(), location: step.location, }; @@ -729,3 +730,13 @@ export class StepSkipError extends Error { } const stepSymbol = Symbol('step'); + +function toReportedParams(params: Record | undefined): Record | undefined { + if (!params) + return undefined; + try { + return JSON.parse(JSON.stringify(params)); + } catch { + return undefined; + } +} diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index f89fd4c330a8f..fd7b78e2c26d0 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -25,6 +25,7 @@ export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: export type GitHubReporterOptions = { omitTags?: boolean }; export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean }; export type JsonReporterOptions = { outputFile?: string }; +export type ChromeTraceReporterOptions = { outputFile?: string }; export type HtmlReporterOptions = { outputFolder?: string; open?: 'always' | 'never' | 'on-failure'; @@ -46,6 +47,7 @@ export type ReporterDescription = Readonly< ['github'] | ['github', GitHubReporterOptions] | ['junit'] | ['junit', JUnitReporterOptions] | ['json'] | ['json', JsonReporterOptions] | + ['chrome-trace'] | ['chrome-trace', ChromeTraceReporterOptions] | ['html'] | ['html', HtmlReporterOptions] | ['null'] | [string] | [string, any] @@ -917,7 +919,7 @@ interface TestConfig { * ``` * */ - reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html', string> | ReporterDescription[]; + reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'chrome-trace', string> | ReporterDescription[]; /** * Global options for all tests, for example * [testOptions.browserName](https://playwright.dev/docs/api/class-testoptions#test-options-browser-name). Learn more @@ -6725,7 +6727,7 @@ export interface TestType { * @param body Step body. * @param options */ - (title: string, body: (step: TestStepInfo) => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + (title: string, body: (step: TestStepInfo) => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise; /** * Mark a test step as "skip" to temporarily disable its execution, useful for steps that are currently failing and * planned for a near-term fix. Playwright will not run the step. See also @@ -6753,7 +6755,7 @@ export interface TestType { * @param body Step body. * @param options */ - skip(title: string, body: (step: TestStepInfo) => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + skip(title: string, body: (step: TestStepInfo) => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise; } /** * `expect` function can be used to create test assertions. Read more about [test assertions](https://playwright.dev/docs/test-assertions). diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 550b27c86090a..0b84a322939e1 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -875,6 +875,34 @@ export interface TestStep { */ location?: Location; + /** + * Step-dependent parameters, when available. For example, steps produced by the Playwright API calls contain the + * target `locator` and the call arguments such as `url`, while + * [test.step(title, body[, options])](https://playwright.dev/docs/api/class-test#test-step) steps contain the + * parameters passed by the test author. + * + * ```js + * // { locator: 'getByRole(\'button\')' } + * await page.getByRole('button').click(); + * + * // { url: 'https://example.com' } + * await page.goto('https://example.com'); + * + * // { locator: 'getByLabel(\'Password\')', value: 'secret' } + * await page.getByLabel('Password').fill('secret'); + * + * // { orderId: 42 } + * await test.step('checkout', async () => { + * // ... + * }, { params: { orderId: 42 } }); + * ``` + * + * To keep the reports small, Playwright API calls only report a curated set of arguments per call, and long string + * values are truncated. Unbounded arguments such as the page content, evaluated expressions or request bodies are + * never reported. + */ + params?: { [key: string]: any; }; + /** * Parent step, if any. */ diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts index b7847e7834f24..4ed1a3874a161 100644 --- a/tests/playwright-test/playwright.trace.spec.ts +++ b/tests/playwright-test/playwright.trace.spec.ts @@ -1540,3 +1540,22 @@ test('should record test annotations in trace', async ({ runInlineTest }, testIn { type: 'note3' }, ]); }); + +test('should record step params in trace', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({}) => { + expect(1).toBe(1); + await test.step('my step', async () => {}, { params: { foo: 'bar' } }); + }); + `, + }, { trace: 'on' }); + + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + const trace = await parseTrace(testInfo.outputPath('test-results', 'a-pass', 'trace.zip')); + const actionByTitle = (title: string) => trace.model.actions.find(a => a.title === title)!; + expect(actionByTitle('my step').params).toEqual({ foo: 'bar' }); + expect(actionByTitle('Expect "toBe"').params).toEqual({ expected: '1' }); +}); diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 0f3590b0e01f8..3a6912791a597 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -1256,6 +1256,40 @@ test('preserve steps in html report', async ({ runInlineTest, mergeReports, show await expect(page.getByText('Expect "toBe"')).toBeVisible(); }); +test('preserve step params', async ({ runInlineTest, mergeReports }) => { + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'params-reporter.js': ` + class ParamsReporter { + onStepEnd(test, result, step) { + if (step.category === 'test.step' || step.title.startsWith('Navigate')) + console.log('%%' + step.title + ' | ' + JSON.stringify(step.params)); + } + } + module.exports = ParamsReporter; + `, + 'playwright.config.ts': ` + module.exports = { + reporter: [['blob']] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('test 1', async ({ page }) => { + await page.goto('about:blank'); + await test.step('my step', async () => {}, { params: { foo: 'bar', count: 7 } }); + }); + `, + }; + await runInlineTest(files); + const { exitCode, outputLines } = await mergeReports(reportDir, undefined, { additionalArgs: ['--reporter', './params-reporter.js'] }); + expect(exitCode).toBe(0); + expect(outputLines).toEqual([ + `Navigate to "about:blank" | {"url":"about:blank"}`, + `my step | {"foo":"bar","count":7}`, + ]); +}); + test('support fileName option', async ({ runInlineTest, mergeReports }) => { const files = (fileSuffix: string) => ({ 'playwright.config.ts': ` diff --git a/tests/playwright-test/reporter-chrome-trace.spec.ts b/tests/playwright-test/reporter-chrome-trace.spec.ts new file mode 100644 index 0000000000000..6a9193b6364f4 --- /dev/null +++ b/tests/playwright-test/reporter-chrome-trace.spec.ts @@ -0,0 +1,292 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import * as zlib from 'zlib'; +import { test, expect } from './playwright-test-fixtures'; + +type TraceEvent = { + name: string; + cat: string; + ph: 'X' | 'M' | 'i'; + ts: number; + dur?: number; + pid: number; + tid: number; + cname?: string; + args?: any; +}; + +function readTrace(baseDir: string, fileName: string = 'test-results/chrome-trace.json') { + const file = path.join(baseDir, fileName); + const content = fileName.endsWith('.gz') ? zlib.gunzipSync(fs.readFileSync(file)).toString('utf8') : fs.readFileSync(file, 'utf8'); + return JSON.parse(content) as { + traceEvents: TraceEvent[], + displayTimeUnit: string, + metadata: any, + }; +} + +function slices(events: TraceEvent[]) { + return events.filter(e => e.ph === 'X'); +} + +function threadNames(events: TraceEvent[]) { + return events.filter(e => e.ph === 'M' && e.name === 'thread_name').map(e => e.args.name); +} + +function findSlice(events: TraceEvent[], name: string) { + return slices(events).find(e => e.name === name); +} + +const testFiles = { + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test.beforeAll(async () => {}); + test.beforeEach(async () => {}); + test.describe('suite', () => { + test('passing @smoke', { annotation: { type: 'issue', description: 'flaky' } }, async ({}) => { + console.log('hello from the test'); + await test.step('outer', async () => { + await test.step('inner', async () => { + expect(1).toBe(1); + }); + }); + }); + }); + test('failing', async ({}) => { + expect(1).toBe(2); + }); + `, +}; + +test('should write a chrome tracing report', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest(testFiles, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(1); + + const report = readTrace(testInfo.outputPath()); + expect(report.displayTimeUnit).toBe('ms'); + expect(report.metadata.status).toBe('failed'); + + const events = report.traceEvents; + expect(events.filter(e => e.ph === 'M' && e.name === 'process_name')[0].args.name).toBe('Playwright Test'); + expect(threadNames(events)).toEqual(['Worker 0']); + + const passing = findSlice(events, 'passing @smoke')!; + expect(passing.cat).toBe('test'); + expect(passing.cname).toBe('good'); + expect(passing.dur).toBeGreaterThan(0); + expect(passing.args).toEqual(expect.objectContaining({ + status: 'passed', + expectedStatus: 'passed', + title: 'suite › passing @smoke', + workerIndex: 0, + parallelIndex: 0, + timeout: 30000, + tags: '@smoke', + annotations: ['issue: flaky'], + stdout: 'hello from the test\n', + })); + expect(passing.args.location).toContain('a.test.ts:'); + + const failing = findSlice(events, 'failing')!; + expect(failing.cname).toBe('bad'); + expect(failing.args.status).toBe('failed'); + expect(failing.args.errors[0]).toContain('expect(received).toBe(expected)'); + + // Hooks, fixtures and steps are all rendered as slices. + expect(findSlice(events, 'Before Hooks')!.cat).toBe('hook'); + expect(findSlice(events, 'beforeAll hook')!.cat).toBe('hook'); + expect(findSlice(events, 'beforeEach hook')!.cat).toBe('hook'); + expect(findSlice(events, 'After Hooks')!.cat).toBe('hook'); + expect(findSlice(events, 'Expect "toBe"')!.cat).toBe('expect'); + + const outer = findSlice(events, 'outer')!; + expect(outer.cat).toBe('test.step'); + expect(outer.args.location).toContain('a.test.ts:'); +}); + +test('should nest steps within the test slice', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest(testFiles, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(1); + + const events = slices(readTrace(testInfo.outputPath()).traceEvents); + // Complete events on the same thread must form a proper stack. + const byThread = new Map(); + for (const event of events) { + let list = byThread.get(event.tid); + if (!list) { + list = []; + byThread.set(event.tid, list); + } + list.push(event); + } + for (const list of byThread.values()) { + list.sort((a, b) => a.ts - b.ts || b.dur! - a.dur!); + const stack: TraceEvent[] = []; + for (const event of list) { + while (stack.length && stack[stack.length - 1].ts + stack[stack.length - 1].dur! <= event.ts) + stack.pop(); + if (stack.length) { + const parent = stack[stack.length - 1]; + expect(event.ts + event.dur!, `${event.name} inside ${parent.name}`).toBeLessThanOrEqual(parent.ts + parent.dur!); + } + stack.push(event); + } + } + + const outer = findSlice(events, 'outer')!; + const inner = findSlice(events, 'inner')!; + expect(inner.ts).toBeGreaterThanOrEqual(outer.ts); + expect(inner.ts + inner.dur!).toBeLessThanOrEqual(outer.ts + outer.dur!); +}); + +test('should use a lane per worker', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { await new Promise(f => setTimeout(f, 500)); }); + `, + 'b.test.ts': ` + import { test, expect } from '@playwright/test'; + test('two', async ({}) => { await new Promise(f => setTimeout(f, 500)); }); + `, + }, { reporter: 'chrome-trace', workers: 2 }); + expect(result.exitCode).toBe(0); + + const events = readTrace(testInfo.outputPath()).traceEvents; + expect(threadNames(events)).toEqual(['Worker 0', 'Worker 1']); + expect(findSlice(events, 'one')!.tid).not.toBe(findSlice(events, 'two')!.tid); +}); + +test('should report attachment files', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import fs from 'fs'; + import { test, expect } from '@playwright/test'; + test('one', async ({}) => { + const file = test.info().outputPath('note.txt'); + fs.writeFileSync(file, 'hello'); + await test.info().attach('inline', { body: 'body' }); + await test.info().attach('file', { path: file }); + }); + `, + }, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(0); + + const one = findSlice(readTrace(testInfo.outputPath()).traceEvents, 'one')!; + expect(one.args.attachments).toEqual(['inline', expect.stringMatching(/^test-results\/a-one\/attachments\/file-.*\.txt$/)]); +}); + +test('should report step params', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({ page }) => { + await page.goto('about:blank'); + await page.setContent(''); + await page.getByRole('button').click(); + await expect(page.getByRole('button')).toBeVisible(); + await test.step('my step', async () => {}, { params: { foo: 'bar', count: 7 } }); + }); + `, + }, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(0); + + const events = slices(readTrace(testInfo.outputPath()).traceEvents); + expect(findSlice(events, 'Navigate to "about:blank"')!.args.params).toEqual({ url: 'about:blank' }); + expect(findSlice(events, 'Set content')!.args.params).toBe(undefined); + expect(findSlice(events, `Click getByRole('button')`)!.args.params).toEqual({ locator: `getByRole('button')` }); + expect(findSlice(events, `Expect "toBeVisible" getByRole('button')`)!.args.params).toEqual({ locator: `getByRole('button')` }); + expect(findSlice(events, 'my step')!.args.params).toEqual({ foo: 'bar', count: 7 }); +}); + +test('should respect outputFile option', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { reporter: [['chrome-trace', { outputFile: 'reports/my-trace.json' }]] }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => {}); + `, + }); + expect(result.exitCode).toBe(0); + expect(findSlice(readTrace(testInfo.outputPath(), 'reports/my-trace.json').traceEvents, 'one')).toBeTruthy(); +}); + +test('should gzip the report when output file ends with .gz', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { reporter: [['chrome-trace', { outputFile: 'chrome-trace.json.gz' }]] }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => {}); + `, + }); + expect(result.exitCode).toBe(0); + + const gzipped = fs.readFileSync(testInfo.outputPath('chrome-trace.json.gz')); + expect(gzipped.subarray(0, 2)).toEqual(Buffer.from([0x1f, 0x8b])); + expect(findSlice(readTrace(testInfo.outputPath(), 'chrome-trace.json.gz').traceEvents, 'one')).toBeTruthy(); +}); + +test('should respect PLAYWRIGHT_CHROME_TRACE_OUTPUT_FILE', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => {}); + `, + }, { reporter: 'chrome-trace' }, { PLAYWRIGHT_CHROME_TRACE_OUTPUT_FILE: testInfo.outputPath('env-trace.json') }); + expect(result.exitCode).toBe(0); + expect(findSlice(readTrace(testInfo.outputPath(), 'env-trace.json').traceEvents, 'one')).toBeTruthy(); +}); + +test('should report retries as separate slices', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('flaky', async ({}, testInfo) => { + expect(testInfo.retry).toBe(1); + }); + `, + }, { reporter: 'chrome-trace', retries: 1 }); + expect(result.exitCode).toBe(0); + + const flaky = slices(readTrace(testInfo.outputPath()).traceEvents).filter(e => e.name === 'flaky'); + expect(flaky).toHaveLength(2); + expect(flaky.map(e => e.args.status)).toEqual(['failed', 'passed']); + expect(flaky[1].args.retry).toBe(1); +}); + +test('should report global errors as instant events', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('one', async ({}) => {}); + `, + 'b.test.ts': ` + throw new Error('Oh my!'); + `, + }, { reporter: 'chrome-trace' }); + expect(result.exitCode).toBe(1); + + const errors = readTrace(testInfo.outputPath()).traceEvents.filter(e => e.ph === 'i'); + expect(errors).toHaveLength(1); + expect(errors[0].args.error).toContain('Oh my!'); +}); diff --git a/tests/playwright-test/test-step.spec.ts b/tests/playwright-test/test-step.spec.ts index 0a994c6d8576f..da821b6f25109 100644 --- a/tests/playwright-test/test-step.spec.ts +++ b/tests/playwright-test/test-step.spec.ts @@ -1861,3 +1861,118 @@ fixture | Fixture "context" pw:api | Close context `); }); + +test('should report step params', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter'; + export default class MyReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + if (step.location?.file.endsWith('a.test.ts')) + console.log('%%' + step.category + ' | ' + step.title + ' | ' + JSON.stringify(step.params)); + } + } + `, + 'playwright.config.ts': ` + module.exports = { reporter: './reporter' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.goto('about:blank'); + await page.setContent(''); + await page.getByRole('button').click(); + await expect(page.getByRole('button')).toBeVisible(); + await test.step('my step', async () => {}, { params: { foo: 'bar', count: 7 } }); + }); + ` + }, { reporter: '' }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + `pw:api | Navigate to "about:blank" | {"url":"about:blank"}`, + `pw:api | Set content | undefined`, + `pw:api | Click getByRole('button') | {"locator":"getByRole('button')"}`, + `expect | Expect "toBeVisible" getByRole('button') | {"locator":"getByRole('button')"}`, + `test.step | my step | {"foo":"bar","count":7}`, + ]); +}); + +test('should report input step params', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter'; + export default class MyReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + if (step.location?.file.endsWith('a.test.ts')) + console.log('%%' + step.title + ' | ' + JSON.stringify(step.params)); + } + } + `, + 'playwright.config.ts': ` + module.exports = { reporter: './reporter' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.setContent(''); + await page.locator('#i').fill('value'); + await page.locator('#i').press('Enter'); + await page.keyboard.type('typed'); + await page.getByRole('button').click({ button: 'right', clickCount: 2, modifiers: ['Shift'], position: { x: 3, y: 4 } }); + await page.mouse.move(10, 20); + await page.mouse.wheel(0, 100); + await page.locator('select').selectOption('b'); + await page.dispatchEvent('#i', 'focus'); + await page.locator('#i').waitFor({ state: 'visible' }); + await page.setViewportSize({ width: 800, height: 600 }); + await page.dragAndDrop('#i', 'select'); + await page.evaluate(() => 1); + }); + ` + }, { reporter: '' }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + `Set content | undefined`, + `Fill "value" locator('#i') | {"locator":"locator('#i')","value":"value"}`, + `Press "Enter" locator('#i') | {"locator":"locator('#i')","key":"Enter"}`, + `Type "typed" | {"text":"typed"}`, + `Click getByRole('button') | {"locator":"getByRole('button')","button":"right","clickCount":2,"modifiers":["Shift"],"position":{"x":3,"y":4}}`, + `Mouse move | {"x":10,"y":20}`, + `Mouse wheel | {"deltaX":0,"deltaY":100}`, + `Select option locator('select') | {"locator":"locator('select')","options":[{"valueOrLabel":"b"}]}`, + `Dispatch "focus" locator('#i') | {"locator":"locator('#i')","type":"focus"}`, + `Wait for selector locator('#i') | {"locator":"locator('#i')","state":"visible"}`, + `Set viewport size | {"width":800,"height":600}`, + `Drag and drop | {"source":"locator('#i')","target":"locator('select')"}`, + `Evaluate | undefined`, + ]); +}); + +test('should truncate long step params', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + import type { Reporter, TestCase, TestResult, TestStep } from '@playwright/test/reporter'; + export default class MyReporter implements Reporter { + onStepEnd(test: TestCase, result: TestResult, step: TestStep) { + if (step.params?.value) + console.log('%%' + step.params.value); + } + } + `, + 'playwright.config.ts': ` + module.exports = { reporter: './reporter' }; + `, + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + await page.setContent(''); + await page.locator('#i').fill('x'.repeat(1000)); + }); + ` + }, { reporter: '' }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual(['x'.repeat(200) + '\u2026']); +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index ee28e42b068aa..f8d7829f7ee89 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -24,6 +24,7 @@ export type ListReporterOptions = { printSteps?: boolean, printFailuresInline?: export type GitHubReporterOptions = { omitTags?: boolean }; export type JUnitReporterOptions = { outputFile?: string, stripANSIControlSequences?: boolean, includeProjectInTestName?: boolean, includeRetries?: boolean, omitTags?: boolean }; export type JsonReporterOptions = { outputFile?: string }; +export type ChromeTraceReporterOptions = { outputFile?: string }; export type HtmlReporterOptions = { outputFolder?: string; open?: 'always' | 'never' | 'on-failure'; @@ -45,6 +46,7 @@ export type ReporterDescription = Readonly< ['github'] | ['github', GitHubReporterOptions] | ['junit'] | ['junit', JUnitReporterOptions] | ['json'] | ['json', JsonReporterOptions] | + ['chrome-trace'] | ['chrome-trace', ChromeTraceReporterOptions] | ['html'] | ['html', HtmlReporterOptions] | ['null'] | [string] | [string, any] @@ -67,7 +69,7 @@ type LiteralUnion = T | (U & { zz_IGNORE_ME?: never }); interface TestConfig { projects?: Project[]; - reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html', string> | ReporterDescription[]; + reporter?: LiteralUnion<'list'|'dot'|'line'|'github'|'json'|'junit'|'null'|'html'|'chrome-trace', string> | ReporterDescription[]; use?: UseOptions; webServer?: TestConfigWebServer | TestConfigWebServer[]; } @@ -195,8 +197,8 @@ export interface TestType { afterAll(title: string, inner: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | any): void; use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void; step: { - (title: string, body: (step: TestStepInfo) => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; - skip(title: string, body: (step: TestStepInfo) => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + (title: string, body: (step: TestStepInfo) => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise; + skip(title: string, body: (step: TestStepInfo) => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number, params?: { [key: string]: any } }): Promise; } expect: Expect<{}>; extend(fixtures: Fixtures): TestType;