From b592dfc7d64ec1f2c9ea9b8c440d78859e507048 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 1 Jul 2026 14:16:42 +0200 Subject: [PATCH 1/8] feat(test-runner): add Reporter.preprocessSuite() hook for test filtering (#41100) --- docs/src/test-reporter-api/class-reporter.md | 23 + docs/src/test-reporter-api/class-suite.md | 38 ++ docs/src/test-reporter-api/class-testcase.md | 38 ++ packages/playwright/src/common/ipc.ts | 1 + packages/playwright/src/common/test.ts | 70 ++- .../playwright/src/isomorphic/teleReceiver.ts | 26 ++ .../src/reporters/internalReporter.ts | 11 +- .../playwright/src/reporters/multiplexer.ts | 16 + .../playwright/src/reporters/reporterV2.ts | 5 + packages/playwright/src/runner/dispatcher.ts | 2 +- packages/playwright/src/runner/loadUtils.ts | 3 +- packages/playwright/src/worker/workerMain.ts | 2 + packages/playwright/types/testReporter.d.ts | 84 ++++ .../reporter-preprocess-suite.spec.ts | 438 ++++++++++++++++++ .../overrides-testReporter.d.ts | 1 + 15 files changed, 754 insertions(+), 4 deletions(-) create mode 100644 tests/playwright-test/reporter-preprocess-suite.spec.ts diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index 8cc8e48676d85..26f8d619e22ca 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -297,3 +297,26 @@ Result of the test run. - returns: <[boolean]> Whether this reporter uses stdio for reporting. When it does not, Playwright Test could add some output to enhance user experience. If your reporter does not print to the terminal, it is strongly recommended to return `false`. + +## optional async method: Reporter.preprocessSuite +* since: v1.61 +- `result` ?<[Object]> + - `implementsSharding` ?<[boolean]> When `true`, Playwright skips its built-in shard filter for this run, leaving sharding to the reporter (typically implemented by calling [`method: TestCase.exclude`] on out-of-shard tests). + +Called after the configuration has been resolved and before [`method: Reporter.onBegin`]. Allows a reporter to mark individual tests as skipped, excluded, fixed or failing. + +### param: Reporter.preprocessSuite.config +* since: v1.61 +- `config` <[FullConfig]> + +Resolved configuration. + +### param: Reporter.preprocessSuite.suite +* since: v1.61 +- `suite` <[Suite]> + +The root suite that contains the projects, files and test cases that will run. + +The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. It contains only the top-level projects being run — setup and dependency projects are not included and cannot be excluded from here. + +The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocessSuite`] returns, unless the returned `implementsSharding` is `true`. diff --git a/docs/src/test-reporter-api/class-suite.md b/docs/src/test-reporter-api/class-suite.md index 1d458c842b3f0..9f93dcabd6f4c 100644 --- a/docs/src/test-reporter-api/class-suite.md +++ b/docs/src/test-reporter-api/class-suite.md @@ -85,3 +85,41 @@ Returns a list of titles from the root down to this suite. Returns the type of the suite. The Suites form the following hierarchy: `root` -> `project` -> `file` -> `describe` -> ...`describe` -> `test`. + +## method: Suite.skip +* since: v1.61 + +Must be called from inside [`method: Reporter.preprocessSuite`]. Mark every [TestCase] of this suite as skipped, see [`method: TestCase.skip`]. + +### param: Suite.skip.reason +* since: v1.61 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: Suite.fixme +* since: v1.61 + +Must be called from inside [`method: Reporter.preprocessSuite`]. Mark every [TestCase] of this suite as fixme, see [`method: TestCase.fixme`]. + +### param: Suite.fixme.reason +* since: v1.61 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: Suite.fail +* since: v1.61 + +Must be called from inside [`method: Reporter.preprocessSuite`]. Mark every [TestCase] of this suite as expected-to-fail, see [`method: TestCase.fail`]. + +### param: Suite.fail.reason +* since: v1.61 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: Suite.exclude +* since: v1.61 + +Must be called from inside [`method: Reporter.preprocessSuite`], exclude this suite from the run. Excluded tests do not appear in the report and their body is not executed. diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index 22a8588fb0934..95e6052e32ae7 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -107,3 +107,41 @@ Returns a list of titles from the root down to this test. - returns: <[TestCaseType]<"test">> Returns "test". Useful for detecting test cases in [`method: Suite.entries`]. + +## method: TestCase.skip +* since: v1.61 + +Must be called from inside [`method: Reporter.preprocessSuite`], skip this test. The test body is not executed and the test is reported as skipped. + +### param: TestCase.skip.reason +* since: v1.61 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: TestCase.fixme +* since: v1.61 + +Must be called from inside [`method: Reporter.preprocessSuite`], mark this test as fixme. The test body is not executed and the test is reported as skipped, with the intention to fix it. + +### param: TestCase.fixme.reason +* since: v1.61 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: TestCase.fail +* since: v1.61 + +Must be called from inside [`method: Reporter.preprocessSuite`], mark this test as "should fail". Playwright runs the test and ensures it is actually failing, useful for documenting broken functionality until it is fixed. + +### param: TestCase.fail.reason +* since: v1.61 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: TestCase.exclude +* since: v1.61 + +Must be called from inside [`method: Reporter.preprocessSuite`], exclude this test from the run. Excluded tests do not appear in the report and their body is not executed. diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 3fdaeccfcb6d5..27fa4287fe552 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -142,6 +142,7 @@ export type StepEndPayload = { export type TestEntry = { testId: string; retry: number; + planAnnotations: { type: string, description?: string, location?: { file: string, line: number, column: number } }[]; }; export type RunPayload = { diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index fa8d8c20032d0..991ac8304ef0d 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -16,7 +16,7 @@ import { rootTestType } from './testType'; import { computeTestCaseOutcome } from '../isomorphic/teleReceiver'; - +import { wrapFunctionWithLocation } from '../transform/transform'; import type { FixturesWithLocation, FullProjectInternal } from './config'; import type { FixturePool } from './fixtures'; import type { TestTypeImpl } from './testType'; @@ -58,11 +58,19 @@ export class Suite extends Base { _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; _fullProject: FullProjectInternal | undefined; _fileId: string | undefined; + _preprocessing = false; readonly _type: 'root' | 'project' | 'file' | 'describe'; + skip: (reason?: string) => void; + fixme: (reason?: string) => void; + fail: (reason?: string) => void; + constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') { super(title); this._type = type; + this.skip = wrapFunctionWithLocation((location, reason?: string) => this._modifier('skip', location, reason)); + this.fixme = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fixme', location, reason)); + this.fail = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fail', location, reason)); } get type(): 'root' | 'project' | 'file' | 'describe' { @@ -96,6 +104,14 @@ export class Suite extends Base { this._entries.unshift(suite); } + _detach(child: Suite | TestCase) { + const idx = this._entries.indexOf(child); + if (idx !== -1) + this._entries.splice(idx, 1); + if (this._entries.length === 0) + this.parent?._detach(this); + } + allTests(): TestCase[] { const result: TestCase[] = []; const visit = (suite: Suite) => { @@ -252,6 +268,25 @@ export class Suite extends Base { project(): FullProject | undefined { return this._fullProject?.project || this.parent?.project(); } + + private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { + if (!this._rootSuite()._preprocessing) + throw new Error(`Suite.${type}() can only be called from Reporter.preprocessSuite().`); + for (const test of this.allTests()) + test._applyPlanAnnotation({ type, description: reason, location }); + } + + exclude(): void { + if (!this._rootSuite()._preprocessing) + throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`); + if (!this.parent) + throw new Error(`Suite.exclude() cannot be called on the root suite.`); + this.parent._detach(this); + } + + _rootSuite(): Suite { + return this.parent?._rootSuite() ?? this; + } } export class TestCase extends Base implements reporterTypes.TestCase { @@ -275,12 +310,20 @@ export class TestCase extends Base implements reporterTypes.TestCase { _projectId = ''; // Explicitly declared tags that are not a part of the title. _tags: string[] = []; + _planAnnotations: TestAnnotation[] = []; + + skip: (reason?: string) => void; + fixme: (reason?: string) => void; + fail: (reason?: string) => void; constructor(title: string, fn: Function, testType: TestTypeImpl, location: Location) { super(title); this.fn = fn; this._testType = testType; this.location = location; + this.skip = wrapFunctionWithLocation((location, reason?: string) => this._modifier('skip', location, reason)); + this.fixme = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fixme', location, reason)); + this.fail = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fail', location, reason)); } titlePath(): string[] { @@ -309,6 +352,31 @@ export class TestCase extends Base implements reporterTypes.TestCase { ]; } + private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { + if (!this._rootSuite()._preprocessing) + throw new Error(`TestCase.${type}() can only be called from Reporter.preprocessSuite().`); + this._applyPlanAnnotation({ type, description: reason, location }); + } + + _applyPlanAnnotation(annotation: TestAnnotation): void { + this.annotations.push(annotation); + this._planAnnotations.push(annotation); + if (annotation.type === 'skip' || annotation.type === 'fixme') + this.expectedStatus = 'skipped'; + else if (annotation.type === 'fail' && this.expectedStatus !== 'skipped') + this.expectedStatus = 'failed'; + } + + exclude(): void { + if (!this._rootSuite()._preprocessing) + throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`); + this.parent._detach(this); + } + + _rootSuite(): Suite { + return this.parent._rootSuite(); + } + _serialize(): any { return { kind: 'test', diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index bfde074d8d1c1..b8a15723b8408 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -648,6 +648,19 @@ export class TeleSuite implements reporterTypes.Suite { suite.parent = this; this._entries.push(suite); } + + skip(_reason?: string): void { + throw new Error('Disposition methods are not supported on a TeleSuite (read-only).'); + } + fixme(_reason?: string): void { + throw new Error('Disposition methods are not supported on a TeleSuite (read-only).'); + } + fail(_reason?: string): void { + throw new Error('Disposition methods are not supported on a TeleSuite (read-only).'); + } + exclude(): void { + throw new Error('Disposition methods are not supported on a TeleSuite (read-only).'); + } } export class TeleTestCase implements reporterTypes.TestCase { @@ -693,6 +706,19 @@ export class TeleTestCase implements reporterTypes.TestCase { this.results.push(result); return result; } + + skip(_reason?: string): void { + throw new Error('Disposition methods are not supported on a TeleTestCase (read-only).'); + } + fixme(_reason?: string): void { + throw new Error('Disposition methods are not supported on a TeleTestCase (read-only).'); + } + fail(_reason?: string): void { + throw new Error('Disposition methods are not supported on a TeleTestCase (read-only).'); + } + exclude(): void { + throw new Error('Disposition methods are not supported on a TeleTestCase (read-only).'); + } } class TeleTestStep implements reporterTypes.TestStep { diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 31d5df1ed1540..7766ec11f6f8b 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -54,6 +54,15 @@ export class InternalReporter implements ReporterV2 { this._reporter.onConfigure?.(config); } + async preprocessSuite(config: FullConfig, suite: testNs.Suite) { + suite._preprocessing = true; + try { + return await this._reporter.preprocessSuite?.(config, suite); + } finally { + suite._preprocessing = false; + } + } + onBegin(suite: testNs.Suite) { this._didBegin = true; this._reporter.onBegin?.(suite); @@ -112,7 +121,7 @@ export class InternalReporter implements ReporterV2 { } printsToStdio() { - return this._reporter.printsToStdio ? this._reporter.printsToStdio() : true; + return this._reporter.printsToStdio?.() ?? true; } private _addSnippetToTestErrors(test: TestCase, result: TestResult) { diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts index e3c5122f8357f..9a57c13ed72ca 100644 --- a/packages/playwright/src/reporters/multiplexer.ts +++ b/packages/playwright/src/reporters/multiplexer.ts @@ -41,6 +41,22 @@ export class Multiplexer implements ReporterV2 { this._wrap(() => reporter.onConfigure?.(config)); } + async preprocessSuite(config: FullConfig, suite: test.Suite) { + // Unlike other reporter callbacks, `preprocessSuite` errors are NOT swallowed — + // they propagate so the run aborts before onBegin. Reporters use preprocessSuite + // to mutate the corpus; silently dropping a planning error would let + // an inconsistent (partial-mutation) state reach the workers. + const shardingReporters: ReporterV2[] = []; + for (const reporter of this._reporters) { + const result = await reporter.preprocessSuite?.(config, suite); + if (result?.implementsSharding) + shardingReporters.push(reporter); + } + if (shardingReporters.length > 1) + throw new Error(`Multiple reporters declare 'implementsSharding': ${shardingReporters.map(r => r.constructor?.name ?? 'reporter').join(', ')}. Only one reporter may handle sharding.`); + return { implementsSharding: shardingReporters.length > 0 }; + } + onBegin(suite: test.Suite) { for (const reporter of this._reporters) this._wrap(() => reporter.onBegin?.(suite)); diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts index 23cffcb4bb916..3d5a65a53d46a 100644 --- a/packages/playwright/src/reporters/reporterV2.ts +++ b/packages/playwright/src/reporters/reporterV2.ts @@ -28,6 +28,7 @@ export interface ReportEndParams { export interface ReporterV2 { onConfigure?(config: FullConfig): void; + preprocessSuite?(config: FullConfig, suite: Suite): { implementsSharding?: boolean } | Promise<{ implementsSharding?: boolean } | undefined | void> | void; onBegin?(suite: Suite): void; onTestBegin?(test: TestCase, result: TestResult): void; onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; @@ -79,6 +80,10 @@ class ReporterV2Wrapper implements ReporterV2 { this._config = config; } + async preprocessSuite(config: FullConfig, suite: Suite) { + return await this._reporter.preprocessSuite?.(config, suite); + } + onBegin(suite: Suite) { this._reporter.onBegin?.(this._config, suite); diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 77cb0aa86cdab..aa30f5fb85579 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -566,7 +566,7 @@ class JobDispatcher { const runPayload: ipc.RunPayload = { file: this.job.requireFile, entries: this.job.tests.map(test => { - return { testId: test.id, retry: test.results.length }; + return { testId: test.id, retry: test.results.length, planAnnotations: test._planAnnotations }; }), }; worker.runTestGroup(runPayload); diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index a2cce3b386459..9485a923ba5a1 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -165,8 +165,9 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } } + const preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite); // Shard only the top-level projects. - if (config.config.shard) { + if (config.config.shard && !preprocessResult?.implementsSharding) { // Create test groups for top-level projects. const testGroups: TestGroup[] = []; for (const projectSuite of rootSuite.suites) { diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts index a566e37f217b9..162f062004bca 100644 --- a/packages/playwright/src/worker/workerMain.ts +++ b/packages/playwright/src/worker/workerMain.ts @@ -223,6 +223,8 @@ export class WorkerMain extends ProcessRunner { suiteUtils.applyRepeatEachIndex(this._project, suite, this._params.repeatEachIndex); suiteUtils.filterTestsRemoveEmptySuites(suite, test => entries.has(test.id)); const tests = suite.allTests(); + for (const test of tests) + test.annotations.push(...entries.get(test.id)!.planAnnotations); // Collect test IDs that were not found in the worker // (e.g. test titles changed between runner and worker). diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index e5bd52c42995e..785f8800cc504 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -145,6 +145,23 @@ export interface FullResult { * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin). */ export interface Reporter { + /** + * Called after the configuration has been resolved and before + * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin). Allows a + * reporter to mark individual tests as skipped, excluded, fixed or failing. + * @param config Resolved configuration. + * @param suite The root suite that contains the projects, files and test cases that will run. + * + * The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that + * match the current invocation. It contains only the top-level projects being run — setup and dependency projects are + * not included and cannot be excluded from here. + * + * The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its + * built-in sharding after + * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite) + * returns, unless the returned `implementsSharding` is `true`. + */ + preprocessSuite?(config: FullConfig, suite: Suite): Promise<{ implementsSharding?: boolean } | undefined | void> | { implementsSharding?: boolean } | void; /** * Called after all tests have been run, or testing has been interrupted. Note that this method may return a [Promise] * and Playwright Test will await it. Reporter is allowed to override the status and hence affect the exit code of the @@ -368,11 +385,45 @@ export interface Suite { */ entries(): Array; + /** + * Must be called from inside + * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), + * exclude this suite from the run. Excluded tests do not appear in the report and their body is not executed. + */ + exclude(): void; + + /** + * Must be called from inside + * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite). + * Mark every [TestCase](https://playwright.dev/docs/api/class-testcase) of this suite as expected-to-fail, see + * [testCase.fail([reason])](https://playwright.dev/docs/api/class-testcase#test-case-fail). + * @param reason Optional explanation surfaced as the annotation description. + */ + fail(reason?: string): void; + + /** + * Must be called from inside + * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite). + * Mark every [TestCase](https://playwright.dev/docs/api/class-testcase) of this suite as fixme, see + * [testCase.fixme([reason])](https://playwright.dev/docs/api/class-testcase#test-case-fixme). + * @param reason Optional explanation surfaced as the annotation description. + */ + fixme(reason?: string): void; + /** * Configuration of the project this suite belongs to, or [void] for the root suite. */ project(): FullProject|undefined; + /** + * Must be called from inside + * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite). + * Mark every [TestCase](https://playwright.dev/docs/api/class-testcase) of this suite as skipped, see + * [testCase.skip([reason])](https://playwright.dev/docs/api/class-testcase#test-case-skip). + * @param reason Optional explanation surfaced as the annotation description. + */ + skip(reason?: string): void; + /** * Returns a list of titles from the root down to this suite. */ @@ -427,6 +478,31 @@ export interface Suite { * projects' suites. */ export interface TestCase { + /** + * Must be called from inside + * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), + * exclude this test from the run. Excluded tests do not appear in the report and their body is not executed. + */ + exclude(): void; + + /** + * Must be called from inside + * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), + * mark this test as "should fail". Playwright runs the test and ensures it is actually failing, useful for + * documenting broken functionality until it is fixed. + * @param reason Optional explanation surfaced as the annotation description. + */ + fail(reason?: string): void; + + /** + * Must be called from inside + * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), + * mark this test as fixme. The test body is not executed and the test is reported as skipped, with the intention to + * fix it. + * @param reason Optional explanation surfaced as the annotation description. + */ + fixme(reason?: string): void; + /** * Whether the test is considered running fine. Non-ok tests fail the test run with non-zero exit code. */ @@ -440,6 +516,14 @@ export interface TestCase { */ outcome(): "skipped"|"expected"|"unexpected"|"flaky"; + /** + * Must be called from inside + * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), + * skip this test. The test body is not executed and the test is reported as skipped. + * @param reason Optional explanation surfaced as the annotation description. + */ + skip(reason?: string): void; + /** * Returns a list of titles from the root down to this test. */ diff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess-suite.spec.ts new file mode 100644 index 0000000000000..7e2f16342851e --- /dev/null +++ b/tests/playwright-test/reporter-preprocess-suite.spec.ts @@ -0,0 +1,438 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +test('preprocessSuite sees the filtered corpus, can skip tests, and records the caller location', async ({ runInlineTest }) => { + // preprocessSuite runs between project setup and onBegin and sees the .only-narrowed corpus. + const only = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + console.log('%% plan: ' + suite.allTests().map(t => t.title).join(',')); + for (const t of suite.allTests()) + if (t.title.includes('skip-me')) t.skip('planned skip'); + } + onBegin(config, suite) { + console.log('%% onBegin: ' + suite.allTests().map(t => t.title).join(',')); + } + onTestEnd(test, result) { + const a = test.annotations.find(a => a.type === 'skip'); + const loc = a && a.location ? require('path').basename(a.location.file) + ':' + a.location.line : 'none'; + console.log('%% end ' + test.title + ' status=' + result.status + ' expected=' + test.expectedStatus + ' ann=' + test.annotations.map(a => a.type + ':' + (a.description || '')).join(',') + ' loc=' + loc); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': `module.exports = { reporter: './reporter.ts' };`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('ignored-by-only', async () => {}); + test.only('run-me', async () => {}); + test.only('skip-me', async () => { throw new Error('should not run'); }); + `, + }, { reporter: '', workers: 1 }); + + expect(only.exitCode).toBe(0); + expect(only.outputLines).toEqual([ + 'plan: run-me,skip-me', + 'onBegin: run-me,skip-me', + 'end run-me status=passed expected=passed ann= loc=none', + // The skip annotation location points at the reporter's `t.skip(...)` call (line 6 of reporter.ts). + 'end skip-me status=skipped expected=skipped ann=skip:planned skip loc=reporter.ts:6', + ]); + + // preprocessSuite respects --grep. + const grep = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + console.log('%% plan: ' + suite.allTests().map(t => t.title).join(',')); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': `module.exports = { reporter: './reporter.ts' };`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('foo-one', async () => {}); + test('bar-two', async () => {}); + `, + }, { reporter: '', workers: 1, grep: 'foo' }); + expect(grep.exitCode).toBe(0); + expect(grep.outputLines).toEqual(['plan: foo-one']); + + // preprocessSuite respects --project. + const project = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(',')); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter.ts', + projects: [{ name: 'one' }, { name: 'two' }], + }; + `, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('t', async () => {}); + `, + }, { reporter: '', workers: 1, project: 'one' }); + expect(project.exitCode).toBe(0); + expect(project.outputLines).toEqual(['plan projects: one']); +}); + +test('TestCase.exclude and Suite.exclude remove entries from the run and report', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + for (const t of suite.allTests()) + if (t.title === 'excluded-test') t.exclude(); + const visit = (s) => { + if (s.title === 'excluded-suite') s.exclude(); + else for (const child of s.suites || []) visit(child); + }; + visit(suite); + } + onBegin(config, suite) { + console.log('%% begin: ' + suite.allTests().map(t => t.title).join(',')); + } + onTestEnd(test, result) { + console.log('%% ran ' + test.title); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': `module.exports = { reporter: './reporter.ts' };`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('kept', async () => {}); + test('excluded-test', async () => { throw new Error('should not run'); }); + test.describe('excluded-suite', () => { + test('doomed', async () => { throw new Error('should not run'); }); + }); + `, + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + 'begin: kept', + 'ran kept', + ]); +}); + +test('Suite.skip cascades to all descendants', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + const visit = (s) => { + if (s.title === 'doomed') s.skip('whole group'); + for (const child of s.suites || []) visit(child); + }; + visit(suite); + } + onTestEnd(test, result) { + console.log('%% ' + test.title + ':' + result.status + ':' + test.expectedStatus); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': `module.exports = { reporter: './reporter.ts' };`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test.describe('doomed', () => { + test('one', async () => { throw new Error('nope'); }); + test('two', async () => { throw new Error('nope'); }); + }); + test('keep', async () => {}); + `, + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines.sort()).toEqual([ + 'keep:passed:passed', + 'one:skipped:skipped', + 'two:skipped:skipped', + ]); +}); + +test('disposition methods throw when called outside preprocessSuite, and the root suite cannot be excluded', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + // Excluding the root suite is banned even during preprocessSuite. + try { + suite.exclude(); + console.log('%% root-exclude: no-throw'); + } catch (e) { + console.log('%% root-exclude: ' + e.message); + } + } + onBegin(config, suite) { + const testCase = suite.allTests()[0]; + const fileSuite = testCase.parent; + for (const [label, obj] of [['TestCase', testCase], ['Suite', fileSuite]]) { + for (const method of ['skip', 'fixme', 'fail', 'exclude']) { + try { + obj[method](); + console.log('%% ' + label + '.' + method + ': no-throw'); + } catch (e) { + console.log('%% ' + label + '.' + method + ': ' + e.message); + } + } + } + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': `module.exports = { reporter: './reporter.ts' };`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('t', async () => {}); + `, + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + 'root-exclude: Suite.exclude() cannot be called on the root suite.', + 'TestCase.skip: TestCase.skip() can only be called from Reporter.preprocessSuite().', + 'TestCase.fixme: TestCase.fixme() can only be called from Reporter.preprocessSuite().', + 'TestCase.fail: TestCase.fail() can only be called from Reporter.preprocessSuite().', + 'TestCase.exclude: TestCase.exclude() can only be called from Reporter.preprocessSuite().', + 'Suite.skip: Suite.skip() can only be called from Reporter.preprocessSuite().', + 'Suite.fixme: Suite.fixme() can only be called from Reporter.preprocessSuite().', + 'Suite.fail: Suite.fail() can only be called from Reporter.preprocessSuite().', + 'Suite.exclude: Suite.exclude() can only be called from Reporter.preprocessSuite().', + ]); +}); + +test('preprocessSuite throwing aborts the run before onBegin', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + throw new Error('plan-aborted'); + } + onBegin(config, suite) { + console.log('%% onBegin: ' + suite.allTests().length); + } + onError(err) { + console.log('%% error: ' + err.message); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': `module.exports = { reporter: './reporter.ts' };`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('one', async () => {}); + `, + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).not.toBe(0); + expect(result.outputLines).toContain('error: Error: plan-aborted'); + // Synthetic empty-suite onBegin is OK; the real onBegin (size 1) must NOT happen. + expect(result.outputLines).not.toContain('onBegin: 1'); +}); + +test('multiple reporters: preprocessSuite called in order, annotations accumulate, exclude prunes for next reporter', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'first.ts': ` + class R { + async preprocessSuite(config, suite) { + console.log('%% first plan sees: ' + suite.allTests().map(t => t.title).join(',')); + for (const t of suite.allTests()) { + if (t.title === 'gone') t.exclude(); + else t.fail('first reason'); + } + } + onTestEnd(test, result) { + console.log('%% first onTestEnd: ' + test.expectedStatus + ' ann=' + test.annotations.map(a => a.type).join(',')); + } + } + module.exports = R; + `, + 'second.ts': ` + class R { + async preprocessSuite(config, suite) { + console.log('%% second plan sees: ' + suite.allTests().map(t => t.title).join(',')); + suite.allTests()[0].skip('second reason'); + } + } + module.exports = R; + `, + 'playwright.config.ts': `module.exports = { reporter: [['./first.ts'], ['./second.ts']] };`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('kept', async () => {}); + test('gone', async () => { throw new Error('should not run'); }); + `, + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + // skip beats fail in expectedStatus, both annotations accumulate. + expect(result.outputLines).toEqual([ + 'first plan sees: kept,gone', + 'second plan sees: kept', + 'first onTestEnd: skipped ann=fail,skip', + ]); +}); + +test('multiple reporters: a later reporter observes an earlier reporter Suite.skip on the tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'first.ts': ` + class R { + async preprocessSuite(config, suite) { + suite.allTests()[0].parent.skip('first reason'); + } + } + module.exports = R; + `, + 'second.ts': ` + class R { + async preprocessSuite(config, suite) { + const skipped = suite.allTests().filter(t => t.expectedStatus === 'skipped').map(t => t.title); + console.log('%% second sees skipped: ' + skipped.join(',')); + } + } + module.exports = R; + `, + 'playwright.config.ts': `module.exports = { reporter: [['./first.ts'], ['./second.ts']] };`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test.describe('group', () => { + test('one', async () => {}); + test('two', async () => {}); + }); + `, + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + // Suite.skip from the first reporter is applied before the second reporter runs. + expect(result.outputLines).toContain('second sees skipped: one,two'); +}); + +test('implementsSharding disables the built-in shard filter; preprocessSuite sees the full corpus', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class R { + async preprocessSuite(config, suite) { + // preprocessSuite observes the full, un-sharded corpus regardless of --shard. + console.log('%% plan: ' + suite.allTests().map(t => t.title).join(',')); + let i = 0; + for (const t of suite.allTests()) { + if (i++ % 2 === 1) t.exclude(); + } + return { implementsSharding: true }; + } + onBegin(config, suite) { + console.log('%% begin: ' + suite.allTests().map(t => t.title).join(',')); + } + } + module.exports = R; + `, + 'playwright.config.ts': `module.exports = { reporter: './reporter.ts', shard: { current: 1, total: 2 } };`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + for (let i = 0; i < 4; i++) + test('t' + i, async () => {}); + `, + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).toBe(0); + // preprocessSuite sees all four tests even though --shard=1/2 was configured. + expect(result.outputLines).toContain('plan: t0,t1,t2,t3'); + // The reporter's own exclusions define the shard; the built-in shard filter did NOT run + // (it would have produced a different split), so t0,t2 remain. + expect(result.outputLines).toContain('begin: t0,t2'); +}); + +test('multiple reporters declaring implementsSharding throws', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter-a.ts': ` + class A { + preprocessSuite() { return { implementsSharding: true }; } + onError(err) { console.log('%% error: ' + err.message); } + } + module.exports = A; + `, + 'reporter-b.ts': ` + class B { preprocessSuite() { return { implementsSharding: true }; } } + module.exports = B; + `, + 'playwright.config.ts': `module.exports = { reporter: [['./reporter-a.ts'], ['./reporter-b.ts']] };`, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('t', async () => {}); + `, + }, { reporter: '', workers: 1 }); + + expect(result.exitCode).not.toBe(0); + expect(result.outputLines.join('\n')).toContain(`Multiple reporters declare 'implementsSharding'`); +}); + +test('plan.suite contains only top-level projects, not dependency/setup projects', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + // The suite only exposes top-level projects, so a reporter has no handle on + // setup/dependency project tests and therefore cannot exclude them. + console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(',')); + console.log('%% plan tests: ' + suite.allTests().map(t => t.title).join(',')); + } + onTestEnd(test, result) { + console.log('%% ran ' + test.parent.project().name + '/' + test.title); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter.ts', + projects: [ + { name: 'setup', testMatch: /a\\.setup\\.ts/ }, + { name: 'main', testMatch: /a\\.test\\.ts/, dependencies: ['setup'] }, + ], + }; + `, + 'a.setup.ts': ` + import { test } from '@playwright/test'; + test('setup-test', async () => {}); + `, + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('main-test', async () => {}); + `, + }, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['--project=main'] }); + + expect(result.exitCode).toBe(0); + // plan only sees the top-level 'main' project; the 'setup' dependency is prepended afterwards. + expect(result.outputLines).toContain('plan projects: main'); + // 'setup-test' is absent from the plan suite, proving setup/dependency tests are not exposed. + expect(result.outputLines).toContain('plan tests: main-test'); + // Both the dependency and the main project still run. + expect(result.outputLines).toContain('ran setup/setup-test'); + expect(result.outputLines).toContain('ran main/main-test'); +}); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index b67b850603af1..a4376aabca5b0 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -42,6 +42,7 @@ export interface FullResult { } export interface Reporter { + preprocessSuite?(config: FullConfig, suite: Suite): Promise<{ implementsSharding?: boolean } | undefined | void> | { implementsSharding?: boolean } | void; onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; } From 3dc8451c4104c16003e36f76b77624d5d15a000f Mon Sep 17 00:00:00 2001 From: "microsoft-playwright-automation[bot]" <203992400+microsoft-playwright-automation[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 14:44:28 +0200 Subject: [PATCH 2/8] feat(webkit): roll to r2318 (#41556) Co-authored-by: microsoft-playwright-automation[bot] <203992400+microsoft-playwright-automation[bot]@users.noreply.github.com> --- packages/playwright-core/browsers.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playwright-core/browsers.json b/packages/playwright-core/browsers.json index 5261809f8319b..6fa27ab5af4c3 100644 --- a/packages/playwright-core/browsers.json +++ b/packages/playwright-core/browsers.json @@ -45,7 +45,7 @@ }, { "name": "webkit", - "revision": "2317", + "revision": "2318", "installByDefault": true, "revisionOverrides": { "mac14": "2251", From 25e41a6c2191b90511aa268b21c4886db3a284d3 Mon Sep 17 00:00:00 2001 From: Alexander Kireyev Date: Wed, 1 Jul 2026 19:44:47 +0700 Subject: [PATCH 3/8] fix(routing): match ws(s) baseURL rewrite case-insensitively (#41557) --- packages/isomorphic/urlMatch.ts | 7 ++++--- tests/library/route-web-socket.spec.ts | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/packages/isomorphic/urlMatch.ts b/packages/isomorphic/urlMatch.ts index 455147898098f..30cd9a95694d0 100644 --- a/packages/isomorphic/urlMatch.ts +++ b/packages/isomorphic/urlMatch.ts @@ -202,9 +202,10 @@ export function resolveGlobToRegexPattern(baseURL: string | undefined, glob: str } function toWebSocketBaseUrl(baseURL: string | undefined) { - // Allow http(s) baseURL to match ws(s) urls. - if (baseURL && /^https?:\/\//.test(baseURL)) - baseURL = baseURL.replace(/^http/, 'ws'); + // Allow http(s) baseURL to match ws(s) urls. Schemes are case-insensitive, + // same as elsewhere in this file, so 'HTTP://...' should be rewritten too. + if (baseURL && /^https?:\/\//i.test(baseURL)) + baseURL = baseURL.replace(/^https?/i, scheme => scheme.toLowerCase() === 'https' ? 'wss' : 'ws'); return baseURL; } diff --git a/tests/library/route-web-socket.spec.ts b/tests/library/route-web-socket.spec.ts index d968889490b4a..21b1b7efc1c45 100644 --- a/tests/library/route-web-socket.spec.ts +++ b/tests/library/route-web-socket.spec.ts @@ -646,6 +646,30 @@ test('should work with baseURL', async ({ contextFactory, server }) => { ]); }); +test('should work with baseURL regardless of scheme casing', async ({ contextFactory, server }) => { + // baseURL schemes are case-insensitive, same as everywhere else in URL matching. + const context = await contextFactory({ baseURL: 'HTTP://' + server.HOST }); + const page = await context.newPage(); + + await page.routeWebSocket('/ws', ws => { + ws.onMessage(message => { + ws.send(message); + }); + }); + + await setupWS(page, server, 'blob'); + + await page.evaluate(async () => { + await window.wsOpened; + window.ws.send('echo'); + }); + + await expect.poll(() => page.evaluate(() => window.log)).toEqual([ + 'open', + `message: data=echo origin=ws://${server.HOST} lastEventId=`, + ]); +}); + test('should expose protocols to the route handler', async ({ page, server }) => { const routes: WebSocketRoute[] = []; await page.routeWebSocket(/.*/, ws => { From 2af447df86fd1a8c42644d95e214d90a78ffcba5 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 1 Jul 2026 14:58:59 +0200 Subject: [PATCH 4/8] docs(elementhandle): deprecate inputValue timeout option (#41568) Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Skn0tt <14912729+Skn0tt@users.noreply.github.com> Co-authored-by: Simon Knott --- docs/src/api/class-elementhandle.md | 7 +++---- packages/playwright-client/types/types.d.ts | 5 +---- packages/playwright-core/types/types.d.ts | 5 +---- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/docs/src/api/class-elementhandle.md b/docs/src/api/class-elementhandle.md index a14f91156a7de..7f3a147ccde72 100644 --- a/docs/src/api/class-elementhandle.md +++ b/docs/src/api/class-elementhandle.md @@ -650,11 +650,10 @@ Returns `input.value` for the selected `` or `