Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/src/test-reporter-api/class-reporter.md
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,6 @@ Resolved configuration.

The root suite that contains the projects, files and test cases that will run.

The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. It contains only the top-level projects being run — setup and dependency projects are not included and cannot be excluded from here.
The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here.

The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocessSuite`] returns, unless the returned `implementsSharding` is `true`.
2 changes: 1 addition & 1 deletion packages/playwright-core/browsers.json
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
},
{
"name": "webkit",
"revision": "2328",
"revision": "2330",
"installByDefault": true,
"revisionOverrides": {
"mac14": "2251",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ export class DispatcherConnection {
return;
}
if (method === '__abort__') {
await this._activeProgressControllers.get(`call@${params.id}`)?.abort(new AbortError(undefined, { cause: params.reason }));
await this._activeProgressControllers.get(`call@${params.id}`)?.abort(new AbortError(params.reason));
return;
}
if (!dispatcher) {
Expand Down
5 changes: 1 addition & 4 deletions packages/playwright-core/src/server/frames.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
*/

import yaml from 'yaml';
import { assertionAbortedMessage } from '@isomorphic/abortSignal';
import { parseAriaSnapshotUnsafe } from '@isomorphic/ariaSnapshot';
import { isInvalidSelectorError } from '@isomorphic/selectorParser';
import { ManualPromise } from '@isomorphic/manualPromise';
Expand All @@ -29,7 +28,7 @@ import { makeWaitForNextTask } from '@utils/task';
import { createGuid } from '@utils/crypto';
import { BrowserContext } from './browserContext';
import * as dom from './dom';
import { TimeoutError, AbortError, isTargetClosedError } from './errors';
import { TimeoutError, isTargetClosedError } from './errors';
import { prepareFilesForUpload } from './fileUploadUtils';
import { FrameSelectors } from './frameSelectors';
import { helper } from './helper';
Expand Down Expand Up @@ -1546,8 +1545,6 @@ export class Frame extends SdkObject<FrameEventMap> {
progress.log(e.message);
if (e instanceof TimeoutError)
details.timedOut = true;
if (e instanceof AbortError)
details.customErrorMessage = assertionAbortedMessage(e.cause);
throw new ExpectError(details);
}
}
Expand Down
4 changes: 3 additions & 1 deletion packages/playwright-core/src/server/progress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,14 +62,16 @@ export class ProgressController {
});
}


async abort(error: Error) {
const logMessage = `operation was aborted: ${error.message}`;
if (this._state === 'running') {
this.metadata.log.push(logMessage);
(error as any)[kAbortErrorSymbol] = true;
this._state = { error };
this._forceAbortPromise.reject(error);
this._controller.abort(error);
} else if (this._state === 'before') {
this.metadata.log.push(logMessage);
(error as any)[kAbortErrorSymbol] = true;
this._pendingAbortError = error;
}
Expand Down
30 changes: 19 additions & 11 deletions packages/playwright/src/common/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ export class Suite extends Base {
_parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none';
_fullProject: FullProjectInternal | undefined;
_fileId: string | undefined;
_preprocessing = false;
_preprocessMode: 'editable' | 'readonly' | undefined = undefined;
readonly _type: 'root' | 'project' | 'file' | 'describe';

skip: (reason?: string) => void;
Expand Down Expand Up @@ -270,22 +270,28 @@ export class Suite extends Base {
}

private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void {
if (!this._rootSuite()._preprocessing)
const mode = this._resolvePreprocessMode();
if (!mode)
throw new Error(`Suite.${type}() can only be called from Reporter.preprocessSuite().`);
if (mode === 'readonly')
throw new Error(`Suite.${type}() cannot be called on a setup or teardown project; these always run in full.`);
for (const test of this.allTests())
test._applyPlanAnnotation({ type, description: reason, location });
}

exclude(): void {
if (!this._rootSuite()._preprocessing)
const mode = this._resolvePreprocessMode();
if (!mode)
throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`);
if (!this.parent)
throw new Error(`Suite.exclude() cannot be called on the root suite.`);
if (mode === 'readonly')
throw new Error(`Suite.exclude() cannot be called on a setup or teardown project; these always run in full.`);
this.parent._detach(this);
}

_rootSuite(): Suite {
return this.parent?._rootSuite() ?? this;
_resolvePreprocessMode(): 'editable' | 'readonly' | undefined {
return this._preprocessMode ?? this.parent?._resolvePreprocessMode();
}
}

Expand Down Expand Up @@ -353,8 +359,11 @@ export class TestCase extends Base implements reporterTypes.TestCase {
}

private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void {
if (!this._rootSuite()._preprocessing)
const mode = this.parent._resolvePreprocessMode();
if (!mode)
throw new Error(`TestCase.${type}() can only be called from Reporter.preprocessSuite().`);
if (mode === 'readonly')
throw new Error(`TestCase.${type}() cannot be called on a setup or teardown project test; these always run in full.`);
this._applyPlanAnnotation({ type, description: reason, location });
}

Expand All @@ -368,15 +377,14 @@ export class TestCase extends Base implements reporterTypes.TestCase {
}

exclude(): void {
if (!this._rootSuite()._preprocessing)
const mode = this.parent._resolvePreprocessMode();
if (!mode)
throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`);
if (mode === 'readonly')
throw new Error(`TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.`);
this.parent._detach(this);
}

_rootSuite(): Suite {
return this.parent._rootSuite();
}

_serialize(): any {
return {
kind: 'test',
Expand Down
7 changes: 1 addition & 6 deletions packages/playwright/src/reporters/internalReporter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,7 @@ export class InternalReporter implements ReporterV2 {
}

async preprocessSuite(config: FullConfig, suite: testNs.Suite) {
suite._preprocessing = true;
try {
return await this._reporter.preprocessSuite?.(config, suite);
} finally {
suite._preprocessing = false;
}
return await this._reporter.preprocessSuite?.(config, suite);
}

onBegin(suite: testNs.Suite) {
Expand Down
36 changes: 28 additions & 8 deletions packages/playwright/src/runner/loadUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,30 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
}
}

const preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite);
// Temporarily prepend unfiltered dependency projects for preprocessing.
const dependencySuites = new Map<commonConfig.FullProjectInternal, testNs.Suite>();
for (const [project, type] of projectClosure) {
if (type !== 'dependency')
continue;
const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!);
dependencySuite._preprocessMode = 'readonly';
dependencySuites.set(project, dependencySuite);
rootSuite._prependSuite(dependencySuite);
}

rootSuite._preprocessMode = 'editable';
let preprocessResult: Awaited<ReturnType<typeof testRun.reporter.preprocessSuite>>;
try {
preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite);
} finally {
// Continue the existing sharding and filtering pipeline with top-level projects only.
rootSuite._preprocessMode = undefined;
for (const dependencySuite of dependencySuites.values()) {
dependencySuite._preprocessMode = undefined;
rootSuite._detach(dependencySuite);
}
}

// Shard only the top-level projects.
if (config.config.shard && !preprocessResult?.implementsSharding) {
// Create test groups for top-level projects.
Expand Down Expand Up @@ -193,16 +216,13 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => testRun.postShardTestFilters.every(filter => filter(test)));

const topLevelProjects = [];
// Now prepend dependency projects without filtration.
{
// Filtering 'only' and sharding might have reduced the number of top-level projects.
// Build the project closure to only include dependencies that are still needed.
const projectClosure = new Map(buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!)));

// Clone file suites for dependency projects.
for (const [project, level] of projectClosure.entries()) {
if (level === 'dependency')
rootSuite._prependSuite(buildProjectSuite(project, projectSuites.get(project)!));
const finalProjectClosure = buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!));
for (const [project, type] of finalProjectClosure) {
if (type === 'dependency')
rootSuite._prependSuite(dependencySuites.get(project)!);
else
topLevelProjects.push(project);
}
Expand Down
3 changes: 1 addition & 2 deletions packages/playwright/types/testReporter.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,7 @@ export interface Reporter {
* @param suite The root suite that contains the projects, files and test cases that will run.
*
* The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that
* match the current invocation. It contains only the top-level projects being run — setup and dependency projects are
* not included and cannot be excluded from here.
* match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here.
*
* The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its
* built-in sharding after
Expand Down
5 changes: 3 additions & 2 deletions tests/page/expect-timeout.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,12 @@ test('should fail like a timeout when the signal is aborted mid-assertion', asyn

Locator: locator('span')
Expected: visible
Error: The assertion was aborted: stop it
Error: element(s) not found

Call log:
- Expect "toBeVisible" with timeout 5000ms
- waiting for locator('span')`);
- waiting for locator('span')
- operation was aborted: stop it`);
});

test('should fail like a timeout when toHaveText is aborted mid-assertion', async ({ page }) => {
Expand Down
7 changes: 4 additions & 3 deletions tests/page/page-click.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1368,11 +1368,11 @@ it('should abort via signal', async ({ page }) => {
// Give the action time to start and emit call log entries before aborting.
await page.waitForTimeout(500);

const reason = new Error('Aborted by user');
const reason = new Error('foo bar');
controller.abort(reason);
const error = await promise;
expect(error.message).toContain('The operation was aborted');
expect(error.message).toContain(`Call log:`);
expect(error.message).toContain('locator.click: foo bar');
expect(error.message).toMatch(/Call log:[\s\S]*operation was aborted: foo bar/);
expect(error.name).toBe('AbortError');
expect(error.cause).toBe(reason);
});
Expand All @@ -1384,6 +1384,7 @@ it('should throw an Error when aborted in-flight with a string reason', async ({
controller.abort('aborted by user');
const error = await promise.catch(e => e);
expect(error).toBeInstanceOf(Error);
expect(error.message).toContain('locator.click: aborted by user');
expect(error.name).toBe('AbortError');
expect(error.cause).toBe('aborted by user');
});
Expand Down
3 changes: 0 additions & 3 deletions tests/page/workers.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -383,8 +383,6 @@ it('should resolve worker script allHeaders in main frame', {
it('should resolve worker script allHeaders in iframe', {
annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright/issues/39948' },
}, async function({ page, server, browserName }) {
it.fixme(browserName === 'chromium', 'https://github.com/microsoft/playwright/issues/39948');

const [request] = await Promise.all([
page.waitForEvent('requestfinished', request => request.url() === server.PREFIX + '/worker/worker.js'),
attachFrame(page, 'frame1', server.PREFIX + '/worker/worker.html'),
Expand All @@ -401,7 +399,6 @@ it('should resolve worker script allHeaders in nested worker inside iframe', {
}, async function({ page, server, browserName }) {
it.fixme(browserName === 'webkit', 'cannot evaluate in nested worker');
it.fixme(browserName === 'firefox', 'nested worker script request is not reported at all');
it.fixme(browserName === 'chromium', 'https://github.com/microsoft/playwright/issues/39948');

const [worker] = await Promise.all([
page.waitForEvent('worker'),
Expand Down
Loading
Loading